diff options
90 files changed, 1700 insertions, 618 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index dacfba658b..777eebd2c3 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -13,6 +13,7 @@ environment: QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21' QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed' + VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74' cache: - C:\tools\vcpkg\installed -> build_msvc\vcpkg-packages.txt - C:\Users\appveyor\clcache -> .appveyor.yml, build_msvc\**, **\Makefile.am, **\*.vcxproj.in @@ -25,7 +26,7 @@ install: # 1. Check whether the vcpkg install directory exists (note that updating the vcpkg-packages.txt file # will cause the appveyor cache rules to invalidate the directory) # 2. If the directory is missing: -# a. Update the vcpkg source (including port files) and build the vcpkg binary, +# a. Checkout the vcpkg source (including port files) for the specific checkout and build the vcpkg binary, # b. Install the missing packages. - ps: | $env:PACKAGES = Get-Content -Path build_msvc\vcpkg-packages.txt @@ -34,6 +35,7 @@ install: cd c:\tools\vcpkg $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead. git pull origin master + git checkout $env:VCPKG_COMMIT_ID .\bootstrap-vcpkg.bat Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)" .\vcpkg install --triplet $env:PLATFORM-windows-static $env:PACKAGES.split() > $null diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index 942c3dc13a..3178f2a3d8 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -53,9 +53,6 @@ /* define if the Boost::Filesystem library is available */ #define HAVE_BOOST_FILESYSTEM /**/ -/* define if the Boost::PROGRAM_OPTIONS library is available */ -#define HAVE_BOOST_PROGRAM_OPTIONS /**/ - /* define if the Boost::System library is available */ #define HAVE_BOOST_SYSTEM /**/ @@ -183,72 +180,6 @@ /* Define to 1 if you have the <inttypes.h> header file. */ #define HAVE_INTTYPES_H 1 -/* Define to 1 if you have the `advapi32' library (-ladvapi32). */ -#define HAVE_LIBADVAPI32 1 - -/* Define to 1 if you have the `comctl32' library (-lcomctl32). */ -#define HAVE_LIBCOMCTL32 1 - -/* Define to 1 if you have the `comdlg32' library (-lcomdlg32). */ -#define HAVE_LIBCOMDLG32 1 - -/* Define to 1 if you have the `crypt32' library (-lcrypt32). */ -#define HAVE_LIBCRYPT32 1 - -/* Define to 1 if you have the `gdi32' library (-lgdi32). */ -#define HAVE_LIBGDI32 1 - -/* Define to 1 if you have the `imm32' library (-limm32). */ -#define HAVE_LIBIMM32 1 - -/* Define to 1 if you have the `iphlpapi' library (-liphlpapi). */ -#define HAVE_LIBIPHLPAPI 1 - -/* Define to 1 if you have the `kernel32' library (-lkernel32). */ -#define HAVE_LIBKERNEL32 1 - -/* Define to 1 if you have the `mingwthrd' library (-lmingwthrd). */ -#define HAVE_LIBMINGWTHRD 1 - -/* Define to 1 if you have the `mswsock' library (-lmswsock). */ -#define HAVE_LIBMSWSOCK 1 - -/* Define to 1 if you have the `ole32' library (-lole32). */ -#define HAVE_LIBOLE32 1 - -/* Define to 1 if you have the `oleaut32' library (-loleaut32). */ -#define HAVE_LIBOLEAUT32 1 - -/* Define to 1 if you have the `rpcrt4' library (-lrpcrt4). */ -#define HAVE_LIBRPCRT4 1 - -/* Define to 1 if you have the `shell32' library (-lshell32). */ -#define HAVE_LIBSHELL32 1 - -/* Define to 1 if you have the `shlwapi' library (-lshlwapi). */ -#define HAVE_LIBSHLWAPI 1 - -/* Define to 1 if you have the `ssp' library (-lssp). */ -#define HAVE_LIBSSP 1 - -/* Define to 1 if you have the `user32' library (-luser32). */ -#define HAVE_LIBUSER32 1 - -/* Define to 1 if you have the `uuid' library (-luuid). */ -#define HAVE_LIBUUID 1 - -/* Define to 1 if you have the `winmm' library (-lwinmm). */ -#define HAVE_LIBWINMM 1 - -/* Define to 1 if you have the `winspool' library (-lwinspool). */ -#define HAVE_LIBWINSPOOL 1 - -/* Define to 1 if you have the `ws2_32' library (-lws2_32). */ -#define HAVE_LIBWS2_32 1 - -/* Define to 1 if you have the `z ' library (-lz ). */ -#define HAVE_LIBZ_ 1 - /* Define this symbol if you have malloc_info */ /* #undef HAVE_MALLOC_INFO */ diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index 722a8647bd..c09997d39d 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -116,7 +116,7 @@ <Link> <SubSystem>Console</SubSystem> <GenerateDebugInformation>true</GenerateDebugInformation> - <AdditionalDependencies>crypt32.lib;Iphlpapi.lib;ws2_32.lib;Shlwapi.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies>Iphlpapi.lib;ws2_32.lib;Shlwapi.lib;kernel32.lib;user32.lib;gdi32.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> <Lib> <AdditionalOptions>/ignore:4221</AdditionalOptions> diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index 96d20a2720..5c4b777d51 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -61,7 +61,9 @@ </PropertyGroup> <ItemGroup> <JsonTestFile Include="..\..\src\test\data\*.json" /> + <RawTestFile Include="..\..\src\test\data\*.raw" /> </ItemGroup> + <HeaderFromHexdump RawFilePath="%(RawTestFile.FullPath)" HeaderFilePath="%(RawTestFile.FullPath).h" SourceHeader="static unsigned const char %(RawTestFile.Filename)_raw[] = {" SourceFooter="};" /> <HeaderFromHexdump RawFilePath="%(JsonTestFile.FullPath)" HeaderFilePath="%(JsonTestFile.FullPath).h" SourceHeader="namespace json_tests{ static unsigned const char %(JsonTestFile.Filename)[] = {" SourceFooter="};}" /> </Target> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> diff --git a/configure.ac b/configure.ac index a6420a49fc..18f3104acb 100644 --- a/configure.ac +++ b/configure.ac @@ -483,29 +483,24 @@ use_pkgconfig=yes case $host in *mingw*) - #pkgconfig does more harm than good with MinGW + dnl pkgconfig does more harm than good with MinGW use_pkgconfig=no TARGET_OS=windows - AC_CHECK_LIB([mingwthrd], [main],, AC_MSG_ERROR(libmingwthrd missing)) - AC_CHECK_LIB([kernel32], [main],, AC_MSG_ERROR(libkernel32 missing)) - AC_CHECK_LIB([user32], [main],, AC_MSG_ERROR(libuser32 missing)) - AC_CHECK_LIB([gdi32], [main],, AC_MSG_ERROR(libgdi32 missing)) - AC_CHECK_LIB([comdlg32], [main],, AC_MSG_ERROR(libcomdlg32 missing)) - AC_CHECK_LIB([winspool], [main],, AC_MSG_ERROR(libwinspool missing)) - AC_CHECK_LIB([winmm], [main],, AC_MSG_ERROR(libwinmm missing)) - AC_CHECK_LIB([shell32], [main],, AC_MSG_ERROR(libshell32 missing)) - AC_CHECK_LIB([comctl32], [main],, AC_MSG_ERROR(libcomctl32 missing)) - AC_CHECK_LIB([ole32], [main],, AC_MSG_ERROR(libole32 missing)) - AC_CHECK_LIB([oleaut32], [main],, AC_MSG_ERROR(liboleaut32 missing)) - AC_CHECK_LIB([uuid], [main],, AC_MSG_ERROR(libuuid missing)) - AC_CHECK_LIB([rpcrt4], [main],, AC_MSG_ERROR(librpcrt4 missing)) - AC_CHECK_LIB([advapi32], [main],, AC_MSG_ERROR(libadvapi32 missing)) - AC_CHECK_LIB([ws2_32], [main],, AC_MSG_ERROR(libws2_32 missing)) - AC_CHECK_LIB([mswsock], [main],, AC_MSG_ERROR(libmswsock missing)) - AC_CHECK_LIB([shlwapi], [main],, AC_MSG_ERROR(libshlwapi missing)) - AC_CHECK_LIB([iphlpapi], [main],, AC_MSG_ERROR(libiphlpapi missing)) - AC_CHECK_LIB([crypt32], [main],, AC_MSG_ERROR(libcrypt32 missing)) + AC_CHECK_LIB([kernel32], [GetModuleFileNameA],, AC_MSG_ERROR(libkernel32 missing)) + AC_CHECK_LIB([user32], [main],, AC_MSG_ERROR(libuser32 missing)) + AC_CHECK_LIB([gdi32], [main],, AC_MSG_ERROR(libgdi32 missing)) + AC_CHECK_LIB([comdlg32], [main],, AC_MSG_ERROR(libcomdlg32 missing)) + AC_CHECK_LIB([winmm], [main],, AC_MSG_ERROR(libwinmm missing)) + AC_CHECK_LIB([shell32], [SHGetSpecialFolderPathW],, AC_MSG_ERROR(libshell32 missing)) + AC_CHECK_LIB([comctl32], [main],, AC_MSG_ERROR(libcomctl32 missing)) + AC_CHECK_LIB([ole32], [CoCreateInstance],, AC_MSG_ERROR(libole32 missing)) + AC_CHECK_LIB([oleaut32], [main],, AC_MSG_ERROR(liboleaut32 missing)) + AC_CHECK_LIB([uuid], [main],, AC_MSG_ERROR(libuuid missing)) + AC_CHECK_LIB([advapi32], [CryptAcquireContextW],, AC_MSG_ERROR(libadvapi32 missing)) + AC_CHECK_LIB([ws2_32], [WSAStartup],, AC_MSG_ERROR(libws2_32 missing)) + AC_CHECK_LIB([shlwapi], [PathRemoveFileSpecW],, AC_MSG_ERROR(libshlwapi missing)) + AC_CHECK_LIB([iphlpapi], [GetAdaptersAddresses],, AC_MSG_ERROR(libiphlpapi missing)) dnl -static is interpreted by libtool, where it has a different meaning. dnl In libtool-speak, it's -all-static. diff --git a/contrib/devtools/test_deterministic_coverage.sh b/contrib/devtools/test_deterministic_coverage.sh index 88ac850021..f5cd05a2c3 100755 --- a/contrib/devtools/test_deterministic_coverage.sh +++ b/contrib/devtools/test_deterministic_coverage.sh @@ -21,8 +21,8 @@ NON_DETERMINISTIC_TESTS=( "miner_tests/CreateNewBlock_validity" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) "scheduler_tests/manythreads" # scheduler.cpp: CScheduler::serviceQueue() "scheduler_tests/singlethreadedscheduler_ordered" # scheduler.cpp: CScheduler::serviceQueue() - "tx_validationcache_tests/checkinputs_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) - "tx_validationcache_tests/tx_mempool_block_doublespend" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "txvalidationcache_tests/checkinputs_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "txvalidationcache_tests/tx_mempool_block_doublespend" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) "txindex_tests/txindex_initial_sync" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) "txvalidation_tests/tx_mempool_reject_coinbase" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) "validation_block_tests/processnewblock_signals_ordering" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 4dfa1729a5..46d755886c 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -62,15 +62,16 @@ Likewise, to perform a bootstrapped build (takes even longer): export ADDITIONAL_GUIX_ENVIRONMENT_FLAGS='--bootstrap --no-substitutes' ``` -### Using the right Guix +### Using a version of Guix with `guix time-machine` capabilities -Once Guix is installed, deploy our patched version into your current Guix -profile. The changes there are slowly being upstreamed. +> Note: This entire section can be skipped if you are already using a version of +> Guix that has [the `guix time-machine` command][guix/time-machine]. + +Once Guix is installed, if it doesn't have the `guix time-machine` command, pull +the latest `guix`. ```sh -guix pull --url=https://github.com/dongcarl/guix.git \ - --commit=82c77e52b8b46e0a3aad2cb12307c2e30547deec \ - --max-jobs=4 # change accordingly +guix pull --max-jobs=4 # change number of jobs accordingly ``` Make sure that you are using your current profile. (You are prompted to do this @@ -80,9 +81,6 @@ at the end of the `guix pull`) export PATH="${HOME}/.config/guix/current/bin${PATH:+:}$PATH" ``` -> Note: There is ongoing work to eliminate this entire section using Guix -> [inferiors][guix/inferiors] and [channels][guix/channels]. - ## Usage ### As a Development Environment @@ -224,6 +222,7 @@ repository and will likely put one up soon. [guix/substitute-server-auth]: https://www.gnu.org/software/guix/manual/en/html_node/Substitute-Server-Authorization.html [guix/inferiors]: https://www.gnu.org/software/guix/manual/en/html_node/Inferiors.html [guix/channels]: https://www.gnu.org/software/guix/manual/en/html_node/Channels.html +[guix/time-machine]: https://guix.gnu.org/manual/en/html_node/Invoking-guix-time_002dmachine.html [debian/guix-package]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=850644 [fanquake/guix-docker]: https://github.com/fanquake/core-review/tree/master/guix diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index f8ba8c7ed2..5e0c681f29 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -13,6 +13,12 @@ make -C "${PWD}/depends" -j"$MAX_JOBS" download ${V:+V=1} ${SOURCES_PATH:+SOURCE # Determine the reference time used for determinism (overridable by environment) SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" +time-machine() { + guix time-machine --url=https://github.com/dongcarl/guix.git \ + --commit=b3a7c72c8b2425f8ddb0fc6e3b1caeed40f86dee \ + -- "$@" +} + # Deterministically build Bitcoin Core for HOSTs (overriable by environment) for host in ${HOSTS=i686-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu}; do @@ -22,18 +28,18 @@ for host in ${HOSTS=i686-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64- # Run the build script 'contrib/guix/libexec/build.sh' in the build # container specified by 'contrib/guix/manifest.scm' # shellcheck disable=SC2086 - guix environment --manifest="${PWD}/contrib/guix/manifest.scm" \ - --container \ - --pure \ - --no-cwd \ - --share="$PWD"=/bitcoin \ - ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ - ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ - -- env HOST="$host" \ - MAX_JOBS="$MAX_JOBS" \ - SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ - ${V:+V=1} \ - ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ - bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + MAX_JOBS="$MAX_JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" done diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index f232bb62c2..744b8ee70f 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -184,3 +184,16 @@ ... fun:_ZN5BCLog6Logger12StartLoggingEv } +{ + Suppress BCLog::Logger::StartLogging() still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + ... + fun:_ZN5BCLog6Logger12StartLoggingEv +} +{ + Suppress rest_blockhash_by_height Conditional jump or move depends on uninitialised value(s) + Memcheck:Cond + fun:_ZL24rest_blockhash_by_heightP11HTTPRequestRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE +} diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 50e9251b8d..c34ca4cb59 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -7,11 +7,8 @@ describe how to use it with AFL and libFuzzer. ## Preparing fuzzing -AFL needs an input directory with examples, and an output directory where it -will place examples that it found. These can be anywhere in the file system, -we'll define environment variables to make it easy to reference them. - -libFuzzer will use the input directory as output directory. +The fuzzer needs some inputs to work on, but the inputs or seeds can be used +interchangeably between libFuzzer and AFL. Extract the example seeds (or other starting inputs) into the inputs directory before starting fuzzing. @@ -21,13 +18,19 @@ git clone https://github.com/bitcoin-core/qa-assets export DIR_FUZZ_IN=$PWD/qa-assets/fuzz_seed_corpus ``` -Only for AFL: +AFL needs an input directory with examples, and an output directory where it +will place examples that it found. These can be anywhere in the file system, +we'll define environment variables to make it easy to reference them. + +So, only for AFL you need to configure the outputs path: ``` mkdir outputs export AFLOUT=$PWD/outputs ``` +libFuzzer will use the input directory as output directory. + ## AFL ### Building AFL @@ -41,6 +44,9 @@ make export AFLPATH=$PWD ``` +For macOS you may need to ignore x86 compilation checks when running `make`: +`AFL_NO_X86=1 make`. + ### Instrumentation To build Bitcoin Core using AFL instrumentation (this assumes that the @@ -48,9 +54,15 @@ To build Bitcoin Core using AFL instrumentation (this assumes that the ``` ./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++ export AFL_HARDEN=1 -cd src/ make ``` + +If you are using clang you will need to substitute `afl-gcc` with `afl-clang` +and `afl-g++` with `afl-clang++`, so the first line above becomes: +``` +./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz CC=${AFLPATH}/afl-clang CXX=${AFLPATH}/afl-clang++ +``` + We disable ccache because we don't want to pollute the ccache with instrumented objects, and similarly don't want to use non-instrumented cached objects linked in. @@ -60,25 +72,32 @@ The fuzzing can be sped up significantly (~200x) by using `afl-clang-fast` and compiling using `afl-clang-fast`/`afl-clang-fast++` the resulting binary will be instrumented in such a way that the AFL features "persistent mode" and "deferred forkserver" can be used. See -https://github.com/mcarpenter/afl/tree/master/llvm_mode for details. +https://github.com/google/AFL/tree/master/llvm_mode for details. ### Fuzzing To start the actual fuzzing use: ``` -export FUZZ_TARGET=fuzz_target_foo # Pick a fuzz_target +export FUZZ_TARGET=bech32 # Pick a fuzz_target mkdir ${AFLOUT}/${FUZZ_TARGET} -$AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- test/fuzz/${FUZZ_TARGET} +$AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- src/test/fuzz/${FUZZ_TARGET} ``` You may have to change a few kernel parameters to test optimally - `afl-fuzz` will print an error and suggestion if so. +On macOS you may need to set `AFL_NO_FORKSRV=1` to get the target to run. +``` +export FUZZ_TARGET=bech32 # Pick a fuzz_target +mkdir ${AFLOUT}/${FUZZ_TARGET} +AFL_NO_FORKSRV=1 $AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- src/test/fuzz/${FUZZ_TARGET} +``` + ## libFuzzer -A recent version of `clang`, the address/undefined sanitizers (ASan/UBSan) and libFuzzer is needed (all -found in the `compiler-rt` runtime libraries package). +A recent version of `clang`, the address/undefined sanitizers (ASan/UBSan) and +libFuzzer is needed (all found in the `compiler-rt` runtime libraries package). To build all fuzz targets with libFuzzer, run @@ -87,11 +106,33 @@ To build all fuzz targets with libFuzzer, run make ``` -The fuzzer needs some inputs to work on, but the inputs or seeds can be used -interchangeably between libFuzzer and AFL. - See https://llvm.org/docs/LibFuzzer.html#running on how to run the libFuzzer instrumented executable. -Alternatively run the script in `./test/fuzz/test_runner.py` and provide it -with the `${DIR_FUZZ_IN}` created earlier. +Alternatively, you can run the script through the fuzzing test harness (only +libFuzzer supported so far). You need to pass it the inputs directory and +the specific test target you want to run. + +``` +./test/fuzz/test_runner.py ${DIR_FUZZ_IN} bech32 +``` + +### macOS hints for libFuzzer + +The default clang/llvm version supplied by Apple on macOS does not include +fuzzing libraries, so macOS users will need to install a full version, for +example using `brew install llvm`. + +Should you run into problems with the address sanitizer, it is possible you +may need to run `./configure` with `--disable-asm` to avoid errors +with certain assembly code from Bitcoin Core's code. See [developer notes on sanitizers](https://github.com/bitcoin/bitcoin/blob/master/doc/developer-notes.md#sanitizers) +for more information. + +You may also need to take care of giving the correct path for clang and +clang++, like `CC=/path/to/clang CXX=/path/to/clang++` if the non-systems +clang does not come first in your path. + +Full configure that was tested on macOS Catalina with `brew` installed `llvm`: +``` +./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=/usr/local/opt/llvm/bin/clang CXX=/usr/local/opt/llvm/bin/clang++ --disable-asm +``` diff --git a/src/Makefile.am b/src/Makefile.am index 821553579a..e58a89ca03 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -210,6 +210,7 @@ BITCOIN_CORE_H = \ txmempool.h \ ui_interface.h \ undo.h \ + util/asmap.h \ util/bip32.h \ util/bytevectorhash.h \ util/check.h \ @@ -510,6 +511,7 @@ libbitcoin_util_a_SOURCES = \ support/cleanse.cpp \ sync.cpp \ threadinterrupt.cpp \ + util/asmap.cpp \ util/bip32.cpp \ util/bytevectorhash.cpp \ util/error.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 0975551995..9d782e7a04 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -54,6 +54,7 @@ FUZZ_TARGETS = \ test/fuzz/script_flags \ test/fuzz/service_deserialize \ test/fuzz/spanparsing \ + test/fuzz/strprintf \ test/fuzz/sub_net_deserialize \ test/fuzz/transaction \ test/fuzz/tx_in \ @@ -82,7 +83,8 @@ JSON_TEST_FILES = \ test/data/tx_invalid.json \ test/data/tx_valid.json -RAW_TEST_FILES = +RAW_TEST_FILES = \ + test/data/asmap.raw GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h) @@ -535,6 +537,12 @@ test_fuzz_spanparsing_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_spanparsing_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_spanparsing_SOURCES = $(FUZZ_SUITE) test/fuzz/spanparsing.cpp +test_fuzz_strprintf_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_strprintf_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_strprintf_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_strprintf_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_strprintf_SOURCES = $(FUZZ_SUITE) test/fuzz/strprintf.cpp + test_fuzz_sub_net_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSUB_NET_DESERIALIZE=1 test_fuzz_sub_net_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_sub_net_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -635,3 +643,12 @@ endif echo "};};"; \ } > "$@.new" && mv -f "$@.new" "$@" @echo "Generated $@" + +%.raw.h: %.raw + @$(MKDIR_P) $(@D) + @{ \ + echo "static unsigned const char $(*F)_raw[] = {" && \ + $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ + echo "};"; \ + } > "$@.new" && mv -f "$@.new" "$@" + @echo "Generated $@" diff --git a/src/addrman.cpp b/src/addrman.cpp index 065fdbe4c6..121ae4bf7e 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -7,20 +7,27 @@ #include <hash.h> #include <serialize.h> +#include <logging.h> -int CAddrInfo::GetTriedBucket(const uint256& nKey) const +int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool> &asmap) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetCheapHash(); - uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); - return hash2 % ADDRMAN_TRIED_BUCKET_COUNT; + uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); + int tried_bucket = hash2 % ADDRMAN_TRIED_BUCKET_COUNT; + uint32_t mapped_as = GetMappedAS(asmap); + LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to tried bucket %i.\n", ToStringIP(), mapped_as, tried_bucket); + return tried_bucket; } -int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src) const +int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector<bool> &asmap) const { - std::vector<unsigned char> vchSourceGroupKey = src.GetGroup(); - uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << vchSourceGroupKey).GetCheapHash(); + std::vector<unsigned char> vchSourceGroupKey = src.GetGroup(asmap); + uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << vchSourceGroupKey).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetCheapHash(); - return hash2 % ADDRMAN_NEW_BUCKET_COUNT; + int new_bucket = hash2 % ADDRMAN_NEW_BUCKET_COUNT; + uint32_t mapped_as = GetMappedAS(asmap); + LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to new bucket %i.\n", ToStringIP(), mapped_as, new_bucket); + return new_bucket; } int CAddrInfo::GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const @@ -153,7 +160,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) assert(info.nRefCount == 0); // which tried bucket to move the entry to - int nKBucket = info.GetTriedBucket(nKey); + int nKBucket = info.GetTriedBucket(nKey, m_asmap); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); // first make space to add it (the existing tried entry there is moved to new, deleting whatever is there). @@ -169,7 +176,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) nTried--; // find which new bucket it belongs to - int nUBucket = infoOld.GetNewBucket(nKey); + int nUBucket = infoOld.GetNewBucket(nKey, m_asmap); int nUBucketPos = infoOld.GetBucketPosition(nKey, true, nUBucket); ClearNew(nUBucket, nUBucketPos); assert(vvNew[nUBucket][nUBucketPos] == -1); @@ -233,7 +240,7 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime return; // which tried bucket to move the entry to - int tried_bucket = info.GetTriedBucket(nKey); + int tried_bucket = info.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket); // Will moving this address into tried evict another entry? @@ -301,7 +308,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP fNew = true; } - int nUBucket = pinfo->GetNewBucket(nKey, source); + int nUBucket = pinfo->GetNewBucket(nKey, source, m_asmap); int nUBucketPos = pinfo->GetBucketPosition(nKey, true, nUBucket); if (vvNew[nUBucket][nUBucketPos] != nId) { bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; @@ -439,7 +446,7 @@ int CAddrMan::Check_() if (vvTried[n][i] != -1) { if (!setTried.count(vvTried[n][i])) return -11; - if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey) != n) + if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey, m_asmap) != n) return -17; if (mapInfo[vvTried[n][i]].GetBucketPosition(nKey, false, n) != i) return -18; @@ -545,7 +552,7 @@ void CAddrMan::ResolveCollisions_() CAddrInfo& info_new = mapInfo[id_new]; // Which tried bucket to move the entry to. - int tried_bucket = info_new.GetTriedBucket(nKey); + int tried_bucket = info_new.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = info_new.GetBucketPosition(nKey, false, tried_bucket); if (!info_new.IsValid()) { // id_new may no longer map to a valid address erase_collision = true; @@ -609,10 +616,33 @@ CAddrInfo CAddrMan::SelectTriedCollision_() CAddrInfo& newInfo = mapInfo[id_new]; // which tried bucket to move the entry to - int tried_bucket = newInfo.GetTriedBucket(nKey); + int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket); int id_old = vvTried[tried_bucket][tried_bucket_pos]; return mapInfo[id_old]; } + +std::vector<bool> CAddrMan::DecodeAsmap(fs::path path) +{ + std::vector<bool> bits; + FILE *filestr = fsbridge::fopen(path, "rb"); + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + LogPrintf("Failed to open asmap file from disk.\n"); + return bits; + } + fseek(filestr, 0, SEEK_END); + int length = ftell(filestr); + LogPrintf("Opened asmap file %s (%d bytes) from disk.\n", path, length); + fseek(filestr, 0, SEEK_SET); + char cur_byte; + for (int i = 0; i < length; ++i) { + file >> cur_byte; + for (int bit = 0; bit < 8; ++bit) { + bits.push_back((cur_byte >> bit) & 1); + } + } + return bits; +} diff --git a/src/addrman.h b/src/addrman.h index 8685ea5049..5901611bee 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -12,11 +12,17 @@ #include <sync.h> #include <timedata.h> #include <util/system.h> +#include <clientversion.h> #include <map> #include <set> #include <stdint.h> #include <vector> +#include <iostream> +#include <streams.h> +#include <fs.h> +#include <hash.h> + /** * Extended statistics about a CAddress @@ -68,15 +74,15 @@ public: } //! Calculate in which "tried" bucket this entry belongs - int GetTriedBucket(const uint256 &nKey) const; + int GetTriedBucket(const uint256 &nKey, const std::vector<bool> &asmap) const; //! Calculate in which "new" bucket this entry belongs, given a certain source - int GetNewBucket(const uint256 &nKey, const CNetAddr& src) const; + int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector<bool> &asmap) const; //! Calculate in which "new" bucket this entry belongs, using its default source - int GetNewBucket(const uint256 &nKey) const + int GetNewBucket(const uint256 &nKey, const std::vector<bool> &asmap) const { - return GetNewBucket(nKey, source); + return GetNewBucket(nKey, source, asmap); } //! Calculate in which position of a bucket to store this entry. @@ -170,6 +176,7 @@ static const int64_t ADDRMAN_TEST_WINDOW = 40*60; // 40 minutes */ class CAddrMan { +friend class CAddrManTest; protected: //! critical section to protect the inner data structures mutable RecursiveMutex cs; @@ -264,9 +271,29 @@ protected: void SetServices_(const CService &addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); public: + // Compressed IP->ASN mapping, loaded from a file when a node starts. + // Should be always empty if no file was provided. + // This mapping is then used for bucketing nodes in Addrman. + // + // If asmap is provided, nodes will be bucketed by + // AS they belong to, in order to make impossible for a node + // to connect to several nodes hosted in a single AS. + // This is done in response to Erebus attack, but also to generally + // diversify the connections every node creates, + // especially useful when a large fraction of nodes + // operate under a couple of cloud providers. + // + // If a new asmap was provided, the existing records + // would be re-bucketed accordingly. + std::vector<bool> m_asmap; + + // Read asmap from provided binary file + static std::vector<bool> DecodeAsmap(fs::path path); + + /** * serialized format: - * * version byte (currently 1) + * * version byte (1 for pre-asmap files, 2 for files including asmap version) * * 0x20 + nKey (serialized as if it were a vector, for backward compatibility) * * nNew * * nTried @@ -298,7 +325,7 @@ public: { LOCK(cs); - unsigned char nVersion = 1; + unsigned char nVersion = 2; s << nVersion; s << ((unsigned char)32); s << nKey; @@ -341,6 +368,13 @@ public: } } } + // Store asmap version after bucket entries so that it + // can be ignored by older clients for backward compatibility. + uint256 asmap_version; + if (m_asmap.size() != 0) { + asmap_version = SerializeHash(m_asmap); + } + s << asmap_version; } template<typename Stream> @@ -349,7 +383,6 @@ public: LOCK(cs); Clear(); - unsigned char nVersion; s >> nVersion; unsigned char nKeySize; @@ -379,16 +412,6 @@ public: mapAddr[info] = n; info.nRandomPos = vRandom.size(); vRandom.push_back(n); - if (nVersion != 1 || nUBuckets != ADDRMAN_NEW_BUCKET_COUNT) { - // In case the new table data cannot be used (nVersion unknown, or bucket count wrong), - // immediately try to give them a reference based on their primary source address. - int nUBucket = info.GetNewBucket(nKey); - int nUBucketPos = info.GetBucketPosition(nKey, true, nUBucket); - if (vvNew[nUBucket][nUBucketPos] == -1) { - vvNew[nUBucket][nUBucketPos] = n; - info.nRefCount++; - } - } } nIdCount = nNew; @@ -397,7 +420,7 @@ public: for (int n = 0; n < nTried; n++) { CAddrInfo info; s >> info; - int nKBucket = info.GetTriedBucket(nKey); + int nKBucket = info.GetTriedBucket(nKey, m_asmap); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); if (vvTried[nKBucket][nKBucketPos] == -1) { info.nRandomPos = vRandom.size(); @@ -413,7 +436,9 @@ public: } nTried -= nLost; - // Deserialize positions in the new table (if possible). + // Store positions in the new table buckets to apply later (if possible). + std::map<int, int> entryToBucket; // Represents which entry belonged to which bucket when serializing + for (int bucket = 0; bucket < nUBuckets; bucket++) { int nSize = 0; s >> nSize; @@ -421,12 +446,38 @@ public: int nIndex = 0; s >> nIndex; if (nIndex >= 0 && nIndex < nNew) { - CAddrInfo &info = mapInfo[nIndex]; - int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); - if (nVersion == 1 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS) { - info.nRefCount++; - vvNew[bucket][nUBucketPos] = nIndex; - } + entryToBucket[nIndex] = bucket; + } + } + } + + uint256 supplied_asmap_version; + if (m_asmap.size() != 0) { + supplied_asmap_version = SerializeHash(m_asmap); + } + uint256 serialized_asmap_version; + if (nVersion > 1) { + s >> serialized_asmap_version; + } + + for (int n = 0; n < nNew; n++) { + CAddrInfo &info = mapInfo[n]; + int bucket = entryToBucket[n]; + int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); + if (nVersion == 2 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && + info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS && serialized_asmap_version == supplied_asmap_version) { + // Bucketing has not changed, using existing bucket positions for the new table + vvNew[bucket][nUBucketPos] = n; + info.nRefCount++; + } else { + // In case the new table data cannot be used (nVersion unknown, bucket count wrong or new asmap), + // try to give them a reference based on their primary source address. + LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); + bucket = info.GetNewBucket(nKey, m_asmap); + nUBucketPos = info.GetBucketPosition(nKey, true, bucket); + if (vvNew[bucket][nUBucketPos] == -1) { + vvNew[bucket][nUBucketPos] = n; + info.nRefCount++; } } } diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index d1b2b938ff..5cf7e43f4b 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -126,6 +126,7 @@ void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double } if (!std::regex_match(p.first, baseMatch, reFilter)) { + g_testing_setup = nullptr; continue; } diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index de8e2e5e8f..d6d5e67c5b 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -31,7 +31,8 @@ static void CoinSelection(benchmark::State& state) { NodeContext node; auto chain = interfaces::MakeChain(node); - const CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + wallet.SetupLegacyScriptPubKeyMan(); std::vector<std::unique_ptr<CWalletTx>> wtxs; LOCK(wallet.cs_wallet); @@ -64,7 +65,7 @@ static void CoinSelection(benchmark::State& state) typedef std::set<CInputCoin> CoinSet; static NodeContext testNode; static auto testChain = interfaces::MakeChain(testNode); -static const CWallet testWallet(testChain.get(), WalletLocation(), WalletDatabase::CreateDummy()); +static CWallet testWallet(testChain.get(), WalletLocation(), WalletDatabase::CreateDummy()); std::vector<std::unique_ptr<CWalletTx>> wtxn; // Copied from src/wallet/test/coinselector_tests.cpp @@ -93,6 +94,7 @@ static CAmount make_hard_case(int utxos, std::vector<OutputGroup>& utxo_pool) static void BnBExhaustion(benchmark::State& state) { // Setup + testWallet.SetupLegacyScriptPubKeyMan(); std::vector<OutputGroup> utxo_pool; CoinSet selection; CAmount value_ret = 0; diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index da94afd62b..62568a9da5 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -20,6 +20,7 @@ static void WalletBalance(benchmark::State& state, const bool set_dirty, const b std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(node); CWallet wallet{chain.get(), WalletLocation(), WalletDatabase::CreateMock()}; { + wallet.SetupLegacyScriptPubKeyMan(); bool first_run; if (wallet.LoadWallet(first_run) != DBErrors::LOAD_OK) assert(false); wallet.handleNotifications(); diff --git a/src/checkqueue.h b/src/checkqueue.h index 978e23a7c4..9dab5a09ac 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -90,8 +90,7 @@ private: nTotal--; bool fRet = fAllOk; // reset the status for new work later - if (fMaster) - fAllOk = true; + fAllOk = true; // return the current status return fRet; } diff --git a/src/coins.h b/src/coins.h index 68f7596745..e71c8a47bc 100644 --- a/src/coins.h +++ b/src/coins.h @@ -61,7 +61,7 @@ public: assert(!IsSpent()); uint32_t code = nHeight * 2 + fCoinBase; ::Serialize(s, VARINT(code)); - ::Serialize(s, CTxOutCompressor(REF(out))); + ::Serialize(s, Using<TxOutCompression>(out)); } template<typename Stream> @@ -70,7 +70,7 @@ public: ::Unserialize(s, VARINT(code)); nHeight = code >> 1; fCoinBase = code & 1; - ::Unserialize(s, CTxOutCompressor(out)); + ::Unserialize(s, Using<TxOutCompression>(out)); } bool IsSpent() const { diff --git a/src/compressor.h b/src/compressor.h index c1eda503c8..7bb60d311e 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -11,10 +11,6 @@ #include <serialize.h> #include <span.h> -class CKeyID; -class CPubKey; -class CScriptID; - bool CompressScript(const CScript& script, std::vector<unsigned char> &out); unsigned int GetSpecialScriptSize(unsigned int nSize); bool DecompressScript(CScript& script, unsigned int nSize, const std::vector<unsigned char> &out); @@ -33,9 +29,8 @@ uint64_t DecompressAmount(uint64_t nAmount); * Other scripts up to 121 bytes require 1 byte + script length. Above * that, scripts up to 16505 bytes require 2 bytes + script length. */ -class CScriptCompressor +struct ScriptCompression { -private: /** * make this static for now (there are only 6 special scripts defined) * this can potentially be extended together with a new nVersion for @@ -44,12 +39,8 @@ private: */ static const unsigned int nSpecialScripts = 6; - CScript &script; -public: - explicit CScriptCompressor(CScript &scriptIn) : script(scriptIn) { } - template<typename Stream> - void Serialize(Stream &s) const { + void Ser(Stream &s, const CScript& script) { std::vector<unsigned char> compr; if (CompressScript(script, compr)) { s << MakeSpan(compr); @@ -61,7 +52,7 @@ public: } template<typename Stream> - void Unserialize(Stream &s) { + void Unser(Stream &s, CScript& script) { unsigned int nSize = 0; s >> VARINT(nSize); if (nSize < nSpecialScripts) { @@ -82,30 +73,24 @@ public: } }; -/** wrapper for CTxOut that provides a more compact serialization */ -class CTxOutCompressor +struct AmountCompression { -private: - CTxOut &txout; - -public: - explicit CTxOutCompressor(CTxOut &txoutIn) : txout(txoutIn) { } - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - if (!ser_action.ForRead()) { - uint64_t nVal = CompressAmount(txout.nValue); - READWRITE(VARINT(nVal)); - } else { - uint64_t nVal = 0; - READWRITE(VARINT(nVal)); - txout.nValue = DecompressAmount(nVal); - } - CScriptCompressor cscript(REF(txout.scriptPubKey)); - READWRITE(cscript); + template<typename Stream, typename I> void Ser(Stream& s, I val) + { + s << VARINT(CompressAmount(val)); + } + template<typename Stream, typename I> void Unser(Stream& s, I& val) + { + uint64_t v; + s >> VARINT(v); + val = DecompressAmount(v); } }; +/** wrapper for CTxOut that provides a more compact serialization */ +struct TxOutCompression +{ + FORMATTER_METHODS(CTxOut, obj) { READWRITE(Using<AmountCompression>(obj.nValue), Using<ScriptCompression>(obj.scriptPubKey)); } +}; + #endif // BITCOIN_COMPRESSOR_H diff --git a/src/init.cpp b/src/init.cpp index 18c32b34ed..49f4727169 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -52,7 +52,11 @@ #include <util/threadnames.h> #include <util/translation.h> #include <util/validation.h> +#include <util/asmap.h> #include <validation.h> +#include <hash.h> + + #include <validationinterface.h> #include <walletinitinterface.h> @@ -98,6 +102,8 @@ static constexpr int DUMP_BANS_INTERVAL = 60 * 15; static const char* FEE_ESTIMATES_FILENAME="fee_estimates.dat"; +static const char* DEFAULT_ASMAP_FILENAME="ip_asn.map"; + /** * The PID file facilities. */ @@ -431,6 +437,7 @@ void SetupServerArgs() gArgs.AddArg("-peertimeout=<n>", strprintf("Specify p2p connection timeout in seconds. This option determines the amount of time a peer may be inactive before the connection to it is dropped. (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); gArgs.AddArg("-torcontrol=<ip>:<port>", strprintf("Tor control port to use if onion listening enabled (default: %s)", DEFAULT_TOR_CONTROL), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-torpassword=<pass>", "Tor control port password (default: empty)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + gArgs.AddArg("-asmap=<file>", "Specify asn mapping used for bucketing of the peers. Path should be relative to the -datadir path.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); #ifdef USE_UPNP #if USE_UPNP gArgs.AddArg("-upnp", "Use UPnP to map the listening port (default: 1 when listening and no -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -521,7 +528,7 @@ void SetupServerArgs() gArgs.AddArg("-minrelaytxfee=<amt>", strprintf("Fees (in %s/kB) smaller than this are considered zero fee for relaying, mining and transaction creation (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_MIN_RELAY_TX_FEE)), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); gArgs.AddArg("-whitelistforcerelay", strprintf("Add 'forcerelay' permission to whitelisted inbound peers with default permissions. This will relay transactions even if the transactions were already in the mempool or violate local relay policy. (default: %d)", DEFAULT_WHITELISTFORCERELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); - gArgs.AddArg("-whitelistrelay", strprintf("Add 'relay' permission to whitelisted inbound peers with default permissions. The will accept relayed transactions even when not relaying transactions (default: %d)", DEFAULT_WHITELISTRELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); + gArgs.AddArg("-whitelistrelay", strprintf("Add 'relay' permission to whitelisted inbound peers with default permissions. This will accept relayed transactions even when not relaying transactions (default: %d)", DEFAULT_WHITELISTRELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); gArgs.AddArg("-blockmaxweight=<n>", strprintf("Set maximum BIP141 block weight (default: %d)", DEFAULT_BLOCK_MAX_WEIGHT), ArgsManager::ALLOW_ANY, OptionsCategory::BLOCK_CREATION); @@ -1813,6 +1820,25 @@ bool AppInitMain(NodeContext& node) return false; } + // Read asmap file if configured + if (gArgs.IsArgSet("-asmap")) { + std::string asmap_file = gArgs.GetArg("-asmap", ""); + if (asmap_file.empty()) { + asmap_file = DEFAULT_ASMAP_FILENAME; + } + const fs::path asmap_path = GetDataDir() / asmap_file; + std::vector<bool> asmap = CAddrMan::DecodeAsmap(asmap_path); + if (asmap.size() == 0) { + InitError(strprintf(_("Could not find or parse specified asmap: '%s'").translated, asmap_path)); + return false; + } + node.connman->SetAsmap(asmap); + const uint256 asmap_version = SerializeHash(asmap); + LogPrintf("Using asmap version %s for IP bucketing.\n", asmap_version.ToString()); + } else { + LogPrintf("Using /16 prefix for IP bucketing.\n"); + } + // ********************************************************* Step 13: finished SetRPCWarmupFinished(); diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index 568ab43ac0..baea71d0bb 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -119,7 +119,7 @@ public: } bool getPubKey(const CScript& script, const CKeyID& address, CPubKey& pub_key) override { - const SigningProvider* provider = m_wallet->GetSigningProvider(script); + std::unique_ptr<SigningProvider> provider = m_wallet->GetSigningProvider(script); if (provider) { return provider->GetPubKey(address, pub_key); } @@ -127,7 +127,7 @@ public: } bool getPrivKey(const CScript& script, const CKeyID& address, CKey& key) override { - const SigningProvider* provider = m_wallet->GetSigningProvider(script); + std::unique_ptr<SigningProvider> provider = m_wallet->GetSigningProvider(script); if (provider) { return provider->GetKey(address, key); } @@ -180,7 +180,6 @@ public: } return result; } - void learnRelatedScripts(const CPubKey& key, OutputType type) override { m_wallet->GetLegacyScriptPubKeyMan()->LearnRelatedScripts(key, type); } bool addDestData(const CTxDestination& dest, const std::string& key, const std::string& value) override { LOCK(m_wallet->cs_wallet); diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index de53b16c0c..d4280e8091 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -108,10 +108,6 @@ public: //! Get wallet address list. virtual std::vector<WalletAddress> getAddresses() = 0; - //! Add scripts to key store so old so software versions opening the wallet - //! database can detect payments to newer address types. - virtual void learnRelatedScripts(const CPubKey& key, OutputType type) = 0; - //! Add dest data. virtual bool addDestData(const CTxDestination& dest, const std::string& key, const std::string& value) = 0; diff --git a/src/net.cpp b/src/net.cpp index 68764bf5cb..9cd2d30d9d 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -498,12 +498,13 @@ void CNode::SetAddrLocal(const CService& addrLocalIn) { #undef X #define X(name) stats.name = name -void CNode::copyStats(CNodeStats &stats) +void CNode::copyStats(CNodeStats &stats, std::vector<bool> &m_asmap) { stats.nodeid = this->GetId(); X(nServices); X(addr); X(addrBind); + stats.m_mapped_as = addr.GetMappedAS(m_asmap); if (m_tx_relay != nullptr) { LOCK(m_tx_relay->cs_filter); stats.fRelayTxes = m_tx_relay->fRelayTxes; @@ -1769,7 +1770,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // but inbound and addnode peers do not use our outbound slots. Inbound peers // also have the added issue that they're attacker controlled and could be used // to prevent us from connecting to particular hosts if we used them here. - setConnected.insert(pnode->addr.GetGroup()); + setConnected.insert(pnode->addr.GetGroup(addrman.m_asmap)); if (pnode->m_tx_relay == nullptr) { nOutboundBlockRelay++; } else if (!pnode->fFeeler) { @@ -1817,7 +1818,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) } // Require outbound connections, other than feelers, to be to distinct network groups - if (!fFeeler && setConnected.count(addr.GetGroup())) { + if (!fFeeler && setConnected.count(addr.GetGroup(addrman.m_asmap))) { break; } @@ -2502,7 +2503,7 @@ void CConnman::GetNodeStats(std::vector<CNodeStats>& vstats) vstats.reserve(vNodes.size()); for (CNode* pnode : vNodes) { vstats.emplace_back(); - pnode->copyStats(vstats.back()); + pnode->copyStats(vstats.back(), addrman.m_asmap); } } @@ -2789,7 +2790,7 @@ CSipHasher CConnman::GetDeterministicRandomizer(uint64_t id) const uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& ad) const { - std::vector<unsigned char> vchNetGroup(ad.GetGroup()); + std::vector<unsigned char> vchNetGroup(ad.GetGroup(addrman.m_asmap)); return GetDeterministicRandomizer(RANDOMIZER_ID_NETGROUP).Write(vchNetGroup.data(), vchNetGroup.size()).Finalize(); } @@ -153,6 +153,7 @@ public: bool m_use_addrman_outgoing = true; std::vector<std::string> m_specified_outgoing; std::vector<std::string> m_added_nodes; + std::vector<bool> m_asmap; }; void Init(const Options& connOptions) { @@ -330,6 +331,8 @@ public: */ int64_t PoissonNextSendInbound(int64_t now, int average_interval_seconds); + void SetAsmap(std::vector<bool> asmap) { addrman.m_asmap = asmap; } + private: struct ListenSocket { public: @@ -603,6 +606,7 @@ public: CAddress addr; // Bind address of our side of the connection CAddress addrBind; + uint32_t m_mapped_as; }; @@ -979,7 +983,7 @@ public: void CloseSocketDisconnect(); - void copyStats(CNodeStats &stats); + void copyStats(CNodeStats &stats, std::vector<bool> &m_asmap); ServiceFlags GetLocalServices() const { diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 2ee4ae3ee3..ce3e17197e 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -6,6 +6,7 @@ #include <netaddress.h> #include <hash.h> #include <util/strencodings.h> +#include <util/asmap.h> #include <tinyformat.h> static const unsigned char pchIPv4[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff }; @@ -400,6 +401,39 @@ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const return true; } +uint32_t CNetAddr::GetNetClass() const { + uint32_t net_class = NET_IPV6; + if (IsLocal()) { + net_class = 255; + } + if (IsInternal()) { + net_class = NET_INTERNAL; + } else if (!IsRoutable()) { + net_class = NET_UNROUTABLE; + } else if (IsIPv4() || IsRFC6145() || IsRFC6052() || IsRFC3964() || IsRFC4380()) { + net_class = NET_IPV4; + } else if (IsTor()) { + net_class = NET_ONION; + } + return net_class; +} + +uint32_t CNetAddr::GetMappedAS(const std::vector<bool> &asmap) const { + uint32_t net_class = GetNetClass(); + if (asmap.size() == 0 || (net_class != NET_IPV4 && net_class != NET_IPV6)) { + return 0; // Indicates not found, safe because AS0 is reserved per RFC7607. + } + std::vector<bool> ip_bits(128); + for (int8_t byte_i = 0; byte_i < 16; ++byte_i) { + uint8_t cur_byte = GetByte(15 - byte_i); + for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { + ip_bits[byte_i * 8 + bit_i] = (cur_byte >> (7 - bit_i)) & 1; + } + } + uint32_t mapped_as = Interpret(asmap, ip_bits); + return mapped_as; +} + /** * Get the canonical identifier of our network group * @@ -410,56 +444,61 @@ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const * @note No two connections will be attempted to addresses with the same network * group. */ -std::vector<unsigned char> CNetAddr::GetGroup() const +std::vector<unsigned char> CNetAddr::GetGroup(const std::vector<bool> &asmap) const { std::vector<unsigned char> vchRet; - int nClass = NET_IPV6; + uint32_t net_class = GetNetClass(); + // If non-empty asmap is supplied and the address is IPv4/IPv6, + // return ASN to be used for bucketing. + uint32_t asn = GetMappedAS(asmap); + if (asn != 0) { // Either asmap was empty, or address has non-asmappable net class (e.g. TOR). + vchRet.push_back(NET_IPV6); // IPv4 and IPv6 with same ASN should be in the same bucket + for (int i = 0; i < 4; i++) { + vchRet.push_back((asn >> (8 * i)) & 0xFF); + } + return vchRet; + } + + vchRet.push_back(net_class); int nStartByte = 0; int nBits = 16; // all local addresses belong to the same group if (IsLocal()) { - nClass = 255; nBits = 0; } // all internal-usage addresses get their own group if (IsInternal()) { - nClass = NET_INTERNAL; nStartByte = sizeof(g_internal_prefix); nBits = (sizeof(ip) - sizeof(g_internal_prefix)) * 8; } // all other unroutable addresses belong to the same group else if (!IsRoutable()) { - nClass = NET_UNROUTABLE; nBits = 0; } // for IPv4 addresses, '1' + the 16 higher-order bits of the IP // includes mapped IPv4, SIIT translated IPv4, and the well-known prefix else if (IsIPv4() || IsRFC6145() || IsRFC6052()) { - nClass = NET_IPV4; nStartByte = 12; } // for 6to4 tunnelled addresses, use the encapsulated IPv4 address else if (IsRFC3964()) { - nClass = NET_IPV4; nStartByte = 2; } // for Teredo-tunnelled IPv6 addresses, use the encapsulated IPv4 address else if (IsRFC4380()) { - vchRet.push_back(NET_IPV4); vchRet.push_back(GetByte(3) ^ 0xFF); vchRet.push_back(GetByte(2) ^ 0xFF); return vchRet; } else if (IsTor()) { - nClass = NET_ONION; nStartByte = 6; nBits = 4; } @@ -470,8 +509,6 @@ std::vector<unsigned char> CNetAddr::GetGroup() const else nBits = 32; - vchRet.push_back(nClass); - // push our ip onto vchRet byte by byte... while (nBits >= 8) { diff --git a/src/netaddress.h b/src/netaddress.h index dcb492da0d..078234595c 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -78,7 +78,14 @@ class CNetAddr unsigned int GetByte(int n) const; uint64_t GetHash() const; bool GetInAddr(struct in_addr* pipv4Addr) const; - std::vector<unsigned char> GetGroup() const; + uint32_t GetNetClass() const; + + // The AS on the BGP path to the node we use to diversify + // peers in AddrMan bucketing based on the AS infrastructure. + // The ip->AS mapping depends on how asmap is constructed. + uint32_t GetMappedAS(const std::vector<bool> &asmap) const; + + std::vector<unsigned char> GetGroup(const std::vector<bool> &asmap) const; int GetReachabilityFrom(const CNetAddr *paddrPartner = nullptr) const; explicit CNetAddr(const struct in6_addr& pipv6Addr, const uint32_t scope = 0); diff --git a/src/node/psbt.cpp b/src/node/psbt.cpp index 34c6866a5c..8678b33cf3 100644 --- a/src/node/psbt.cpp +++ b/src/node/psbt.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <amount.h> #include <coins.h> #include <consensus/tx_verify.h> #include <node/psbt.h> @@ -31,9 +32,17 @@ PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx) // Check for a UTXO CTxOut utxo; if (psbtx.GetInputUTXO(utxo, i)) { + if (!MoneyRange(utxo.nValue) || !MoneyRange(in_amt + utxo.nValue)) { + result.SetInvalid(strprintf("PSBT is not valid. Input %u has invalid value", i)); + return result; + } in_amt += utxo.nValue; input_analysis.has_utxo = true; } else { + if (input.non_witness_utxo && psbtx.tx->vin[i].prevout.n >= input.non_witness_utxo->vout.size()) { + result.SetInvalid(strprintf("PSBT is not valid. Input %u specifies invalid prevout", i)); + return result; + } input_analysis.has_utxo = false; input_analysis.is_final = false; input_analysis.next = PSBTRole::UPDATER; @@ -85,9 +94,16 @@ PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx) // Get the output amount CAmount out_amt = std::accumulate(psbtx.tx->vout.begin(), psbtx.tx->vout.end(), CAmount(0), [](CAmount a, const CTxOut& b) { + if (!MoneyRange(a) || !MoneyRange(b.nValue) || !MoneyRange(a + b.nValue)) { + return CAmount(-1); + } return a += b.nValue; } ); + if (!MoneyRange(out_amt)) { + result.SetInvalid(strprintf("PSBT is not valid. Output amount invalid")); + return result; + } // Get the fee CAmount fee = in_amt - out_amt; diff --git a/src/outputtype.cpp b/src/outputtype.cpp index 85ceb03aa6..567eecb5c9 100644 --- a/src/outputtype.cpp +++ b/src/outputtype.cpp @@ -19,6 +19,8 @@ static const std::string OUTPUT_TYPE_STRING_LEGACY = "legacy"; static const std::string OUTPUT_TYPE_STRING_P2SH_SEGWIT = "p2sh-segwit"; static const std::string OUTPUT_TYPE_STRING_BECH32 = "bech32"; +const std::array<OutputType, 3> OUTPUT_TYPES = {OutputType::LEGACY, OutputType::P2SH_SEGWIT, OutputType::BECH32}; + bool ParseOutputType(const std::string& type, OutputType& output_type) { if (type == OUTPUT_TYPE_STRING_LEGACY) { @@ -80,22 +82,30 @@ CTxDestination AddAndGetDestinationForScript(FillableSigningProvider& keystore, { // Add script to keystore keystore.AddCScript(script); + ScriptHash sh(script); // Note that scripts over 520 bytes are not yet supported. switch (type) { case OutputType::LEGACY: - return ScriptHash(script); + keystore.AddCScript(GetScriptForDestination(sh)); + return sh; case OutputType::P2SH_SEGWIT: case OutputType::BECH32: { CTxDestination witdest = WitnessV0ScriptHash(script); CScript witprog = GetScriptForDestination(witdest); // Check if the resulting program is solvable (i.e. doesn't use an uncompressed key) - if (!IsSolvable(keystore, witprog)) return ScriptHash(script); + if (!IsSolvable(keystore, witprog)) { + // Since the wsh is invalid, add and return the sh instead. + keystore.AddCScript(GetScriptForDestination(sh)); + return sh; + } // Add the redeemscript, so that P2WSH and P2SH-P2WSH outputs are recognized as ours. keystore.AddCScript(witprog); if (type == OutputType::BECH32) { return witdest; } else { - return ScriptHash(witprog); + ScriptHash sh_w = ScriptHash(witprog); + keystore.AddCScript(GetScriptForDestination(sh_w)); + return sh_w; } } default: assert(false); diff --git a/src/outputtype.h b/src/outputtype.h index b91082ddc0..1438f65844 100644 --- a/src/outputtype.h +++ b/src/outputtype.h @@ -10,6 +10,7 @@ #include <script/signingprovider.h> #include <script/standard.h> +#include <array> #include <string> #include <vector> @@ -27,6 +28,8 @@ enum class OutputType { CHANGE_AUTO, }; +extern const std::array<OutputType, 3> OUTPUT_TYPES; + NODISCARD bool ParseOutputType(const std::string& str, OutputType& output_type); const std::string& FormatOutputType(OutputType type); @@ -47,4 +50,3 @@ std::vector<CTxDestination> GetAllDestinationsForKey(const CPubKey& key); CTxDestination AddAndGetDestinationForScript(FillableSigningProvider& keystore, const CScript& script, OutputType); #endif // BITCOIN_OUTPUTTYPE_H - diff --git a/src/psbt.cpp b/src/psbt.cpp index c23b78b3ee..e6b6285652 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -66,8 +66,11 @@ bool PartiallySignedTransaction::AddOutput(const CTxOut& txout, const PSBTOutput bool PartiallySignedTransaction::GetInputUTXO(CTxOut& utxo, int input_index) const { PSBTInput input = inputs[input_index]; - int prevout_index = tx->vin[input_index].prevout.n; + uint32_t prevout_index = tx->vin[input_index].prevout.n; if (input.non_witness_utxo) { + if (prevout_index >= input.non_witness_utxo->vout.size()) { + return false; + } utxo = input.non_witness_utxo->vout[prevout_index]; } else if (!input.witness_utxo.IsNull()) { utxo = input.witness_utxo; @@ -255,6 +258,9 @@ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& if (input.non_witness_utxo) { // If we're taking our information from a non-witness UTXO, verify that it matches the prevout. COutPoint prevout = tx.vin[index].prevout; + if (prevout.n >= input.non_witness_utxo->vout.size()) { + return false; + } if (input.non_witness_utxo->GetHash() != prevout.hash) { return false; } diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index f22d33bf0a..5fab267610 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -318,8 +318,8 @@ void BitcoinGUI::createActions() verifyMessageAction = new QAction(tr("&Verify message..."), this); verifyMessageAction->setStatusTip(tr("Verify messages to ensure they were signed with specified Bitcoin addresses")); - openRPCConsoleAction = new QAction(tr("&Debug window"), this); - openRPCConsoleAction->setStatusTip(tr("Open debugging and diagnostic console")); + openRPCConsoleAction = new QAction(tr("Node window"), this); + openRPCConsoleAction->setStatusTip(tr("Open node debugging and diagnostic console")); // initially disable the debug window menu item openRPCConsoleAction->setEnabled(false); openRPCConsoleAction->setObjectName("openRPCConsoleAction"); @@ -648,6 +648,10 @@ void BitcoinGUI::addWallet(WalletModel* walletModel) void BitcoinGUI::removeWallet(WalletModel* walletModel) { if (!walletFrame) return; + + labelWalletHDStatusIcon->hide(); + labelWalletEncryptionIcon->hide(); + int index = m_wallet_selector->findData(QVariant::fromValue(walletModel)); m_wallet_selector->removeItem(index); if (m_wallet_selector->count() == 0) { @@ -659,8 +663,6 @@ void BitcoinGUI::removeWallet(WalletModel* walletModel) rpcConsole->removeWallet(walletModel); walletFrame->removeWallet(walletModel); updateWindowTitle(); - labelWalletHDStatusIcon->hide(); - labelWalletEncryptionIcon->hide(); } void BitcoinGUI::setCurrentWallet(WalletModel* wallet_model) diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index be807b20c0..ebb6bbd4f5 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -11,7 +11,7 @@ </rect> </property> <property name="windowTitle"> - <string>Debug window</string> + <string>Node window</string> </property> <layout class="QVBoxLayout" name="verticalLayout_2"> <item> diff --git a/src/qt/forms/modaloverlay.ui b/src/qt/forms/modaloverlay.ui index da19a6fa2e..d2e7ca8f06 100644 --- a/src/qt/forms/modaloverlay.ui +++ b/src/qt/forms/modaloverlay.ui @@ -351,6 +351,9 @@ QLabel { color: rgb(40,40,40); }</string> <property name="text"> <string>Hide</string> </property> + <property name="shortcut"> + <string>Esc</string> + </property> <property name="focusPolicy"> <enum>Qt::StrongFocus</enum> </property> diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 53f0c3a108..ad21dfc3ef 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -12,6 +12,7 @@ #include <qt/guiconstants.h> #include <qt/guiutil.h> +#include <qt/optionsmodel.h> #include <interfaces/node.h> #include <util/system.h> @@ -22,9 +23,6 @@ #include <cmath> -/* Total required space (in GB) depending on user choice (prune, not prune) */ -static uint64_t requiredSpace; - /* Check free space asynchronously to prevent hanging the UI thread. Up to one request to check a path is in flight to this thread; when the check() @@ -109,14 +107,24 @@ void FreespaceChecker::check() Q_EMIT reply(replyStatus, replyMessage, freeBytesAvailable); } +namespace { +//! Return pruning size that will be used if automatic pruning is enabled. +int GetPruneTargetGB() +{ + int64_t prune_target_mib = gArgs.GetArg("-prune", 0); + // >1 means automatic pruning is enabled by config, 1 means manual pruning, 0 means no pruning. + return prune_target_mib > 1 ? PruneMiBtoGB(prune_target_mib) : DEFAULT_PRUNE_TARGET_GB; +} +} // namespace -Intro::Intro(QWidget *parent, uint64_t blockchain_size, uint64_t chain_state_size) : +Intro::Intro(QWidget *parent, int64_t blockchain_size_gb, int64_t chain_state_size_gb) : QDialog(parent), ui(new Ui::Intro), thread(nullptr), signalled(false), - m_blockchain_size(blockchain_size), - m_chain_state_size(chain_state_size) + m_blockchain_size_gb(blockchain_size_gb), + m_chain_state_size_gb(chain_state_size_gb), + m_prune_target_gb{GetPruneTargetGB()} { ui->setupUi(this); ui->welcomeLabel->setText(ui->welcomeLabel->text().arg(PACKAGE_NAME)); @@ -124,37 +132,24 @@ Intro::Intro(QWidget *parent, uint64_t blockchain_size, uint64_t chain_state_siz ui->lblExplanation1->setText(ui->lblExplanation1->text() .arg(PACKAGE_NAME) - .arg(m_blockchain_size) + .arg(m_blockchain_size_gb) .arg(2009) .arg(tr("Bitcoin")) ); ui->lblExplanation2->setText(ui->lblExplanation2->text().arg(PACKAGE_NAME)); - uint64_t pruneTarget = std::max<int64_t>(0, gArgs.GetArg("-prune", 0)); - if (pruneTarget > 1) { // -prune=1 means enabled, above that it's a size in MB + if (gArgs.GetArg("-prune", 0) > 1) { // -prune=1 means enabled, above that it's a size in MiB ui->prune->setChecked(true); ui->prune->setEnabled(false); } - ui->prune->setText(tr("Discard blocks after verification, except most recent %1 GB (prune)").arg(pruneTarget ? pruneTarget / 1000 : DEFAULT_PRUNE_TARGET_GB)); - requiredSpace = m_blockchain_size; - QString storageRequiresMsg = tr("At least %1 GB of data will be stored in this directory, and it will grow over time."); - if (pruneTarget) { - uint64_t prunedGBs = std::ceil(pruneTarget * 1024 * 1024.0 / GB_BYTES); - if (prunedGBs <= requiredSpace) { - requiredSpace = prunedGBs; - storageRequiresMsg = tr("Approximately %1 GB of data will be stored in this directory."); - } - ui->lblExplanation3->setVisible(true); - } else { - ui->lblExplanation3->setVisible(false); - } - requiredSpace += m_chain_state_size; - ui->sizeWarningLabel->setText( - tr("%1 will download and store a copy of the Bitcoin block chain.").arg(PACKAGE_NAME) + " " + - storageRequiresMsg.arg(requiredSpace) + " " + - tr("The wallet will also be stored in this directory.") - ); - this->adjustSize(); + ui->prune->setText(tr("Discard blocks after verification, except most recent %1 GB (prune)").arg(m_prune_target_gb)); + UpdatePruneLabels(ui->prune->isChecked()); + + connect(ui->prune, &QCheckBox::toggled, [this](bool prune_checked) { + UpdatePruneLabels(prune_checked); + UpdateFreeSpaceLabel(); + }); + startThread(); } @@ -270,25 +265,31 @@ void Intro::setStatus(int status, const QString &message, quint64 bytesAvailable { ui->freeSpace->setText(""); } else { - QString freeString = tr("%n GB of free space available", "", bytesAvailable/GB_BYTES); - if(bytesAvailable < requiredSpace * GB_BYTES) - { - freeString += " " + tr("(of %n GB needed)", "", requiredSpace); - ui->freeSpace->setStyleSheet("QLabel { color: #800000 }"); - ui->prune->setChecked(true); - } else if (bytesAvailable / GB_BYTES - requiredSpace < 10) { - freeString += " " + tr("(%n GB needed for full chain)", "", requiredSpace); - ui->freeSpace->setStyleSheet("QLabel { color: #999900 }"); - ui->prune->setChecked(true); - } else { - ui->freeSpace->setStyleSheet(""); + m_bytes_available = bytesAvailable; + if (ui->prune->isEnabled()) { + ui->prune->setChecked(m_bytes_available < (m_blockchain_size_gb + m_chain_state_size_gb + 10) * GB_BYTES); } - ui->freeSpace->setText(freeString + "."); + UpdateFreeSpaceLabel(); } /* Don't allow confirm in ERROR state */ ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(status != FreespaceChecker::ST_ERROR); } +void Intro::UpdateFreeSpaceLabel() +{ + QString freeString = tr("%n GB of free space available", "", m_bytes_available / GB_BYTES); + if (m_bytes_available < m_required_space_gb * GB_BYTES) { + freeString += " " + tr("(of %n GB needed)", "", m_required_space_gb); + ui->freeSpace->setStyleSheet("QLabel { color: #800000 }"); + } else if (m_bytes_available / GB_BYTES - m_required_space_gb < 10) { + freeString += " " + tr("(%n GB needed for full chain)", "", m_required_space_gb); + ui->freeSpace->setStyleSheet("QLabel { color: #999900 }"); + } else { + ui->freeSpace->setStyleSheet(""); + } + ui->freeSpace->setText(freeString + "."); +} + void Intro::on_dataDirectory_textChanged(const QString &dataDirStr) { /* Disable OK button until check result comes in */ @@ -349,3 +350,20 @@ QString Intro::getPathToCheck() mutex.unlock(); return retval; } + +void Intro::UpdatePruneLabels(bool prune_checked) +{ + m_required_space_gb = m_blockchain_size_gb + m_chain_state_size_gb; + QString storageRequiresMsg = tr("At least %1 GB of data will be stored in this directory, and it will grow over time."); + if (prune_checked && m_prune_target_gb <= m_blockchain_size_gb) { + m_required_space_gb = m_prune_target_gb + m_chain_state_size_gb; + storageRequiresMsg = tr("Approximately %1 GB of data will be stored in this directory."); + } + ui->lblExplanation3->setVisible(prune_checked); + ui->sizeWarningLabel->setText( + tr("%1 will download and store a copy of the Bitcoin block chain.").arg(PACKAGE_NAME) + " " + + storageRequiresMsg.arg(m_required_space_gb) + " " + + tr("The wallet will also be stored in this directory.") + ); + this->adjustSize(); +} diff --git a/src/qt/intro.h b/src/qt/intro.h index 41da06141f..732393246e 100644 --- a/src/qt/intro.h +++ b/src/qt/intro.h @@ -31,7 +31,7 @@ class Intro : public QDialog public: explicit Intro(QWidget *parent = nullptr, - uint64_t blockchain_size = 0, uint64_t chain_state_size = 0); + int64_t blockchain_size_gb = 0, int64_t chain_state_size_gb = 0); ~Intro(); QString getDataDirectory(); @@ -67,12 +67,18 @@ private: QMutex mutex; bool signalled; QString pathToCheck; - uint64_t m_blockchain_size; - uint64_t m_chain_state_size; + const int64_t m_blockchain_size_gb; + const int64_t m_chain_state_size_gb; + //! Total required space (in GB) depending on user choice (prune or not prune). + int64_t m_required_space_gb{0}; + uint64_t m_bytes_available{0}; + const int64_t m_prune_target_gb; void startThread(); void checkPath(const QString &dataDir); QString getPathToCheck(); + void UpdatePruneLabels(bool prune_checked); + void UpdateFreeSpaceLabel(); friend class FreespaceChecker; }; diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index b4b5b32311..977076c4c2 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -240,9 +240,8 @@ void OptionsModel::SetPruneEnabled(bool prune, bool force) { QSettings settings; settings.setValue("bPrune", prune); - // Convert prune size from GB to MiB: - const uint64_t nPruneSizeMiB = (settings.value("nPruneSize").toInt() * GB_BYTES) >> 20; - std::string prune_val = prune ? std::to_string(nPruneSizeMiB) : "0"; + const int64_t prune_target_mib = PruneGBtoMiB(settings.value("nPruneSize").toInt()); + std::string prune_val = prune ? std::to_string(prune_target_mib) : "0"; if (force) { m_node.forceSetArg("-prune", prune_val); return; diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 524fe268b9..b3260349e7 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -6,6 +6,7 @@ #define BITCOIN_QT_OPTIONSMODEL_H #include <amount.h> +#include <qt/guiconstants.h> #include <QAbstractListModel> @@ -16,6 +17,16 @@ class Node; extern const char *DEFAULT_GUI_PROXY_HOST; static constexpr unsigned short DEFAULT_GUI_PROXY_PORT = 9050; +/** + * Convert configured prune target MiB to displayed GB. Round up to avoid underestimating max disk usage. + */ +static inline int PruneMiBtoGB(int64_t mib) { return (mib * 1024 * 1024 + GB_BYTES - 1) / GB_BYTES; } + +/** + * Convert displayed prune target GB to configured MiB. Round down so roundtrip GB -> MiB -> GB conversion is stable. + */ +static inline int64_t PruneGBtoMiB(int gb) { return gb * GB_BYTES / 1024 / 1024; } + /** Interface from Qt to configuration data structure for Bitcoin client. To Qt, the options are presented as a list with the different options laid out vertically. diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 176aa7902b..0f082802cc 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -59,6 +59,7 @@ void TestAddAddressesToSendBook(interfaces::Node& node) { TestChain100Setup test; std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), WalletLocation(), WalletDatabase::CreateMock()); + wallet->SetupLegacyScriptPubKeyMan(); bool firstRun; wallet->LoadWallet(firstRun); diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index b4cd7f6bac..c1a0f63f73 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -143,10 +143,9 @@ void TestGUI(interfaces::Node& node) bool firstRun; wallet->LoadWallet(firstRun); { - auto spk_man = wallet->GetLegacyScriptPubKeyMan(); + auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); auto locked_chain = wallet->chain().lock(); - LOCK(wallet->cs_wallet); - AssertLockHeld(spk_man->cs_wallet); + LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); wallet->SetAddressBook(GetDestinationForKey(test.coinbaseKey.GetPubKey(), wallet->m_default_address_type), "", "receive"); spk_man->AddKeyPubKey(test.coinbaseKey, test.coinbaseKey.GetPubKey()); wallet->SetLastBlockProcessed(105, ::ChainActive().Tip()->GetBlockHash()); diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index e0c1976f1a..1ce49709b2 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -83,6 +83,7 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) " \"addr\":\"host:port\", (string) The IP address and port of the peer\n" " \"addrbind\":\"ip:port\", (string) Bind address of the connection to the peer\n" " \"addrlocal\":\"ip:port\", (string) Local address as reported by the peer\n" + " \"mapped_as\":\"mapped_as\", (string) The AS in the BGP route to the peer used for diversifying peer selection\n" " \"services\":\"xxxxxxxxxxxxxxxx\", (string) The services offered\n" " \"servicesnames\":[ (array) the services offered, in human-readable form\n" " \"SERVICE_NAME\", (string) the service name if it is recognised\n" @@ -152,6 +153,9 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) obj.pushKV("addrlocal", stats.addrLocal); if (stats.addrBind.IsValid()) obj.pushKV("addrbind", stats.addrBind.ToString()); + if (stats.m_mapped_as != 0) { + obj.pushKV("mapped_as", uint64_t(stats.m_mapped_as)); + } obj.pushKV("services", strprintf("%016x", stats.nServices)); obj.pushKV("servicesnames", GetServicesNames(stats.nServices)); obj.pushKV("relaytxes", stats.fRelayTxes); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 5be7acce1c..cea59b2c7a 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -1079,7 +1079,12 @@ UniValue decodepsbt(const JSONRPCRequest& request) UniValue out(UniValue::VOBJ); out.pushKV("amount", ValueFromAmount(txout.nValue)); - total_in += txout.nValue; + if (MoneyRange(txout.nValue) && MoneyRange(total_in + txout.nValue)) { + total_in += txout.nValue; + } else { + // Hack to just not show fee later + have_all_utxos = false; + } UniValue o(UniValue::VOBJ); ScriptToUniv(txout.scriptPubKey, o, true); @@ -1089,7 +1094,13 @@ UniValue decodepsbt(const JSONRPCRequest& request) UniValue non_wit(UniValue::VOBJ); TxToUniv(*input.non_witness_utxo, uint256(), non_wit, false); in.pushKV("non_witness_utxo", non_wit); - total_in += input.non_witness_utxo->vout[psbtx.tx->vin[i].prevout.n].nValue; + CAmount utxo_val = input.non_witness_utxo->vout[psbtx.tx->vin[i].prevout.n].nValue; + if (MoneyRange(utxo_val) && MoneyRange(total_in + utxo_val)) { + total_in += utxo_val; + } else { + // Hack to just not show fee later + have_all_utxos = false; + } } else { have_all_utxos = false; } @@ -1205,7 +1216,12 @@ UniValue decodepsbt(const JSONRPCRequest& request) outputs.push_back(out); // Fee calculation - output_value += psbtx.tx->vout[i].nValue; + if (MoneyRange(psbtx.tx->vout[i].nValue) && MoneyRange(output_value + psbtx.tx->vout[i].nValue)) { + output_value += psbtx.tx->vout[i].nValue; + } else { + // Hack to just not show fee later + have_all_utxos = false; + } } result.pushKV("outputs", outputs); if (have_all_utxos) { diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index b919046ab6..d0865d2793 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1281,13 +1281,11 @@ uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn return ss.GetHash(); } - static const uint256 one(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); - // Check for invalid use of SIGHASH_SINGLE if ((nHashType & 0x1f) == SIGHASH_SINGLE) { if (nIn >= txTo.vout.size()) { // nOut out of range - return one; + return UINT256_ONE(); } } diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 8791d1542a..58eae3ce96 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -144,8 +144,13 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator ret.push_back(valtype()); // workaround CHECKMULTISIG bug for (size_t i = 1; i < vSolutions.size() - 1; ++i) { CPubKey pubkey = CPubKey(vSolutions[i]); - if (ret.size() < required + 1 && CreateSig(creator, sigdata, provider, sig, pubkey, scriptPubKey, sigversion)) { - ret.push_back(std::move(sig)); + // We need to always call CreateSig in order to fill sigdata with all + // possible signatures that we can create. This will allow further PSBT + // processing to work as it needs all possible signature and pubkey pairs + if (CreateSig(creator, sigdata, provider, sig, pubkey, scriptPubKey, sigversion)) { + if (ret.size() < required + 1) { + ret.push_back(std::move(sig)); + } } } bool ok = ret.size() == required + 1; diff --git a/src/serialize.h b/src/serialize.h index 56c324c527..7fa669ebdb 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -200,6 +200,30 @@ template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; } } /** + * Implement the Ser and Unser methods needed for implementing a formatter (see Using below). + * + * Both Ser and Unser are delegated to a single static method SerializationOps, which is polymorphic + * in the serialized/deserialized type (allowing it to be const when serializing, and non-const when + * deserializing). + * + * Example use: + * struct FooFormatter { + * FORMATTER_METHODS(Class, obj) { READWRITE(obj.val1, VARINT(obj.val2)); } + * } + * would define a class FooFormatter that defines a serialization of Class objects consisting + * of serializing its val1 member using the default serialization, and its val2 member using + * VARINT serialization. That FooFormatter can then be used in statements like + * READWRITE(Using<FooFormatter>(obj.bla)). + */ +#define FORMATTER_METHODS(cls, obj) \ + template<typename Stream> \ + static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, CSerActionSerialize()); } \ + template<typename Stream> \ + static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, CSerActionUnserialize()); } \ + template<typename Stream, typename Type, typename Operation> \ + static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \ + +/** * Implement the Serialize and Unserialize methods by delegating to a single templated * static method that takes the to-be-(de)serialized object as a parameter. This approach * has the advantage that the constness of the object becomes a template parameter, and @@ -211,17 +235,15 @@ template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; } void Serialize(Stream& s) const \ { \ static_assert(std::is_same<const cls&, decltype(*this)>::value, "Serialize type mismatch"); \ - SerializationOps(*this, s, CSerActionSerialize()); \ + Ser(s, *this); \ } \ template<typename Stream> \ void Unserialize(Stream& s) \ { \ static_assert(std::is_same<cls&, decltype(*this)>::value, "Unserialize type mismatch"); \ - SerializationOps(*this, s, CSerActionUnserialize()); \ + Unser(s, *this); \ } \ - template<typename Stream, typename Type, typename Operation> \ - static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \ - + FORMATTER_METHODS(cls, obj) #ifndef CHAR_EQUALS_INT8 template<typename Stream> inline void Serialize(Stream& s, char a ) { ser_writedata8(s, a); } // TODO Get rid of bare char diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index f44ed712d9..07cebeb35a 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -5,6 +5,8 @@ #include <test/util/setup_common.h> #include <string> #include <boost/test/unit_test.hpp> +#include <util/asmap.h> +#include <test/data/asmap.raw.h> #include <hash.h> #include <netbase.h> @@ -12,13 +14,18 @@ class CAddrManTest : public CAddrMan { +private: + bool deterministic; public: - explicit CAddrManTest(bool makeDeterministic = true) + explicit CAddrManTest(bool makeDeterministic = true, + std::vector<bool> asmap = std::vector<bool>()) { if (makeDeterministic) { // Set addrman addr placement to be deterministic. MakeDeterministic(); } + deterministic = makeDeterministic; + m_asmap = asmap; } //! Ensure that bucket placement is always the same for testing purposes. @@ -46,6 +53,21 @@ public: CAddrMan::Delete(nId); } + // Used to test deserialization + std::pair<int, int> GetBucketAndEntry(const CAddress& addr) + { + LOCK(cs); + int nId = mapAddr[addr]; + for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; ++bucket) { + for (int entry = 0; entry < ADDRMAN_BUCKET_SIZE; ++entry) { + if (nId == vvNew[bucket][entry]) { + return std::pair<int, int>(bucket, entry); + } + } + } + return std::pair<int, int>(-1, -1); + } + // Simulates connection failure so that we can test eviction of offline nodes void SimConnFail(CService& addr) { @@ -57,6 +79,16 @@ public: int64_t nLastTry = GetAdjustedTime()-61; Attempt(addr, count_failure, nLastTry); } + + void Clear() + { + CAddrMan::Clear(); + if (deterministic) { + nKey.SetNull(); + insecure_rand = FastRandomContext(true); + } + } + }; static CNetAddr ResolveIP(const std::string& ip) @@ -74,6 +106,18 @@ static CService ResolveService(const std::string& ip, const int port = 0) } +static std::vector<bool> FromBytes(const unsigned char* source, int vector_size) { + std::vector<bool> result(vector_size); + for (int byte_i = 0; byte_i < vector_size / 8; ++byte_i) { + unsigned char cur_byte = source[byte_i]; + for (int bit_i = 0; bit_i < 8; ++bit_i) { + result[byte_i * 8 + bit_i] = (cur_byte >> bit_i) & 1; + } + } + return result; +} + + BOOST_FIXTURE_TEST_SUITE(addrman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(addrman_simple) @@ -400,7 +444,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) } -BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) +BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) { CAddrManTest addrman; @@ -415,30 +459,31 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + std::vector<bool> asmap; // use /16 - BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1), 40); + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, asmap), 40); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetTriedBucket(nKey1) != info1.GetTriedBucket(nKey2)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info1.GetTriedBucket(nKey2, asmap)); // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. CAddrInfo info2 = CAddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK(info1.GetTriedBucket(nKey1) != info2.GetTriedBucket(nKey1)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { CAddrInfo infoi = CAddrInfo( CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), ResolveIP("250.1.1." + std::to_string(i))); - int bucket = infoi.GetTriedBucket(nKey1); + int bucket = infoi.GetTriedBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the same group (\16 prefix for IPv4) should - // never get more than 8 buckets + // Test: IP addresses in the same /16 prefix should + // never get more than 8 buckets with legacy grouping BOOST_CHECK_EQUAL(buckets.size(), 8U); buckets.clear(); @@ -446,15 +491,15 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) CAddrInfo infoj = CAddrInfo( CAddress(ResolveService("250." + std::to_string(j) + ".1.1"), NODE_NONE), ResolveIP("250." + std::to_string(j) + ".1.1")); - int bucket = infoj.GetTriedBucket(nKey1); + int bucket = infoj.GetTriedBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the different groups should map to more than - // 8 buckets. + // Test: IP addresses in the different /16 prefix should map to more than + // 8 buckets with legacy grouping BOOST_CHECK_EQUAL(buckets.size(), 160U); } -BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) +BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) { CAddrManTest addrman; @@ -468,25 +513,27 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + std::vector<bool> asmap; // use /16 + // Test: Make sure the buckets are what we expect - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), 786); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, asmap), 786); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetNewBucket(nKey1) != info1.GetNewBucket(nKey2)); + BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); // Test: Ports should not affect bucket placement in the addr CAddrInfo info2 = CAddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), info2.GetNewBucket(nKey1)); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { CAddrInfo infoi = CAddrInfo( CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), ResolveIP("250.1.1." + std::to_string(i))); - int bucket = infoi.GetNewBucket(nKey1); + int bucket = infoi.GetNewBucket(nKey1, asmap); buckets.insert(bucket); } // Test: IP addresses in the same group (\16 prefix for IPv4) should @@ -499,10 +546,10 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) ResolveService( std::to_string(250 + (j / 255)) + "." + std::to_string(j % 256) + ".1.1"), NODE_NONE), ResolveIP("251.4.1.1")); - int bucket = infoj.GetNewBucket(nKey1); + int bucket = infoj.GetNewBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the same source groups should map to no more + // Test: IP addresses in the same source groups should map to NO MORE // than 64 buckets. BOOST_CHECK(buckets.size() <= 64); @@ -511,14 +558,226 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) CAddrInfo infoj = CAddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("250." + std::to_string(p) + ".1.1")); - int bucket = infoj.GetNewBucket(nKey1); + int bucket = infoj.GetNewBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the different source groups should map to more + // Test: IP addresses in the different source groups should map to MORE // than 64 buckets. BOOST_CHECK(buckets.size() > 64); } +// The following three test cases use asmap.raw +// We use an artificial minimal mock mapping +// 250.0.0.0/8 AS1000 +// 101.1.0.0/16 AS1 +// 101.2.0.0/16 AS2 +// 101.3.0.0/16 AS3 +// 101.4.0.0/16 AS4 +// 101.5.0.0/16 AS5 +// 101.6.0.0/16 AS6 +// 101.7.0.0/16 AS7 +// 101.8.0.0/16 AS8 +BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) +{ + CAddrManTest addrman; + + CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE); + + CNetAddr source1 = ResolveIP("250.1.1.1"); + + + CAddrInfo info1 = CAddrInfo(addr1, source1); + + uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); + uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + + std::vector<bool> asmap = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, asmap), 236); + + // Test: Make sure key actually randomizes bucket placement. A fail on + // this test could be a security issue. + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info1.GetTriedBucket(nKey2, asmap)); + + // Test: Two addresses with same IP but different ports can map to + // different buckets because they have different keys. + CAddrInfo info2 = CAddrInfo(addr2, source1); + + BOOST_CHECK(info1.GetKey() != info2.GetKey()); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); + + std::set<int> buckets; + for (int j = 0; j < 255; j++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("101." + std::to_string(j) + ".1.1"), NODE_NONE), + ResolveIP("101." + std::to_string(j) + ".1.1")); + int bucket = infoj.GetTriedBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different /16 prefix MAY map to more than + // 8 buckets. + BOOST_CHECK(buckets.size() > 8); + + buckets.clear(); + for (int j = 0; j < 255; j++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250." + std::to_string(j) + ".1.1"), NODE_NONE), + ResolveIP("250." + std::to_string(j) + ".1.1")); + int bucket = infoj.GetTriedBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different /16 prefix MAY NOT map to more than + // 8 buckets. + BOOST_CHECK(buckets.size() == 8); +} + +BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) +{ + CAddrManTest addrman; + + CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); + + CNetAddr source1 = ResolveIP("250.1.2.1"); + + CAddrInfo info1 = CAddrInfo(addr1, source1); + + uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); + uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + + std::vector<bool> asmap = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + + // Test: Make sure the buckets are what we expect + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), 795); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, asmap), 795); + + // Test: Make sure key actually randomizes bucket placement. A fail on + // this test could be a security issue. + BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); + + // Test: Ports should not affect bucket placement in the addr + CAddrInfo info2 = CAddrInfo(addr2, source1); + BOOST_CHECK(info1.GetKey() != info2.GetKey()); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); + + std::set<int> buckets; + for (int i = 0; i < 255; i++) { + CAddrInfo infoi = CAddrInfo( + CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), + ResolveIP("250.1.1." + std::to_string(i))); + int bucket = infoi.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the same /16 prefix + // usually map to the same bucket. + BOOST_CHECK_EQUAL(buckets.size(), 1U); + + buckets.clear(); + for (int j = 0; j < 4 * 255; j++) { + CAddrInfo infoj = CAddrInfo(CAddress( + ResolveService( + std::to_string(250 + (j / 255)) + "." + std::to_string(j % 256) + ".1.1"), NODE_NONE), + ResolveIP("251.4.1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the same source /16 prefix should not map to more + // than 64 buckets. + BOOST_CHECK(buckets.size() <= 64); + + buckets.clear(); + for (int p = 0; p < 255; p++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250.1.1.1"), NODE_NONE), + ResolveIP("101." + std::to_string(p) + ".1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different source /16 prefixes usually map to MORE + // than 1 bucket. + BOOST_CHECK(buckets.size() > 1); + + buckets.clear(); + for (int p = 0; p < 255; p++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250.1.1.1"), NODE_NONE), + ResolveIP("250." + std::to_string(p) + ".1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different source /16 prefixes sometimes map to NO MORE + // than 1 bucket. + BOOST_CHECK(buckets.size() == 1); + +} + +BOOST_AUTO_TEST_CASE(addrman_serialization) +{ + std::vector<bool> asmap1 = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + + CAddrManTest addrman_asmap1(true, asmap1); + CAddrManTest addrman_asmap1_dup(true, asmap1); + CAddrManTest addrman_noasmap; + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + + CAddress addr = CAddress(ResolveService("250.1.1.1"), NODE_NONE); + CNetAddr default_source; + + + addrman_asmap1.Add(addr, default_source); + + stream << addrman_asmap1; + // serizalizing/deserializing addrman with the same asmap + stream >> addrman_asmap1_dup; + + std::pair<int, int> bucketAndEntry_asmap1 = addrman_asmap1.GetBucketAndEntry(addr); + std::pair<int, int> bucketAndEntry_asmap1_dup = addrman_asmap1_dup.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_asmap1.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1_dup.second != -1); + + BOOST_CHECK(bucketAndEntry_asmap1.first == bucketAndEntry_asmap1_dup.first); + BOOST_CHECK(bucketAndEntry_asmap1.second == bucketAndEntry_asmap1_dup.second); + + // deserializing asmaped peers.dat to non-asmaped addrman + stream << addrman_asmap1; + stream >> addrman_noasmap; + std::pair<int, int> bucketAndEntry_noasmap = addrman_noasmap.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_noasmap.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1.first != bucketAndEntry_noasmap.first); + BOOST_CHECK(bucketAndEntry_asmap1.second != bucketAndEntry_noasmap.second); + + // deserializing non-asmaped peers.dat to asmaped addrman + addrman_asmap1.Clear(); + addrman_noasmap.Clear(); + addrman_noasmap.Add(addr, default_source); + stream << addrman_noasmap; + stream >> addrman_asmap1; + std::pair<int, int> bucketAndEntry_asmap1_deser = addrman_asmap1.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_asmap1_deser.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1_deser.first != bucketAndEntry_noasmap.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser.first == bucketAndEntry_asmap1_dup.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser.second == bucketAndEntry_asmap1_dup.second); + + // used to map to different buckets, now maps to the same bucket. + addrman_asmap1.Clear(); + addrman_noasmap.Clear(); + CAddress addr1 = CAddress(ResolveService("250.1.1.1"), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.2.1.1"), NODE_NONE); + addrman_noasmap.Add(addr, default_source); + addrman_noasmap.Add(addr2, default_source); + std::pair<int, int> bucketAndEntry_noasmap_addr1 = addrman_noasmap.GetBucketAndEntry(addr1); + std::pair<int, int> bucketAndEntry_noasmap_addr2 = addrman_noasmap.GetBucketAndEntry(addr2); + BOOST_CHECK(bucketAndEntry_noasmap_addr1.first != bucketAndEntry_noasmap_addr2.first); + BOOST_CHECK(bucketAndEntry_noasmap_addr1.second != bucketAndEntry_noasmap_addr2.second); + stream << addrman_noasmap; + stream >> addrman_asmap1; + std::pair<int, int> bucketAndEntry_asmap1_deser_addr1 = addrman_asmap1.GetBucketAndEntry(addr1); + std::pair<int, int> bucketAndEntry_asmap1_deser_addr2 = addrman_asmap1.GetBucketAndEntry(addr2); + BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.first == bucketAndEntry_asmap1_deser_addr2.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.second != bucketAndEntry_asmap1_deser_addr2.second); +} + BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) { diff --git a/src/test/data/asmap.raw b/src/test/data/asmap.raw Binary files differnew file mode 100644 index 0000000000..3dcf1f3940 --- /dev/null +++ b/src/test/data/asmap.raw diff --git a/src/test/fuzz/FuzzedDataProvider.h b/src/test/fuzz/FuzzedDataProvider.h index 1b5b4bb012..3e069eba69 100644 --- a/src/test/fuzz/FuzzedDataProvider.h +++ b/src/test/fuzz/FuzzedDataProvider.h @@ -13,11 +13,10 @@ #ifndef LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_ #define LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_ -#include <limits.h> -#include <stddef.h> -#include <stdint.h> - #include <algorithm> +#include <climits> +#include <cstddef> +#include <cstdint> #include <cstring> #include <initializer_list> #include <string> @@ -25,8 +24,10 @@ #include <utility> #include <vector> +// In addition to the comments below, the API is also briefly documented at +// https://github.com/google/fuzzing/blob/master/docs/split-inputs.md#fuzzed-data-provider class FuzzedDataProvider { -public: + public: // |data| is an array of length |size| that the FuzzedDataProvider wraps to // provide more granular access. |data| must outlive the FuzzedDataProvider. FuzzedDataProvider(const uint8_t *data, size_t size) @@ -143,9 +144,9 @@ public: return ConsumeBytes<T>(remaining_bytes_); } + // Returns a std::string containing all remaining bytes of the input data. // Prefer using |ConsumeRemainingBytes| unless you actually need a std::string // object. - // Returns a std::vector containing all remaining bytes of the input data. std::string ConsumeRemainingBytesAsString() { return ConsumeBytesAsString(remaining_bytes_); } @@ -161,7 +162,7 @@ public: // Reads one byte and returns a bool, or false when no data remains. bool ConsumeBool() { return 1 & ConsumeIntegral<uint8_t>(); } - // Returns a copy of a value selected from a fixed-size |array|. + // Returns a copy of the value selected from the given fixed-size |array|. template <typename T, size_t size> T PickValueInArray(const T (&array)[size]) { static_assert(size > 0, "The array must be non empty."); @@ -170,11 +171,14 @@ public: template <typename T> T PickValueInArray(std::initializer_list<const T> list) { - // static_assert(list.size() > 0, "The array must be non empty."); + // TODO(Dor1s): switch to static_assert once C++14 is allowed. + if (!list.size()) + abort(); + return *(list.begin() + ConsumeIntegralInRange<size_t>(0, list.size() - 1)); } - // Return an enum value. The enum must start at 0 and be contiguous. It must + // Returns an enum value. The enum must start at 0 and be contiguous. It must // also contain |kMaxValue| aliased to its largest (inclusive) value. Such as: // enum class Foo { SomeValue, OtherValue, kMaxValue = OtherValue }; template <typename T> T ConsumeEnum() { @@ -183,10 +187,60 @@ public: 0, static_cast<uint32_t>(T::kMaxValue))); } + // Returns a floating point number in the range [0.0, 1.0]. If there's no + // input data left, always returns 0. + template <typename T> T ConsumeProbability() { + static_assert(std::is_floating_point<T>::value, + "A floating point type is required."); + + // Use different integral types for different floating point types in order + // to provide better density of the resulting values. + using IntegralType = + typename std::conditional<(sizeof(T) <= sizeof(uint32_t)), uint32_t, + uint64_t>::type; + + T result = static_cast<T>(ConsumeIntegral<IntegralType>()); + result /= static_cast<T>(std::numeric_limits<IntegralType>::max()); + return result; + } + + // Returns a floating point value in the range [Type's lowest, Type's max] by + // consuming bytes from the input data. If there's no input data left, always + // returns approximately 0. + template <typename T> T ConsumeFloatingPoint() { + return ConsumeFloatingPointInRange<T>(std::numeric_limits<T>::lowest(), + std::numeric_limits<T>::max()); + } + + // Returns a floating point value in the given range by consuming bytes from + // the input data. If there's no input data left, returns |min|. Note that + // |min| must be less than or equal to |max|. + template <typename T> T ConsumeFloatingPointInRange(T min, T max) { + if (min > max) + abort(); + + T range = .0; + T result = min; + constexpr T zero(.0); + if (max > zero && min < zero && max > min + std::numeric_limits<T>::max()) { + // The diff |max - min| would overflow the given floating point type. Use + // the half of the diff as the range and consume a bool to decide whether + // the result is in the first of the second part of the diff. + range = (max / 2.0) - (min / 2.0); + if (ConsumeBool()) { + result += range; + } + } else { + range = max - min; + } + + return result + range * ConsumeProbability<T>(); + } + // Reports the remaining bytes available for fuzzed input. size_t remaining_bytes() { return remaining_bytes_; } -private: + private: FuzzedDataProvider(const FuzzedDataProvider &) = delete; FuzzedDataProvider &operator=(const FuzzedDataProvider &) = delete; @@ -209,6 +263,12 @@ private: // which seems to be a natural choice for other implementations as well. // To increase the odds even more, we also call |shrink_to_fit| below. std::vector<T> result(size); + if (size == 0) { + if (num_bytes_to_consume != 0) + abort(); + return result; + } + std::memcpy(result.data(), data_ptr_, num_bytes_to_consume); Advance(num_bytes_to_consume); @@ -230,9 +290,9 @@ private: // Avoid using implementation-defined unsigned to signer conversions. // To learn more, see https://stackoverflow.com/questions/13150449. - if (value <= std::numeric_limits<TS>::max()) + if (value <= std::numeric_limits<TS>::max()) { return static_cast<TS>(value); - else { + } else { constexpr auto TS_min = std::numeric_limits<TS>::min(); return TS_min + static_cast<char>(value - TS_min); } diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index bd05283b78..f06f339b9d 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -206,7 +206,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) DeserializeFromFuzzingInput(buffer, dbi); #elif TXOUTCOMPRESSOR_DESERIALIZE CTxOut to; - CTxOutCompressor toc(to); + auto toc = Using<TxOutCompression>(to); DeserializeFromFuzzingInput(buffer, toc); #elif BLOCKTRANSACTIONS_DESERIALIZE BlockTransactions bt; diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index a6ab620e21..a085e36911 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -12,6 +12,7 @@ const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; +#if defined(__AFL_COMPILER) static bool read_stdin(std::vector<uint8_t>& data) { uint8_t buffer[1024]; @@ -23,6 +24,7 @@ static bool read_stdin(std::vector<uint8_t>& data) } return length == 0; } +#endif // Default initialization: Override using a non-weak initialize(). __attribute__((weak)) void initialize() @@ -44,9 +46,9 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) return 0; } -// Declare main(...) "weak" to allow for libFuzzer linking. libFuzzer provides -// the main(...) function. -__attribute__((weak)) int main(int argc, char** argv) +// Generally, the fuzzer will provide main(), except for AFL +#if defined(__AFL_COMPILER) +int main(int argc, char** argv) { initialize(); #ifdef __AFL_INIT @@ -74,3 +76,4 @@ __attribute__((weak)) int main(int argc, char** argv) #endif return 0; } +#endif diff --git a/src/test/fuzz/strprintf.cpp b/src/test/fuzz/strprintf.cpp new file mode 100644 index 0000000000..0de21f0e7c --- /dev/null +++ b/src/test/fuzz/strprintf.cpp @@ -0,0 +1,147 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <tinyformat.h> +#include <util/strencodings.h> + +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const std::string format_string = fuzzed_data_provider.ConsumeRandomLengthString(64); + + const int digits_in_format_specifier = std::count_if(format_string.begin(), format_string.end(), IsDigit); + + // Avoid triggering the following crash bug: + // * strprintf("%987654321000000:", 1); + // + // Avoid triggering the following OOM bug: + // * strprintf("%.222222200000000$", 1.1); + // + // Upstream bug report: https://github.com/c42f/tinyformat/issues/70 + if (format_string.find("%") != std::string::npos && digits_in_format_specifier >= 7) { + return; + } + + // Avoid triggering the following crash bug: + // * strprintf("%1$*1$*", -11111111); + // + // Upstream bug report: https://github.com/c42f/tinyformat/issues/70 + if (format_string.find("%") != std::string::npos && format_string.find("$") != std::string::npos && format_string.find("*") != std::string::npos && digits_in_format_specifier > 0) { + return; + } + + // Avoid triggering the following crash bug: + // * strprintf("%.1s", (char*)nullptr); + // + // (void)strprintf(format_string, (char*)nullptr); + // + // Upstream bug report: https://github.com/c42f/tinyformat/issues/70 + + try { + (void)strprintf(format_string, (signed char*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (unsigned char*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (void*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (bool*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (float*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (double*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (int16_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (uint16_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (int32_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (uint32_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (int64_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (uint64_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + + try { + switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 13)) { + case 0: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeRandomLengthString(32)); + break; + case 1: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeRandomLengthString(32).c_str()); + break; + case 2: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<signed char>()); + break; + case 3: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<unsigned char>()); + break; + case 4: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<char>()); + break; + case 5: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeBool()); + break; + case 6: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeFloatingPoint<float>()); + break; + case 7: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeFloatingPoint<double>()); + break; + case 8: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int16_t>()); + break; + case 9: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint16_t>()); + break; + case 10: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int32_t>()); + break; + case 11: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint32_t>()); + break; + case 12: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int64_t>()); + break; + case 13: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + break; + default: + assert(false); + } + } catch (const tinyformat::format_error&) { + } +} diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 58e0565bda..9730b40580 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -286,23 +286,23 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_AUTO_TEST_CASE(netbase_getgroup) { - - BOOST_CHECK(ResolveIP("127.0.0.1").GetGroup() == std::vector<unsigned char>({0})); // Local -> !Routable() - BOOST_CHECK(ResolveIP("257.0.0.1").GetGroup() == std::vector<unsigned char>({0})); // !Valid -> !Routable() - BOOST_CHECK(ResolveIP("10.0.0.1").GetGroup() == std::vector<unsigned char>({0})); // RFC1918 -> !Routable() - BOOST_CHECK(ResolveIP("169.254.1.1").GetGroup() == std::vector<unsigned char>({0})); // RFC3927 -> !Routable() - BOOST_CHECK(ResolveIP("1.2.3.4").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // IPv4 - BOOST_CHECK(ResolveIP("::FFFF:0:102:304").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6145 - BOOST_CHECK(ResolveIP("64:FF9B::102:304").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6052 - BOOST_CHECK(ResolveIP("2002:102:304:9999:9999:9999:9999:9999").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC3964 - BOOST_CHECK(ResolveIP("2001:0:9999:9999:9999:9999:FEFD:FCFB").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC4380 - BOOST_CHECK(ResolveIP("FD87:D87E:EB43:edb1:8e4:3588:e546:35ca").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_ONION, 239})); // Tor - BOOST_CHECK(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 4, 112, 175})); //he.net - BOOST_CHECK(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 32, 1})); //IPv6 + std::vector<bool> asmap; // use /16 + BOOST_CHECK(ResolveIP("127.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // Local -> !Routable() + BOOST_CHECK(ResolveIP("257.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // !Valid -> !Routable() + BOOST_CHECK(ResolveIP("10.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // RFC1918 -> !Routable() + BOOST_CHECK(ResolveIP("169.254.1.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // RFC3927 -> !Routable() + BOOST_CHECK(ResolveIP("1.2.3.4").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // IPv4 + BOOST_CHECK(ResolveIP("::FFFF:0:102:304").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6145 + BOOST_CHECK(ResolveIP("64:FF9B::102:304").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6052 + BOOST_CHECK(ResolveIP("2002:102:304:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC3964 + BOOST_CHECK(ResolveIP("2001:0:9999:9999:9999:9999:FEFD:FCFB").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC4380 + BOOST_CHECK(ResolveIP("FD87:D87E:EB43:edb1:8e4:3588:e546:35ca").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_ONION, 239})); // Tor + BOOST_CHECK(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 4, 112, 175})); //he.net + BOOST_CHECK(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 32, 1})); //IPv6 // baz.net sha256 hash: 12929400eb4607c4ac075f087167e75286b179c693eb059a01774b864e8fe505 std::vector<unsigned char> internal_group = {NET_INTERNAL, 0x12, 0x92, 0x94, 0x00, 0xeb, 0x46, 0x07, 0xc4, 0xac, 0x07}; - BOOST_CHECK(CreateInternal("baz.net").GetGroup() == internal_group); + BOOST_CHECK(CreateInternal("baz.net").GetGroup(asmap) == internal_group); } BOOST_AUTO_TEST_CASE(netbase_parsenetwork) diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 2c56bbdbb0..bcc4a46873 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -26,10 +26,9 @@ extern UniValue read_json(const std::string& jsondata); // Old script.cpp SignatureHash function uint256 static SignatureHashOld(CScript scriptCode, const CTransaction& txTo, unsigned int nIn, int nHashType) { - static const uint256 one(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); if (nIn >= txTo.vin.size()) { - return one; + return UINT256_ONE(); } CMutableTransaction txTmp(txTo); @@ -59,7 +58,7 @@ uint256 static SignatureHashOld(CScript scriptCode, const CTransaction& txTo, un unsigned int nOut = nIn; if (nOut >= txTmp.vout.size()) { - return one; + return UINT256_ONE(); } txTmp.vout.resize(nOut+1); for (unsigned int i = 0; i < nOut; i++) diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 0939803953..fb45ce0ee6 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -831,6 +831,7 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); BOOST_CHECK_EQUAL(reason, "bare-multisig"); + fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index c14cd64766..7842594b80 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -15,7 +15,7 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks); -BOOST_AUTO_TEST_SUITE(tx_validationcache_tests) +BOOST_AUTO_TEST_SUITE(txvalidationcache_tests) BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) { diff --git a/src/test/util/wallet.cpp b/src/test/util/wallet.cpp index 226d2df6e4..fd6012e9fe 100644 --- a/src/test/util/wallet.cpp +++ b/src/test/util/wallet.cpp @@ -27,8 +27,7 @@ std::string getnewaddress(CWallet& w) void importaddress(CWallet& wallet, const std::string& address) { auto spk_man = wallet.GetLegacyScriptPubKeyMan(); - LOCK(wallet.cs_wallet); - AssertLockHeld(spk_man->cs_wallet); + LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); const auto dest = DecodeDestination(address); assert(IsValidDestination(dest)); const auto script = GetScriptForDestination(dest); diff --git a/src/txdb.cpp b/src/txdb.cpp index 9568251149..35bbdab00d 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -336,7 +336,7 @@ public: vout.assign(vAvail.size(), CTxOut()); for (unsigned int i = 0; i < vAvail.size(); i++) { if (vAvail[i]) - ::Unserialize(s, CTxOutCompressor(vout[i])); + ::Unserialize(s, Using<TxOutCompression>(vout[i])); } // coinbase height ::Unserialize(s, VARINT(nHeight, VarIntMode::NONNEGATIVE_SIGNED)); diff --git a/src/uint256.cpp b/src/uint256.cpp index 6398d6326f..a943e71062 100644 --- a/src/uint256.cpp +++ b/src/uint256.cpp @@ -75,3 +75,8 @@ template std::string base_blob<256>::GetHex() const; template std::string base_blob<256>::ToString() const; template void base_blob<256>::SetHex(const char*); template void base_blob<256>::SetHex(const std::string&); + +uint256& UINT256_ONE() { + static uint256* one = new uint256(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); + return *one; +} diff --git a/src/uint256.h b/src/uint256.h index ff0b74e117..b36598f572 100644 --- a/src/uint256.h +++ b/src/uint256.h @@ -144,4 +144,6 @@ inline uint256 uint256S(const std::string& str) return rv; } +uint256& UINT256_ONE(); + #endif // BITCOIN_UINT256_H diff --git a/src/undo.h b/src/undo.h index 3f50f4caad..2009c721ab 100644 --- a/src/undo.h +++ b/src/undo.h @@ -32,7 +32,7 @@ public: // Required to maintain compatibility with older undo format. ::Serialize(s, (unsigned char)0); } - ::Serialize(s, CTxOutCompressor(REF(txout->out))); + ::Serialize(s, Using<TxOutCompression>(REF(txout->out))); } explicit TxInUndoSerializer(const Coin* coin) : txout(coin) {} @@ -56,7 +56,7 @@ public: unsigned int nVersionDummy; ::Unserialize(s, VARINT(nVersionDummy)); } - ::Unserialize(s, CTxOutCompressor(REF(txout->out))); + ::Unserialize(s, Using<TxOutCompression>(REF(txout->out))); } explicit TxInUndoDeserializer(Coin* coin) : txout(coin) {} diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp new file mode 100644 index 0000000000..ac230e9ee5 --- /dev/null +++ b/src/util/asmap.cpp @@ -0,0 +1,97 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <vector> +#include <assert.h> +#include <crypto/common.h> + +namespace { + +uint32_t DecodeBits(std::vector<bool>::const_iterator& bitpos, uint8_t minval, const std::vector<uint8_t> &bit_sizes) +{ + uint32_t val = minval; + bool bit; + for (std::vector<uint8_t>::const_iterator bit_sizes_it = bit_sizes.begin(); + bit_sizes_it != bit_sizes.end(); ++bit_sizes_it) { + if (bit_sizes_it + 1 != bit_sizes.end()) { + bit = *bitpos; + bitpos++; + } else { + bit = 0; + } + if (bit) { + val += (1 << *bit_sizes_it); + } else { + for (int b = 0; b < *bit_sizes_it; b++) { + bit = *bitpos; + bitpos++; + val += bit << (*bit_sizes_it - 1 - b); + } + return val; + } + } + return -1; +} + +const std::vector<uint8_t> TYPE_BIT_SIZES{0, 0, 1}; +uint32_t DecodeType(std::vector<bool>::const_iterator& bitpos) +{ + return DecodeBits(bitpos, 0, TYPE_BIT_SIZES); +} + +const std::vector<uint8_t> ASN_BIT_SIZES{15, 16, 17, 18, 19, 20, 21, 22, 23, 24}; +uint32_t DecodeASN(std::vector<bool>::const_iterator& bitpos) +{ + return DecodeBits(bitpos, 1, ASN_BIT_SIZES); +} + + +const std::vector<uint8_t> MATCH_BIT_SIZES{1, 2, 3, 4, 5, 6, 7, 8}; +uint32_t DecodeMatch(std::vector<bool>::const_iterator& bitpos) +{ + return DecodeBits(bitpos, 2, MATCH_BIT_SIZES); +} + + +const std::vector<uint8_t> JUMP_BIT_SIZES{5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30}; +uint32_t DecodeJump(std::vector<bool>::const_iterator& bitpos) +{ + return DecodeBits(bitpos, 17, JUMP_BIT_SIZES); +} + +} + +uint32_t Interpret(const std::vector<bool> &asmap, const std::vector<bool> &ip) +{ + std::vector<bool>::const_iterator pos = asmap.begin(); + uint8_t bits = ip.size(); + uint8_t default_asn = 0; + uint32_t opcode, jump, match, matchlen; + while (1) { + assert(pos != asmap.end()); + opcode = DecodeType(pos); + if (opcode == 0) { + return DecodeASN(pos); + } else if (opcode == 1) { + jump = DecodeJump(pos); + if (ip[ip.size() - bits]) { + pos += jump; + } + bits--; + } else if (opcode == 2) { + match = DecodeMatch(pos); + matchlen = CountBits(match) - 1; + for (uint32_t bit = 0; bit < matchlen; bit++) { + if ((ip[ip.size() - bits]) != ((match >> (matchlen - 1 - bit)) & 1)) { + return default_asn; + } + bits--; + } + } else if (opcode == 3) { + default_asn = DecodeASN(pos); + } else { + assert(0); + } + } +} diff --git a/src/util/asmap.h b/src/util/asmap.h new file mode 100644 index 0000000000..a0e14013c5 --- /dev/null +++ b/src/util/asmap.h @@ -0,0 +1,10 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_ASMAP_H +#define BITCOIN_UTIL_ASMAP_H + +uint32_t Interpret(const std::vector<bool> &asmap, const std::vector<bool> &ip); + +#endif // BITCOIN_UTIL_ASMAP_H diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index dd0d2ffbd7..2ebc9aba39 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -46,7 +46,7 @@ void WalletInit::AddWalletOptions() const gArgs.AddArg("-fallbackfee=<amt>", strprintf("A fee rate (in %s/kB) that will be used when fee estimation has insufficient data. 0 to entirely disable the fallbackfee feature. (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_FALLBACK_FEE)), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); - gArgs.AddArg("-keypool=<n>", strprintf("Set key pool size to <n> (default: %u)", DEFAULT_KEYPOOL_SIZE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); + gArgs.AddArg("-keypool=<n>", strprintf("Set key pool size to <n> (default: %u). Warning: Smaller sizes may increase the risk of losing funds when restoring from an old backup, if none of the addresses in the original keypool have been used.", DEFAULT_KEYPOOL_SIZE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-maxtxfee=<amt>", strprintf("Maximum total fees (in %s) to use in a single wallet transaction; setting this too low may abort large transactions (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE)), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-mintxfee=<amt>", strprintf("Fees (in %s/kB) smaller than this are considered zero fee for transaction creation (default: %s)", diff --git a/src/wallet/psbtwallet.cpp b/src/wallet/psbtwallet.cpp index 96c1ad8d3f..d995fb06d4 100644 --- a/src/wallet/psbtwallet.cpp +++ b/src/wallet/psbtwallet.cpp @@ -44,6 +44,9 @@ TransactionError FillPSBT(const CWallet* pwallet, PartiallySignedTransaction& ps if (!input.witness_utxo.IsNull()) { script = input.witness_utxo.scriptPubKey; } else if (input.non_witness_utxo) { + if (txin.prevout.n >= input.non_witness_utxo->vout.size()) { + return TransactionError::MISSING_INPUTS; + } script = input.non_witness_utxo->vout[txin.prevout.n].scriptPubKey; } else { // There's no UTXO so we can just skip this now @@ -52,21 +55,21 @@ TransactionError FillPSBT(const CWallet* pwallet, PartiallySignedTransaction& ps } SignatureData sigdata; input.FillSignatureData(sigdata); - const SigningProvider* provider = pwallet->GetSigningProvider(script, sigdata); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(script, sigdata); if (!provider) { complete = false; continue; } - complete &= SignPSBTInput(HidingSigningProvider(provider, !sign, !bip32derivs), psbtx, i, sighash_type); + complete &= SignPSBTInput(HidingSigningProvider(provider.get(), !sign, !bip32derivs), psbtx, i, sighash_type); } // Fill in the bip32 keypaths and redeemscripts for the outputs so that hardware wallets can identify change for (unsigned int i = 0; i < psbtx.tx->vout.size(); ++i) { const CTxOut& out = psbtx.tx->vout.at(i); - const SigningProvider* provider = pwallet->GetSigningProvider(out.scriptPubKey); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(out.scriptPubKey); if (provider) { - UpdatePSBTOutput(HidingSigningProvider(provider, true, !bip32derivs), psbtx, i); + UpdatePSBTOutput(HidingSigningProvider(provider.get(), true, !bip32derivs), psbtx, i); } } diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 633ac1b16d..b730d4a4dd 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -125,7 +125,7 @@ UniValue importprivkey(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import private keys to a wallet with private keys disabled"); } - EnsureLegacyScriptPubKeyMan(*wallet); + EnsureLegacyScriptPubKeyMan(*wallet, true); WalletRescanReserver reserver(pwallet); bool fRescan = true; @@ -253,7 +253,7 @@ UniValue importaddress(const JSONRPCRequest& request) }, }.Check(request); - EnsureLegacyScriptPubKeyMan(*pwallet); + EnsureLegacyScriptPubKeyMan(*pwallet, true); std::string strLabel; if (!request.params[1].isNull()) @@ -454,7 +454,7 @@ UniValue importpubkey(const JSONRPCRequest& request) }, }.Check(request); - EnsureLegacyScriptPubKeyMan(*wallet); + EnsureLegacyScriptPubKeyMan(*wallet, true); std::string strLabel; if (!request.params[1].isNull()) @@ -538,7 +538,7 @@ UniValue importwallet(const JSONRPCRequest& request) }, }.Check(request); - EnsureLegacyScriptPubKeyMan(*wallet); + EnsureLegacyScriptPubKeyMan(*wallet, true); if (pwallet->chain().havePruned()) { // Exit early and print an error. @@ -700,7 +700,7 @@ UniValue dumpprivkey(const JSONRPCRequest& request) LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*wallet); auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); + LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); EnsureWalletIsUnlocked(pwallet); @@ -751,8 +751,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*wallet); auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); - AssertLockHeld(spk_man.cs_wallet); + LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); EnsureWalletIsUnlocked(pwallet); @@ -1335,7 +1334,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) RPCTypeCheck(mainRequest.params, {UniValue::VARR, UniValue::VOBJ}); - EnsureLegacyScriptPubKeyMan(*wallet); + EnsureLegacyScriptPubKeyMan(*wallet, true); const UniValue& requests = mainRequest.params[0]; diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 05719b4754..df71f97e85 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -124,9 +124,13 @@ void EnsureWalletIsUnlocked(const CWallet* pwallet) } } -LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet) +// also_create should only be set to true only when the RPC is expected to add things to a blank wallet and make it no longer blank +LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_create) { LegacyScriptPubKeyMan* spk_man = wallet.GetLegacyScriptPubKeyMan(); + if (!spk_man && also_create) { + spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); + } if (!spk_man) { throw JSONRPCError(RPC_WALLET_ERROR, "This type of wallet does not support this command"); } @@ -561,7 +565,7 @@ static UniValue signmessage(const JSONRPCRequest& request) } CScript script_pub_key = GetScriptForDestination(*pkhash); - const SigningProvider* provider = pwallet->GetSigningProvider(script_pub_key); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(script_pub_key); if (!provider) { throw JSONRPCError(RPC_WALLET_ERROR, "Private key not available"); } @@ -983,7 +987,7 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet); auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); + LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); std::string label; if (!request.params[2].isNull()) @@ -2944,7 +2948,7 @@ static UniValue listunspent(const JSONRPCRequest& request) entry.pushKV("label", i->second.name); } - const SigningProvider* provider = pwallet->GetSigningProvider(scriptPubKey); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(scriptPubKey); if (provider) { if (scriptPubKey.IsPayToScriptHash()) { const CScriptID& hash = CScriptID(boost::get<ScriptHash>(address)); @@ -2984,7 +2988,7 @@ static UniValue listunspent(const JSONRPCRequest& request) entry.pushKV("spendable", out.fSpendable); entry.pushKV("solvable", out.fSolvable); if (out.fSolvable) { - const SigningProvider* provider = pwallet->GetSigningProvider(scriptPubKey); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(scriptPubKey); if (provider) { auto descriptor = InferDescriptor(scriptPubKey, *provider); entry.pushKV("desc", descriptor->ToString()); @@ -3297,21 +3301,21 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request) // Parse the prevtxs array ParsePrevouts(request.params[1], nullptr, coins); - std::set<const SigningProvider*> providers; + std::set<std::shared_ptr<SigningProvider>> providers; for (const std::pair<COutPoint, Coin> coin_pair : coins) { - const SigningProvider* provider = pwallet->GetSigningProvider(coin_pair.second.out.scriptPubKey); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(coin_pair.second.out.scriptPubKey); if (provider) { providers.insert(std::move(provider)); } } if (providers.size() == 0) { - // When there are no available providers, use DUMMY_SIGNING_PROVIDER so we can check if the tx is complete - providers.insert(&DUMMY_SIGNING_PROVIDER); + // When there are no available providers, use a dummy SigningProvider so we can check if the tx is complete + providers.insert(std::make_shared<SigningProvider>()); } UniValue result(UniValue::VOBJ); - for (const SigningProvider* provider : providers) { - SignTransaction(mtx, provider, coins, request.params[2], result); + for (std::shared_ptr<SigningProvider> provider : providers) { + SignTransaction(mtx, provider.get(), coins, request.params[2], result); } return result; } @@ -3697,12 +3701,12 @@ static UniValue DescribeWalletAddress(CWallet* pwallet, const CTxDestination& de UniValue ret(UniValue::VOBJ); UniValue detail = DescribeAddress(dest); CScript script = GetScriptForDestination(dest); - const SigningProvider* provider = nullptr; + std::unique_ptr<SigningProvider> provider = nullptr; if (pwallet) { provider = pwallet->GetSigningProvider(script); } ret.pushKVs(detail); - ret.pushKVs(boost::apply_visitor(DescribeWalletAddressVisitor(provider), dest)); + ret.pushKVs(boost::apply_visitor(DescribeWalletAddressVisitor(provider.get()), dest)); return ret; } @@ -3800,7 +3804,7 @@ UniValue getaddressinfo(const JSONRPCRequest& request) CScript scriptPubKey = GetScriptForDestination(dest); ret.pushKV("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end())); - const SigningProvider* provider = pwallet->GetSigningProvider(scriptPubKey); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(scriptPubKey); isminetype mine = pwallet->IsMine(dest); ret.pushKV("ismine", bool(mine & ISMINE_SPENDABLE)); @@ -4003,7 +4007,7 @@ UniValue sethdseed(const JSONRPCRequest& request) }, }.Check(request); - LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet); + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet, true); if (pwallet->chain().isInitialBlockDownload()) { throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Cannot set a new HD seed while still in Initial Block Download"); @@ -4014,7 +4018,7 @@ UniValue sethdseed(const JSONRPCRequest& request) } auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); + LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); // Do not do anything to non-HD wallets if (!pwallet->CanSupportFeature(FEATURE_HD)) { diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index becca455f6..2813fa2bfc 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -41,7 +41,7 @@ std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& reques std::string HelpRequiringPassphrase(const CWallet*); void EnsureWalletIsUnlocked(const CWallet*); bool EnsureWalletIsAvailable(const CWallet*, bool avoidException); -LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet); +LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_create = false); UniValue getaddressinfo(const JSONRPCRequest& request); UniValue signrawtransactionwithwallet(const JSONRPCRequest& request); diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index be8a71da97..4c9d88973e 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -9,10 +9,10 @@ #include <util/strencodings.h> #include <util/translation.h> #include <wallet/scriptpubkeyman.h> -#include <wallet/wallet.h> bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) { + LOCK(cs_KeyStore); error.clear(); // Generate a new key that is added to wallet @@ -238,7 +238,6 @@ bool LegacyScriptPubKeyMan::CheckDecryptionKey(const CKeyingMaterial& master_key bool LegacyScriptPubKeyMan::Encrypt(const CKeyingMaterial& master_key, WalletBatch* batch) { - AssertLockHeld(cs_wallet); LOCK(cs_KeyStore); encrypted_batch = batch; if (!mapCryptedKeys.empty()) { @@ -269,6 +268,7 @@ bool LegacyScriptPubKeyMan::Encrypt(const CKeyingMaterial& master_key, WalletBat bool LegacyScriptPubKeyMan::GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool) { + LOCK(cs_KeyStore); if (!CanGetAddresses(internal)) { return false; } @@ -282,7 +282,7 @@ bool LegacyScriptPubKeyMan::GetReservedDestination(const OutputType type, bool i void LegacyScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); // extract addresses and check if they match with an unused keypool key for (const auto& keyid : GetAffectedKeys(script, *this)) { std::map<CKeyID, int64_t>::const_iterator mi = m_pool_key_to_index.find(keyid); @@ -299,7 +299,7 @@ void LegacyScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) void LegacyScriptPubKeyMan::UpgradeKeyMetadata() { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); if (m_storage.IsLocked() || m_storage.IsWalletFlagSet(WALLET_FLAG_KEY_ORIGIN_METADATA)) { return; } @@ -352,7 +352,7 @@ bool LegacyScriptPubKeyMan::IsHDEnabled() const bool LegacyScriptPubKeyMan::CanGetAddresses(bool internal) { - LOCK(cs_wallet); + LOCK(cs_KeyStore); // Check if the keypool has keys bool keypool_has_keys; if (internal && m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) { @@ -369,7 +369,7 @@ bool LegacyScriptPubKeyMan::CanGetAddresses(bool internal) bool LegacyScriptPubKeyMan::Upgrade(int prev_version, std::string& error) { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); error = ""; bool hd_upgrade = false; bool split_upgrade = false; @@ -383,7 +383,7 @@ bool LegacyScriptPubKeyMan::Upgrade(int prev_version, std::string& error) hd_upgrade = true; } // Upgrade to HD chain split if necessary - if (m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) { + if (m_storage.CanSupportFeature(FEATURE_HD_SPLIT) && CHDChain::VERSION_HD_CHAIN_SPLIT) { WalletLogPrintf("Upgrading wallet to use HD chain split\n"); m_storage.SetMinVersion(FEATURE_PRE_SPLIT_KEYPOOL); split_upgrade = FEATURE_HD_SPLIT > prev_version; @@ -410,7 +410,7 @@ bool LegacyScriptPubKeyMan::HavePrivateKeys() const void LegacyScriptPubKeyMan::RewriteDB() { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); setInternalKeyPool.clear(); setExternalKeyPool.clear(); m_pool_key_to_index.clear(); @@ -435,7 +435,7 @@ static int64_t GetOldestKeyTimeInPool(const std::set<int64_t>& setKeyPool, Walle int64_t LegacyScriptPubKeyMan::GetOldestKeyPoolTime() { - LOCK(cs_wallet); + LOCK(cs_KeyStore); WalletBatch batch(m_storage.GetDatabase()); @@ -453,25 +453,53 @@ int64_t LegacyScriptPubKeyMan::GetOldestKeyPoolTime() size_t LegacyScriptPubKeyMan::KeypoolCountExternalKeys() { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); return setExternalKeyPool.size() + set_pre_split_keypool.size(); } unsigned int LegacyScriptPubKeyMan::GetKeyPoolSize() const { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); return setInternalKeyPool.size() + setExternalKeyPool.size() + set_pre_split_keypool.size(); } int64_t LegacyScriptPubKeyMan::GetTimeFirstKey() const { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); return nTimeFirstKey; } +std::unique_ptr<SigningProvider> LegacyScriptPubKeyMan::GetSigningProvider(const CScript& script) const +{ + return MakeUnique<LegacySigningProvider>(*this); +} + +bool LegacyScriptPubKeyMan::CanProvide(const CScript& script, SignatureData& sigdata) +{ + if (IsMine(script) != ISMINE_NO) { + // If it IsMine, we can always provide in some way + return true; + } else if (HaveCScript(CScriptID(script))) { + // We can still provide some stuff if we have the script, but IsMine failed because we don't have keys + return true; + } else { + // If, given the stuff in sigdata, we could make a valid sigature, then we can provide for this script + ProduceSignature(*this, DUMMY_SIGNATURE_CREATOR, script, sigdata); + if (!sigdata.signatures.empty()) { + // If we could make signatures, make sure we have a private key to actually make a signature + bool has_privkeys = false; + for (const auto& key_sig_pair : sigdata.signatures) { + has_privkeys |= HaveKey(key_sig_pair.first); + } + return has_privkeys; + } + return false; + } +} + const CKeyMetadata* LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); CKeyID key_id = GetKeyForDestination(*this, dest); if (!key_id.IsNull()) { @@ -490,13 +518,18 @@ const CKeyMetadata* LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& des return nullptr; } +uint256 LegacyScriptPubKeyMan::GetID() const +{ + return UINT256_ONE(); +} + /** * Update wallet first key creation time. This should be called whenever keys * are added to the wallet, with the oldest key creation time. */ void LegacyScriptPubKeyMan::UpdateTimeFirstKey(int64_t nCreateTime) { - AssertLockHeld(cs_wallet); + AssertLockHeld(cs_KeyStore); if (nCreateTime <= 1) { // Cannot determine birthday information, so set the wallet birthday to // the beginning of time. @@ -513,13 +546,14 @@ bool LegacyScriptPubKeyMan::LoadKey(const CKey& key, const CPubKey &pubkey) bool LegacyScriptPubKeyMan::AddKeyPubKey(const CKey& secret, const CPubKey &pubkey) { + LOCK(cs_KeyStore); WalletBatch batch(m_storage.GetDatabase()); return LegacyScriptPubKeyMan::AddKeyPubKeyWithDB(batch, secret, pubkey); } bool LegacyScriptPubKeyMan::AddKeyPubKeyWithDB(WalletBatch& batch, const CKey& secret, const CPubKey& pubkey) { - AssertLockHeld(cs_wallet); + AssertLockHeld(cs_KeyStore); // Make sure we aren't adding private keys to private key disabled wallets assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); @@ -574,14 +608,14 @@ bool LegacyScriptPubKeyMan::LoadCScript(const CScript& redeemScript) void LegacyScriptPubKeyMan::LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata& meta) { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); UpdateTimeFirstKey(meta.nCreateTime); mapKeyMetadata[keyID] = meta; } void LegacyScriptPubKeyMan::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata& meta) { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); UpdateTimeFirstKey(meta.nCreateTime); m_script_metadata[script_id] = meta; } @@ -630,7 +664,7 @@ bool LegacyScriptPubKeyMan::AddCryptedKey(const CPubKey &vchPubKey, if (!AddCryptedKeyInner(vchPubKey, vchCryptedSecret)) return false; { - LOCK(cs_wallet); + LOCK(cs_KeyStore); if (encrypted_batch) return encrypted_batch->WriteCryptedKey(vchPubKey, vchCryptedSecret, @@ -663,7 +697,6 @@ static bool ExtractPubKey(const CScript &dest, CPubKey& pubKeyOut) bool LegacyScriptPubKeyMan::RemoveWatchOnly(const CScript &dest) { - AssertLockHeld(cs_wallet); { LOCK(cs_KeyStore); setWatchOnly.erase(dest); @@ -734,7 +767,7 @@ bool LegacyScriptPubKeyMan::AddWatchOnly(const CScript& dest, int64_t nCreateTim void LegacyScriptPubKeyMan::SetHDChain(const CHDChain& chain, bool memonly) { - LOCK(cs_wallet); + LOCK(cs_KeyStore); if (!memonly && !WalletBatch(m_storage.GetDatabase()).WriteHDChain(chain)) throw std::runtime_error(std::string(__func__) + ": writing chain failed"); @@ -771,7 +804,7 @@ bool LegacyScriptPubKeyMan::GetKeyOrigin(const CKeyID& keyID, KeyOriginInfo& inf { CKeyMetadata meta; { - LOCK(cs_wallet); + LOCK(cs_KeyStore); auto it = mapKeyMetadata.find(keyID); if (it != mapKeyMetadata.end()) { meta = it->second; @@ -821,7 +854,7 @@ CPubKey LegacyScriptPubKeyMan::GenerateNewKey(WalletBatch &batch, bool internal) { assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)); - AssertLockHeld(cs_wallet); + AssertLockHeld(cs_KeyStore); bool fCompressed = m_storage.CanSupportFeature(FEATURE_COMPRPUBKEY); // default to compressed public keys if we want 0.6.0 wallets CKey secret; @@ -913,7 +946,7 @@ void LegacyScriptPubKeyMan::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& void LegacyScriptPubKeyMan::LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); if (keypool.m_pre_split) { set_pre_split_keypool.insert(nIndex); } else if (keypool.fInternal) { @@ -935,7 +968,7 @@ void LegacyScriptPubKeyMan::LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) bool LegacyScriptPubKeyMan::CanGenerateKeys() { // A wallet can generate keys if it has an HD seed (IsHDEnabled) or it is a non-HD wallet (pre FEATURE_HD) - LOCK(cs_wallet); + LOCK(cs_KeyStore); return IsHDEnabled() || !m_storage.CanSupportFeature(FEATURE_HD); } @@ -962,7 +995,7 @@ CPubKey LegacyScriptPubKeyMan::DeriveNewSeed(const CKey& key) metadata.hd_seed_id = seed.GetID(); { - LOCK(cs_wallet); + LOCK(cs_KeyStore); // mem store the metadata mapKeyMetadata[seed.GetID()] = metadata; @@ -977,7 +1010,7 @@ CPubKey LegacyScriptPubKeyMan::DeriveNewSeed(const CKey& key) void LegacyScriptPubKeyMan::SetHDSeed(const CPubKey& seed) { - LOCK(cs_wallet); + LOCK(cs_KeyStore); // store the keyid (hash160) together with // the child index counter in the database // as a hdchain object @@ -1000,7 +1033,7 @@ bool LegacyScriptPubKeyMan::NewKeyPool() return false; } { - LOCK(cs_wallet); + LOCK(cs_KeyStore); WalletBatch batch(m_storage.GetDatabase()); for (const int64_t nIndex : setInternalKeyPool) { @@ -1034,7 +1067,7 @@ bool LegacyScriptPubKeyMan::TopUp(unsigned int kpSize) return false; } { - LOCK(cs_wallet); + LOCK(cs_KeyStore); if (m_storage.IsLocked()) return false; @@ -1076,7 +1109,7 @@ bool LegacyScriptPubKeyMan::TopUp(unsigned int kpSize) void LegacyScriptPubKeyMan::AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch) { - LOCK(cs_wallet); + LOCK(cs_KeyStore); assert(m_max_keypool_index < std::numeric_limits<int64_t>::max()); // How in the hell did you use so many keys? int64_t index = ++m_max_keypool_index; if (!batch.WritePool(index, CKeyPool(pubkey, internal))) { @@ -1107,7 +1140,7 @@ void LegacyScriptPubKeyMan::ReturnDestination(int64_t nIndex, bool fInternal, co { // Return to key pool { - LOCK(cs_wallet); + LOCK(cs_KeyStore); if (fInternal) { setInternalKeyPool.insert(nIndex); } else if (!set_pre_split_keypool.empty()) { @@ -1131,7 +1164,7 @@ bool LegacyScriptPubKeyMan::GetKeyFromPool(CPubKey& result, const OutputType typ CKeyPool keypool; { - LOCK(cs_wallet); + LOCK(cs_KeyStore); int64_t nIndex; if (!ReserveKeyFromKeyPool(nIndex, keypool, internal) && !m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { if (m_storage.IsLocked()) return false; @@ -1150,7 +1183,7 @@ bool LegacyScriptPubKeyMan::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& key nIndex = -1; keypool.vchPubKey = CPubKey(); { - LOCK(cs_wallet); + LOCK(cs_KeyStore); bool fReturningInternal = fRequestedInternal; fReturningInternal &= (IsHDEnabled() && m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) || m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); @@ -1210,7 +1243,7 @@ void LegacyScriptPubKeyMan::LearnAllRelatedScripts(const CPubKey& key) void LegacyScriptPubKeyMan::MarkReserveKeysAsUsed(int64_t keypool_id) { - AssertLockHeld(cs_wallet); + AssertLockHeld(cs_KeyStore); bool internal = setInternalKeyPool.count(keypool_id); if (!internal) assert(setExternalKeyPool.count(keypool_id) || set_pre_split_keypool.count(keypool_id)); std::set<int64_t> *setKeyPool = internal ? &setInternalKeyPool : (set_pre_split_keypool.empty() ? &setExternalKeyPool : &set_pre_split_keypool); @@ -1281,7 +1314,7 @@ bool LegacyScriptPubKeyMan::AddCScriptWithDB(WalletBatch& batch, const CScript& bool LegacyScriptPubKeyMan::AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info) { - LOCK(cs_wallet); + LOCK(cs_KeyStore); std::copy(info.fingerprint, info.fingerprint + 4, mapKeyMetadata[pubkey.GetID()].key_origin.fingerprint); mapKeyMetadata[pubkey.GetID()].key_origin.path = info.path; mapKeyMetadata[pubkey.GetID()].has_key_origin = true; @@ -1393,13 +1426,3 @@ std::set<CKeyID> LegacyScriptPubKeyMan::GetKeys() const } return set_address; } - -// Temporary CWallet accessors and aliases. -LegacyScriptPubKeyMan::LegacyScriptPubKeyMan(CWallet& wallet) - : ScriptPubKeyMan(wallet), - m_wallet(wallet), - cs_wallet(wallet.cs_wallet) {} - -void LegacyScriptPubKeyMan::NotifyWatchonlyChanged(bool fHaveWatchOnly) const { return m_wallet.NotifyWatchonlyChanged(fHaveWatchOnly); } -void LegacyScriptPubKeyMan::NotifyCanGetAddressesChanged() const { return m_wallet.NotifyCanGetAddressesChanged(); } -template<typename... Params> void LegacyScriptPubKeyMan::WalletLogPrintf(const std::string& fmt, const Params&... parameters) const { return m_wallet.WalletLogPrintf(fmt, parameters...); } diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 8b50711280..7b1c023bc9 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -74,6 +74,11 @@ std::vector<CKeyID> GetAffectedKeys(const CScript& spk, const SigningProvider& p * keys (by default 1000) ahead of the last used key and scans for the * addresses of those keys. This avoids the risk of not seeing transactions * involving the wallet's addresses, or of re-using the same address. + * In the unlikely case where none of the addresses in the `gap limit` are + * used on-chain, the look-ahead will not be incremented to keep + * a constant size and addresses beyond this range will not be detected by an + * old backup. For this reason, it is not recommended to decrease keypool size + * lower than default value. * * The HD-split wallet feature added a second keypool (commit: 02592f4c). There * is an external keypool (for addresses to hand out) and an internal keypool @@ -196,8 +201,28 @@ public: virtual int64_t GetTimeFirstKey() const { return 0; } - //! Return address metadata virtual const CKeyMetadata* GetMetadata(const CTxDestination& dest) const { return nullptr; } + + virtual std::unique_ptr<SigningProvider> GetSigningProvider(const CScript& script) const { return nullptr; } + + /** Whether this ScriptPubKeyMan can provide a SigningProvider (via GetSigningProvider) that, combined with + * sigdata, can produce a valid signature. + */ + virtual bool CanProvide(const CScript& script, SignatureData& sigdata) { return false; } + + virtual uint256 GetID() const { return uint256(); } + + /** Prepends the wallet name in logging output to ease debugging in multi-wallet use cases */ + template<typename... Params> + void WalletLogPrintf(std::string fmt, Params... parameters) const { + LogPrintf(("%s " + fmt).c_str(), m_storage.GetDisplayName(), parameters...); + }; + + /** Watch-only address added */ + boost::signals2::signal<void (bool fHaveWatchOnly)> NotifyWatchonlyChanged; + + /** Keypool has new keys */ + boost::signals2::signal<void ()> NotifyCanGetAddressesChanged; }; class LegacyScriptPubKeyMan : public ScriptPubKeyMan, public FillableSigningProvider @@ -209,7 +234,7 @@ private: using WatchOnlySet = std::set<CScript>; using WatchKeyMap = std::map<CKeyID, CPubKey>; - WalletBatch *encrypted_batch GUARDED_BY(cs_wallet) = nullptr; + WalletBatch *encrypted_batch GUARDED_BY(cs_KeyStore) = nullptr; using CryptedKeyMap = std::map<CKeyID, std::pair<CPubKey, std::vector<unsigned char>>>; @@ -217,7 +242,7 @@ private: WatchOnlySet setWatchOnly GUARDED_BY(cs_KeyStore); WatchKeyMap mapWatchKeys GUARDED_BY(cs_KeyStore); - int64_t nTimeFirstKey GUARDED_BY(cs_wallet) = 0; + int64_t nTimeFirstKey GUARDED_BY(cs_KeyStore) = 0; bool AddKeyPubKeyInner(const CKey& key, const CPubKey &pubkey); bool AddCryptedKeyInner(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); @@ -231,14 +256,14 @@ private: * of the other AddWatchOnly which accepts a timestamp and sets * nTimeFirstKey more intelligently for more efficient rescans. */ - bool AddWatchOnly(const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddWatchOnly(const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); bool AddWatchOnlyInMem(const CScript &dest); //! Adds a watch-only address to the store, and saves it to disk. - bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); //! Adds a key to the store, and saves it to disk. - bool AddKeyPubKeyWithDB(WalletBatch &batch,const CKey& key, const CPubKey &pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddKeyPubKeyWithDB(WalletBatch &batch,const CKey& key, const CPubKey &pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); void AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch); @@ -252,12 +277,12 @@ private: CHDChain hdChain; /* HD derive new child key (on internal or external chain) */ - void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); - std::set<int64_t> setInternalKeyPool GUARDED_BY(cs_wallet); - std::set<int64_t> setExternalKeyPool GUARDED_BY(cs_wallet); - std::set<int64_t> set_pre_split_keypool GUARDED_BY(cs_wallet); - int64_t m_max_keypool_index GUARDED_BY(cs_wallet) = 0; + std::set<int64_t> setInternalKeyPool GUARDED_BY(cs_KeyStore); + std::set<int64_t> setExternalKeyPool GUARDED_BY(cs_KeyStore); + std::set<int64_t> set_pre_split_keypool GUARDED_BY(cs_KeyStore); + int64_t m_max_keypool_index GUARDED_BY(cs_KeyStore) = 0; std::map<CKeyID, int64_t> m_pool_key_to_index; // Tracks keypool indexes to CKeyIDs of keys that have been taken out of the keypool but may be returned to it std::map<int64_t, CKeyID> m_index_to_reserved_key; @@ -282,6 +307,8 @@ private: bool ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal); public: + using ScriptPubKeyMan::ScriptPubKeyMan; + bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) override; isminetype IsMine(const CScript& script) const override; @@ -297,7 +324,7 @@ public: void MarkUnusedAddresses(const CScript& script) override; //! Upgrade stored CKeyMetadata objects to store key origin info as KeyOriginInfo - void UpgradeKeyMetadata() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void UpgradeKeyMetadata(); bool IsHDEnabled() const override; @@ -310,7 +337,7 @@ public: void RewriteDB() override; int64_t GetOldestKeyPoolTime() override; - size_t KeypoolCountExternalKeys() override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + size_t KeypoolCountExternalKeys() override; unsigned int GetKeyPoolSize() const override; int64_t GetTimeFirstKey() const override; @@ -319,28 +346,34 @@ public: bool CanGetAddresses(bool internal = false) override; + std::unique_ptr<SigningProvider> GetSigningProvider(const CScript& script) const override; + + bool CanProvide(const CScript& script, SignatureData& sigdata) override; + + uint256 GetID() const override; + // Map from Key ID to key metadata. - std::map<CKeyID, CKeyMetadata> mapKeyMetadata GUARDED_BY(cs_wallet); + std::map<CKeyID, CKeyMetadata> mapKeyMetadata GUARDED_BY(cs_KeyStore); // Map from Script ID to key metadata (for watch-only keys). - std::map<CScriptID, CKeyMetadata> m_script_metadata GUARDED_BY(cs_wallet); + std::map<CScriptID, CKeyMetadata> m_script_metadata GUARDED_BY(cs_KeyStore); //! Adds a key to the store, and saves it to disk. - bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override; //! Adds a key to the store, without saving it to disk (used by LoadWallet) bool LoadKey(const CKey& key, const CPubKey &pubkey); //! Adds an encrypted key to the store, and saves it to disk. bool AddCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); //! Adds an encrypted key to the store, without saving it to disk (used by LoadWallet) bool LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); - void UpdateTimeFirstKey(int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void UpdateTimeFirstKey(int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); //! Adds a CScript to the store bool LoadCScript(const CScript& redeemScript); //! Load metadata (used by LoadWallet) - void LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &metadata); + void LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &metadata); //! Generate a new key - CPubKey GenerateNewKey(WalletBatch& batch, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + CPubKey GenerateNewKey(WalletBatch& batch, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); /* Set the HD chain model (chain child index counters) */ void SetHDChain(const CHDChain& chain, bool memonly); @@ -353,8 +386,8 @@ public: //! Returns whether there are any watch-only things in the wallet bool HaveWatchOnly() const; //! Remove a watch only script from the keystore - bool RemoveWatchOnly(const CScript &dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool AddWatchOnly(const CScript& dest, int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool RemoveWatchOnly(const CScript &dest); + bool AddWatchOnly(const CScript& dest, int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); //! Fetches a pubkey from mapWatchKeys if it exists there bool GetWatchPubKey(const CKeyID &address, CPubKey &pubkey_out) const; @@ -367,14 +400,14 @@ public: bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override; //! Load a keypool entry - void LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void LoadKeyPool(int64_t nIndex, const CKeyPool &keypool); bool NewKeyPool(); - void MarkPreSplitKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void MarkPreSplitKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); - bool ImportScripts(const std::set<CScript> scripts, int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool ImportScriptPubKeys(const std::set<CScript>& script_pub_keys, const bool have_solving_data, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool ImportScripts(const std::set<CScript> scripts, int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + bool ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + bool ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + bool ImportScriptPubKeys(const std::set<CScript>& script_pub_keys, const bool have_solving_data, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); /* Returns true if the wallet can generate new keys */ bool CanGenerateKeys(); @@ -408,19 +441,26 @@ public: /** * Marks all keys in the keypool up to and including reserve_key as used. */ - void MarkReserveKeysAsUsed(int64_t keypool_id) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void MarkReserveKeysAsUsed(int64_t keypool_id) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); const std::map<CKeyID, int64_t>& GetAllReserveKeys() const { return m_pool_key_to_index; } std::set<CKeyID> GetKeys() const override; - // Temporary CWallet accessors and aliases. - friend class CWallet; - friend class ReserveDestination; - LegacyScriptPubKeyMan(CWallet& wallet); - void NotifyWatchonlyChanged(bool fHaveWatchOnly) const; - void NotifyCanGetAddressesChanged() const; - template<typename... Params> void WalletLogPrintf(const std::string& fmt, const Params&... parameters) const; - CWallet& m_wallet; - RecursiveMutex& cs_wallet; +}; + +/** Wraps a LegacyScriptPubKeyMan so that it can be returned in a new unique_ptr */ +class LegacySigningProvider : public SigningProvider +{ +private: + const LegacyScriptPubKeyMan& m_spk_man; +public: + LegacySigningProvider(const LegacyScriptPubKeyMan& spk_man) : m_spk_man(spk_man) {} + + bool GetCScript(const CScriptID &scriptid, CScript& script) const override { return m_spk_man.GetCScript(scriptid, script); } + bool HaveCScript(const CScriptID &scriptid) const override { return m_spk_man.HaveCScript(scriptid); } + bool GetPubKey(const CKeyID &address, CPubKey& pubkey) const override { return m_spk_man.GetPubKey(address, pubkey); } + bool GetKey(const CKeyID &address, CKey& key) const override { return m_spk_man.GetKey(address, key); } + bool HaveKey(const CKeyID &address) const override { return m_spk_man.HaveKey(address); } + bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override { return m_spk_man.GetKeyOrigin(keyid, info); } }; #endif // BITCOIN_WALLET_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 0e0f06c64c..d65a0e9075 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -136,6 +136,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) { LOCK(testWallet.cs_wallet); + testWallet.SetupLegacyScriptPubKeyMan(); // Setup std::vector<CInputCoin> utxo_pool; @@ -278,6 +279,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) std::unique_ptr<CWallet> wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()); bool firstRun; wallet->LoadWallet(firstRun); + wallet->SetupLegacyScriptPubKeyMan(); LOCK(wallet->cs_wallet); add_coin(*wallet, 5 * CENT, 6 * 24, false, 0, true); add_coin(*wallet, 3 * CENT, 6 * 24, false, 0, true); @@ -299,6 +301,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) bool bnb_used; LOCK(testWallet.cs_wallet); + testWallet.SetupLegacyScriptPubKeyMan(); // test multiple times to allow for differences in the shuffle order for (int i = 0; i < RUN_TESTS; i++) @@ -578,6 +581,7 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) bool bnb_used; LOCK(testWallet.cs_wallet); + testWallet.SetupLegacyScriptPubKeyMan(); empty_wallet(); @@ -596,6 +600,8 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) // Tests that with the ideal conditions, the coin selector will always be able to find a solution that can pay the target value BOOST_AUTO_TEST_CASE(SelectCoins_test) { + testWallet.SetupLegacyScriptPubKeyMan(); + // Random generator stuff std::default_random_engine generator; std::exponential_distribution<double> distribution (100); diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index 76c3639d16..4c0e4dc653 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -36,7 +36,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK compressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(pubkeys[0]); // Keystore does not have key @@ -52,7 +53,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK uncompressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(uncompressedPubkey); // Keystore does not have key @@ -68,7 +70,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH compressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(pubkeys[0])); // Keystore does not have key @@ -84,7 +87,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH uncompressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(uncompressedPubkey)); // Keystore does not have key @@ -100,7 +104,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript redeemScript = GetScriptForDestination(PKHash(pubkeys[0])); scriptPubKey = GetScriptForDestination(ScriptHash(redeemScript)); @@ -123,7 +128,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2SH (invalid) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript redeemscript_inner = GetScriptForDestination(PKHash(pubkeys[0])); CScript redeemscript = GetScriptForDestination(ScriptHash(redeemscript_inner)); @@ -140,7 +146,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2WSH (invalid) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript redeemscript = GetScriptForDestination(PKHash(pubkeys[0])); CScript witnessscript = GetScriptForDestination(ScriptHash(redeemscript)); @@ -157,7 +164,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH inside P2WSH (invalid) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript witnessscript = GetScriptForDestination(WitnessV0KeyHash(PKHash(pubkeys[0]))); scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessscript)); @@ -172,7 +180,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2WSH inside P2WSH (invalid) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript witnessscript_inner = GetScriptForDestination(PKHash(pubkeys[0])); CScript witnessscript = GetScriptForDestination(WitnessV0ScriptHash(witnessscript_inner)); @@ -189,7 +198,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH compressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(PKHash(pubkeys[0]))); @@ -203,7 +213,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH uncompressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(PKHash(uncompressedPubkey))); @@ -221,7 +232,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // scriptPubKey multisig { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForMultisig(2, {uncompressedPubkey, pubkeys[1]}); @@ -251,7 +263,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH multisig { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); @@ -271,7 +284,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with compressed keys { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); @@ -296,7 +310,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with uncompressed key { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); @@ -321,7 +336,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig wrapped in P2SH { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript witnessScript = GetScriptForMultisig(2, {pubkeys[0], pubkeys[1]}); CScript redeemScript = GetScriptForDestination(WitnessV0ScriptHash(witnessScript)); @@ -347,7 +363,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // OP_RETURN { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); @@ -360,7 +377,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // witness unspendable { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); @@ -373,7 +391,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // witness unknown { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); @@ -386,7 +405,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Nonstandard { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index d930ca6bea..f923de6178 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -16,8 +16,8 @@ BOOST_FIXTURE_TEST_SUITE(psbt_wallet_tests, WalletTestingSetup) BOOST_AUTO_TEST_CASE(psbt_updater_test) { - auto spk_man = m_wallet.GetLegacyScriptPubKeyMan(); - LOCK(m_wallet.cs_wallet); + auto spk_man = m_wallet.GetOrCreateLegacyScriptPubKeyMan(); + LOCK2(m_wallet.cs_wallet, spk_man->cs_KeyStore); // Create prevtxs and add to wallet CDataStream s_prev_tx1(ParseHex("0200000000010158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88702483045022100a22edcc6e5bc511af4cc4ae0de0fcd75c7e04d8c1c3a8aa9d820ed4b967384ec02200642963597b9b1bc22c75e9f3e117284a962188bf5e8a74c895089046a20ad770121035509a48eb623e10aace8bfd0212fdb8a8e5af3c94b0b133b95e114cab89e4f7965000000"), SER_NETWORK, PROTOCOL_VERSION); @@ -68,6 +68,15 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) ssTx << psbtx; std::string final_hex = HexStr(ssTx.begin(), ssTx.end()); BOOST_CHECK_EQUAL(final_hex, "70736274ff01009a020000000258e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd750000000000ffffffff838d0427d0ec650a68aa46bb0b098aea4422c071b2ca78352a077959d07cea1d0100000000ffffffff0270aaf00800000000160014d85c2b71d0060b09c9886aeb815e50991dda124d00e1f5050000000016001400aea9a2e5f0f876a588df5546e8742d1d87008f00000000000100bb0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f6187650000000104475221029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f2102dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d752ae2206029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f10d90c6a4f000000800000008000000080220602dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d710d90c6a4f0000008000000080010000800001012000c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88701042200208c2353173743b595dfb4a07b72ba8e42e3797da74e87fe7d9d7497e3b2028903010547522103089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc21023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7352ae2206023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7310d90c6a4f000000800000008003000080220603089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc10d90c6a4f00000080000000800200008000220203a9a4c37f5996d3aa25dbac6b570af0650394492942460b354753ed9eeca5877110d90c6a4f000000800000008004000080002202027f6399757d2eff55a136ad02c684b1838b6556e5f1b6b34282a94b6b5005109610d90c6a4f00000080000000800500008000"); + + // Mutate the transaction so that one of the inputs is invalid + psbtx.tx->vin[0].prevout.n = 2; + + // Try to sign the mutated input + SignatureData sigdata; + psbtx.inputs[0].FillSignatureData(sigdata); + const std::unique_ptr<SigningProvider> provider = m_wallet.GetSigningProvider(ws1, sigdata); + BOOST_CHECK(!SignPSBTInput(*provider, psbtx, 0, SIGHASH_ALL)); } BOOST_AUTO_TEST_CASE(parse_hd_keypath) diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 2f21b2439b..a487e9e2e0 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -28,9 +28,8 @@ BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) static void AddKey(CWallet& wallet, const CKey& key) { - auto spk_man = wallet.GetLegacyScriptPubKeyMan(); - LOCK(wallet.cs_wallet); - AssertLockHeld(spk_man->cs_wallet); + auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); + LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); spk_man->AddKeyPubKey(key, key.GetPubKey()); } @@ -152,6 +151,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) // after. { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + wallet->SetupLegacyScriptPubKeyMan(); AddWallet(wallet); UniValue keys; keys.setArray(); @@ -216,9 +216,8 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Import key into wallet and call dumpwallet to create backup file. { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - auto spk_man = wallet->GetLegacyScriptPubKeyMan(); - LOCK(wallet->cs_wallet); - AssertLockHeld(spk_man->cs_wallet); + auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); + LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); spk_man->mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); @@ -234,6 +233,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // were scanned, and no prior blocks were scanned. { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + wallet->SetupLegacyScriptPubKeyMan(); JSONRPCRequest request; request.params.setArray(); @@ -267,13 +267,12 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) auto chain = interfaces::MakeChain(node); CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - auto spk_man = wallet.GetLegacyScriptPubKeyMan(); + auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); CWalletTx wtx(&wallet, m_coinbase_txns.back()); auto locked_chain = chain->lock(); LockAssertion lock(::cs_main); - LOCK(wallet.cs_wallet); - AssertLockHeld(spk_man->cs_wallet); + LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, ::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash(), 0); @@ -283,7 +282,7 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) // cache the current immature credit amount, which is 0. BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(), 0); - // Invalidate the cached vanue, add the key, and make sure a new immature + // Invalidate the cached value, add the key, and make sure a new immature // credit amount is calculated. wtx.MarkDirty(); BOOST_CHECK(spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey())); @@ -377,7 +376,7 @@ static void TestWatchOnlyPubKey(LegacyScriptPubKeyMan* spk_man, const CPubKey& a CScript p2pk = GetScriptForRawPubKey(add_pubkey); CKeyID add_address = add_pubkey.GetID(); CPubKey found_pubkey; - LOCK(spk_man->cs_wallet); + LOCK(spk_man->cs_KeyStore); // all Scripts (i.e. also all PubKeys) are added to the general watch-only set BOOST_CHECK(!spk_man->HaveWatchOnly(p2pk)); @@ -394,7 +393,6 @@ static void TestWatchOnlyPubKey(LegacyScriptPubKeyMan* spk_man, const CPubKey& a BOOST_CHECK(found_pubkey == CPubKey()); // passed key is unchanged } - AssertLockHeld(spk_man->cs_wallet); spk_man->RemoveWatchOnly(p2pk); BOOST_CHECK(!spk_man->HaveWatchOnly(p2pk)); @@ -419,7 +417,7 @@ BOOST_AUTO_TEST_CASE(WatchOnlyPubKeys) { CKey key; CPubKey pubkey; - LegacyScriptPubKeyMan* spk_man = m_wallet.GetLegacyScriptPubKeyMan(); + LegacyScriptPubKeyMan* spk_man = m_wallet.GetOrCreateLegacyScriptPubKeyMan(); BOOST_CHECK(!spk_man->HaveWatchOnly()); @@ -581,6 +579,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) NodeContext node; auto chain = interfaces::MakeChain(node); std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + wallet->SetupLegacyScriptPubKeyMan(); wallet->SetMinVersion(FEATURE_LATEST); wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); BOOST_CHECK(!wallet->TopUpKeyPool(1000)); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 724997a36d..4a38571dfc 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -56,6 +56,7 @@ bool AddWallet(const std::shared_ptr<CWallet>& wallet) std::vector<std::shared_ptr<CWallet>>::const_iterator i = std::find(vpwallets.begin(), vpwallets.end(), wallet); if (i != vpwallets.end()) return false; vpwallets.push_back(wallet); + wallet->ConnectScriptPubKeyManNotifiers(); return true; } @@ -219,7 +220,8 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& // Set a seed for the wallet { - if (auto spk_man = wallet->m_spk_man.get()) { + LOCK(wallet->cs_wallet); + for (auto spk_man : wallet->GetActiveScriptPubKeyMans()) { if (!spk_man->SetupGeneration()) { error = "Unable to generate initial keys"; return WalletCreationStatus::CREATION_FAILED; @@ -237,7 +239,7 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& return WalletCreationStatus::SUCCESS; } -const uint256 CWalletTx::ABANDON_HASH(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); +const uint256 CWalletTx::ABANDON_HASH(UINT256_ONE()); /** @defgroup mapWallet * @@ -264,10 +266,12 @@ void CWallet::UpgradeKeyMetadata() return; } - if (m_spk_man) { - AssertLockHeld(m_spk_man->cs_wallet); - m_spk_man->UpgradeKeyMetadata(); + auto spk_man = GetLegacyScriptPubKeyMan(); + if (!spk_man) { + return; } + + spk_man->UpgradeKeyMetadata(); SetWalletFlag(WALLET_FLAG_KEY_ORIGIN_METADATA); } @@ -548,7 +552,8 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) } encrypted_batch->WriteMasterKey(nMasterKeyMaxID, kMasterKey); - if (auto spk_man = m_spk_man.get()) { + for (const auto& spk_man_pair : m_spk_managers) { + auto spk_man = spk_man_pair.second.get(); if (!spk_man->Encrypt(_vMasterKey, encrypted_batch)) { encrypted_batch->TxnAbort(); delete encrypted_batch; @@ -577,7 +582,7 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) Unlock(strWalletPassphrase); // if we are using HD, replace the HD seed with a new one - if (auto spk_man = m_spk_man.get()) { + if (auto spk_man = GetLegacyScriptPubKeyMan()) { if (spk_man->IsHDEnabled()) { if (!spk_man->SetupGeneration(true)) { return false; @@ -922,8 +927,8 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Co // loop though all outputs for (const CTxOut& txout: tx.vout) { - if (auto spk_man = m_spk_man.get()) { - spk_man->MarkUnusedAddresses(txout.scriptPubKey); + for (const auto& spk_man_pair : m_spk_managers) { + spk_man_pair.second->MarkUnusedAddresses(txout.scriptPubKey); } } @@ -1194,8 +1199,8 @@ isminetype CWallet::IsMine(const CTxDestination& dest) const isminetype CWallet::IsMine(const CScript& script) const { isminetype result = ISMINE_NO; - if (auto spk_man = m_spk_man.get()) { - result = spk_man->IsMine(script); + for (const auto& spk_man_pair : m_spk_managers) { + result = std::max(result, spk_man_pair.second->IsMine(script)); } return result; } @@ -1314,16 +1319,18 @@ CAmount CWallet::GetChange(const CTransaction& tx) const bool CWallet::IsHDEnabled() const { bool result = true; - if (auto spk_man = m_spk_man.get()) { - result &= spk_man->IsHDEnabled(); + for (const auto& spk_man_pair : m_spk_managers) { + result &= spk_man_pair.second->IsHDEnabled(); } return result; } bool CWallet::CanGetAddresses(bool internal) { - { - auto spk_man = m_spk_man.get(); + LOCK(cs_wallet); + if (m_spk_managers.empty()) return false; + for (OutputType t : OUTPUT_TYPES) { + auto spk_man = GetScriptPubKeyMan(t, internal); if (spk_man && spk_man->CanGetAddresses(internal)) { return true; } @@ -1392,7 +1399,7 @@ bool CWallet::DummySignInput(CTxIn &tx_in, const CTxOut &txout, bool use_max_sig const CScript& scriptPubKey = txout.scriptPubKey; SignatureData sigdata; - const SigningProvider* provider = GetSigningProvider(scriptPubKey); + std::unique_ptr<SigningProvider> provider = GetSigningProvider(scriptPubKey); if (!provider) { // We don't know about this scriptpbuKey; return false; @@ -1427,7 +1434,7 @@ bool CWallet::ImportScripts(const std::set<CScript> scripts, int64_t timestamp) if (!spk_man) { return false; } - AssertLockHeld(spk_man->cs_wallet); + LOCK(spk_man->cs_KeyStore); return spk_man->ImportScripts(scripts, timestamp); } @@ -1437,7 +1444,7 @@ bool CWallet::ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const in if (!spk_man) { return false; } - AssertLockHeld(spk_man->cs_wallet); + LOCK(spk_man->cs_KeyStore); return spk_man->ImportPrivKeys(privkey_map, timestamp); } @@ -1447,7 +1454,7 @@ bool CWallet::ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const st if (!spk_man) { return false; } - AssertLockHeld(spk_man->cs_wallet); + LOCK(spk_man->cs_KeyStore); return spk_man->ImportPubKeys(ordered_pubkeys, pubkey_map, key_origins, add_keypool, internal, timestamp); } @@ -1457,7 +1464,7 @@ bool CWallet::ImportScriptPubKeys(const std::string& label, const std::set<CScri if (!spk_man) { return false; } - AssertLockHeld(spk_man->cs_wallet); + LOCK(spk_man->cs_KeyStore); if (!spk_man->ImportScriptPubKeys(script_pub_keys, have_solving_data, timestamp)) { return false; } @@ -2156,7 +2163,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< continue; } - const SigningProvider* provider = GetSigningProvider(wtx.tx->vout[i].scriptPubKey); + std::unique_ptr<SigningProvider> provider = GetSigningProvider(wtx.tx->vout[i].scriptPubKey); bool solvable = provider ? IsSolvable(*provider, wtx.tx->vout[i].scriptPubKey) : false; bool spendable = ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || (((mine & ISMINE_WATCH_ONLY) != ISMINE_NO) && (coinControl && coinControl->fAllowWatchOnly && solvable)); @@ -2410,7 +2417,7 @@ bool CWallet::SignTransaction(CMutableTransaction& tx) const CAmount& amount = mi->second.tx->vout[input.prevout.n].nValue; SignatureData sigdata; - const SigningProvider* provider = GetSigningProvider(scriptPubKey); + std::unique_ptr<SigningProvider> provider = GetSigningProvider(scriptPubKey); if (!provider) { // We don't know about this scriptpbuKey; return false; @@ -2879,7 +2886,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std const CScript& scriptPubKey = coin.txout.scriptPubKey; SignatureData sigdata; - const SigningProvider* provider = GetSigningProvider(scriptPubKey); + std::unique_ptr<SigningProvider> provider = GetSigningProvider(scriptPubKey); if (!provider || !ProduceSignature(*provider, MutableTransactionSignatureCreator(&txNew, nIn, coin.txout.nValue, SIGHASH_ALL), scriptPubKey, sigdata)) { strFailReason = _("Signing transaction failed").translated; @@ -2986,17 +2993,17 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) { if (database->Rewrite("\x04pool")) { - if (auto spk_man = m_spk_man.get()) { - spk_man->RewriteDB(); + for (const auto& spk_man_pair : m_spk_managers) { + spk_man_pair.second->RewriteDB(); } } } - { - LOCK(cs_KeyStore); - // This wallet is in its first run if all of these are empty - fFirstRunRet = mapKeys.empty() && mapCryptedKeys.empty() && mapWatchKeys.empty() && setWatchOnly.empty() && mapScripts.empty() - && !IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET); + // This wallet is in its first run if there are no ScriptPubKeyMans and it isn't blank or no privkeys + fFirstRunRet = m_spk_managers.empty() && !IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET); + if (fFirstRunRet) { + assert(m_external_spk_managers.empty()); + assert(m_internal_spk_managers.empty()); } if (nLoadWalletRet != DBErrors::LOAD_OK) @@ -3020,8 +3027,8 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 { if (database->Rewrite("\x04pool")) { - if (auto spk_man = m_spk_man.get()) { - spk_man->RewriteDB(); + for (const auto& spk_man_pair : m_spk_managers) { + spk_man_pair.second->RewriteDB(); } } } @@ -3041,8 +3048,8 @@ DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx) { if (database->Rewrite("\x04pool")) { - if (auto spk_man = m_spk_man.get()) { - spk_man->RewriteDB(); + for (const auto& spk_man_pair : m_spk_managers) { + spk_man_pair.second->RewriteDB(); } } } @@ -3102,8 +3109,7 @@ size_t CWallet::KeypoolCountExternalKeys() AssertLockHeld(cs_wallet); unsigned int count = 0; - if (auto spk_man = m_spk_man.get()) { - AssertLockHeld(spk_man->cs_wallet); + for (auto spk_man : GetActiveScriptPubKeyMans()) { count += spk_man->KeypoolCountExternalKeys(); } @@ -3115,7 +3121,7 @@ unsigned int CWallet::GetKeyPoolSize() const AssertLockHeld(cs_wallet); unsigned int count = 0; - if (auto spk_man = m_spk_man.get()) { + for (auto spk_man : GetActiveScriptPubKeyMans()) { count += spk_man->GetKeyPoolSize(); } return count; @@ -3123,8 +3129,9 @@ unsigned int CWallet::GetKeyPoolSize() const bool CWallet::TopUpKeyPool(unsigned int kpSize) { + LOCK(cs_wallet); bool res = true; - if (auto spk_man = m_spk_man.get()) { + for (auto spk_man : GetActiveScriptPubKeyMans()) { res &= spk_man->TopUp(kpSize); } return res; @@ -3135,7 +3142,7 @@ bool CWallet::GetNewDestination(const OutputType type, const std::string label, LOCK(cs_wallet); error.clear(); bool result = false; - auto spk_man = m_spk_man.get(); + auto spk_man = GetScriptPubKeyMan(type, false /* internal */); if (spk_man) { spk_man->TopUp(); result = spk_man->GetNewDestination(type, dest, error); @@ -3149,6 +3156,7 @@ bool CWallet::GetNewDestination(const OutputType type, const std::string label, bool CWallet::GetNewChangeDestination(const OutputType type, CTxDestination& dest, std::string& error) { + LOCK(cs_wallet); error.clear(); ReserveDestination reservedest(this, type); @@ -3163,9 +3171,10 @@ bool CWallet::GetNewChangeDestination(const OutputType type, CTxDestination& des int64_t CWallet::GetOldestKeyPoolTime() { + LOCK(cs_wallet); int64_t oldestKey = std::numeric_limits<int64_t>::max(); - if (auto spk_man = m_spk_man.get()) { - oldestKey = spk_man->GetOldestKeyPoolTime(); + for (const auto& spk_man_pair : m_spk_managers) { + oldestKey = std::min(oldestKey, spk_man_pair.second->GetOldestKeyPoolTime()); } return oldestKey; } @@ -3334,7 +3343,7 @@ std::set<CTxDestination> CWallet::GetLabelAddresses(const std::string& label) co bool ReserveDestination::GetReservedDestination(CTxDestination& dest, bool internal) { - m_spk_man = pwallet->GetLegacyScriptPubKeyMan(); + m_spk_man = pwallet->GetScriptPubKeyMan(type, internal); if (!m_spk_man) { return false; } @@ -3416,7 +3425,7 @@ void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<C LegacyScriptPubKeyMan* spk_man = GetLegacyScriptPubKeyMan(); assert(spk_man != nullptr); - AssertLockHeld(spk_man->cs_wallet); + LOCK(spk_man->cs_KeyStore); // get birth times for keys with metadata for (const auto& entry : spk_man->mapKeyMetadata) { @@ -3711,7 +3720,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, return nullptr; } - if (auto spk_man = walletInstance->m_spk_man.get()) { + for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { if (!spk_man->Upgrade(prev_version, error)) { return nullptr; } @@ -3724,8 +3733,13 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, walletInstance->SetMinVersion(FEATURE_LATEST); walletInstance->SetWalletFlags(wallet_creation_flags, false); + + // Always create LegacyScriptPubKeyMan for now + walletInstance->SetupLegacyScriptPubKeyMan(); + if (!(wallet_creation_flags & (WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET))) { - if (auto spk_man = walletInstance->m_spk_man.get()) { + LOCK(walletInstance->cs_wallet); + for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { if (!spk_man->SetupGeneration()) { error = _("Unable to generate initial keys").translated; return nullptr; @@ -3740,9 +3754,10 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, error = strprintf(_("Error loading %s: Private keys can only be disabled during creation").translated, walletFile); return NULL; } else if (walletInstance->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - if (walletInstance->m_spk_man) { - if (walletInstance->m_spk_man->HavePrivateKeys()) { + for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { + if (spk_man->HavePrivateKeys()) { warnings.push_back(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys").translated, walletFile)); + break; } } } @@ -3896,7 +3911,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // No need to read and scan block if block was created before // our wallet birthday (as adjusted for block time variability) Optional<int64_t> time_first_key; - if (auto spk_man = walletInstance->m_spk_man.get()) { + for (auto spk_man : walletInstance->GetAllScriptPubKeyMans()) { int64_t time = spk_man->GetTimeFirstKey(); if (!time_first_key || time < *time_first_key) time_first_key = time; } @@ -4064,7 +4079,7 @@ bool CWallet::IsLocked() const if (!IsCrypted()) { return false; } - LOCK(cs_KeyStore); + LOCK(cs_wallet); return vMasterKey.empty(); } @@ -4074,7 +4089,7 @@ bool CWallet::Lock() return false; { - LOCK(cs_KeyStore); + LOCK(cs_wallet); vMasterKey.clear(); } @@ -4085,9 +4100,9 @@ bool CWallet::Lock() bool CWallet::Unlock(const CKeyingMaterial& vMasterKeyIn, bool accept_no_keys) { { - LOCK(cs_KeyStore); - if (m_spk_man) { - if (!m_spk_man->CheckDecryptionKey(vMasterKeyIn, accept_no_keys)) { + LOCK(cs_wallet); + for (const auto& spk_man_pair : m_spk_managers) { + if (!spk_man_pair.second->CheckDecryptionKey(vMasterKeyIn, accept_no_keys)) { return false; } } @@ -4097,24 +4112,102 @@ bool CWallet::Unlock(const CKeyingMaterial& vMasterKeyIn, bool accept_no_keys) return true; } +std::set<ScriptPubKeyMan*> CWallet::GetActiveScriptPubKeyMans() const +{ + std::set<ScriptPubKeyMan*> spk_mans; + for (bool internal : {false, true}) { + for (OutputType t : OUTPUT_TYPES) { + auto spk_man = GetScriptPubKeyMan(t, internal); + if (spk_man) { + spk_mans.insert(spk_man); + } + } + } + return spk_mans; +} + +std::set<ScriptPubKeyMan*> CWallet::GetAllScriptPubKeyMans() const +{ + std::set<ScriptPubKeyMan*> spk_mans; + for (const auto& spk_man_pair : m_spk_managers) { + spk_mans.insert(spk_man_pair.second.get()); + } + return spk_mans; +} + +ScriptPubKeyMan* CWallet::GetScriptPubKeyMan(const OutputType& type, bool internal) const +{ + const std::map<OutputType, ScriptPubKeyMan*>& spk_managers = internal ? m_internal_spk_managers : m_external_spk_managers; + std::map<OutputType, ScriptPubKeyMan*>::const_iterator it = spk_managers.find(type); + if (it == spk_managers.end()) { + WalletLogPrintf("%s scriptPubKey Manager for output type %d does not exist\n", internal ? "Internal" : "External", static_cast<int>(type)); + return nullptr; + } + return it->second; +} + ScriptPubKeyMan* CWallet::GetScriptPubKeyMan(const CScript& script) const { - return m_spk_man.get(); + SignatureData sigdata; + for (const auto& spk_man_pair : m_spk_managers) { + if (spk_man_pair.second->CanProvide(script, sigdata)) { + return spk_man_pair.second.get(); + } + } + return nullptr; } -const SigningProvider* CWallet::GetSigningProvider(const CScript& script) const +ScriptPubKeyMan* CWallet::GetScriptPubKeyMan(const uint256& id) const { - return m_spk_man.get(); + if (m_spk_managers.count(id) > 0) { + return m_spk_managers.at(id).get(); + } + return nullptr; } -const SigningProvider* CWallet::GetSigningProvider(const CScript& script, SignatureData& sigdata) const +std::unique_ptr<SigningProvider> CWallet::GetSigningProvider(const CScript& script) const { - return m_spk_man.get(); + SignatureData sigdata; + return GetSigningProvider(script, sigdata); +} + +std::unique_ptr<SigningProvider> CWallet::GetSigningProvider(const CScript& script, SignatureData& sigdata) const +{ + for (const auto& spk_man_pair : m_spk_managers) { + if (spk_man_pair.second->CanProvide(script, sigdata)) { + return spk_man_pair.second->GetSigningProvider(script); + } + } + return nullptr; } LegacyScriptPubKeyMan* CWallet::GetLegacyScriptPubKeyMan() const { - return m_spk_man.get(); + // Legacy wallets only have one ScriptPubKeyMan which is a LegacyScriptPubKeyMan. + // Everything in m_internal_spk_managers and m_external_spk_managers point to the same legacyScriptPubKeyMan. + auto it = m_internal_spk_managers.find(OutputType::LEGACY); + if (it == m_internal_spk_managers.end()) return nullptr; + return dynamic_cast<LegacyScriptPubKeyMan*>(it->second); +} + +LegacyScriptPubKeyMan* CWallet::GetOrCreateLegacyScriptPubKeyMan() +{ + SetupLegacyScriptPubKeyMan(); + return GetLegacyScriptPubKeyMan(); +} + +void CWallet::SetupLegacyScriptPubKeyMan() +{ + if (!m_internal_spk_managers.empty() || !m_external_spk_managers.empty() || !m_spk_managers.empty()) { + return; + } + + auto spk_manager = std::unique_ptr<ScriptPubKeyMan>(new LegacyScriptPubKeyMan(*this)); + for (const auto& type : OUTPUT_TYPES) { + m_internal_spk_managers[type] = spk_manager.get(); + m_external_spk_managers[type] = spk_manager.get(); + } + m_spk_managers[spk_manager->GetID()] = std::move(spk_manager); } const CKeyingMaterial& CWallet::GetEncryptionKey() const @@ -4126,3 +4219,11 @@ bool CWallet::HasEncryptionKeys() const { return !mapMasterKeys.empty(); } + +void CWallet::ConnectScriptPubKeyManNotifiers() +{ + for (const auto& spk_man : GetActiveScriptPubKeyMans()) { + spk_man->NotifyWatchonlyChanged.connect(NotifyWatchonlyChanged); + spk_man->NotifyCanGetAddressesChanged.connect(NotifyCanGetAddressesChanged); + } +} diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 44bfa20612..a918bb8833 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -606,7 +606,7 @@ class WalletRescanReserver; //forward declarations for ScanForWalletTransactions class CWallet final : public WalletStorage, private interfaces::Chain::Notifications { private: - CKeyingMaterial vMasterKey GUARDED_BY(cs_KeyStore); + CKeyingMaterial vMasterKey GUARDED_BY(cs_wallet); bool Unlock(const CKeyingMaterial& vMasterKeyIn, bool accept_no_keys = false); @@ -702,6 +702,13 @@ private: */ int m_last_block_processed_height GUARDED_BY(cs_wallet) = -1; + std::map<OutputType, ScriptPubKeyMan*> m_external_spk_managers; + std::map<OutputType, ScriptPubKeyMan*> m_internal_spk_managers; + + // Indexed by a unique identifier produced by each ScriptPubKeyMan using + // ScriptPubKeyMan::GetID. In many cases it will be the hash of an internal structure + std::map<uint256, std::unique_ptr<ScriptPubKeyMan>> m_spk_managers; + public: /* * Main wallet lock. @@ -1132,28 +1139,34 @@ public: LogPrintf(("%s " + fmt).c_str(), GetDisplayName(), parameters...); }; + //! Returns all unique ScriptPubKeyMans in m_internal_spk_managers and m_external_spk_managers + std::set<ScriptPubKeyMan*> GetActiveScriptPubKeyMans() const; + + //! Returns all unique ScriptPubKeyMans + std::set<ScriptPubKeyMan*> GetAllScriptPubKeyMans() const; + + //! Get the ScriptPubKeyMan for the given OutputType and internal/external chain. + ScriptPubKeyMan* GetScriptPubKeyMan(const OutputType& type, bool internal) const; + //! Get the ScriptPubKeyMan for a script ScriptPubKeyMan* GetScriptPubKeyMan(const CScript& script) const; + //! Get the ScriptPubKeyMan by id + ScriptPubKeyMan* GetScriptPubKeyMan(const uint256& id) const; //! Get the SigningProvider for a script - const SigningProvider* GetSigningProvider(const CScript& script) const; - const SigningProvider* GetSigningProvider(const CScript& script, SignatureData& sigdata) const; + std::unique_ptr<SigningProvider> GetSigningProvider(const CScript& script) const; + std::unique_ptr<SigningProvider> GetSigningProvider(const CScript& script, SignatureData& sigdata) const; + //! Get the LegacyScriptPubKeyMan which is used for all types, internal, and external. LegacyScriptPubKeyMan* GetLegacyScriptPubKeyMan() const; + LegacyScriptPubKeyMan* GetOrCreateLegacyScriptPubKeyMan(); + + //! Make a LegacyScriptPubKeyMan and set it for all types, internal, and external. + void SetupLegacyScriptPubKeyMan(); const CKeyingMaterial& GetEncryptionKey() const override; bool HasEncryptionKeys() const override; - // Temporary LegacyScriptPubKeyMan accessors and aliases. - friend class LegacyScriptPubKeyMan; - std::unique_ptr<LegacyScriptPubKeyMan> m_spk_man = MakeUnique<LegacyScriptPubKeyMan>(*this); - RecursiveMutex& cs_KeyStore = m_spk_man->cs_KeyStore; - LegacyScriptPubKeyMan::KeyMap& mapKeys GUARDED_BY(cs_KeyStore) = m_spk_man->mapKeys; - LegacyScriptPubKeyMan::ScriptMap& mapScripts GUARDED_BY(cs_KeyStore) = m_spk_man->mapScripts; - LegacyScriptPubKeyMan::CryptedKeyMap& mapCryptedKeys GUARDED_BY(cs_KeyStore) = m_spk_man->mapCryptedKeys; - LegacyScriptPubKeyMan::WatchOnlySet& setWatchOnly GUARDED_BY(cs_KeyStore) = m_spk_man->setWatchOnly; - LegacyScriptPubKeyMan::WatchKeyMap& mapWatchKeys GUARDED_BY(cs_KeyStore) = m_spk_man->mapWatchKeys; - /** Get last block processed height */ int GetLastBlockHeight() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { @@ -1168,6 +1181,9 @@ public: m_last_block_processed_height = block_height; m_last_block_processed = block_hash; }; + + //! Connect the signals from ScriptPubKeyMans to the signals in CWallet + void ConnectScriptPubKeyManNotifiers(); }; /** diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 7d04b04764..a1928f45c4 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -196,7 +196,7 @@ public: static bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, - CWalletScanState &wss, std::string& strType, std::string& strErr) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet, pwallet->GetLegacyScriptPubKeyMan()->cs_wallet) + CWalletScanState &wss, std::string& strType, std::string& strErr) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { try { // Unserialize @@ -251,7 +251,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, char fYes; ssValue >> fYes; if (fYes == '1') { - pwallet->GetLegacyScriptPubKeyMan()->LoadWatchOnly(script); + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadWatchOnly(script); } } else if (strType == DBKeys::KEY) { CPubKey vchPubKey; @@ -303,7 +303,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, strErr = "Error reading wallet database: CPrivKey corrupt"; return false; } - if (!pwallet->GetLegacyScriptPubKeyMan()->LoadKey(key, vchPubKey)) + if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKey(key, vchPubKey)) { strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadKey failed"; return false; @@ -334,7 +334,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, ssValue >> vchPrivKey; wss.nCKeys++; - if (!pwallet->GetLegacyScriptPubKeyMan()->LoadCryptedKey(vchPubKey, vchPrivKey)) + if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadCryptedKey(vchPubKey, vchPrivKey)) { strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadCryptedKey failed"; return false; @@ -346,14 +346,14 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, CKeyMetadata keyMeta; ssValue >> keyMeta; wss.nKeyMeta++; - pwallet->GetLegacyScriptPubKeyMan()->LoadKeyMetadata(vchPubKey.GetID(), keyMeta); + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKeyMetadata(vchPubKey.GetID(), keyMeta); } else if (strType == DBKeys::WATCHMETA) { CScript script; ssKey >> script; CKeyMetadata keyMeta; ssValue >> keyMeta; wss.nKeyMeta++; - pwallet->GetLegacyScriptPubKeyMan()->LoadScriptMetadata(CScriptID(script), keyMeta); + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadScriptMetadata(CScriptID(script), keyMeta); } else if (strType == DBKeys::DEFAULTKEY) { // We don't want or need the default key, but if there is one set, // we want to make sure that it is valid so that we can detect corruption @@ -369,13 +369,13 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, CKeyPool keypool; ssValue >> keypool; - pwallet->GetLegacyScriptPubKeyMan()->LoadKeyPool(nIndex, keypool); + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKeyPool(nIndex, keypool); } else if (strType == DBKeys::CSCRIPT) { uint160 hash; ssKey >> hash; CScript script; ssValue >> script; - if (!pwallet->GetLegacyScriptPubKeyMan()->LoadCScript(script)) + if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadCScript(script)) { strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadCScript failed"; return false; @@ -391,7 +391,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, } else if (strType == DBKeys::HDCHAIN) { CHDChain chain; ssValue >> chain; - pwallet->GetLegacyScriptPubKeyMan()->SetHDChain(chain, true); + pwallet->GetOrCreateLegacyScriptPubKeyMan()->SetHDChain(chain, true); } else if (strType == DBKeys::FLAGS) { uint64_t flags; ssValue >> flags; @@ -434,7 +434,6 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) DBErrors result = DBErrors::LOAD_OK; LOCK(pwallet->cs_wallet); - AssertLockHeld(pwallet->GetLegacyScriptPubKeyMan()->cs_wallet); try { int nMinVersion = 0; if (m_batch.Read(DBKeys::MINVERSION, nMinVersion)) { @@ -516,8 +515,9 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) // nTimeFirstKey is only reliable if all keys have metadata if ((wss.nKeys + wss.nCKeys + wss.nWatchKeys) != wss.nKeyMeta) { - auto spk_man = pwallet->GetLegacyScriptPubKeyMan(); + auto spk_man = pwallet->GetOrCreateLegacyScriptPubKeyMan(); if (spk_man) { + LOCK(spk_man->cs_KeyStore); spk_man->UpdateTimeFirstKey(1); } } @@ -713,7 +713,6 @@ bool WalletBatch::RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, C { // Required in LoadKeyMetadata(): LOCK(dummyWallet->cs_wallet); - AssertLockHeld(dummyWallet->GetLegacyScriptPubKeyMan()->cs_wallet); fReadOK = ReadKeyValue(dummyWallet, ssKey, ssValue, dummyWss, strType, strErr); } diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index dc0cac60bd..fbfdf9dd6b 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -27,6 +27,7 @@ static std::shared_ptr<CWallet> CreateWallet(const std::string& name, const fs:: } // dummy chain interface std::shared_ptr<CWallet> wallet_instance(new CWallet(nullptr /* chain */, WalletLocation(name), WalletDatabase::Create(path)), WalletToolReleaseWallet); + LOCK(wallet_instance->cs_wallet); bool first_run = true; DBErrors load_wallet_ret = wallet_instance->LoadWallet(first_run); if (load_wallet_ret != DBErrors::LOAD_OK) { @@ -37,7 +38,7 @@ static std::shared_ptr<CWallet> CreateWallet(const std::string& name, const fs:: wallet_instance->SetMinVersion(FEATURE_HD_SPLIT); // generate a new HD seed - auto spk_man = wallet_instance->GetLegacyScriptPubKeyMan(); + auto spk_man = wallet_instance->GetOrCreateLegacyScriptPubKeyMan(); CPubKey seed = spk_man->GenerateNewSeed(); spk_man->SetHDSeed(seed); diff --git a/test/functional/feature_abortnode.py b/test/functional/feature_abortnode.py index 62c3eca07d..80c3cab5e1 100755 --- a/test/functional/feature_abortnode.py +++ b/test/functional/feature_abortnode.py @@ -40,7 +40,7 @@ class AbortNodeTest(BitcoinTestFramework): # Check that node0 aborted self.log.info("Waiting for crash") - wait_until(lambda: self.nodes[0].is_node_stopped(), timeout=60) + wait_until(lambda: self.nodes[0].is_node_stopped(), timeout=200) self.log.info("Node crashed - now verifying restart fails") self.nodes[0].assert_start_raises_init_error() diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index c7e98bd4db..95905f477b 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -1401,7 +1401,7 @@ class FullBlockTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() self.bootstrap_p2p(timeout=timeout) - def send_blocks(self, blocks, success=True, reject_reason=None, force_send=False, reconnect=False, timeout=60): + def send_blocks(self, blocks, success=True, reject_reason=None, force_send=False, reconnect=False, timeout=960): """Sends blocks to test node. Syncs and verifies that tip has advanced to most recent block. Call with success = False if the tip shouldn't advance to the most recent block.""" diff --git a/test/functional/feature_help.py b/test/functional/feature_help.py index ed1d25c0d6..e3e2456183 100755 --- a/test/functional/feature_help.py +++ b/test/functional/feature_help.py @@ -17,7 +17,7 @@ class HelpTest(BitcoinTestFramework): # Don't start the node def get_node_output(self, *, ret_code_expected): - ret_code = self.nodes[0].process.wait(timeout=5) + ret_code = self.nodes[0].process.wait(timeout=60) assert_equal(ret_code, ret_code_expected) self.nodes[0].stdout.seek(0) self.nodes[0].stderr.seek(0) diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index 123f0b4c28..3b148d5cf0 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -76,7 +76,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): spend_101_id = self.nodes[0].sendrawtransaction(spend_101_raw) spend_102_1_id = self.nodes[0].sendrawtransaction(spend_102_1_raw) - self.sync_all() + self.sync_all(timeout=360) assert_equal(set(self.nodes[0].getrawmempool()), {spend_101_id, spend_102_1_id, timelock_tx_id}) @@ -91,7 +91,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): for node in self.nodes: node.invalidateblock(new_blocks[0]) - self.sync_all() + self.sync_all(timeout=360) # mempool should be empty. assert_equal(set(self.nodes[0].getrawmempool()), set()) diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 07eacf410d..9876d749ff 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -78,7 +78,7 @@ class InvalidMessagesTest(BitcoinTestFramework): # Peer 1, despite serving up a bunch of nonsense, should still be connected. self.log.info("Waiting for node to drop junk messages.") - node.p2p.sync_with_ping(timeout=320) + node.p2p.sync_with_ping(timeout=400) assert node.p2p.is_connected # diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 2cc9650cb2..33af819d34 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -422,5 +422,20 @@ class PSBTTest(BitcoinTestFramework): assert_equal(analysis['next'], 'creator') assert_equal(analysis['error'], 'PSBT is not valid. Input 0 spends unspendable output') + self.log.info("PSBT with invalid values should have error message and Creator as next") + analysis = self.nodes[0].analyzepsbt('cHNidP8BAHECAAAAAfA00BFgAm6tp86RowwH6BMImQNL5zXUcTT97XoLGz0BAAAAAAD/////AgD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XL87QKVAAAAABYAFPck4gF7iL4NL4wtfRAKgQbghiTUAAAAAAABAR8AgIFq49AHABYAFJUDtxf2PHo641HEOBOAIvFMNTr2AAAA') + assert_equal(analysis['next'], 'creator') + assert_equal(analysis['error'], 'PSBT is not valid. Input 0 has invalid value') + + analysis = self.nodes[0].analyzepsbt('cHNidP8BAHECAAAAAfA00BFgAm6tp86RowwH6BMImQNL5zXUcTT97XoLGz0BAAAAAAD/////AgCAgWrj0AcAFgAUKNw0x8HRctAgmvoevm4u1SbN7XL87QKVAAAAABYAFPck4gF7iL4NL4wtfRAKgQbghiTUAAAAAAABAR8A8gUqAQAAABYAFJUDtxf2PHo641HEOBOAIvFMNTr2AAAA') + assert_equal(analysis['next'], 'creator') + assert_equal(analysis['error'], 'PSBT is not valid. Output amount invalid') + + analysis = self.nodes[0].analyzepsbt('cHNidP8BAJoCAAAAAkvEW8NnDtdNtDpsmze+Ht2LH35IJcKv00jKAlUs21RrAwAAAAD/////S8Rbw2cO1020OmybN74e3Ysffkglwq/TSMoCVSzbVGsBAAAAAP7///8CwLYClQAAAAAWABSNJKzjaUb3uOxixsvh1GGE3fW7zQD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XIAAAAAAAEAnQIAAAACczMa321tVHuN4GKWKRncycI22aX3uXgwSFUKM2orjRsBAAAAAP7///9zMxrfbW1Ue43gYpYpGdzJwjbZpfe5eDBIVQozaiuNGwAAAAAA/v///wIA+QKVAAAAABl2qRT9zXUVA8Ls5iVqynLHe5/vSe1XyYisQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAAAAAQEfQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAA==') + assert_equal(analysis['next'], 'creator') + assert_equal(analysis['error'], 'PSBT is not valid. Input 0 specifies invalid prevout') + + assert_raises_rpc_error(-25, 'Missing inputs', self.nodes[0].walletprocesspsbt, 'cHNidP8BAJoCAAAAAkvEW8NnDtdNtDpsmze+Ht2LH35IJcKv00jKAlUs21RrAwAAAAD/////S8Rbw2cO1020OmybN74e3Ysffkglwq/TSMoCVSzbVGsBAAAAAP7///8CwLYClQAAAAAWABSNJKzjaUb3uOxixsvh1GGE3fW7zQD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XIAAAAAAAEAnQIAAAACczMa321tVHuN4GKWKRncycI22aX3uXgwSFUKM2orjRsBAAAAAP7///9zMxrfbW1Ue43gYpYpGdzJwjbZpfe5eDBIVQozaiuNGwAAAAAA/v///wIA+QKVAAAAABl2qRT9zXUVA8Ls5iVqynLHe5/vSe1XyYisQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAAAAAQEfQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAA==') + if __name__ == '__main__': PSBTTest().main() diff --git a/test/functional/rpc_setban.py b/test/functional/rpc_setban.py index b1d2b6f431..1cc1fb164b 100755 --- a/test/functional/rpc_setban.py +++ b/test/functional/rpc_setban.py @@ -26,7 +26,7 @@ class SetBanTests(BitcoinTestFramework): self.nodes[1].setban("127.0.0.1", "add") # Node 0 should not be able to reconnect - with self.nodes[1].assert_debug_log(expected_msgs=['dropped (banned)\n'], timeout=5): + with self.nodes[1].assert_debug_log(expected_msgs=['dropped (banned)\n'], timeout=50): self.restart_node(1, []) self.nodes[0].addnode("127.0.0.1:" + str(p2p_port(1)), "onetry") diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index 53edf710b9..a39dfc7895 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -137,7 +137,7 @@ class WalletDumpTest(BitcoinTestFramework): # encrypt wallet, restart, unlock and dump self.nodes[0].encryptwallet('test') - self.nodes[0].walletpassphrase('test', 10) + self.nodes[0].walletpassphrase('test', 100) # Should be a no-op: self.nodes[0].keypoolrefill() self.nodes[0].dumpwallet(wallet_enc_dump) diff --git a/test/functional/wallet_groups.py b/test/functional/wallet_groups.py index 3cf8aaf3dc..f2fa1d3e40 100755 --- a/test/functional/wallet_groups.py +++ b/test/functional/wallet_groups.py @@ -16,7 +16,7 @@ class WalletGroupTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 3 self.extra_args = [[], [], ['-avoidpartialspends']] - self.rpc_timeout = 120 + self.rpc_timeout = 240 def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index 9027311a8b..6b687060e2 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -39,6 +39,7 @@ FUZZERS_MISSING_CORPORA = [ "psbt_output_deserialize", "pub_key_deserialize", "script_deserialize", + "strprintf", "sub_net_deserialize", "tx_in", "tx_in_deserialize", diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index ee17e7912d..e769039682 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -25,7 +25,6 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "wallet/fees -> wallet/wallet -> wallet/fees" "wallet/wallet -> wallet/walletdb -> wallet/wallet" "policy/fees -> txmempool -> validation -> policy/fees" - "wallet/scriptpubkeyman -> wallet/wallet -> wallet/scriptpubkeyman" ) EXIT_CODE=0 diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index cc24a0b609..2870432bff 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -20,10 +20,10 @@ FALSE_POSITIVES = [ ("src/validationinterface.cpp", "LogPrint(BCLog::VALIDATION, fmt \"\\n\", __VA_ARGS__)"), ("src/wallet/wallet.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), ("src/wallet/wallet.h", "LogPrintf((\"%s \" + fmt).c_str(), GetDisplayName(), parameters...)"), + ("src/wallet/scriptpubkeyman.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), + ("src/wallet/scriptpubkeyman.h", "LogPrintf((\"%s \" + fmt).c_str(), m_storage.GetDisplayName(), parameters...)"), ("src/logging.h", "LogPrintf(const char* fmt, const Args&... args)"), ("src/wallet/scriptpubkeyman.h", "WalletLogPrintf(const std::string& fmt, const Params&... parameters)"), - ("src/wallet/scriptpubkeyman.cpp", "WalletLogPrintf(fmt, parameters...)"), - ("src/wallet/scriptpubkeyman.cpp", "WalletLogPrintf(const std::string& fmt, const Params&... parameters)"), ] diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh index 6cb486689b..184c3682c8 100755 --- a/test/lint/lint-format-strings.sh +++ b/test/lint/lint-format-strings.sh @@ -34,7 +34,7 @@ if ! python3 -m doctest test/lint/lint-format-strings.py; then fi for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do IFS="," read -r FUNCTION_NAME SKIP_ARGUMENTS <<< "${S}" - for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|tinyformat|univalue)"); do + for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do MATCHING_FILES+=("${MATCHING_FILE}") done if ! test/lint/lint-format-strings.py --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then |