diff options
75 files changed, 1527 insertions, 350 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 31c3aba8c6..cb6a920060 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,12 +7,12 @@ clone_depth: 5 environment: APPVEYOR_SAVE_CACHE_ON_ERROR: true CLCACHE_SERVER: 1 - PACKAGES: boost-filesystem boost-signals2 boost-test libevent openssl zeromq berkeleydb + PACKAGES: berkeleydb boost-filesystem boost-signals2 boost-test libevent openssl rapidcheck zeromq PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%' PYTHONUTF8: 1 cache: -- C:\tools\vcpkg\installed -- C:\Users\appveyor\clcache +- C:\tools\vcpkg\installed -> appveyor.yml +- C:\Users\appveyor\clcache -> appveyor.yml, build_msvc\**, **\Makefile.am, **\*.vcxproj.in install: - cmd: pip install --quiet git+https://github.com/frerich/clcache.git@v4.2.0 # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. @@ -28,7 +28,6 @@ before_build: ${content} = (Get-Content ${files}[${i}]); ${content} = ${content}.Replace("</RuntimeLibrary>", "</RuntimeLibrary><DebugInformationFormat>None</DebugInformationFormat>"); ${content} = ${content}.Replace("<WholeProgramOptimization>true", "<WholeProgramOptimization>false"); - ${content} = ${content}.Replace("NDEBUG;", ""); Set-Content ${files}[${i}] ${content}; } - ps: Start-Process clcache-server diff --git a/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj b/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj index 32f0354fad..6c82b1e7de 100644 --- a/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj +++ b/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj @@ -111,7 +111,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -163,7 +163,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj b/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj index 86fd614990..c52dfdb28c 100644 --- a/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj +++ b/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj @@ -114,7 +114,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -166,7 +166,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj index 12fda65904..a83529c782 100644 --- a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj +++ b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj @@ -132,7 +132,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -184,7 +184,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/bitcoind/bitcoind.vcxproj b/build_msvc/bitcoind/bitcoind.vcxproj index 9a071f3bdd..9a42f141c9 100644 --- a/build_msvc/bitcoind/bitcoind.vcxproj +++ b/build_msvc/bitcoind/bitcoind.vcxproj @@ -113,7 +113,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -136,7 +136,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/libbitcoin_cli/libbitcoin_cli.vcxproj.in b/build_msvc/libbitcoin_cli/libbitcoin_cli.vcxproj.in index b7265054fb..1b24acd8ce 100644 --- a/build_msvc/libbitcoin_cli/libbitcoin_cli.vcxproj.in +++ b/build_msvc/libbitcoin_cli/libbitcoin_cli.vcxproj.in @@ -97,7 +97,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -146,7 +146,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in b/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in index b51cdb2991..459e81eb0c 100644 --- a/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in +++ b/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in @@ -127,7 +127,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <SuppressStartupBanner>false</SuppressStartupBanner> @@ -149,7 +149,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <SuppressStartupBanner>false</SuppressStartupBanner> diff --git a/build_msvc/libbitcoin_crypto/libbitcoin_crypto.vcxproj.in b/build_msvc/libbitcoin_crypto/libbitcoin_crypto.vcxproj.in index a05125723a..ad183d4904 100644 --- a/build_msvc/libbitcoin_crypto/libbitcoin_crypto.vcxproj.in +++ b/build_msvc/libbitcoin_crypto/libbitcoin_crypto.vcxproj.in @@ -119,7 +119,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -138,7 +138,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj index 43bfb271fb..73ba90aa88 100644 --- a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj +++ b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj @@ -873,7 +873,7 @@ <Optimization>Disabled</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>false</IntrinsicFunctions> - <PreprocessorDefinitions>_X86_;WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_X86_;WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>.\GeneratedFiles;..\..\src;..\..\src\univalue\include;.\QtGenerated\mocheaders</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -922,7 +922,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>.\GeneratedFiles;..\..\src;..\..\src\univalue\include;.\QtGenerated\mocheaders</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in index dc17c98e98..855c9353fc 100644 --- a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in +++ b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in @@ -132,7 +132,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -156,7 +156,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in b/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in index 1bb7be6f7f..b19f1e2396 100644 --- a/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in +++ b/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in @@ -129,7 +129,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\wallet;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -149,7 +149,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\wallet;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj.in b/build_msvc/libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj.in index b9cf68aee0..2e32c25762 100644 --- a/build_msvc/libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj.in +++ b/build_msvc/libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj.in @@ -129,7 +129,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -149,7 +149,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/libbitcoin_zmq/libbitcoin_zmq.vcxproj.in b/build_msvc/libbitcoin_zmq/libbitcoin_zmq.vcxproj.in index e396c1ad0c..7aba987cd1 100644 --- a/build_msvc/libbitcoin_zmq/libbitcoin_zmq.vcxproj.in +++ b/build_msvc/libbitcoin_zmq/libbitcoin_zmq.vcxproj.in @@ -129,7 +129,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -149,7 +149,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj b/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj index 961225f9ba..32ea070a05 100644 --- a/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj +++ b/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj @@ -160,7 +160,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>false</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <ExceptionHandling>Sync</ExceptionHandling> @@ -181,7 +181,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>false</SDLCheck> <AdditionalIncludeDirectories>..\..\src;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <ExceptionHandling>Sync</ExceptionHandling> diff --git a/build_msvc/libleveldb/libleveldb.vcxproj b/build_msvc/libleveldb/libleveldb.vcxproj index 6de3a2b282..545508001e 100644 --- a/build_msvc/libleveldb/libleveldb.vcxproj +++ b/build_msvc/libleveldb/libleveldb.vcxproj @@ -162,7 +162,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;NOMINMAX;LEVELDB_PLATFORM_WINDOWS;LEVELDB_ATOMIC_PRESENT;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;NOMINMAX;LEVELDB_PLATFORM_WINDOWS;LEVELDB_ATOMIC_PRESENT;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src\leveldb;..\..\src\leveldb\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -181,7 +181,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;NOMINMAX;LEVELDB_PLATFORM_WINDOWS;LEVELDB_ATOMIC_PRESENT;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;NOMINMAX;LEVELDB_PLATFORM_WINDOWS;LEVELDB_ATOMIC_PRESENT;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src\leveldb;..\..\src\leveldb\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/libsecp256k1/libsecp256k1.vcxproj b/build_msvc/libsecp256k1/libsecp256k1.vcxproj index 699c7d6871..b4c9ec28ee 100644 --- a/build_msvc/libsecp256k1/libsecp256k1.vcxproj +++ b/build_msvc/libsecp256k1/libsecp256k1.vcxproj @@ -122,7 +122,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>ENABLE_MODULE_ECDH;ENABLE_MODULE_RECOVERY;HAVE_CONFIG_H;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>ENABLE_MODULE_ECDH;ENABLE_MODULE_RECOVERY;HAVE_CONFIG_H;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src\secp256k1;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -141,7 +141,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>ENABLE_MODULE_ECDH;ENABLE_MODULE_RECOVERY;HAVE_CONFIG_H;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>ENABLE_MODULE_ECDH;ENABLE_MODULE_RECOVERY;HAVE_CONFIG_H;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src\secp256k1;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/libunivalue/libunivalue.vcxproj b/build_msvc/libunivalue/libunivalue.vcxproj index 57d469debf..c3799b6408 100644 --- a/build_msvc/libunivalue/libunivalue.vcxproj +++ b/build_msvc/libunivalue/libunivalue.vcxproj @@ -125,7 +125,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -144,7 +144,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index b2d4c118f3..a5d666c114 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -89,7 +89,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> </ClCompile> <Link> @@ -132,7 +132,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> </ClCompile> <Link> diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index 8f39132a22..03db97c647 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -21,6 +21,8 @@ </ItemGroup> <ItemGroup> <ClCompile Include="..\..\src\test\*_tests.cpp" /> + <ClCompile Include="..\..\src\test\*_properties.cpp" /> + <ClCompile Include="..\..\src\test\gen\*_gen.cpp" /> <ClCompile Include="..\..\src\wallet\test\*_tests.cpp" /> <ClCompile Include="..\..\src\test\test_bitcoin.cpp" /> <ClCompile Include="..\..\src\test\test_bitcoin_main.cpp" /> diff --git a/doc/README.md b/doc/README.md index 51950d4a13..8876ffdd72 100644 --- a/doc/README.md +++ b/doc/README.md @@ -51,6 +51,7 @@ Development The Bitcoin repo's [root README](/README.md) contains relevant information on the development process and automated testing. - [Developer Notes](developer-notes.md) +- [Productivity Notes](productivity.md) - [Release Notes](release-notes.md) - [Release Process](release-process.md) - [Source Code Documentation (External Link)](https://dev.visucore.com/bitcoin/doxygen/) diff --git a/doc/REST-interface.md b/doc/REST-interface.md index d21df36130..02a665008b 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -20,7 +20,8 @@ Supported API Given a transaction hash: returns a transaction in binary, hex-encoded binary, or JSON formats. -For full TX query capability, one must enable the transaction index via "txindex=1" command line / configuration option. +By default, this endpoint will only search the mempool. +To query for a confirmed transaction, enable the transaction index via "txindex=1" command line / configuration option. #### Blocks `GET /rest/block/<BLOCK-HASH>.<bin|hex|json>` diff --git a/doc/descriptors.md b/doc/descriptors.md index de4d4e574f..5dbcd95e1d 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -39,7 +39,7 @@ Output descriptors currently support: ## Reference -Descriptors consist of several types of expressions. The top level expression is always a `SCRIPT`. +Descriptors consist of several types of expressions. The top level expression is either a `SCRIPT`, or `SCRIPT#CHECKSUM` where `CHECKSUM` is an 8-character alphanumeric descriptor checksum. `SCRIPT` expressions: - `sh(SCRIPT)` (top level only): P2SH embed the argument. @@ -169,3 +169,20 @@ existing Bitcoin Core wallets, a convenience function `combo` is provided, which takes as input a public key, and describes a set of P2PK, P2PKH, P2WPKH, and P2SH-P2WPH scripts for that key. In case the key is uncompressed, the set only includes P2PK and P2PKH scripts. + +### Checksums + +Descriptors can optionally be suffixed with a checksum to protect against +typos or copy-paste errors. + +These checksums consist of 8 alphanumeric characters. As long as errors are +restricted to substituting characters in `0123456789()[],'/*abcdefgh@:$%{}` +for others in that set and changes in letter case, up to 4 errors will always +be detected in descriptors up to 501 characters, and up to 3 errors in longer +ones. For larger numbers of errors, or other types of errors, there is a +roughly 1 in a trillion chance of not detecting the errors. + +All RPCs in Bitcoin Core will include the checksum in their output. Only +certain RPCs require checksums on input, including `deriveaddress` and +`importmulti`. The checksum for a descriptor without one can be computed +using the `getdescriptorinfo` RPC. diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 1deb5d791a..f765346cd8 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -844,54 +844,6 @@ would be to revert the upstream fix before applying the updates to Bitcoin's copy of LevelDB. In general you should be wary of any upstream changes affecting what data is returned from LevelDB queries. -Git and GitHub tips ---------------------- - -- For resolving merge/rebase conflicts, it can be useful to enable diff3 style using - `git config merge.conflictstyle diff3`. Instead of - - <<< - yours - === - theirs - >>> - - you will see - - <<< - yours - ||| - original - === - theirs - >>> - - This may make it much clearer what caused the conflict. In this style, you can often just look - at what changed between *original* and *theirs*, and mechanically apply that to *yours* (or the other way around). - -- When reviewing patches which change indentation in C++ files, use `git diff -w` and `git show -w`. This makes - the diff algorithm ignore whitespace changes. This feature is also available on github.com, by adding `?w=1` - at the end of any URL which shows a diff. - -- When reviewing patches that change symbol names in many places, use `git diff --word-diff`. This will instead - of showing the patch as deleted/added *lines*, show deleted/added *words*. - -- When reviewing patches that move code around, try using - `git diff --patience commit~:old/file.cpp commit:new/file/name.cpp`, and ignoring everything except the - moved body of code which should show up as neither `+` or `-` lines. In case it was not a pure move, this may - even work when combined with the `-w` or `--word-diff` options described above. - -- When looking at other's pull requests, it may make sense to add the following section to your `.git/config` - file: - - [remote "upstream-pull"] - fetch = +refs/pull/*:refs/remotes/upstream-pull/* - url = git@github.com:bitcoin/bitcoin.git - - This will add an `upstream-pull` remote to your git repository, which can be fetched using `git fetch --all` - or `git fetch upstream-pull`. Afterwards, you can use `upstream-pull/NUMBER/head` in arguments to `git show`, - `git checkout` and anywhere a commit id would be acceptable to see the changes from pull request NUMBER. - Scripted diffs -------------- diff --git a/doc/productivity.md b/doc/productivity.md new file mode 100644 index 0000000000..862017290d --- /dev/null +++ b/doc/productivity.md @@ -0,0 +1,161 @@ +Productivity Notes +================== + +Table of Contents +----------------- + +* [General](#general) + * [Cache compilations with `ccache`](#cache-compilations-with-ccache) + * [Disable features with `./configure`](#disable-features-with-configure) + * [Make use of your threads with `make -j`](#make-use-of-your-threads-with-make--j) + * [Multiple working directories with `git worktrees`](#multiple-working-directories-with-git-worktrees) +* [Writing code](#writing-code) + * [Format C/C++/Protobuf diffs with `clang-format-diff.py`](#format-ccprotobuf-diffs-with-clang-format-diffpy) + * [Format Python diffs with `yapf-diff.py`](#format-python-diffs-with-yapf-diffpy) +* [Rebasing/Merging code](#rebasingmerging-code) + * [More conflict context with `merge.conflictstyle diff3`](#more-conflict-context-with-mergeconflictstyle-diff3) +* [Reviewing code](#reviewing-code) + * [Reduce mental load with `git diff` options](#reduce-mental-load-with-git-diff-options) + * [Reference PRs easily with `refspec`s](#reference-prs-easily-with-refspecs) + * [Diff the diffs with `git range-diff`](#diff-the-diffs-with-git-range-diff) + +General +------ + +### Cache compilations with `ccache` + +The easiest way to faster compile times is to cache compiles. `ccache` is a way to do so, from its description at the time of writing: + +> ccache is a compiler cache. It speeds up recompilation by caching the result of previous compilations and detecting when the same compilation is being done again. Supported languages are C, C++, Objective-C and Objective-C++. + +Install `ccache` through your distribution's package manager, and run `./configure` with your normal flags to pick it up. + +To use ccache for all your C/C++ projects, follow the symlinks method [here](https://ccache.samba.org/manual/latest.html#_run_modes) to set it up. + +### Disable features with `./configure` + +After running `./autogen.sh`, which generates the `./configure` file, use `./configure --help` to identify features that you can disable to save on compilation time. A few common flags: + +```sh +--without-miniupnpc +--disable-bench +--disable-wallet +--without-gui +``` + +### Make use of your threads with `make -j` + +If you have multiple threads on your machine, you can tell `make` to utilize all of them with: + +```sh +make -j"$(($(nproc)+1))" +``` + +### Multiple working directories with `git worktrees` + +If you work with multiple branches or multiple copies of the repository, you should try `git worktrees`. + +To create a new branch that lives under a new working directory without disrupting your current working directory (useful for creating pull requests): +```sh +git worktree add -b my-shiny-new-branch ../living-at-my-new-working-directory based-on-my-crufty-old-commit-ish +``` + +To simply check out a commit-ish under a new working directory without disrupting your current working directory (useful for reviewing pull requests): +```sh +git worktree add --checkout ../where-my-checkout-commit-ish-will-live my-checkout-commit-ish +``` + +----- + +This synergizes well with [`ccache`](#cache-compilations-with-ccache) as objects resulting from unchanged code will most likely hit the cache and won't need to be recompiled. + +You can also set up [upstream refspecs](#reference-prs-easily-with-refspecs) to refer to pull requests easier in the above `git worktree` commands. + +Writing code +------------ + +### Format C/C++/Protobuf diffs with `clang-format-diff.py` + +See [contrib/devtools/README.md](contrib/devtools/README.md#clang-format-diff.py). + +### Format Python diffs with `yapf-diff.py` + +Usage is exactly the same as [`clang-format-diff.py`](#format-ccprotobuf-diffs-with-clang-format-diffpy). You can get it [here](https://github.com/MarcoFalke/yapf-diff). + +Rebasing/Merging code +------------- + +### More conflict context with `merge.conflictstyle diff3` + +For resolving merge/rebase conflicts, it can be useful to enable diff3 style using `git config merge.conflictstyle diff3`. Instead of + +```diff +<<< +yours +=== +theirs +>>> +``` + + you will see + +```diff +<<< +yours +||| +original +=== +theirs +>>> +``` + +This may make it much clearer what caused the conflict. In this style, you can often just look at what changed between *original* and *theirs*, and mechanically apply that to *yours* (or the other way around). + +Reviewing code +-------------- + +### Reduce mental load with `git diff` options + +When reviewing patches which change indentation in C++ files, use `git diff -w` and `git show -w`. This makes the diff algorithm ignore whitespace changes. This feature is also available on github.com, by adding `?w=1` at the end of any URL which shows a diff. + +When reviewing patches that change symbol names in many places, use `git diff --word-diff`. This will instead of showing the patch as deleted/added *lines*, show deleted/added *words*. + +When reviewing patches that move code around, try using `git diff --patience commit~:old/file.cpp commit:new/file/name.cpp`, and ignoring everything except the moved body of code which should show up as neither `+` or `-` lines. In case it was not a pure move, this may even work when combined with the `-w` or `--word-diff` options described above. `--color-moved=dimmed-zebra` will also dim the coloring of moved hunks in the diff on compatible terminals. + +### Reference PRs easily with `refspec`s + +When looking at other's pull requests, it may make sense to add the following section to your `.git/config` file: + +``` +[remote "upstream-pull"] + fetch = +refs/pull/*:refs/remotes/upstream-pull/* + url = git@github.com:bitcoin/bitcoin.git +``` + +This will add an `upstream-pull` remote to your git repository, which can be fetched using `git fetch --all` or `git fetch upstream-pull`. Afterwards, you can use `upstream-pull/NUMBER/head` in arguments to `git show`, `git checkout` and anywhere a commit id would be acceptable to see the changes from pull request NUMBER. + +### Diff the diffs with `git range-diff` + +It is very common for contributors to rebase their pull requests, or make changes to commits (perhaps in response to review) that are not at the head of their branch. This poses a problem for reviewers as when the contributor force pushes, the reviewer is no longer sure that his previous reviews of commits are still valid (as the commit hashes can now be different even though the diff is semantically the same). `git range-diff` can help solve this problem by diffing the diffs. + +For example, to identify the differences between your previously reviewed diffs P1-5, and the new diffs P1-2,N3-4 as illustrated below: +``` + P1--P2--P3--P4--P5 <-- previously-reviewed-head + / +...--m <-- master + \ + P1--P2--N3--N4--N5 <-- new-head (with P3 slightly modified) +``` + +You can do: +```sh +git range-diff master previously-reviewed-head new-head +``` + +Note that `git range-diff` also work for rebases. + +----- + +`git range-diff` also accepts normal `git diff` options, see [Reduce mental load with `git diff` options](#reduce-mental-load-with-git-diff-options) for useful `git diff` options. + +You can also set up [upstream refspecs](#reference-prs-easily-with-refspecs) to refer to pull requests easier in the above `git range-diff` commands. diff --git a/doc/release-notes.md b/doc/release-notes.md index 113b8c07d0..a6408cf1e6 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -263,6 +263,9 @@ in the Low-level Changes section below. - See the [Mining](#mining) section for changes to `getblocktemplate`. +- The `getmininginfo` RPC now omits `currentblockweight` and `currentblocktx` + when a block was never assembled via RPC on this node. + - The `getrawtransaction` RPC & REST endpoints no longer check the unspent UTXO set for a transaction. The remaining behaviors are as follows: 1. If a blockhash is provided, check the corresponding block. diff --git a/src/Makefile.am b/src/Makefile.am index e3f5734c0b..d491530ca1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -118,6 +118,7 @@ BITCOIN_CORE_H = \ clientversion.h \ coins.h \ compat.h \ + compat/assumptions.h \ compat/byteswap.h \ compat/endian.h \ compat/sanity.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index f40a205a73..84bc326cfe 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -20,6 +20,7 @@ FUZZ_TARGETS = \ test/fuzz/inv_deserialize \ test/fuzz/messageheader_deserialize \ test/fuzz/netaddr_deserialize \ + test/fuzz/script_flags \ test/fuzz/service_deserialize \ test/fuzz/transaction_deserialize \ test/fuzz/txoutcompressor_deserialize \ @@ -172,7 +173,7 @@ test_test_bitcoin_LDADD += $(ZMQ_LIBS) endif if ENABLE_FUZZ -test_fuzz_block_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_block_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_block_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_DESERIALIZE=1 test_fuzz_block_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_block_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -189,7 +190,7 @@ test_fuzz_block_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_block_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_transaction_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_transaction_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_transaction_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTRANSACTION_DESERIALIZE=1 test_fuzz_transaction_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -206,7 +207,7 @@ test_fuzz_transaction_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_transaction_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_blocklocator_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blocklocator_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blocklocator_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKLOCATOR_DESERIALIZE=1 test_fuzz_blocklocator_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocklocator_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -223,7 +224,7 @@ test_fuzz_blocklocator_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_blocklocator_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_blockmerkleroot_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blockmerkleroot_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockmerkleroot_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKMERKLEROOT=1 test_fuzz_blockmerkleroot_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockmerkleroot_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -240,7 +241,7 @@ test_fuzz_blockmerkleroot_LDADD = \ $(LIBSECP256K1) test_fuzz_blockmerkleroot_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_addrman_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_addrman_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_addrman_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRMAN_DESERIALIZE=1 test_fuzz_addrman_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_addrman_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -257,7 +258,7 @@ test_fuzz_addrman_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_addrman_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_blockheader_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blockheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKHEADER_DESERIALIZE=1 test_fuzz_blockheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -274,7 +275,7 @@ test_fuzz_blockheader_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_blockheader_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_banentry_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_banentry_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_banentry_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBANENTRY_DESERIALIZE=1 test_fuzz_banentry_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_banentry_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -291,7 +292,7 @@ test_fuzz_banentry_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_banentry_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_txundo_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_txundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_txundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXUNDO_DESERIALIZE=1 test_fuzz_txundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_txundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -308,7 +309,7 @@ test_fuzz_txundo_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_txundo_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_blockundo_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blockundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKUNDO_DESERIALIZE=1 test_fuzz_blockundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -325,7 +326,7 @@ test_fuzz_blockundo_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_blockundo_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_coins_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_coins_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_coins_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DCOINS_DESERIALIZE=1 test_fuzz_coins_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_coins_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -342,7 +343,7 @@ test_fuzz_coins_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_coins_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_netaddr_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_netaddr_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_netaddr_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DNETADDR_DESERIALIZE=1 test_fuzz_netaddr_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_netaddr_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -359,7 +360,24 @@ test_fuzz_netaddr_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_netaddr_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_service_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_script_flags_SOURCES = $(FUZZ_SUITE) test/fuzz/script_flags.cpp +test_fuzz_script_flags_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_script_flags_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_script_flags_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_flags_LDADD = \ + $(LIBUNIVALUE) \ + $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_COMMON) \ + $(LIBBITCOIN_UTIL) \ + $(LIBBITCOIN_CONSENSUS) \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBBITCOIN_CRYPTO_SSE41) \ + $(LIBBITCOIN_CRYPTO_AVX2) \ + $(LIBBITCOIN_CRYPTO_SHANI) \ + $(LIBSECP256K1) +test_fuzz_script_flags_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) + +test_fuzz_service_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_service_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSERVICE_DESERIALIZE=1 test_fuzz_service_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_service_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -376,7 +394,7 @@ test_fuzz_service_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_service_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_messageheader_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_messageheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_messageheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGEHEADER_DESERIALIZE=1 test_fuzz_messageheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_messageheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -393,7 +411,7 @@ test_fuzz_messageheader_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_messageheader_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_address_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_address_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_address_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRESS_DESERIALIZE=1 test_fuzz_address_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_address_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -410,7 +428,7 @@ test_fuzz_address_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_address_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_inv_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_inv_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_inv_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DINV_DESERIALIZE=1 test_fuzz_inv_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_inv_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -427,7 +445,7 @@ test_fuzz_inv_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_inv_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_bloomfilter_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_bloomfilter_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_bloomfilter_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOOMFILTER_DESERIALIZE=1 test_fuzz_bloomfilter_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_bloomfilter_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -444,7 +462,7 @@ test_fuzz_bloomfilter_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_bloomfilter_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_diskblockindex_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_diskblockindex_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_diskblockindex_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DDISKBLOCKINDEX_DESERIALIZE=1 test_fuzz_diskblockindex_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_diskblockindex_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -461,7 +479,7 @@ test_fuzz_diskblockindex_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_diskblockindex_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_txoutcompressor_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_txoutcompressor_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_txoutcompressor_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXOUTCOMPRESSOR_DESERIALIZE=1 test_fuzz_txoutcompressor_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_txoutcompressor_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -478,7 +496,7 @@ test_fuzz_txoutcompressor_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_txoutcompressor_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_blocktransactions_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blocktransactions_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blocktransactions_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONS_DESERIALIZE=1 test_fuzz_blocktransactions_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocktransactions_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -495,7 +513,7 @@ test_fuzz_blocktransactions_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_blocktransactions_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_blocktransactionsrequest_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blocktransactionsrequest_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blocktransactionsrequest_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONSREQUEST_DESERIALIZE=1 test_fuzz_blocktransactionsrequest_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocktransactionsrequest_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 8133fd9d65..cd4543c012 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -73,10 +73,12 @@ static void AssembleBlock(benchmark::State& state) boost::thread_group thread_group; CScheduler scheduler; { + LOCK(cs_main); ::pblocktree.reset(new CBlockTreeDB(1 << 20, true)); ::pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); ::pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); - + } + { const CChainParams& chainparams = Params(); thread_group.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); diff --git a/src/bench/duplicate_inputs.cpp b/src/bench/duplicate_inputs.cpp index e0854e2c62..1f6840d813 100644 --- a/src/bench/duplicate_inputs.cpp +++ b/src/bench/duplicate_inputs.cpp @@ -37,10 +37,12 @@ static void DuplicateInputs(benchmark::State& state) CScheduler scheduler; const CChainParams& chainparams = Params(); { + LOCK(cs_main); ::pblocktree.reset(new CBlockTreeDB(1 << 20, true)); ::pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); ::pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); - + } + { thread_group.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); LoadGenesisBlock(chainparams); diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 97b0e05b20..b8e0ea23dd 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -91,10 +91,10 @@ public: consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 1510704000; // November 15th, 2017. // The best chain should have at least this much work. - consensus.nMinimumChainWork = uint256S("0x0000000000000000000000000000000000000000028822fef1c230963535a90d"); + consensus.nMinimumChainWork = uint256S("0x0000000000000000000000000000000000000000051dc8b82f450202ecb3d471"); // By default assume that the signatures in ancestors of this block are valid. - consensus.defaultAssumeValid = uint256S("0x0000000000000000002e63058c023a9a1de233554f28c7b21380b6c9003f36a8"); //534292 + consensus.defaultAssumeValid = uint256S("0x0000000000000000000f1c54590ee18d15ec70e68c8cd4cfbadb1b4f11697eee"); //563378 /** * The message start string is designed to be unlikely to occur in normal data. @@ -162,10 +162,10 @@ public: }; chainTxData = ChainTxData{ - // Data from rpc: getchaintxstats 4096 0000000000000000002e63058c023a9a1de233554f28c7b21380b6c9003f36a8 - /* nTime */ 1532884444, - /* nTxCount */ 331282217, - /* dTxRate */ 2.4 + // Data from rpc: getchaintxstats 4096 0000000000000000000f1c54590ee18d15ec70e68c8cd4cfbadb1b4f11697eee + /* nTime */ 1550374134, + /* nTxCount */ 383732546, + /* dTxRate */ 3.685496590998308 }; /* disable fallback fee on mainnet */ diff --git a/src/compat/assumptions.h b/src/compat/assumptions.h new file mode 100644 index 0000000000..820c9b93d9 --- /dev/null +++ b/src/compat/assumptions.h @@ -0,0 +1,49 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +// Compile-time verification of assumptions we make. + +#ifndef BITCOIN_COMPAT_ASSUMPTIONS_H +#define BITCOIN_COMPAT_ASSUMPTIONS_H + +#include <limits> + +// Assumption: We assume that the macro NDEBUG is not defined. +// Example(s): We use assert(...) extensively with the assumption of it never +// being a noop at runtime. +#if defined(NDEBUG) +# error "Bitcoin cannot be compiled without assertions." +#endif + +// Assumption: We assume the floating-point types to fulfill the requirements of +// IEC 559 (IEEE 754) standard. +// Example(s): Floating-point division by zero in ConnectBlock, CreateTransaction +// and EstimateMedianVal. +static_assert(std::numeric_limits<float>::is_iec559, "IEEE 754 float assumed"); +static_assert(std::numeric_limits<double>::is_iec559, "IEEE 754 double assumed"); + +// Assumption: We assume eight bits per byte (obviously, but remember: don't +// trust -- verify!). +// Example(s): Everywhere :-) +static_assert(std::numeric_limits<unsigned char>::digits == 8, "8-bit byte assumed"); + +// Assumption: We assume floating-point widths. +// Example(s): Type punning in serialization code (ser_{float,double}_to_uint{32,64}). +static_assert(sizeof(float) == 4, "32-bit float assumed"); +static_assert(sizeof(double) == 8, "64-bit double assumed"); + +// Assumption: We assume integer widths. +// Example(s): GetSizeOfCompactSize and WriteCompactSize in the serialization +// code. +static_assert(sizeof(short) == 2, "16-bit short assumed"); +static_assert(sizeof(int) == 4, "32-bit int assumed"); + +// Some important things we are NOT assuming (non-exhaustive list): +// * We are NOT assuming a specific value for sizeof(std::size_t). +// * We are NOT assuming a specific value for std::endian::native. +// * We are NOT assuming a specific value for std::locale("").name(). +// * We are NOT assuming a specific value for std::numeric_limits<char>::is_signed. + +#endif // BITCOIN_COMPAT_ASSUMPTIONS_H diff --git a/src/init.cpp b/src/init.cpp index 0013319ad5..caab4a5674 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -53,6 +53,8 @@ #include <stdio.h> #ifndef WIN32 +#include <attributes.h> +#include <cerrno> #include <signal.h> #include <sys/stat.h> #endif @@ -92,6 +94,30 @@ std::unique_ptr<BanMan> g_banman; static const char* FEE_ESTIMATES_FILENAME="fee_estimates.dat"; +/** + * The PID file facilities. + */ +#ifndef WIN32 +static const char* BITCOIN_PID_FILENAME = "bitcoind.pid"; + +static fs::path GetPidFile() +{ + return AbsPathForConfigVal(fs::path(gArgs.GetArg("-pid", BITCOIN_PID_FILENAME))); +} + +NODISCARD static bool CreatePidFile() +{ + FILE* file = fsbridge::fopen(GetPidFile(), "w"); + if (file) { + fprintf(file, "%d\n", getpid()); + fclose(file); + return true; + } else { + return InitError(strprintf(_("Unable to create the PID file '%s': %s"), GetPidFile().string(), std::strerror(errno))); + } +} +#endif + ////////////////////////////////////////////////////////////////////////////// // // Shutdown @@ -262,9 +288,11 @@ void Shutdown(InitInterfaces& interfaces) #ifndef WIN32 try { - fs::remove(GetPidFile()); + if (!fs::remove(GetPidFile())) { + LogPrintf("%s: Unable to remove PID file: File does not exist\n", __func__); + } } catch (const fs::filesystem_error& e) { - LogPrintf("%s: Unable to remove pidfile: %s\n", __func__, e.what()); + LogPrintf("%s: Unable to remove PID file: %s\n", __func__, e.what()); } #endif interfaces.chain_clients.clear(); @@ -1201,7 +1229,10 @@ bool AppInitMain(InitInterfaces& interfaces) const CChainParams& chainparams = Params(); // ********************************************************* Step 4a: application initialization #ifndef WIN32 - CreatePidFile(GetPidFile(), getpid()); + if (!CreatePidFile()) { + // Detailed error printed inside CreatePidFile(). + return false; + } #endif if (LogInstance().m_print_to_file) { if (gArgs.GetBoolArg("-shrinkdebugfile", LogInstance().DefaultShrinkDebugFile())) { diff --git a/src/miner.cpp b/src/miner.cpp index ef48a86e32..80a2f8f018 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,8 +10,8 @@ #include <chainparams.h> #include <coins.h> #include <consensus/consensus.h> -#include <consensus/tx_verify.h> #include <consensus/merkle.h> +#include <consensus/tx_verify.h> #include <consensus/validation.h> #include <hash.h> #include <net.h> @@ -21,22 +21,14 @@ #include <primitives/transaction.h> #include <script/standard.h> #include <timedata.h> -#include <util/system.h> #include <util/moneystr.h> +#include <util/system.h> #include <validationinterface.h> #include <algorithm> #include <queue> #include <utility> -// Unconfirmed transactions in the memory pool often depend on other -// transactions in the memory pool. When we select transactions from the -// pool, we select by highest fee rate of a transaction combined with all -// its ancestors. - -uint64_t nLastBlockTx = 0; -uint64_t nLastBlockWeight = 0; - int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) { int64_t nOldTime = pblock->nTime; @@ -95,6 +87,9 @@ void BlockAssembler::resetBlock() nFees = 0; } +Optional<int64_t> BlockAssembler::m_last_block_num_txs{nullopt}; +Optional<int64_t> BlockAssembler::m_last_block_weight{nullopt}; + std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& scriptPubKeyIn) { int64_t nTimeStart = GetTimeMicros(); @@ -147,8 +142,8 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc int64_t nTime1 = GetTimeMicros(); - nLastBlockTx = nBlockTx; - nLastBlockWeight = nBlockWeight; + m_last_block_num_txs = nBlockTx; + m_last_block_weight = nBlockWeight; // Create coinbase transaction. CMutableTransaction coinbaseTx; diff --git a/src/miner.h b/src/miner.h index 44c50b01ad..7c4c455072 100644 --- a/src/miner.h +++ b/src/miner.h @@ -1,17 +1,19 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_MINER_H #define BITCOIN_MINER_H +#include <optional.h> #include <primitives/block.h> #include <txmempool.h> #include <validation.h> -#include <stdint.h> #include <memory> +#include <stdint.h> + #include <boost/multi_index_container.hpp> #include <boost/multi_index/ordered_index.hpp> @@ -159,6 +161,9 @@ public: /** Construct a new block template with coinbase to scriptPubKeyIn */ std::unique_ptr<CBlockTemplate> CreateNewBlock(const CScript& scriptPubKeyIn); + static Optional<int64_t> m_last_block_num_txs; + static Optional<int64_t> m_last_block_weight; + private: // utility functions /** Clear the block's state and prepare for assembling a new block */ diff --git a/src/psbt.cpp b/src/psbt.cpp index 81633c0cc7..32fb459dec 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -42,6 +42,40 @@ bool PartiallySignedTransaction::IsSane() const return true; } +bool PartiallySignedTransaction::AddInput(const CTxIn& txin, PSBTInput& psbtin) +{ + if (std::find(tx->vin.begin(), tx->vin.end(), txin) != tx->vin.end()) { + return false; + } + tx->vin.push_back(txin); + psbtin.partial_sigs.clear(); + psbtin.final_script_sig.clear(); + psbtin.final_script_witness.SetNull(); + inputs.push_back(psbtin); + return true; +} + +bool PartiallySignedTransaction::AddOutput(const CTxOut& txout, const PSBTOutput& psbtout) +{ + tx->vout.push_back(txout); + outputs.push_back(psbtout); + return true; +} + +bool PartiallySignedTransaction::GetInputUTXO(CTxOut& utxo, int input_index) const +{ + PSBTInput input = inputs[input_index]; + int prevout_index = tx->vin[input_index].prevout.n; + if (input.non_witness_utxo) { + utxo = input.non_witness_utxo->vout[prevout_index]; + } else if (!input.witness_utxo.IsNull()) { + utxo = input.witness_utxo; + } else { + return false; + } + return true; +} + bool PSBTInput::IsNull() const { return !non_witness_utxo && witness_utxo.IsNull() && partial_sigs.empty() && unknown.empty() && hd_keypaths.empty() && redeem_script.empty() && witness_script.empty(); @@ -171,13 +205,12 @@ void PSBTOutput::Merge(const PSBTOutput& output) if (redeem_script.empty() && !output.redeem_script.empty()) redeem_script = output.redeem_script; if (witness_script.empty() && !output.witness_script.empty()) witness_script = output.witness_script; } - bool PSBTInputSigned(PSBTInput& input) { return !input.final_script_sig.empty() || !input.final_script_witness.IsNull(); } -bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, int sighash) +bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, int sighash, SignatureData* out_sigdata, bool use_dummy) { PSBTInput& input = psbt.inputs.at(index); const CMutableTransaction& tx = *psbt.tx; @@ -217,9 +250,14 @@ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& return false; } - MutableTransactionSignatureCreator creator(&tx, index, utxo.nValue, sighash); sigdata.witness = false; - bool sig_complete = ProduceSignature(provider, creator, utxo.scriptPubKey, sigdata); + bool sig_complete; + if (use_dummy) { + sig_complete = ProduceSignature(provider, DUMMY_SIGNATURE_CREATOR, utxo.scriptPubKey, sigdata); + } else { + MutableTransactionSignatureCreator creator(&tx, index, utxo.nValue, sighash); + sig_complete = ProduceSignature(provider, creator, utxo.scriptPubKey, sigdata); + } // Verify that a witness signature was produced in case one was required. if (require_witness_sig && !sigdata.witness) return false; input.FromSignatureData(sigdata); @@ -230,6 +268,14 @@ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& input.non_witness_utxo = nullptr; } + // Fill in the missing info + if (out_sigdata) { + out_sigdata->missing_pubkeys = sigdata.missing_pubkeys; + out_sigdata->missing_sigs = sigdata.missing_sigs; + out_sigdata->missing_redeem_script = sigdata.missing_redeem_script; + out_sigdata->missing_witness_script = sigdata.missing_witness_script; + } + return sig_complete; } diff --git a/src/psbt.h b/src/psbt.h index e18790322b..27b0aedd05 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -389,9 +389,19 @@ struct PartiallySignedTransaction * same actual Bitcoin transaction.) Returns true if the merge succeeded, false otherwise. */ NODISCARD bool Merge(const PartiallySignedTransaction& psbt); bool IsSane() const; + bool AddInput(const CTxIn& txin, PSBTInput& psbtin); + bool AddOutput(const CTxOut& txout, const PSBTOutput& psbtout); PartiallySignedTransaction() {} PartiallySignedTransaction(const PartiallySignedTransaction& psbt_in) : tx(psbt_in.tx), inputs(psbt_in.inputs), outputs(psbt_in.outputs), unknown(psbt_in.unknown) {} explicit PartiallySignedTransaction(const CMutableTransaction& tx); + /** + * Finds the UTXO for a given input index + * + * @param[out] utxo The UTXO of the input if found + * @param[in] input_index Index of the input to retrieve the UTXO of + * @return Whether the UTXO for the specified input was found + */ + bool GetInputUTXO(CTxOut& utxo, int input_index) const; template <typename Stream> inline void Serialize(Stream& s) const { @@ -542,7 +552,7 @@ struct PartiallySignedTransaction bool PSBTInputSigned(PSBTInput& input); /** Signs a PSBTInput, verifying that all provided data matches what is being signed. */ -bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, int sighash = SIGHASH_ALL); +bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, int sighash = SIGHASH_ALL, SignatureData* out_sigdata = nullptr, bool use_dummy = false); /** * Finalizes a PSBT if possible, combining partial signatures. diff --git a/src/random.cpp b/src/random.cpp index 3277c34d3f..1aa78a9034 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -78,25 +78,119 @@ static inline int64_t GetPerformanceCounter() noexcept } #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) -static bool rdrand_supported = false; +static bool g_rdrand_supported = false; +static bool g_rdseed_supported = false; static constexpr uint32_t CPUID_F1_ECX_RDRAND = 0x40000000; +static constexpr uint32_t CPUID_F7_EBX_RDSEED = 0x00040000; +#ifdef bit_RDRND +static_assert(CPUID_F1_ECX_RDRAND == bit_RDRND, "Unexpected value for bit_RDRND"); +#endif +#ifdef bit_RDSEED +static_assert(CPUID_F7_EBX_RDSEED == bit_RDSEED, "Unexpected value for bit_RDSEED"); +#endif +static void inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) +{ + // We can't use __get_cpuid as it doesn't support subleafs. +#ifdef __GNUC__ + __cpuid_count(leaf, subleaf, a, b, c, d); +#else + __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); +#endif +} + static void InitHardwareRand() { uint32_t eax, ebx, ecx, edx; - if (__get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx & CPUID_F1_ECX_RDRAND)) { - rdrand_supported = true; + GetCPUID(1, 0, eax, ebx, ecx, edx); + if (ecx & CPUID_F1_ECX_RDRAND) { + g_rdrand_supported = true; + } + GetCPUID(7, 0, eax, ebx, ecx, edx); + if (ebx & CPUID_F7_EBX_RDSEED) { + g_rdseed_supported = true; } } static void ReportHardwareRand() { - if (rdrand_supported) { - // This must be done in a separate function, as HWRandInit() may be indirectly called - // from global constructors, before logging is initialized. + // This must be done in a separate function, as HWRandInit() may be indirectly called + // from global constructors, before logging is initialized. + if (g_rdseed_supported) { + LogPrintf("Using RdSeed as additional entropy source\n"); + } + if (g_rdrand_supported) { LogPrintf("Using RdRand as an additional entropy source\n"); } } +/** Read 64 bits of entropy using rdrand. + * + * Must only be called when RdRand is supported. + */ +static uint64_t GetRdRand() noexcept +{ + // RdRand may very rarely fail. Invoke it up to 10 times in a loop to reduce this risk. +#ifdef __i386__ + uint8_t ok; + uint32_t r1, r2; + for (int i = 0; i < 10; ++i) { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdrand %eax + if (ok) break; + } + for (int i = 0; i < 10; ++i) { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r2), "=q"(ok) :: "cc"); // rdrand %eax + if (ok) break; + } + return (((uint64_t)r2) << 32) | r1; +#elif defined(__x86_64__) || defined(__amd64__) + uint8_t ok; + uint64_t r1; + for (int i = 0; i < 10; ++i) { + __asm__ volatile (".byte 0x48, 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdrand %rax + if (ok) break; + } + return r1; +#else +#error "RdRand is only supported on x86 and x86_64" +#endif +} + +/** Read 64 bits of entropy using rdseed. + * + * Must only be called when RdSeed is supported. + */ +static uint64_t GetRdSeed() noexcept +{ + // RdSeed may fail when the HW RNG is overloaded. Loop indefinitely until enough entropy is gathered, + // but pause after every failure. +#ifdef __i386__ + uint8_t ok; + uint32_t r1, r2; + do { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf8; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdseed %eax + if (ok) break; + __asm__ volatile ("pause"); + } while(true); + do { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf8; setc %1" : "=a"(r2), "=q"(ok) :: "cc"); // rdseed %eax + if (ok) break; + __asm__ volatile ("pause"); + } while(true); + return (((uint64_t)r2) << 32) | r1; +#elif defined(__x86_64__) || defined(__amd64__) + uint8_t ok; + uint64_t r1; + do { + __asm__ volatile (".byte 0x48, 0x0f, 0xc7, 0xf8; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdseed %rax + if (ok) break; + __asm__ volatile ("pause"); + } while(true); + return r1; +#else +#error "RdSeed is only supported on x86 and x86_64" +#endif +} + #else /* Access to other hardware random number generators could be added here later, * assuming it is sufficiently fast (in the order of a few hundred CPU cycles). @@ -107,40 +201,40 @@ static void InitHardwareRand() {} static void ReportHardwareRand() {} #endif -static bool GetHardwareRand(unsigned char* ent32) noexcept { +/** Add 64 bits of entropy gathered from hardware to hasher. Do nothing if not supported. */ +static void SeedHardwareFast(CSHA512& hasher) noexcept { #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) - if (rdrand_supported) { - uint8_t ok; - // Not all assemblers support the rdrand instruction, write it in hex. -#ifdef __i386__ - for (int iter = 0; iter < 4; ++iter) { - uint32_t r1, r2; - __asm__ volatile (".byte 0x0f, 0xc7, 0xf0;" // rdrand %eax - ".byte 0x0f, 0xc7, 0xf2;" // rdrand %edx - "setc %2" : - "=a"(r1), "=d"(r2), "=q"(ok) :: "cc"); - if (!ok) return false; - WriteLE32(ent32 + 8 * iter, r1); - WriteLE32(ent32 + 8 * iter + 4, r2); - } -#else - uint64_t r1, r2, r3, r4; - __asm__ volatile (".byte 0x48, 0x0f, 0xc7, 0xf0, " // rdrand %rax - "0x48, 0x0f, 0xc7, 0xf3, " // rdrand %rbx - "0x48, 0x0f, 0xc7, 0xf1, " // rdrand %rcx - "0x48, 0x0f, 0xc7, 0xf2; " // rdrand %rdx - "setc %4" : - "=a"(r1), "=b"(r2), "=c"(r3), "=d"(r4), "=q"(ok) :: "cc"); - if (!ok) return false; - WriteLE64(ent32, r1); - WriteLE64(ent32 + 8, r2); - WriteLE64(ent32 + 16, r3); - WriteLE64(ent32 + 24, r4); + if (g_rdrand_supported) { + uint64_t out = GetRdRand(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + return; + } #endif - return true; +} + +/** Add 256 bits of entropy gathered from hardware to hasher. Do nothing if not supported. */ +static void SeedHardwareSlow(CSHA512& hasher) noexcept { +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) + // When we want 256 bits of entropy, prefer RdSeed over RdRand, as it's + // guaranteed to produce independent randomness on every call. + if (g_rdseed_supported) { + for (int i = 0; i < 4; ++i) { + uint64_t out = GetRdSeed(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + } + return; + } + // When falling back to RdRand, XOR the result of 1024 results. + // This guarantees a reseeding occurs between each. + if (g_rdrand_supported) { + for (int i = 0; i < 4; ++i) { + uint64_t out = 0; + for (int j = 0; j < 1024; ++j) out ^= GetRdRand(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + } + return; } #endif - return false; } static void RandAddSeedPerfmon(CSHA512& hasher) @@ -407,8 +501,7 @@ static void SeedFast(CSHA512& hasher) noexcept hasher.Write((const unsigned char*)&ptr, sizeof(ptr)); // Hardware randomness is very fast when available; use it always. - bool have_hw_rand = GetHardwareRand(buffer); - if (have_hw_rand) hasher.Write(buffer, sizeof(buffer)); + SeedHardwareFast(hasher); // High-precision timestamp SeedTimestamp(hasher); @@ -460,6 +553,9 @@ static void SeedStartup(CSHA512& hasher) noexcept RAND_screen(); #endif + // Gather 256 bits of hardware randomness, if available + SeedHardwareSlow(hasher); + // Everything that the 'slow' seeder includes. SeedSlow(hasher); diff --git a/src/random.h b/src/random.h index 4c73f3822a..1c035f87ba 100644 --- a/src/random.h +++ b/src/random.h @@ -24,7 +24,7 @@ * perform 'fast' seeding, consisting of mixing in: * - A stack pointer (indirectly committing to calling thread and call stack) * - A high-precision timestamp (rdtsc when available, c++ high_resolution_clock otherwise) - * - Hardware RNG (rdrand) when available. + * - 64 bits from the hardware RNG (rdrand) when available. * These entropy sources are very fast, and only designed to protect against situations * where a VM state restore/copy results in multiple systems with the same randomness. * FastRandomContext on the other hand does not protect against this once created, but @@ -48,6 +48,7 @@ * * On first use of the RNG (regardless of what function is called first), all entropy * sources used in the 'slow' seeder are included, but also: + * - 256 bits from the hardware RNG (rdseed or rdrand) when available. * - (On Windows) Performance monitoring data from the OS. * - (On Windows) Through OpenSSL, the screen contents. * diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index c5694e6d55..1cdc9f87a7 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -112,6 +112,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "createpsbt", 2, "locktime" }, { "createpsbt", 3, "replaceable" }, { "combinepsbt", 0, "txs"}, + { "joinpsbts", 0, "txs"}, { "finalizepsbt", 1, "extract"}, { "converttopsbt", 1, "permitsigdata"}, { "converttopsbt", 2, "iswitness"}, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 66a3b1048e..6625a03bbd 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -187,36 +187,36 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) static UniValue getmininginfo(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 0) + if (request.fHelp || request.params.size() != 0) { throw std::runtime_error( RPCHelpMan{"getmininginfo", "\nReturns a json object containing mining-related information.", {}, RPCResult{ - "{\n" - " \"blocks\": nnn, (numeric) The current block\n" - " \"currentblockweight\": nnn, (numeric) The last block weight\n" - " \"currentblocktx\": nnn, (numeric) The last block transaction\n" - " \"difficulty\": xxx.xxxxx (numeric) The current difficulty\n" - " \"networkhashps\": nnn, (numeric) The network hashes per second\n" - " \"pooledtx\": n (numeric) The size of the mempool\n" - " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" - " \"warnings\": \"...\" (string) any network and blockchain warnings\n" - "}\n" + "{\n" + " \"blocks\": nnn, (numeric) The current block\n" + " \"currentblockweight\": nnn, (numeric, optional) The block weight of the last assembled block (only present if a block was ever assembled)\n" + " \"currentblocktx\": nnn, (numeric, optional) The number of block transactions of the last assembled block (only present if a block was ever assembled)\n" + " \"difficulty\": xxx.xxxxx (numeric) The current difficulty\n" + " \"networkhashps\": nnn, (numeric) The network hashes per second\n" + " \"pooledtx\": n (numeric) The size of the mempool\n" + " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" + " \"warnings\": \"...\" (string) any network and blockchain warnings\n" + "}\n" }, RPCExamples{ HelpExampleCli("getmininginfo", "") + HelpExampleRpc("getmininginfo", "") }, }.ToString()); - + } LOCK(cs_main); UniValue obj(UniValue::VOBJ); obj.pushKV("blocks", (int)chainActive.Height()); - obj.pushKV("currentblockweight", (uint64_t)nLastBlockWeight); - obj.pushKV("currentblocktx", (uint64_t)nLastBlockTx); + if (BlockAssembler::m_last_block_weight) obj.pushKV("currentblockweight", *BlockAssembler::m_last_block_weight); + if (BlockAssembler::m_last_block_num_txs) obj.pushKV("currentblocktx", *BlockAssembler::m_last_block_num_txs); obj.pushKV("difficulty", (double)GetDifficulty(chainActive.Tip())); obj.pushKV("networkhashps", getnetworkhashps(request)); obj.pushKV("pooledtx", (uint64_t)mempool.size()); diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 784adfa969..999a307e2b 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -143,6 +143,46 @@ static UniValue createmultisig(const JSONRPCRequest& request) return result; } +UniValue getdescriptorinfo(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 1) { + throw std::runtime_error( + RPCHelpMan{"getdescriptorinfo", + {"\nAnalyses a descriptor.\n"}, + { + {"descriptor", RPCArg::Type::STR, RPCArg::Optional::NO, "The descriptor."}, + }, + RPCResult{ + "{\n" + " \"descriptor\" : \"desc\", (string) The descriptor in canonical form, without private keys\n" + " \"isrange\" : true|false, (boolean) Whether the descriptor is ranged\n" + " \"issolvable\" : true|false, (boolean) Whether the descriptor is solvable\n" + " \"hasprivatekeys\" : true|false, (boolean) Whether the input descriptor contained at least one private key\n" + "}\n" + }, + RPCExamples{ + "Analyse a descriptor\n" + + HelpExampleCli("getdescriptorinfo", "\"wpkh([d34db33f/84h/0h/0h]0279be667ef9dcbbac55a06295Ce870b07029Bfcdb2dce28d959f2815b16f81798)\"") + }}.ToString() + ); + } + + RPCTypeCheck(request.params, {UniValue::VSTR}); + + FlatSigningProvider provider; + auto desc = Parse(request.params[0].get_str(), provider); + if (!desc) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Invalid descriptor")); + } + + UniValue result(UniValue::VOBJ); + result.pushKV("descriptor", desc->ToString()); + result.pushKV("isrange", desc->IsRange()); + result.pushKV("issolvable", desc->IsSolvable()); + result.pushKV("hasprivatekeys", provider.keys.size() > 0); + return result; +} + UniValue deriveaddresses(const JSONRPCRequest& request) { if (request.fHelp || request.params.empty() || request.params.size() > 3) { @@ -167,7 +207,7 @@ UniValue deriveaddresses(const JSONRPCRequest& request) }, RPCExamples{ "First three native segwit receive addresses\n" + - HelpExampleCli("deriveaddresses", "\"wpkh([d34db33f/84h/0h/0h]xpub6DJ2dNUysrn5Vt36jH2KLBT2i1auw1tTSSomg8PhqNiUtx8QX2SvC9nrHu81fT41fvDUnhMjEzQgXnQjKEu3oaqMSzhSrHMxyyoEAmUHQbY/0/*)\" 0 2") + HelpExampleCli("deriveaddresses", "\"wpkh([d34db33f/84h/0h/0h]xpub6DJ2dNUysrn5Vt36jH2KLBT2i1auw1tTSSomg8PhqNiUtx8QX2SvC9nrHu81fT41fvDUnhMjEzQgXnQjKEu3oaqMSzhSrHMxyyoEAmUHQbY/0/*)#trd0mf0l\" 0 2") }}.ToString() ); } @@ -193,7 +233,7 @@ UniValue deriveaddresses(const JSONRPCRequest& request) } FlatSigningProvider provider; - auto desc = Parse(desc_str, provider); + auto desc = Parse(desc_str, provider, /* require_checksum = */ true); if (!desc) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Invalid descriptor")); } @@ -564,6 +604,7 @@ static const CRPCCommand commands[] = { "util", "validateaddress", &validateaddress, {"address"} }, { "util", "createmultisig", &createmultisig, {"nrequired","keys","address_type"} }, { "util", "deriveaddresses", &deriveaddresses, {"descriptor", "begin", "end"} }, + { "util", "getdescriptorinfo", &getdescriptorinfo, {"descriptor"} }, { "util", "verifymessage", &verifymessage, {"address","signature","message"} }, { "util", "signmessagewithprivkey", &signmessagewithprivkey, {"privkey","message"} }, diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 4d8a1b87fc..38e2dc237e 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -7,6 +7,7 @@ #include <coins.h> #include <compat/byteswap.h> #include <consensus/validation.h> +#include <consensus/tx_verify.h> #include <core_io.h> #include <index/txindex.h> #include <init.h> @@ -31,6 +32,8 @@ #include <validation.h> #include <validationinterface.h> + +#include <numeric> #include <stdint.h> #include <univalue.h> @@ -1703,6 +1706,338 @@ UniValue converttopsbt(const JSONRPCRequest& request) return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); } +UniValue utxoupdatepsbt(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 1) { + throw std::runtime_error( + RPCHelpMan{"utxoupdatepsbt", + "\nUpdates a PSBT with witness UTXOs retrieved from the UTXO set or the mempool.\n", + { + {"psbt", RPCArg::Type::STR, RPCArg::Optional::NO, "A base64 string of a PSBT"} + }, + RPCResult { + " \"psbt\" (string) The base64-encoded partially signed transaction with inputs updated\n" + }, + RPCExamples { + HelpExampleCli("utxoupdatepsbt", "\"psbt\"") + }}.ToString()); + } + + RPCTypeCheck(request.params, {UniValue::VSTR}, true); + + // Unserialize the transactions + PartiallySignedTransaction psbtx; + std::string error; + if (!DecodeBase64PSBT(psbtx, request.params[0].get_str(), error)) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("TX decode failed %s", error)); + } + + // Fetch previous transactions (inputs): + CCoinsView viewDummy; + CCoinsViewCache view(&viewDummy); + { + LOCK2(cs_main, mempool.cs); + CCoinsViewCache &viewChain = *pcoinsTip; + CCoinsViewMemPool viewMempool(&viewChain, mempool); + view.SetBackend(viewMempool); // temporarily switch cache backend to db+mempool view + + for (const CTxIn& txin : psbtx.tx->vin) { + view.AccessCoin(txin.prevout); // Load entries from viewChain into view; can fail. + } + + view.SetBackend(viewDummy); // switch back to avoid locking mempool for too long + } + + // Fill the inputs + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + PSBTInput& input = psbtx.inputs.at(i); + + if (input.non_witness_utxo || !input.witness_utxo.IsNull()) { + continue; + } + + const Coin& coin = view.AccessCoin(psbtx.tx->vin[i].prevout); + + std::vector<std::vector<unsigned char>> solutions_data; + txnouttype which_type = Solver(coin.out.scriptPubKey, solutions_data); + if (which_type == TX_WITNESS_V0_SCRIPTHASH || which_type == TX_WITNESS_V0_KEYHASH || which_type == TX_WITNESS_UNKNOWN) { + input.witness_utxo = coin.out; + } + } + + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); +} + +UniValue joinpsbts(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 1) { + throw std::runtime_error( + RPCHelpMan{"joinpsbts", + "\nJoins multiple distinct PSBTs with different inputs and outputs into one PSBT with inputs and outputs from all of the PSBTs\n" + "No input in any of the PSBTs can be in more than one of the PSBTs.\n", + { + {"txs", RPCArg::Type::ARR, RPCArg::Optional::NO, "A json array of base64 strings of partially signed transactions", + { + {"psbt", RPCArg::Type::STR, RPCArg::Optional::NO, "A base64 string of a PSBT"} + }} + }, + RPCResult { + " \"psbt\" (string) The base64-encoded partially signed transaction\n" + }, + RPCExamples { + HelpExampleCli("joinpsbts", "\"psbt\"") + }}.ToString()); + } + + RPCTypeCheck(request.params, {UniValue::VARR}, true); + + // Unserialize the transactions + std::vector<PartiallySignedTransaction> psbtxs; + UniValue txs = request.params[0].get_array(); + + if (txs.size() <= 1) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "At least two PSBTs are required to join PSBTs."); + } + + int32_t best_version = 1; + uint32_t best_locktime = 0xffffffff; + for (unsigned int i = 0; i < txs.size(); ++i) { + PartiallySignedTransaction psbtx; + std::string error; + if (!DecodeBase64PSBT(psbtx, txs[i].get_str(), error)) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("TX decode failed %s", error)); + } + psbtxs.push_back(psbtx); + // Choose the highest version number + if (psbtx.tx->nVersion > best_version) { + best_version = psbtx.tx->nVersion; + } + // Choose the lowest lock time + if (psbtx.tx->nLockTime < best_locktime) { + best_locktime = psbtx.tx->nLockTime; + } + } + + // Create a blank psbt where everything will be added + PartiallySignedTransaction merged_psbt; + merged_psbt.tx = CMutableTransaction(); + merged_psbt.tx->nVersion = best_version; + merged_psbt.tx->nLockTime = best_locktime; + + // Merge + for (auto& psbt : psbtxs) { + for (unsigned int i = 0; i < psbt.tx->vin.size(); ++i) { + if (!merged_psbt.AddInput(psbt.tx->vin[i], psbt.inputs[i])) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Input %s:%d exists in multiple PSBTs", psbt.tx->vin[i].prevout.hash.ToString().c_str(), psbt.tx->vin[i].prevout.n)); + } + } + for (unsigned int i = 0; i < psbt.tx->vout.size(); ++i) { + merged_psbt.AddOutput(psbt.tx->vout[i], psbt.outputs[i]); + } + merged_psbt.unknown.insert(psbt.unknown.begin(), psbt.unknown.end()); + } + + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << merged_psbt; + return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); +} + +UniValue analyzepsbt(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 1) { + throw std::runtime_error( + RPCHelpMan{"analyzepsbt", + "\nAnalyzes and provides information about the current status of a PSBT and its inputs\n", + { + {"psbt", RPCArg::Type::STR, RPCArg::Optional::NO, "A base64 string of a PSBT"} + }, + RPCResult { + "{\n" + " \"inputs\" : [ (array of json objects)\n" + " {\n" + " \"has_utxo\" : true|false (boolean) Whether a UTXO is provided\n" + " \"is_final\" : true|false (boolean) Whether the input is finalized\n" + " \"missing\" : { (json object, optional) Things that are missing that are required to complete this input\n" + " \"pubkeys\" : [ (array)\n" + " \"keyid\" (string) Public key ID, hash160 of the public key, of a public key whose BIP 32 derivation path is missing\n" + " ]\n" + " \"signatures\" : [ (array)\n" + " \"keyid\" (string) Public key ID, hash160 of the public key, of a public key whose signature is missing\n" + " ]\n" + " \"redeemscript\" : \"hash\" (string) Hash160 of the redeemScript that is missing\n" + " \"witnessscript\" : \"hash\" (string) SHA256 of the witnessScript that is missing\n" + " }\n" + " \"next\" : \"role\" (string) Role of the next person that this input needs to go to\n" + " }\n" + " ,...\n" + " ]\n" + " \"estimated_vsize\" : vsize (numeric) Estimated vsize of the final signed transaction\n" + " \"estimated_feerate\" : feerate (numeric, optional) Estimated feerate of the final signed transaction. Shown only if all UTXO slots in the PSBT have been filled.\n" + " \"fee\" : fee (numeric, optional) The transaction fee paid. Shown only if all UTXO slots in the PSBT have been filled.\n" + " \"next\" : \"role\" (string) Role of the next person that this psbt needs to go to\n" + "}\n" + }, + RPCExamples { + HelpExampleCli("analyzepsbt", "\"psbt\"") + }}.ToString()); + } + + RPCTypeCheck(request.params, {UniValue::VSTR}); + + // Unserialize the transaction + PartiallySignedTransaction psbtx; + std::string error; + if (!DecodeBase64PSBT(psbtx, request.params[0].get_str(), error)) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("TX decode failed %s", error)); + } + + // Go through each input and build status + UniValue result(UniValue::VOBJ); + UniValue inputs_result(UniValue::VARR); + bool calc_fee = true; + bool all_final = true; + bool only_missing_sigs = true; + bool only_missing_final = false; + CAmount in_amt = 0; + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + PSBTInput& input = psbtx.inputs[i]; + UniValue input_univ(UniValue::VOBJ); + UniValue missing(UniValue::VOBJ); + + // Check for a UTXO + CTxOut utxo; + if (psbtx.GetInputUTXO(utxo, i)) { + in_amt += utxo.nValue; + input_univ.pushKV("has_utxo", true); + } else { + input_univ.pushKV("has_utxo", false); + input_univ.pushKV("is_final", false); + input_univ.pushKV("next", "updater"); + calc_fee = false; + } + + // Check if it is final + if (!utxo.IsNull() && !PSBTInputSigned(input)) { + input_univ.pushKV("is_final", false); + all_final = false; + + // Figure out what is missing + SignatureData outdata; + bool complete = SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, 1, &outdata); + + // Things are missing + if (!complete) { + if (!outdata.missing_pubkeys.empty()) { + // Missing pubkeys + UniValue missing_pubkeys_univ(UniValue::VARR); + for (const CKeyID& pubkey : outdata.missing_pubkeys) { + missing_pubkeys_univ.push_back(HexStr(pubkey)); + } + missing.pushKV("pubkeys", missing_pubkeys_univ); + } + if (!outdata.missing_redeem_script.IsNull()) { + // Missing redeemScript + missing.pushKV("redeemscript", HexStr(outdata.missing_redeem_script)); + } + if (!outdata.missing_witness_script.IsNull()) { + // Missing witnessScript + missing.pushKV("witnessscript", HexStr(outdata.missing_witness_script)); + } + if (!outdata.missing_sigs.empty()) { + // Missing sigs + UniValue missing_sigs_univ(UniValue::VARR); + for (const CKeyID& pubkey : outdata.missing_sigs) { + missing_sigs_univ.push_back(HexStr(pubkey)); + } + missing.pushKV("signatures", missing_sigs_univ); + } + input_univ.pushKV("missing", missing); + + // If we are only missing signatures and nothing else, then next is signer + if (outdata.missing_pubkeys.empty() && outdata.missing_redeem_script.IsNull() && outdata.missing_witness_script.IsNull() && !outdata.missing_sigs.empty()) { + input_univ.pushKV("next", "signer"); + } else { + only_missing_sigs = false; + input_univ.pushKV("next", "updater"); + } + } else { + only_missing_final = true; + input_univ.pushKV("next", "finalizer"); + } + } else if (!utxo.IsNull()){ + input_univ.pushKV("is_final", true); + } + inputs_result.push_back(input_univ); + } + result.pushKV("inputs", inputs_result); + + if (all_final) { + only_missing_sigs = false; + result.pushKV("next", "extractor"); + } + if (calc_fee) { + // Get the output amount + CAmount out_amt = std::accumulate(psbtx.tx->vout.begin(), psbtx.tx->vout.end(), 0, + [](int a, const CTxOut& b) { + return a += b.nValue; + } + ); + + // Get the fee + CAmount fee = in_amt - out_amt; + + // Estimate the size + CMutableTransaction mtx(*psbtx.tx); + CCoinsView view_dummy; + CCoinsViewCache view(&view_dummy); + bool success = true; + + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + PSBTInput& input = psbtx.inputs[i]; + if (SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, 1, nullptr, true)) { + mtx.vin[i].scriptSig = input.final_script_sig; + mtx.vin[i].scriptWitness = input.final_script_witness; + + Coin newcoin; + if (!psbtx.GetInputUTXO(newcoin.out, i)) { + success = false; + break; + } + newcoin.nHeight = 1; + view.AddCoin(psbtx.tx->vin[i].prevout, std::move(newcoin), true); + } else { + success = false; + break; + } + } + + if (success) { + CTransaction ctx = CTransaction(mtx); + size_t size = GetVirtualTransactionSize(ctx, GetTransactionSigOpCost(ctx, view, STANDARD_SCRIPT_VERIFY_FLAGS)); + result.pushKV("estimated_vsize", (int)size); + // Estimate fee rate + CFeeRate feerate(fee, size); + result.pushKV("estimated_feerate", feerate.ToString()); + } + result.pushKV("fee", ValueFromAmount(fee)); + + if (only_missing_sigs) { + result.pushKV("next", "signer"); + } else if (only_missing_final) { + result.pushKV("next", "finalizer"); + } else if (all_final) { + result.pushKV("next", "extractor"); + } else { + result.pushKV("next", "updater"); + } + } else { + result.pushKV("next", "updater"); + } + return result; +} + // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -1721,6 +2056,9 @@ static const CRPCCommand commands[] = { "rawtransactions", "finalizepsbt", &finalizepsbt, {"psbt", "extract"} }, { "rawtransactions", "createpsbt", &createpsbt, {"inputs","outputs","locktime","replaceable"} }, { "rawtransactions", "converttopsbt", &converttopsbt, {"hexstring","permitsigdata","iswitness"} }, + { "rawtransactions", "utxoupdatepsbt", &utxoupdatepsbt, {"psbt"} }, + { "rawtransactions", "joinpsbts", &joinpsbts, {"txs"} }, + { "rawtransactions", "analyzepsbt", &analyzepsbt, {"psbt"} }, { "blockchain", "gettxoutproof", &gettxoutproof, {"txids", "blockhash"} }, { "blockchain", "verifytxoutproof", &verifytxoutproof, {"proof"} }, diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 532a8028a2..43448d7222 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -21,6 +21,125 @@ namespace { //////////////////////////////////////////////////////////////////////////// +// Checksum // +//////////////////////////////////////////////////////////////////////////// + +// This section implements a checksum algorithm for descriptors with the +// following properties: +// * Mistakes in a descriptor string are measured in "symbol errors". The higher +// the number of symbol errors, the harder it is to detect: +// * An error substituting a character from 0123456789()[],'/*abcdefgh@:$%{} for +// another in that set always counts as 1 symbol error. +// * Note that hex encoded keys are covered by these characters. Xprvs and +// xpubs use other characters too, but already have their own checksum +// mechanism. +// * Function names like "multi()" use other characters, but mistakes in +// these would generally result in an unparseable descriptor. +// * A case error always counts as 1 symbol error. +// * Any other 1 character substitution error counts as 1 or 2 symbol errors. +// * Any 1 symbol error is always detected. +// * Any 2 or 3 symbol error in a descriptor of up to 49154 characters is always detected. +// * Any 4 symbol error in a descriptor of up to 507 characters is always detected. +// * Any 5 symbol error in a descriptor of up to 77 characters is always detected. +// * Is optimized to minimize the chance a 5 symbol error in a descriptor up to 387 characters is undetected +// * Random errors have a chance of 1 in 2**40 of being undetected. +// +// These properties are achieved by expanding every group of 3 (non checksum) characters into +// 4 GF(32) symbols, over which a cyclic code is defined. + +/* + * Interprets c as 8 groups of 5 bits which are the coefficients of a degree 8 polynomial over GF(32), + * multiplies that polynomial by x, computes its remainder modulo a generator, and adds the constant term val. + * + * This generator is G(x) = x^8 + {30}x^7 + {23}x^6 + {15}x^5 + {14}x^4 + {10}x^3 + {6}x^2 + {12}x + {9}. + * It is chosen to define an cyclic error detecting code which is selected by: + * - Starting from all BCH codes over GF(32) of degree 8 and below, which by construction guarantee detecting + * 3 errors in windows up to 19000 symbols. + * - Taking all those generators, and for degree 7 ones, extend them to degree 8 by adding all degree-1 factors. + * - Selecting just the set of generators that guarantee detecting 4 errors in a window of length 512. + * - Selecting one of those with best worst-case behavior for 5 errors in windows of length up to 512. + * + * The generator and the constants to implement it can be verified using this Sage code: + * B = GF(2) # Binary field + * BP.<b> = B[] # Polynomials over the binary field + * F_mod = b**5 + b**3 + 1 + * F.<f> = GF(32, modulus=F_mod, repr='int') # GF(32) definition + * FP.<x> = F[] # Polynomials over GF(32) + * E_mod = x**3 + x + F.fetch_int(8) + * E.<e> = F.extension(E_mod) # Extension field definition + * alpha = e**2743 # Choice of an element in extension field + * for p in divisors(E.order() - 1): # Verify alpha has order 32767. + * assert((alpha**p == 1) == (p % 32767 == 0)) + * G = lcm([(alpha**i).minpoly() for i in [1056,1057,1058]] + [x + 1]) + * print(G) # Print out the generator + * for i in [1,2,4,8,16]: # Print out {1,2,4,8,16}*(G mod x^8), packed in hex integers. + * v = 0 + * for coef in reversed((F.fetch_int(i)*(G % x**8)).coefficients(sparse=True)): + * v = v*32 + coef.integer_representation() + * print("0x%x" % v) + */ +uint64_t PolyMod(uint64_t c, int val) +{ + uint8_t c0 = c >> 35; + c = ((c & 0x7ffffffff) << 5) ^ val; + if (c0 & 1) c ^= 0xf5dee51989; + if (c0 & 2) c ^= 0xa9fdca3312; + if (c0 & 4) c ^= 0x1bab10e32d; + if (c0 & 8) c ^= 0x3706b1677a; + if (c0 & 16) c ^= 0x644d626ffd; + return c; +} + +std::string DescriptorChecksum(const Span<const char>& span) +{ + /** A character set designed such that: + * - The most common 'unprotected' descriptor characters (hex, keypaths) are in the first group of 32. + * - Case errors cause an offset that's a multiple of 32. + * - As many alphabetic characters are in the same group (while following the above restrictions). + * + * If p(x) gives the position of a character c in this character set, every group of 3 characters + * (a,b,c) is encoded as the 4 symbols (p(a) & 31, p(b) & 31, p(c) & 31, (p(a) / 32) + 3 * (p(b) / 32) + 9 * (p(c) / 32). + * This means that changes that only affect the lower 5 bits of the position, or only the higher 2 bits, will just + * affect a single symbol. + * + * As a result, within-group-of-32 errors count as 1 symbol, as do cross-group errors that don't affect + * the position within the groups. + */ + static std::string INPUT_CHARSET = + "0123456789()[],'/*abcdefgh@:$%{}" + "IJKLMNOPQRSTUVWXYZ&+-.;<=>?!^_|~" + "ijklmnopqrstuvwxyzABCDEFGH`#\"\\ "; + + /** The character set for the checksum itself (same as bech32). */ + static std::string CHECKSUM_CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l"; + + uint64_t c = 1; + int cls = 0; + int clscount = 0; + for (auto ch : span) { + auto pos = INPUT_CHARSET.find(ch); + if (pos == std::string::npos) return ""; + c = PolyMod(c, pos & 31); // Emit a symbol for the position inside the group, for every character. + cls = cls * 3 + (pos >> 5); // Accumulate the group numbers + if (++clscount == 3) { + // Emit an extra symbol representing the group numbers, for every 3 characters. + c = PolyMod(c, cls); + cls = 0; + clscount = 0; + } + } + if (clscount > 0) c = PolyMod(c, cls); + for (int j = 0; j < 8; ++j) c = PolyMod(c, 0); // Shift further to determine the checksum. + c ^= 1; // Prevent appending zeroes from not affecting the checksum. + + std::string ret(8, ' '); + for (int j = 0; j < 8; ++j) ret[j] = CHECKSUM_CHARSET[(c >> (5 * (7 - j))) & 31]; + return ret; +} + +std::string AddChecksum(const std::string& str) { return str + "#" + DescriptorChecksum(MakeSpan(str)); } + +//////////////////////////////////////////////////////////////////////////// // Internal representation // //////////////////////////////////////////////////////////////////////////// @@ -273,10 +392,15 @@ public: { std::string ret; ToStringHelper(nullptr, ret, false); - return ret; + return AddChecksum(ret); } - bool ToPrivateString(const SigningProvider& arg, std::string& out) const override final { return ToStringHelper(&arg, out, true); } + bool ToPrivateString(const SigningProvider& arg, std::string& out) const override final + { + bool ret = ToStringHelper(&arg, out, true); + out = AddChecksum(out); + return ret; + } bool ExpandHelper(int pos, const SigningProvider& arg, Span<const unsigned char>* cache_read, std::vector<CScript>& output_scripts, FlatSigningProvider& out, std::vector<unsigned char>* cache_write) const { @@ -751,11 +875,25 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo return MakeUnique<RawDescriptor>(script); } + } // namespace -std::unique_ptr<Descriptor> Parse(const std::string& descriptor, FlatSigningProvider& out) +std::unique_ptr<Descriptor> Parse(const std::string& descriptor, FlatSigningProvider& out, bool require_checksum) { Span<const char> sp(descriptor.data(), descriptor.size()); + + // Checksum checks + auto check_split = Split(sp, '#'); + if (check_split.size() > 2) return nullptr; // Multiple '#' symbols + if (check_split.size() == 1 && require_checksum) return nullptr; // Missing checksum + if (check_split.size() == 2) { + if (check_split[1].size() != 8) return nullptr; // Unexpected length for checksum + auto checksum = DescriptorChecksum(check_split[0]); + if (checksum.empty()) return nullptr; // Invalid characters in payload + if (!std::equal(checksum.begin(), checksum.end(), check_split[1].begin())) return nullptr; // Checksum mismatch + } + sp = check_split[0]; + auto ret = ParseScript(sp, ParseScriptContext::TOP, out); if (sp.size() == 0 && ret) return std::unique_ptr<Descriptor>(std::move(ret)); return nullptr; diff --git a/src/script/descriptor.h b/src/script/descriptor.h index 44f0efca03..907a102284 100644 --- a/src/script/descriptor.h +++ b/src/script/descriptor.h @@ -62,8 +62,15 @@ struct Descriptor { virtual bool ExpandFromCache(int pos, const std::vector<unsigned char>& cache, std::vector<CScript>& output_scripts, FlatSigningProvider& out) const = 0; }; -/** Parse a descriptor string. Included private keys are put in out. Returns nullptr if parsing fails. */ -std::unique_ptr<Descriptor> Parse(const std::string& descriptor, FlatSigningProvider& out); +/** Parse a descriptor string. Included private keys are put in out. + * + * If the descriptor has a checksum, it must be valid. If require_checksum + * is set, the checksum is mandatory - otherwise it is optional. + * + * If a parse error occurs, or the checksum is missing/invalid, or anything + * else is wrong, nullptr is returned. + */ +std::unique_ptr<Descriptor> Parse(const std::string& descriptor, FlatSigningProvider& out, bool require_checksum = false); /** Find a descriptor for the specified script, using information from provider where possible. * diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 9d6a390ea2..320956d0c4 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -83,6 +83,8 @@ static bool CreateSig(const BaseSignatureCreator& creator, SignatureData& sigdat assert(i.second); return true; } + // Could not make signature or signature not found, add keyid to missing + sigdata.missing_sigs.push_back(keyid); return false; } @@ -116,17 +118,24 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator case TX_PUBKEYHASH: { CKeyID keyID = CKeyID(uint160(vSolutions[0])); CPubKey pubkey; - if (!GetPubKey(provider, sigdata, keyID, pubkey)) return false; + if (!GetPubKey(provider, sigdata, keyID, pubkey)) { + // Pubkey could not be found, add to missing + sigdata.missing_pubkeys.push_back(keyID); + return false; + } if (!CreateSig(creator, sigdata, provider, sig, pubkey, scriptPubKey, sigversion)) return false; ret.push_back(std::move(sig)); ret.push_back(ToByteVector(pubkey)); return true; } case TX_SCRIPTHASH: - if (GetCScript(provider, sigdata, uint160(vSolutions[0]), scriptRet)) { + h160 = uint160(vSolutions[0]); + if (GetCScript(provider, sigdata, h160, scriptRet)) { ret.push_back(std::vector<unsigned char>(scriptRet.begin(), scriptRet.end())); return true; } + // Could not find redeemScript, add to missing + sigdata.missing_redeem_script = h160; return false; case TX_MULTISIG: { @@ -154,6 +163,8 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator ret.push_back(std::vector<unsigned char>(scriptRet.begin(), scriptRet.end())); return true; } + // Could not find witnessScript, add to missing + sigdata.missing_witness_script = uint256(vSolutions[0]); return false; default: diff --git a/src/script/sign.h b/src/script/sign.h index 3e9a3b38c6..491fb54c45 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -131,6 +131,10 @@ struct SignatureData { CScriptWitness scriptWitness; ///< The scriptWitness of an input. Contains complete signatures or the traditional partial signatures format. scriptWitness is part of a transaction input per BIP 144. std::map<CKeyID, SigPair> signatures; ///< BIP 174 style partial signatures for the input. May contain all signatures necessary for producing a final scriptSig or scriptWitness. std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>> misc_pubkeys; + std::vector<CKeyID> missing_pubkeys; ///< KeyIDs of pubkeys which could not be found + std::vector<CKeyID> missing_sigs; ///< KeyIDs of pubkeys for signatures which could not be found + uint160 missing_redeem_script; ///< ScriptID of the missing redeemScript (if any) + uint256 missing_witness_script; ///< SHA256 of the missing witnessScript (if any) SignatureData() {} explicit SignatureData(const CScript& script) : scriptSig(script) {} diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 8da8cfc00c..ff2b8d4fc9 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -18,8 +18,8 @@ void CheckUnparsable(const std::string& prv, const std::string& pub) FlatSigningProvider keys_priv, keys_pub; auto parse_priv = Parse(prv, keys_priv); auto parse_pub = Parse(pub, keys_pub); - BOOST_CHECK(!parse_priv); - BOOST_CHECK(!parse_pub); + BOOST_CHECK_MESSAGE(!parse_priv, prv); + BOOST_CHECK_MESSAGE(!parse_pub, pub); } constexpr int DEFAULT = 0; @@ -28,6 +28,18 @@ constexpr int HARDENED = 2; // Derivation needs access to private keys constexpr int UNSOLVABLE = 4; // This descriptor is not expected to be solvable constexpr int SIGNABLE = 8; // We can sign with this descriptor (this is not true when actual BIP32 derivation is used, as that's not integrated in our signing code) +/** Compare two descriptors. If only one of them has a checksum, the checksum is ignored. */ +bool EqualDescriptor(std::string a, std::string b) +{ + bool a_check = (a.size() > 9 && a[a.size() - 9] == '#'); + bool b_check = (b.size() > 9 && b[b.size() - 9] == '#'); + if (a_check != b_check) { + if (a_check) a = a.substr(0, a.size() - 9); + if (b_check) b = b.substr(0, b.size() - 9); + } + return a == b; +} + std::string MaybeUseHInsteadOfApostrophy(std::string ret) { if (InsecureRandBool()) { @@ -35,6 +47,7 @@ std::string MaybeUseHInsteadOfApostrophy(std::string ret) auto it = ret.find("'"); if (it != std::string::npos) { ret[it] = 'h'; + if (ret.size() > 9 && ret[ret.size() - 9] == '#') ret = ret.substr(0, ret.size() - 9); // Changing apostrophe to h breaks the checksum } else { break; } @@ -63,16 +76,16 @@ void Check(const std::string& prv, const std::string& pub, int flags, const std: // Check that both versions serialize back to the public version. std::string pub1 = parse_priv->ToString(); std::string pub2 = parse_pub->ToString(); - BOOST_CHECK_EQUAL(pub, pub1); - BOOST_CHECK_EQUAL(pub, pub2); + BOOST_CHECK(EqualDescriptor(pub, pub1)); + BOOST_CHECK(EqualDescriptor(pub, pub2)); // Check that both can be serialized with private key back to the private version, but not without private key. std::string prv1; BOOST_CHECK(parse_priv->ToPrivateString(keys_priv, prv1)); - BOOST_CHECK_EQUAL(prv, prv1); + BOOST_CHECK(EqualDescriptor(prv, prv1)); BOOST_CHECK(!parse_priv->ToPrivateString(keys_pub, prv1)); BOOST_CHECK(parse_pub->ToPrivateString(keys_priv, prv1)); - BOOST_CHECK_EQUAL(prv, prv1); + BOOST_CHECK(EqualDescriptor(prv, prv1)); BOOST_CHECK(!parse_pub->ToPrivateString(keys_pub, prv1)); // Check whether IsRange on both returns the expected result @@ -210,6 +223,15 @@ BOOST_AUTO_TEST_CASE(descriptor_test) CheckUnparsable("wsh(sh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)))", "wsh(sh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)))"); // Cannot embed P2SH inside P2WSH CheckUnparsable("sh(sh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)))", "sh(sh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)))"); // Cannot embed P2SH inside P2SH CheckUnparsable("wsh(wsh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)))", "wsh(wsh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)))"); // Cannot embed P2WSH inside P2WSH + + // Checksums + Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxfy", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5t", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, {{0x8000006FUL,222},{0}}); + Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, {{0x8000006FUL,222},{0}}); + CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#"); // Empty checksum + CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxfyq", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5tq"); // Too long checksum + CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxf", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5"); // Too short checksum + CheckUnparsable("sh(multi(3,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxfy", "sh(multi(3,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5t"); // Error in payload + CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggssrxfy", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjq09x4t"); // Error in checksum } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/test_bitcoin_fuzzy.cpp b/src/test/fuzz/deserialize.cpp index 859fba0bdc..859fba0bdc 100644 --- a/src/test/test_bitcoin_fuzzy.cpp +++ b/src/test/fuzz/deserialize.cpp diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp new file mode 100644 index 0000000000..2c0bfa360c --- /dev/null +++ b/src/test/fuzz/script_flags.cpp @@ -0,0 +1,72 @@ +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <script/interpreter.h> +#include <script/script.h> +#include <streams.h> +#include <version.h> + +#include <test/fuzz/fuzz.h> + +/** Flags that are not forbidden by an assert */ +static bool IsValidFlagCombination(unsigned flags); + +void test_one_input(std::vector<uint8_t> buffer) +{ + CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); + try { + int nVersion; + ds >> nVersion; + ds.SetVersion(nVersion); + } catch (const std::ios_base::failure&) { + return; + } + + try { + const CTransaction tx(deserialize, ds); + const PrecomputedTransactionData txdata(tx); + + unsigned int verify_flags; + ds >> verify_flags; + + if (!IsValidFlagCombination(verify_flags)) return; + + unsigned int fuzzed_flags; + ds >> fuzzed_flags; + + for (unsigned i = 0; i < tx.vin.size(); ++i) { + CTxOut prevout; + ds >> prevout; + + const TransactionSignatureChecker checker{&tx, i, prevout.nValue, txdata}; + + ScriptError serror; + const bool ret = VerifyScript(tx.vin.at(i).scriptSig, prevout.scriptPubKey, &tx.vin.at(i).scriptWitness, verify_flags, checker, &serror); + assert(ret == (serror == SCRIPT_ERR_OK)); + + // Verify that removing flags from a passing test or adding flags to a failing test does not change the result + if (ret) { + verify_flags &= ~fuzzed_flags; + } else { + verify_flags |= fuzzed_flags; + } + if (!IsValidFlagCombination(verify_flags)) return; + + ScriptError serror_fuzzed; + const bool ret_fuzzed = VerifyScript(tx.vin.at(i).scriptSig, prevout.scriptPubKey, &tx.vin.at(i).scriptWitness, verify_flags, checker, &serror_fuzzed); + assert(ret_fuzzed == (serror_fuzzed == SCRIPT_ERR_OK)); + + assert(ret_fuzzed == ret); + } + } catch (const std::ios_base::failure&) { + return; + } +} + +static bool IsValidFlagCombination(unsigned flags) +{ + if (flags & SCRIPT_VERIFY_CLEANSTACK && ~flags & (SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS)) return false; + if (flags & SCRIPT_VERIFY_WITNESS && ~flags & SCRIPT_VERIFY_P2SH) return false; + return true; +} diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 0301901bf0..d667c26c3c 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -69,7 +69,13 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) } } - txindex.Stop(); // Stop thread before calling destructor + // shutdown sequence (c.f. Shutdown() in init.cpp) + txindex.Stop(); + + threadGroup.interrupt_all(); + threadGroup.join_all(); + + // Rest of shutdown sequence and destructors happen in ~TestingSetup() } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index c467f27836..4d04aae7e9 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -102,7 +102,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) // should fail. // Capture this interaction with the upgraded_nop argument: set it when evaluating // any script flag that is implemented as an upgraded NOP code. -static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t failing_flags, bool add_to_cache) +static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t failing_flags, bool add_to_cache) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { PrecomputedTransactionData txdata(tx); // If we add many more flags, this loop can get too expensive, but we can @@ -219,11 +219,10 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) CBlock block; block = CreateAndProcessBlock({spend_tx}, p2pk_scriptPubKey); + LOCK(cs_main); BOOST_CHECK(chainActive.Tip()->GetBlockHash() == block.GetHash()); BOOST_CHECK(pcoinsTip->GetBestBlock() == block.GetHash()); - LOCK(cs_main); - // Test P2SH: construct a transaction that is valid without P2SH, and // then test validity with P2SH. { diff --git a/src/util/system.cpp b/src/util/system.cpp index 27ed24d012..6e82de743a 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -74,7 +74,6 @@ const int64_t nStartupTime = GetTime(); const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf"; -const char * const BITCOIN_PID_FILENAME = "bitcoind.pid"; ArgsManager gArgs; @@ -965,23 +964,6 @@ std::string ArgsManager::GetChainName() const return CBaseChainParams::MAIN; } -#ifndef WIN32 -fs::path GetPidFile() -{ - return AbsPathForConfigVal(fs::path(gArgs.GetArg("-pid", BITCOIN_PID_FILENAME))); -} - -void CreatePidFile(const fs::path &path, pid_t pid) -{ - FILE* file = fsbridge::fopen(path, "w"); - if (file) - { - fprintf(file, "%d\n", pid); - fclose(file); - } -} -#endif - bool RenameOver(fs::path src, fs::path dest) { #ifdef WIN32 diff --git a/src/util/system.h b/src/util/system.h index 54d4cf2e58..69ae11d1ed 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -16,6 +16,7 @@ #include <attributes.h> #include <compat.h> +#include <compat/assumptions.h> #include <fs.h> #include <logging.h> #include <sync.h> @@ -39,7 +40,6 @@ int64_t GetStartupTime(); extern const char * const BITCOIN_CONF_FILENAME; -extern const char * const BITCOIN_PID_FILENAME; /** Translate a message to the native language of the user. */ const extern std::function<std::string(const char*)> G_TRANSLATION_FUN; @@ -85,10 +85,6 @@ const fs::path &GetBlocksDir(); const fs::path &GetDataDir(bool fNetSpecific = true); void ClearDatadirCache(); fs::path GetConfigFile(const std::string& confPath); -#ifndef WIN32 -fs::path GetPidFile(); -void CreatePidFile(const fs::path &path, pid_t pid); -#endif #ifdef WIN32 fs::path GetSpecialFolderPath(int nFolder, bool fCreate = true); #endif diff --git a/src/validation.h b/src/validation.h index 49f73e4c9b..1975846b69 100644 --- a/src/validation.h +++ b/src/validation.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,8 +14,8 @@ #include <coins.h> #include <crypto/common.h> // for ReadLE64 #include <fs.h> -#include <protocol.h> // For CMessageHeader::MessageStartChars #include <policy/feerate.h> +#include <protocol.h> // For CMessageHeader::MessageStartChars #include <script/script_error.h> #include <sync.h> #include <versionbits.h> @@ -152,8 +152,6 @@ extern CTxMemPool mempool; extern std::atomic_bool g_is_mempool_loaded; typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; extern BlockMap& mapBlockIndex GUARDED_BY(cs_main); -extern uint64_t nLastBlockTx; -extern uint64_t nLastBlockWeight; extern const std::string strMessageMagic; extern Mutex g_best_block_mutex; extern std::condition_variable g_best_block_cv; diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 930274f8a1..f38202a2b8 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -1117,7 +1117,7 @@ static UniValue ProcessImportDescriptor(ImportData& import_data, std::map<CKeyID const std::string& descriptor = data["desc"].get_str(); FlatSigningProvider keys; - auto parsed_desc = Parse(descriptor, keys); + auto parsed_desc = Parse(descriptor, keys, /* require_checksum = */ true); if (!parsed_desc) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Descriptor is invalid"); } @@ -1382,6 +1382,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) {"internal", RPCArg::Type::BOOL, /* default */ "false", "Stating whether matching outputs should be treated as not incoming payments (also known as change)"}, {"watchonly", RPCArg::Type::BOOL, /* default */ "false", "Stating whether matching outputs should be considered watchonly."}, {"label", RPCArg::Type::STR, /* default */ "''", "Label to assign to the address, only allowed with internal=false"}, + {"keypool", RPCArg::Type::BOOL, /* default */ "false", "Stating whether imported public keys should be added to the keypool for when users request new addresses. Only allowed when wallet private keys are disabled"}, }, }, }, diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 22ea592738..37fc88dfd5 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -2758,7 +2758,7 @@ static UniValue listunspent(const JSONRPCRequest& request) " \"scriptPubKey\" : \"key\", (string) the script key\n" " \"amount\" : x.xxx, (numeric) the transaction output amount in " + CURRENCY_UNIT + "\n" " \"confirmations\" : n, (numeric) The number of confirmations\n" - " \"redeemScript\" : \"script\" (string) The redeemScript if scriptPubKey is P2SH" + " \"redeemScript\" : \"script\" (string) The redeemScript if scriptPubKey is P2SH\n" " \"witnessScript\" : \"script\" (string) witnessScript if the scriptPubKey is P2WSH or P2SH-P2WSH\n" " \"spendable\" : xxx, (bool) Whether we have the private keys to spend this output\n" " \"solvable\" : xxx, (bool) Whether we know how to spend this output, ignoring the lack of keys\n" diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index d174e308f0..388422bec8 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -371,6 +371,8 @@ void CWallet::UpgradeKeyMetadata() return; } + std::unique_ptr<WalletBatch> batch = MakeUnique<WalletBatch>(*database); + size_t cnt = 0; for (auto& meta_pair : mapKeyMetadata) { CKeyMetadata& meta = meta_pair.second; if (!meta.hd_seed_id.IsNull() && !meta.has_key_origin && meta.hdKeypath != "s") { // If the hdKeypath is "s", that's the seed and it doesn't have a key origin @@ -392,10 +394,15 @@ void CWallet::UpgradeKeyMetadata() // Write meta to wallet CPubKey pubkey; if (GetPubKey(meta_pair.first, pubkey)) { - WriteKeyMetadata(meta, pubkey, true); + batch->WriteKeyMetadata(meta, pubkey, true); + if (++cnt % 1000 == 0) { + // avoid creating overlarge in-memory batches in case the wallet contains large amounts of keys + batch.reset(new WalletBatch(*database)); + } } } } + batch.reset(); //write before setting the flag SetWalletFlag(WALLET_FLAG_KEY_ORIGIN_METADATA); } diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 1efc50e71f..658a8cd75e 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -38,31 +38,29 @@ def find_spendable_utxo(node, min_value): raise AssertionError("Unspent output equal or higher than %s not found" % min_value) +txs_mined = {} # txindex from txid to blockhash + class SegWitTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation. - # TODO: remove -txindex. Currently required for getrawtransaction call. self.extra_args = [ [ "-rpcserialversion=0", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", - "-txindex" ], [ "-blockversion=4", "-rpcserialversion=1", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", - "-txindex" ], [ "-blockversion=536870915", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", - "-txindex" ], ] @@ -157,10 +155,10 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Verify previous witness txs skipped for mining can now be mined") assert_equal(len(self.nodes[2].getrawmempool()), 4) - block = self.nodes[2].generate(1) # block 432 (first block with new rules; 432 = 144 * 3) + blockhash = self.nodes[2].generate(1)[0] # block 432 (first block with new rules; 432 = 144 * 3) sync_blocks(self.nodes) assert_equal(len(self.nodes[2].getrawmempool()), 0) - segwit_tx_list = self.nodes[2].getblock(block[0])["tx"] + segwit_tx_list = self.nodes[2].getblock(blockhash)["tx"] assert_equal(len(segwit_tx_list), 5) self.log.info("Verify default node can't accept txs with missing witness") @@ -174,15 +172,16 @@ class SegWitTest(BitcoinTestFramework): self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V1][0], False, witness_script(True, self.pubkey[0])) self.log.info("Verify block and transaction serialization rpcs return differing serializations depending on rpc serialization flag") - assert(self.nodes[2].getblock(block[0], False) != self.nodes[0].getblock(block[0], False)) - assert(self.nodes[1].getblock(block[0], False) == self.nodes[2].getblock(block[0], False)) - for i in range(len(segwit_tx_list)): - tx = FromHex(CTransaction(), self.nodes[2].gettransaction(segwit_tx_list[i])["hex"]) - assert(self.nodes[2].getrawtransaction(segwit_tx_list[i]) != self.nodes[0].getrawtransaction(segwit_tx_list[i])) - assert(self.nodes[1].getrawtransaction(segwit_tx_list[i], 0) == self.nodes[2].getrawtransaction(segwit_tx_list[i])) - assert(self.nodes[0].getrawtransaction(segwit_tx_list[i]) != self.nodes[2].gettransaction(segwit_tx_list[i])["hex"]) - assert(self.nodes[1].getrawtransaction(segwit_tx_list[i]) == self.nodes[2].gettransaction(segwit_tx_list[i])["hex"]) - assert(self.nodes[0].getrawtransaction(segwit_tx_list[i]) == bytes_to_hex_str(tx.serialize_without_witness())) + assert(self.nodes[2].getblock(blockhash, False) != self.nodes[0].getblock(blockhash, False)) + assert(self.nodes[1].getblock(blockhash, False) == self.nodes[2].getblock(blockhash, False)) + + for tx_id in segwit_tx_list: + tx = FromHex(CTransaction(), self.nodes[2].gettransaction(tx_id)["hex"]) + assert(self.nodes[2].getrawtransaction(tx_id, False, blockhash) != self.nodes[0].getrawtransaction(tx_id, False, blockhash)) + assert(self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].getrawtransaction(tx_id, False, blockhash)) + assert(self.nodes[0].getrawtransaction(tx_id, False, blockhash) != self.nodes[2].gettransaction(tx_id)["hex"]) + assert(self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].gettransaction(tx_id)["hex"]) + assert(self.nodes[0].getrawtransaction(tx_id, False, blockhash) == bytes_to_hex_str(tx.serialize_without_witness())) self.log.info("Verify witness txs without witness data are invalid after the fork") self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch) (code 64)', wit_ids[NODE_2][WIT_V0][2], sign=False) @@ -538,7 +537,7 @@ class SegWitTest(BitcoinTestFramework): tx.rehash() signresults = self.nodes[0].signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize_without_witness()))['hex'] txid = self.nodes[0].sendrawtransaction(signresults, True) - self.nodes[0].generate(1) + txs_mined[txid] = self.nodes[0].generate(1)[0] sync_blocks(self.nodes) watchcount = 0 spendcount = 0 @@ -581,7 +580,7 @@ class SegWitTest(BitcoinTestFramework): tx = CTransaction() for i in txids: txtmp = CTransaction() - txraw = self.nodes[0].getrawtransaction(i) + txraw = self.nodes[0].getrawtransaction(i, 0, txs_mined[i]) f = BytesIO(hex_str_to_bytes(txraw)) txtmp.deserialize(f) for j in range(len(txtmp.vout)): diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index a47a556406..f33040242c 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -43,8 +43,7 @@ class RESTTest (BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 - # TODO: remove -txindex. Currently required for getrawtransaction call. - self.extra_args = [["-rest", "-txindex"], []] + self.extra_args = [["-rest"], []] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -91,15 +90,17 @@ class RESTTest (BitcoinTestFramework): txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) self.sync_all() - self.nodes[1].generatetoaddress(1, not_related_address) - self.sync_all() - bb_hash = self.nodes[0].getbestblockhash() - assert_equal(self.nodes[1].getbalance(), Decimal("0.1")) - - self.log.info("Load the transaction using the /tx URI") + self.log.info("Test the /tx URI") json_obj = self.test_rest_request("/tx/{}".format(txid)) + assert_equal(json_obj['txid'], txid) + + # Check hex format response + hex_response = self.test_rest_request("/tx/{}".format(txid), req_type=ReqType.HEX, ret_type=RetType.OBJ) + assert_greater_than_or_equal(int(hex_response.getheader('content-length')), + json_obj['size']*2) + spent = (json_obj['vin'][0]['txid'], json_obj['vin'][0]['vout']) # get the vin to later check for utxo (should be spent by then) # get n of 0.1 outpoint n, = filter_output_indices_by_value(json_obj['vout'], Decimal('0.1')) @@ -107,9 +108,14 @@ class RESTTest (BitcoinTestFramework): self.log.info("Query an unspent TXO using the /getutxos URI") - json_obj = self.test_rest_request("/getutxos/{}-{}".format(*spending)) + self.nodes[1].generatetoaddress(1, not_related_address) + self.sync_all() + bb_hash = self.nodes[0].getbestblockhash() + + assert_equal(self.nodes[1].getbalance(), Decimal("0.1")) # Check chainTip response + json_obj = self.test_rest_request("/getutxos/{}-{}".format(*spending)) assert_equal(json_obj['chaintipHash'], bb_hash) # Make sure there is one utxo @@ -274,17 +280,6 @@ class RESTTest (BitcoinTestFramework): json_obj = self.test_rest_request("/headers/5/{}".format(bb_hash)) assert_equal(len(json_obj), 5) # now we should have 5 header objects - self.log.info("Test the /tx URI") - - tx_hash = block_json_obj['tx'][0]['txid'] - json_obj = self.test_rest_request("/tx/{}".format(tx_hash)) - assert_equal(json_obj['txid'], tx_hash) - - # Check hex format response - hex_response = self.test_rest_request("/tx/{}".format(tx_hash), req_type=ReqType.HEX, ret_type=RetType.OBJ) - assert_greater_than_or_equal(int(hex_response.getheader('content-length')), - json_obj['size']*2) - self.log.info("Test tx inclusion in the /mempool and /block URIs") # Make 3 tx and mine them on node 1 diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index 5dafb11ac5..d0bef48cb6 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mining RPCs @@ -11,7 +11,10 @@ import copy from decimal import Decimal -from test_framework.blocktools import create_coinbase +from test_framework.blocktools import ( + create_coinbase, + TIME_GENESIS_BLOCK, +) from test_framework.messages import ( CBlock, CBlockHeader, @@ -25,9 +28,11 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, bytes_to_hex_str as b2x, + connect_nodes_bi, ) from test_framework.script import CScriptNum + def assert_template(node, block, expect, rehash=True): if rehash: block.hashMerkleRoot = block.calc_merkle_root() @@ -38,9 +43,22 @@ def assert_template(node, block, expect, rehash=True): class MiningTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.setup_clean_chain = False + self.setup_clean_chain = True + + def mine_chain(self): + self.log.info('Create some old blocks') + for t in range(TIME_GENESIS_BLOCK, TIME_GENESIS_BLOCK + 200 * 600, 600): + self.nodes[0].setmocktime(t) + self.nodes[0].generate(1) + mining_info = self.nodes[0].getmininginfo() + assert_equal(mining_info['blocks'], 200) + assert_equal(mining_info['currentblocktx'], 0) + assert_equal(mining_info['currentblockweight'], 4000) + self.restart_node(0) + connect_nodes_bi(self.nodes, 0, 1) def run_test(self): + self.mine_chain() node = self.nodes[0] def assert_submitblock(block, result_str_1, result_str_2=None): @@ -53,8 +71,8 @@ class MiningTest(BitcoinTestFramework): mining_info = node.getmininginfo() assert_equal(mining_info['blocks'], 200) assert_equal(mining_info['chain'], 'regtest') - assert_equal(mining_info['currentblocktx'], 0) - assert_equal(mining_info['currentblockweight'], 0) + assert 'currentblocktx' not in mining_info + assert 'currentblockweight' not in mining_info assert_equal(mining_info['difficulty'], Decimal('4.656542373906925E-10')) assert_equal(mining_info['networkhashps'], Decimal('0.003333333333333334')) assert_equal(mining_info['pooledtx'], 0) @@ -166,7 +184,7 @@ class MiningTest(BitcoinTestFramework): self.log.info('submitheader tests') assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='xx' * BLOCK_HEADER_SIZE)) assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='ff' * (BLOCK_HEADER_SIZE-2))) - assert_raises_rpc_error(-25, 'Must submit previous header', lambda: node.submitheader(hexdata=super(CBlock, bad_block).serialize().hex())) + assert_raises_rpc_error(-25, 'Must submit previous header', lambda: node.submitheader(hexdata=b2x(super(CBlock, bad_block).serialize()))) block.nTime += 1 block.solve() diff --git a/test/functional/rpc_deriveaddresses.py b/test/functional/rpc_deriveaddresses.py index 2cc5bc974b..c8b58cfc74 100755 --- a/test/functional/rpc_deriveaddresses.py +++ b/test/functional/rpc_deriveaddresses.py @@ -4,6 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the deriveaddresses rpc call.""" from test_framework.test_framework import BitcoinTestFramework +from test_framework.descriptors import descsum_create from test_framework.util import assert_equal, assert_raises_rpc_error class DeriveaddressesTest(BitcoinTestFramework): @@ -14,36 +15,37 @@ class DeriveaddressesTest(BitcoinTestFramework): def run_test(self): assert_raises_rpc_error(-5, "Invalid descriptor", self.nodes[0].deriveaddresses, "a") - descriptor = "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)" + descriptor = "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)#t6wfjs64" address = "bcrt1qjqmxmkpmxt80xz4y3746zgt0q3u3ferr34acd5" - assert_equal(self.nodes[0].deriveaddresses(descriptor), [address]) - descriptor_pubkey = "wpkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/0)" - address = "bcrt1qjqmxmkpmxt80xz4y3746zgt0q3u3ferr34acd5" + descriptor = descriptor[:-9] + assert_raises_rpc_error(-5, "Invalid descriptor", self.nodes[0].deriveaddresses, descriptor) + descriptor_pubkey = "wpkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/0)#s9ga3alw" + address = "bcrt1qjqmxmkpmxt80xz4y3746zgt0q3u3ferr34acd5" assert_equal(self.nodes[0].deriveaddresses(descriptor_pubkey), [address]) - ranged_descriptor = "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)" + ranged_descriptor = "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)#kft60nuy" assert_equal(self.nodes[0].deriveaddresses(ranged_descriptor, 0, 2), [address, "bcrt1qhku5rq7jz8ulufe2y6fkcpnlvpsta7rq4442dy", "bcrt1qpgptk2gvshyl0s9lqshsmx932l9ccsv265tvaq"]) - assert_raises_rpc_error(-8, "Range should not be specified for an un-ranged descriptor", self.nodes[0].deriveaddresses, "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)", 0, 2) + assert_raises_rpc_error(-8, "Range should not be specified for an un-ranged descriptor", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)"), 0, 2) - assert_raises_rpc_error(-8, "Range must be specified for a ranged descriptor", self.nodes[0].deriveaddresses, "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)") + assert_raises_rpc_error(-8, "Range must be specified for a ranged descriptor", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)")) - assert_raises_rpc_error(-8, "Missing range end parameter", self.nodes[0].deriveaddresses, "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)", 0) + assert_raises_rpc_error(-8, "Missing range end parameter", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), 0) - assert_raises_rpc_error(-8, "Range end should be equal to or greater than begin", self.nodes[0].deriveaddresses, "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)", 2, 0) + assert_raises_rpc_error(-8, "Range end should be equal to or greater than begin", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), 2, 0) - assert_raises_rpc_error(-8, "Range should be greater or equal than 0", self.nodes[0].deriveaddresses, "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)", -1, 0) + assert_raises_rpc_error(-8, "Range should be greater or equal than 0", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), -1, 0) - combo_descriptor = "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)" + combo_descriptor = descsum_create("combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)") assert_equal(self.nodes[0].deriveaddresses(combo_descriptor), ["mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", "mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", address, "2NDvEwGfpEqJWfybzpKPHF2XH3jwoQV3D7x"]) - hardened_without_privkey_descriptor = "wpkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1'/1/0)" + hardened_without_privkey_descriptor = descsum_create("wpkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1'/1/0)") assert_raises_rpc_error(-5, "Cannot derive script without private keys", self.nodes[0].deriveaddresses, hardened_without_privkey_descriptor) - bare_multisig_descriptor = "multi(1, tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/0, tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/1)" + bare_multisig_descriptor = descsum_create("multi(1,tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/0,tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/1)") assert_raises_rpc_error(-5, "Descriptor does not have a corresponding address", self.nodes[0].deriveaddresses, bare_multisig_descriptor) if __name__ == '__main__': diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index c98f105828..e9098e4f5a 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -5,8 +5,9 @@ """Test the Partially Signed Transaction RPCs. """ +from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, find_output, disconnect_nodes, connect_nodes_bi, sync_blocks +from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes_bi, disconnect_nodes, find_output, sync_blocks import json import os @@ -19,8 +20,6 @@ class PSBTTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = False self.num_nodes = 3 - # TODO: remove -txindex. Currently required for getrawtransaction call. - self.extra_args = [[], ["-txindex"], ["-txindex"]] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -160,11 +159,11 @@ class PSBTTest(BitcoinTestFramework): node1_addr = self.nodes[1].getnewaddress() node2_addr = self.nodes[2].getnewaddress() txid1 = self.nodes[0].sendtoaddress(node1_addr, 13) - txid2 =self.nodes[0].sendtoaddress(node2_addr, 13) - self.nodes[0].generate(6) + txid2 = self.nodes[0].sendtoaddress(node2_addr, 13) + blockhash = self.nodes[0].generate(6)[0] self.sync_all() - vout1 = find_output(self.nodes[1], txid1, 13) - vout2 = find_output(self.nodes[2], txid2, 13) + vout1 = find_output(self.nodes[1], txid1, 13, blockhash=blockhash) + vout2 = find_output(self.nodes[2], txid2, 13, blockhash=blockhash) # Create a psbt spending outputs from nodes 1 and 2 psbt_orig = self.nodes[0].createpsbt([{"txid":txid1, "vout":vout1}, {"txid":txid2, "vout":vout2}], {self.nodes[0].getnewaddress():25.999}) @@ -296,5 +295,72 @@ class PSBTTest(BitcoinTestFramework): # Test decoding error: invalid base64 assert_raises_rpc_error(-22, "TX decode failed invalid base64", self.nodes[0].decodepsbt, ";definitely not base64;") + # Send to all types of addresses + addr1 = self.nodes[1].getnewaddress("", "bech32") + txid1 = self.nodes[0].sendtoaddress(addr1, 11) + vout1 = find_output(self.nodes[0], txid1, 11) + addr2 = self.nodes[1].getnewaddress("", "legacy") + txid2 = self.nodes[0].sendtoaddress(addr2, 11) + vout2 = find_output(self.nodes[0], txid2, 11) + addr3 = self.nodes[1].getnewaddress("", "p2sh-segwit") + txid3 = self.nodes[0].sendtoaddress(addr3, 11) + vout3 = find_output(self.nodes[0], txid3, 11) + self.sync_all() + + # Update a PSBT with UTXOs from the node + # Bech32 inputs should be filled with witness UTXO. Other inputs should not be filled because they are non-witness + psbt = self.nodes[1].createpsbt([{"txid":txid1, "vout":vout1},{"txid":txid2, "vout":vout2},{"txid":txid3, "vout":vout3}], {self.nodes[0].getnewaddress():32.999}) + decoded = self.nodes[1].decodepsbt(psbt) + assert "witness_utxo" not in decoded['inputs'][0] and "non_witness_utxo" not in decoded['inputs'][0] + assert "witness_utxo" not in decoded['inputs'][1] and "non_witness_utxo" not in decoded['inputs'][1] + assert "witness_utxo" not in decoded['inputs'][2] and "non_witness_utxo" not in decoded['inputs'][2] + updated = self.nodes[1].utxoupdatepsbt(psbt) + decoded = self.nodes[1].decodepsbt(updated) + assert "witness_utxo" in decoded['inputs'][0] and "non_witness_utxo" not in decoded['inputs'][0] + assert "witness_utxo" not in decoded['inputs'][1] and "non_witness_utxo" not in decoded['inputs'][1] + assert "witness_utxo" not in decoded['inputs'][2] and "non_witness_utxo" not in decoded['inputs'][2] + + # Two PSBTs with a common input should not be joinable + psbt1 = self.nodes[1].createpsbt([{"txid":txid1, "vout":vout1}], {self.nodes[0].getnewaddress():Decimal('10.999')}) + assert_raises_rpc_error(-8, "exists in multiple PSBTs", self.nodes[1].joinpsbts, [psbt1, updated]) + + # Join two distinct PSBTs + addr4 = self.nodes[1].getnewaddress("", "p2sh-segwit") + txid4 = self.nodes[0].sendtoaddress(addr4, 5) + vout4 = find_output(self.nodes[0], txid4, 5) + self.nodes[0].generate(6) + self.sync_all() + psbt2 = self.nodes[1].createpsbt([{"txid":txid4, "vout":vout4}], {self.nodes[0].getnewaddress():Decimal('4.999')}) + psbt2 = self.nodes[1].walletprocesspsbt(psbt2)['psbt'] + psbt2_decoded = self.nodes[0].decodepsbt(psbt2) + assert "final_scriptwitness" in psbt2_decoded['inputs'][0] and "final_scriptSig" in psbt2_decoded['inputs'][0] + joined = self.nodes[0].joinpsbts([psbt, psbt2]) + joined_decoded = self.nodes[0].decodepsbt(joined) + assert len(joined_decoded['inputs']) == 4 and len(joined_decoded['outputs']) == 2 and "final_scriptwitness" not in joined_decoded['inputs'][3] and "final_scriptSig" not in joined_decoded['inputs'][3] + + # Newly created PSBT needs UTXOs and updating + addr = self.nodes[1].getnewaddress("", "p2sh-segwit") + txid = self.nodes[0].sendtoaddress(addr, 7) + addrinfo = self.nodes[1].getaddressinfo(addr) + blockhash = self.nodes[0].generate(6)[0] + self.sync_all() + vout = find_output(self.nodes[0], txid, 7, blockhash=blockhash) + psbt = self.nodes[1].createpsbt([{"txid":txid, "vout":vout}], {self.nodes[0].getnewaddress("", "p2sh-segwit"):Decimal('6.999')}) + analyzed = self.nodes[0].analyzepsbt(psbt) + assert not analyzed['inputs'][0]['has_utxo'] and not analyzed['inputs'][0]['is_final'] and analyzed['inputs'][0]['next'] == 'updater' and analyzed['next'] == 'updater' + + # After update with wallet, only needs signing + updated = self.nodes[1].walletprocesspsbt(psbt, False, 'ALL', True)['psbt'] + analyzed = self.nodes[0].analyzepsbt(updated) + assert analyzed['inputs'][0]['has_utxo'] and not analyzed['inputs'][0]['is_final'] and analyzed['inputs'][0]['next'] == 'signer' and analyzed['next'] == 'signer' and analyzed['inputs'][0]['missing']['signatures'][0] == addrinfo['embedded']['witness_program'] + + # Check fee and size things + assert analyzed['fee'] == Decimal('0.001') and analyzed['estimated_vsize'] == 134 and analyzed['estimated_feerate'] == '0.00746268 BTC/kB' + + # After signing and finalizing, needs extracting + signed = self.nodes[1].walletprocesspsbt(updated)['psbt'] + analyzed = self.nodes[0].analyzepsbt(signed) + assert analyzed['inputs'][0]['has_utxo'] and analyzed['inputs'][0]['is_final'] and analyzed['next'] == 'extractor' + if __name__ == '__main__': PSBTTest().main() diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index a97d753626..c793cf00de 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -42,7 +42,6 @@ class RawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 - # TODO: remove -txindex. Currently required for getrawtransaction call. self.extra_args = [["-addresstype=legacy", "-txindex"], ["-addresstype=legacy", "-txindex"], ["-addresstype=legacy", "-txindex"]] def skip_test_if_missing_module(self): diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index 11b4db6ec5..29074c6af3 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -97,9 +97,9 @@ class ScantxoutsetTest(BitcoinTestFramework): assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1500}])['total_amount'], Decimal("28.672")) # Test the reported descriptors for a few matches - assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0h/0'/*)", "range": 1499}])), ["pkh([0c5f9a1e/0'/0'/0]026dbd8b2315f296d36e6b6920b1579ca75569464875c7ebe869b536a7d9503c8c)", "pkh([0c5f9a1e/0'/0'/1]033e6f25d76c00bedb3a8993c7d5739ee806397f0529b1b31dda31ef890f19a60c)"]) - assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)"])), ["pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)"]) - assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1500}])), ['pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)', 'pkh([0c5f9a1e/1/1/1500]03832901c250025da2aebae2bfb38d5c703a57ab66ad477f9c578bfbcd78abca6f)', 'pkh([0c5f9a1e/1/1/1]030d820fc9e8211c4169be8530efbc632775d8286167afd178caaf1089b77daba7)']) + assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0h/0'/*)", "range": 1499}])), ["pkh([0c5f9a1e/0'/0'/0]026dbd8b2315f296d36e6b6920b1579ca75569464875c7ebe869b536a7d9503c8c)#dzxw429x", "pkh([0c5f9a1e/0'/0'/1]033e6f25d76c00bedb3a8993c7d5739ee806397f0529b1b31dda31ef890f19a60c)#43rvceed"]) + assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)"])), ["pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)#cxmct4w8"]) + assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1500}])), ['pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)#cxmct4w8', 'pkh([0c5f9a1e/1/1/1500]03832901c250025da2aebae2bfb38d5c703a57ab66ad477f9c578bfbcd78abca6f)#vchwd07g', 'pkh([0c5f9a1e/1/1/1]030d820fc9e8211c4169be8530efbc632775d8286167afd178caaf1089b77daba7)#z2t3ypsa']) if __name__ == '__main__': ScantxoutsetTest().main() diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 6b47cae4c3..15f4502994 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Utilities for manipulating blocks and transactions.""" @@ -43,9 +43,13 @@ from io import BytesIO MAX_BLOCK_SIGOPS = 20000 +# Genesis block time (regtest) +TIME_GENESIS_BLOCK = 1296688602 + # From BIP141 WITNESS_COMMITMENT_HEADER = b"\xaa\x21\xa9\xed" + def create_block(hashprev, coinbase, ntime=None, *, version=1): """Create a block (with regtest difficulty).""" block = CBlock() diff --git a/test/functional/test_framework/descriptors.py b/test/functional/test_framework/descriptors.py new file mode 100644 index 0000000000..29482ce01e --- /dev/null +++ b/test/functional/test_framework/descriptors.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Utility functions related to output descriptors""" + +INPUT_CHARSET = "0123456789()[],'/*abcdefgh@:$%{}IJKLMNOPQRSTUVWXYZ&+-.;<=>?!^_|~ijklmnopqrstuvwxyzABCDEFGH`#\"\\ " +CHECKSUM_CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" +GENERATOR = [0xf5dee51989, 0xa9fdca3312, 0x1bab10e32d, 0x3706b1677a, 0x644d626ffd] + +def descsum_polymod(symbols): + """Internal function that computes the descriptor checksum.""" + chk = 1 + for value in symbols: + top = chk >> 35 + chk = (chk & 0x7ffffffff) << 5 ^ value + for i in range(5): + chk ^= GENERATOR[i] if ((top >> i) & 1) else 0 + return chk + +def descsum_expand(s): + """Internal function that does the character to symbol expansion""" + groups = [] + symbols = [] + for c in s: + if not c in INPUT_CHARSET: + return None + v = INPUT_CHARSET.find(c) + symbols.append(v & 31) + groups.append(v >> 5) + if len(groups) == 3: + symbols.append(groups[0] * 9 + groups[1] * 3 + groups[2]) + groups = [] + if len(groups) == 1: + symbols.append(groups[0]) + elif len(groups) == 2: + symbols.append(groups[0] * 3 + groups[1]) + return symbols + +def descsum_create(s): + """Add a checksum to a descriptor without""" + symbols = descsum_expand(s) + [0, 0, 0, 0, 0, 0, 0, 0] + checksum = descsum_polymod(symbols) ^ 1 + return s + '#' + ''.join(CHECKSUM_CHARSET[(checksum >> (5 * (7 - i))) & 31] for i in range(8)) + +def descsum_check(s, require=True): + """Verify that the checksum is correct in a descriptor""" + if not '#' in s: + return not require + if s[-9] != '#': + return False + if not all(x in CHECKSUM_CHARSET for x in s[-8:]): + return False + symbols = descsum_expand(s[:-9]) + [CHECKSUM_CHARSET.find(x) for x in s[-8:]] + return descsum_polymod(symbols) == 1 diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 8c4c0d7226..869b36673e 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -318,6 +318,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): bitcoin_cli=self.options.bitcoincli, mocktime=self.mocktime, coverage_dir=self.options.coveragedir, + cwd=self.options.tmpdir, extra_conf=extra_confs[i], extra_args=extra_args[i], use_cli=self.options.usecli, @@ -469,6 +470,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): bitcoin_cli=self.options.bitcoincli, mocktime=self.mocktime, coverage_dir=None, + cwd=self.options.tmpdir, )) self.nodes[i].args = args self.start_node(i) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 999ea68254..352774914d 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -61,7 +61,7 @@ class TestNode(): To make things easier for the test writer, any unrecognised messages will be dispatched to the RPC connection.""" - def __init__(self, i, datadir, *, rpchost, timewait, bitcoind, bitcoin_cli, mocktime, coverage_dir, extra_conf=None, extra_args=None, use_cli=False, start_perf=False): + def __init__(self, i, datadir, *, rpchost, timewait, bitcoind, bitcoin_cli, mocktime, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False): """ Kwargs: start_perf (bool): If True, begin profiling the node with `perf` as soon as @@ -76,6 +76,7 @@ class TestNode(): self.rpc_timeout = timewait self.binary = bitcoind self.coverage_dir = coverage_dir + self.cwd = cwd if extra_conf is not None: append_config(datadir, extra_conf) # Most callers will just need to add extra args to the standard list below. @@ -171,7 +172,7 @@ class TestNode(): assert self.rpc_connected and self.rpc is not None, self._node_msg("Error: no RPC connection") return getattr(self.rpc, name) - def start(self, extra_args=None, *, stdout=None, stderr=None, **kwargs): + def start(self, extra_args=None, *, cwd=None, stdout=None, stderr=None, **kwargs): """Start the node.""" if extra_args is None: extra_args = self.extra_args @@ -184,6 +185,9 @@ class TestNode(): self.stderr = stderr self.stdout = stdout + if cwd is None: + cwd = self.cwd + # Delete any existing cookie file -- if such a file exists (eg due to # unclean shutdown), it will get overwritten anyway by bitcoind, and # potentially interfere with our attempt to authenticate @@ -192,7 +196,7 @@ class TestNode(): # add environment variable LIBC_FATAL_STDERR_=1 so that libc errors are written to stderr and not the terminal subp_env = dict(os.environ, LIBC_FATAL_STDERR_="1") - self.process = subprocess.Popen(self.args + extra_args, env=subp_env, stdout=stdout, stderr=stderr, **kwargs) + self.process = subprocess.Popen(self.args + extra_args, env=subp_env, stdout=stdout, stderr=stderr, cwd=cwd, **kwargs) self.running = True self.log.debug("bitcoind started, waiting for RPC to come up") diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index d0a78d8dfd..fef9982412 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -410,12 +410,12 @@ def sync_mempools(rpc_connections, *, wait=1, timeout=60, flush_scheduler=True): # Transaction/Block functions ############################# -def find_output(node, txid, amount): +def find_output(node, txid, amount, *, blockhash=None): """ Return index to output of txid with value amount Raises exception if there is none. """ - txdata = node.getrawtransaction(txid, 1) + txdata = node.getrawtransaction(txid, 1, blockhash) for i in range(len(txdata["vout"])): if txdata["vout"][i]["value"] == amount: return i diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index 0c3c247694..b0b151d2d6 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -26,8 +26,7 @@ from test_framework.util import ( class AbandonConflictTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - # TODO: remove -txindex. Currently required for getrawtransaction call. - self.extra_args = [["-minrelaytxfee=0.00001", "-txindex"], []] + self.extra_args = [["-minrelaytxfee=0.00001"], []] def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_address_types.py b/test/functional/wallet_address_types.py index bafa556aad..380dd9c0ad 100755 --- a/test/functional/wallet_address_types.py +++ b/test/functional/wallet_address_types.py @@ -54,6 +54,10 @@ from decimal import Decimal import itertools from test_framework.test_framework import BitcoinTestFramework +from test_framework.descriptors import ( + descsum_create, + descsum_check, +) from test_framework.util import ( assert_equal, assert_greater_than, @@ -167,24 +171,31 @@ class AddressTypeTest(BitcoinTestFramework): assert_equal(deriv['path'][0], 'm') key_descs[deriv['pubkey']] = '[' + deriv['master_fingerprint'] + deriv['path'][1:] + ']' + deriv['pubkey'] + # Verify the descriptor checksum against the Python implementation + assert(descsum_check(info['desc'])) + # Verify that stripping the checksum and recreating it using Python roundtrips + assert(info['desc'] == descsum_create(info['desc'][:-9])) + # Verify that stripping the checksum and feeding it to getdescriptorinfo roundtrips + assert(info['desc'] == self.nodes[0].getdescriptorinfo(info['desc'][:-9])['descriptor']) + if not multisig and typ == 'legacy': # P2PKH - assert_equal(info['desc'], "pkh(%s)" % key_descs[info['pubkey']]) + assert_equal(info['desc'], descsum_create("pkh(%s)" % key_descs[info['pubkey']])) elif not multisig and typ == 'p2sh-segwit': # P2SH-P2WPKH - assert_equal(info['desc'], "sh(wpkh(%s))" % key_descs[info['pubkey']]) + assert_equal(info['desc'], descsum_create("sh(wpkh(%s))" % key_descs[info['pubkey']])) elif not multisig and typ == 'bech32': # P2WPKH - assert_equal(info['desc'], "wpkh(%s)" % key_descs[info['pubkey']]) + assert_equal(info['desc'], descsum_create("wpkh(%s)" % key_descs[info['pubkey']])) elif typ == 'legacy': # P2SH-multisig - assert_equal(info['desc'], "sh(multi(2,%s,%s))" % (key_descs[info['pubkeys'][0]], key_descs[info['pubkeys'][1]])) + assert_equal(info['desc'], descsum_create("sh(multi(2,%s,%s))" % (key_descs[info['pubkeys'][0]], key_descs[info['pubkeys'][1]]))) elif typ == 'p2sh-segwit': # P2SH-P2WSH-multisig - assert_equal(info['desc'], "sh(wsh(multi(2,%s,%s)))" % (key_descs[info['embedded']['pubkeys'][0]], key_descs[info['embedded']['pubkeys'][1]])) + assert_equal(info['desc'], descsum_create("sh(wsh(multi(2,%s,%s)))" % (key_descs[info['embedded']['pubkeys'][0]], key_descs[info['embedded']['pubkeys'][1]]))) elif typ == 'bech32': # P2WSH-multisig - assert_equal(info['desc'], "wsh(multi(2,%s,%s))" % (key_descs[info['pubkeys'][0]], key_descs[info['pubkeys'][1]])) + assert_equal(info['desc'], descsum_create("wsh(multi(2,%s,%s))" % (key_descs[info['pubkeys'][0]], key_descs[info['pubkeys'][1]]))) else: # Unknown type assert(False) diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index fe1a614700..a5df763eb0 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -24,8 +24,6 @@ class WalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 self.setup_clean_chain = True - # TODO: remove -txindex. Currently required for getrawtransaction call. - self.extra_args = [[], [], ["-txindex"], []] def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_importmulti.py b/test/functional/wallet_importmulti.py index 46e3ab77c8..8e25aa7337 100755 --- a/test/functional/wallet_importmulti.py +++ b/test/functional/wallet_importmulti.py @@ -20,6 +20,7 @@ from test_framework.script import ( OP_NOP, ) from test_framework.test_framework import BitcoinTestFramework +from test_framework.descriptors import descsum_create from test_framework.util import ( assert_equal, assert_greater_than, @@ -545,11 +546,22 @@ class ImportMultiTest(BitcoinTestFramework): # Test importing of a P2SH-P2WPKH address via descriptor + private key key = get_key(self.nodes[0]) - self.log.info("Should import a p2sh-p2wpkh address from descriptor and private key") + self.log.info("Should not import a p2sh-p2wpkh address from descriptor without checksum and private key") self.test_importmulti({"desc": "sh(wpkh(" + key.pubkey + "))", "timestamp": "now", "label": "Descriptor import test", "keys": [key.privkey]}, + success=False, + error_code=-5, + error_message="Descriptor is invalid") + + # Test importing of a P2SH-P2WPKH address via descriptor + private key + key = get_key(self.nodes[0]) + self.log.info("Should import a p2sh-p2wpkh address from descriptor and private key") + self.test_importmulti({"desc": descsum_create("sh(wpkh(" + key.pubkey + "))"), + "timestamp": "now", + "label": "Descriptor import test", + "keys": [key.privkey]}, success=True) test_address(self.nodes[1], key.p2sh_p2wpkh_addr, @@ -562,7 +574,7 @@ class ImportMultiTest(BitcoinTestFramework): addresses = ["2N7yv4p8G8yEaPddJxY41kPihnWvs39qCMf", "2MsHxyb2JS3pAySeNUsJ7mNnurtpeenDzLA"] # hdkeypath=m/0'/0'/0' and 1' desc = "sh(wpkh(" + xpriv + "/0'/0'/*'" + "))" self.log.info("Ranged descriptor import should fail without a specified range") - self.test_importmulti({"desc": desc, + self.test_importmulti({"desc": descsum_create(desc), "timestamp": "now"}, success=False, error_code=-8, @@ -570,7 +582,7 @@ class ImportMultiTest(BitcoinTestFramework): # Test importing of a ranged descriptor without keys self.log.info("Should import the ranged descriptor with specified range as solvable") - self.test_importmulti({"desc": desc, + self.test_importmulti({"desc": descsum_create(desc), "timestamp": "now", "range": {"end": 1}}, success=True, @@ -583,7 +595,7 @@ class ImportMultiTest(BitcoinTestFramework): # Test importing of a P2PKH address via descriptor key = get_key(self.nodes[0]) self.log.info("Should import a p2pkh address from descriptor") - self.test_importmulti({"desc": "pkh(" + key.pubkey + ")", + self.test_importmulti({"desc": descsum_create("pkh(" + key.pubkey + ")"), "timestamp": "now", "label": "Descriptor import test"}, True, @@ -597,7 +609,7 @@ class ImportMultiTest(BitcoinTestFramework): # Test import fails if both desc and scriptPubKey are provided key = get_key(self.nodes[0]) self.log.info("Import should fail if both scriptPubKey and desc are provided") - self.test_importmulti({"desc": "pkh(" + key.pubkey + ")", + self.test_importmulti({"desc": descsum_create("pkh(" + key.pubkey + ")"), "scriptPubKey": {"address": key.p2pkh_addr}, "timestamp": "now"}, success=False, @@ -616,7 +628,7 @@ class ImportMultiTest(BitcoinTestFramework): key1 = get_key(self.nodes[0]) key2 = get_key(self.nodes[0]) self.log.info("Should import a 1-of-2 bare multisig from descriptor") - self.test_importmulti({"desc": "multi(1," + key1.pubkey + "," + key2.pubkey + ")", + self.test_importmulti({"desc": descsum_create("multi(1," + key1.pubkey + "," + key2.pubkey + ")"), "timestamp": "now"}, success=True) self.log.info("Should not treat individual keys from the imported bare multisig as watchonly") @@ -635,7 +647,7 @@ class ImportMultiTest(BitcoinTestFramework): pub_fpr = info['hdmasterfingerprint'] result = self.nodes[0].importmulti( [{ - 'desc' : "wpkh([" + pub_fpr + pub_keypath[1:] +"]" + pub + ")", + 'desc' : descsum_create("wpkh([" + pub_fpr + pub_keypath[1:] +"]" + pub + ")"), "timestamp": "now", }] ) @@ -653,7 +665,7 @@ class ImportMultiTest(BitcoinTestFramework): priv_fpr = info['hdmasterfingerprint'] result = self.nodes[0].importmulti( [{ - 'desc' : "wpkh([" + priv_fpr + priv_keypath[1:] + "]" + priv + ")", + 'desc' : descsum_create("wpkh([" + priv_fpr + priv_keypath[1:] + "]" + priv + ")"), "timestamp": "now", }] ) @@ -701,12 +713,12 @@ class ImportMultiTest(BitcoinTestFramework): pub2 = self.nodes[0].getaddressinfo(addr2)['pubkey'] result = wrpc.importmulti( [{ - 'desc': 'wpkh(' + pub1 + ')', + 'desc': descsum_create('wpkh(' + pub1 + ')'), 'keypool': True, "timestamp": "now", }, { - 'desc': 'wpkh(' + pub2 + ')', + 'desc': descsum_create('wpkh(' + pub2 + ')'), 'keypool': True, "timestamp": "now", }] @@ -727,13 +739,13 @@ class ImportMultiTest(BitcoinTestFramework): pub2 = self.nodes[0].getaddressinfo(addr2)['pubkey'] result = wrpc.importmulti( [{ - 'desc': 'wpkh(' + pub1 + ')', + 'desc': descsum_create('wpkh(' + pub1 + ')'), 'keypool': True, 'internal': True, "timestamp": "now", }, { - 'desc': 'wpkh(' + pub2 + ')', + 'desc': descsum_create('wpkh(' + pub2 + ')'), 'keypool': True, 'internal': True, "timestamp": "now", @@ -755,7 +767,7 @@ class ImportMultiTest(BitcoinTestFramework): pub2 = self.nodes[0].getaddressinfo(addr2)['pubkey'] result = wrpc.importmulti( [{ - 'desc': 'wsh(multi(2,' + pub1 + ',' + pub2 + '))', + 'desc': descsum_create('wsh(multi(2,' + pub1 + ',' + pub2 + '))'), 'keypool': True, "timestamp": "now", }] @@ -769,7 +781,7 @@ class ImportMultiTest(BitcoinTestFramework): assert wrpc.getwalletinfo()['private_keys_enabled'] result = wrpc.importmulti( [{ - 'desc': 'wpkh(' + pub1 + ')', + 'desc': descsum_create('wpkh(' + pub1 + ')'), 'keypool': True, "timestamp": "now", }] @@ -792,7 +804,7 @@ class ImportMultiTest(BitcoinTestFramework): ] result = wrpc.importmulti( [{ - 'desc': 'wpkh([80002067/0h/0h]' + xpub + '/*)', + 'desc': descsum_create('wpkh([80002067/0h/0h]' + xpub + '/*)'), 'keypool': True, 'timestamp': 'now', 'range' : {'start': 0, 'end': 4} diff --git a/test/functional/wallet_txn_clone.py b/test/functional/wallet_txn_clone.py index 1c2e0a9cb7..a0a35bf43d 100755 --- a/test/functional/wallet_txn_clone.py +++ b/test/functional/wallet_txn_clone.py @@ -4,13 +4,16 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet accounts properly when there are cloned transactions with malleated scriptsigs.""" +import io from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, + bytes_to_hex_str as b2x, connect_nodes, disconnect_nodes, sync_blocks, ) +from test_framework.messages import CTransaction, COIN class TxnMallTest(BitcoinTestFramework): def set_test_params(self): @@ -72,19 +75,14 @@ class TxnMallTest(BitcoinTestFramework): clone_raw = self.nodes[0].createrawtransaction(clone_inputs, clone_outputs, clone_locktime) # createrawtransaction randomizes the order of its outputs, so swap them if necessary. - # output 0 is at version+#inputs+input+sigstub+sequence+#outputs - # 40 BTC serialized is 00286bee00000000 - pos0 = 2 * (4 + 1 + 36 + 1 + 4 + 1) - hex40 = "00286bee00000000" - output_len = 16 + 2 + 2 * int("0x" + clone_raw[pos0 + 16:pos0 + 16 + 2], 0) - if (rawtx1["vout"][0]["value"] == 40 and clone_raw[pos0:pos0 + 16] != hex40 or rawtx1["vout"][0]["value"] != 40 and clone_raw[pos0:pos0 + 16] == hex40): - output0 = clone_raw[pos0:pos0 + output_len] - output1 = clone_raw[pos0 + output_len:pos0 + 2 * output_len] - clone_raw = clone_raw[:pos0] + output1 + output0 + clone_raw[pos0 + 2 * output_len:] + clone_tx = CTransaction() + clone_tx.deserialize(io.BytesIO(bytes.fromhex(clone_raw))) + if (rawtx1["vout"][0]["value"] == 40 and clone_tx.vout[0].nValue != 40*COIN or rawtx1["vout"][0]["value"] != 40 and clone_tx.vout[0].nValue == 40*COIN): + (clone_tx.vout[0], clone_tx.vout[1]) = (clone_tx.vout[1], clone_tx.vout[0]) # Use a different signature hash type to sign. This creates an equivalent but malleated clone. # Don't send the clone anywhere yet - tx1_clone = self.nodes[0].signrawtransactionwithwallet(clone_raw, None, "ALL|ANYONECANPAY") + tx1_clone = self.nodes[0].signrawtransactionwithwallet(b2x(clone_tx.serialize()), None, "ALL|ANYONECANPAY") assert_equal(tx1_clone["complete"], True) # Have node0 mine a block, if requested: diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index 1534d5ef68..2b6c78c2c8 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -8,6 +8,7 @@ KNOWN_VIOLATIONS=( "src/dbwrapper.cpp:.*vsnprintf" "src/httprpc.cpp.*trim" "src/init.cpp:.*atoi" + "src/init.cpp:.*fprintf" "src/qt/rpcconsole.cpp:.*atoi" "src/rest.cpp:.*strtol" "src/test/dbwrapper_tests.cpp:.*snprintf" @@ -18,7 +19,6 @@ KNOWN_VIOLATIONS=( "src/util/strencodings.cpp:.*strtoul" "src/util/strencodings.h:.*atoi" "src/util/system.cpp:.*atoi" - "src/util/system.cpp:.*fprintf" ) REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/)" |