diff options
40 files changed, 614 insertions, 296 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 6cf91f8752..5bfdcbc9b1 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -73,6 +73,19 @@ task: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV task: + name: 'tidy [jammy]' + << : *GLOBAL_TASK_TEMPLATE + container: + image: ubuntu:jammy + cpu: 2 + memory: 5G + # For faster CI feedback, immediately schedule the linters + << : *CREDITS_TEMPLATE + env: + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh" + +task: name: "Win64 native [msvc]" << : *FILTER_TEMPLATE windows_container: diff --git a/build_msvc/.gitignore b/build_msvc/.gitignore index b0e557bc0c..b2eb9313a0 100644 --- a/build_msvc/.gitignore +++ b/build_msvc/.gitignore @@ -22,6 +22,7 @@ bench_bitcoin/bench_bitcoin.vcxproj libtest_util/libtest_util.vcxproj /bitcoin_config.h +/common.init.vcxproj */Win32 libbitcoin_qt/QtGeneratedFiles/* diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj.in index 0cbe2effd5..182efff233 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj.in @@ -39,7 +39,7 @@ <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration"> <LinkIncremental>false</LinkIncremental> <UseDebugLibraries>false</UseDebugLibraries> - <PlatformToolset>v142</PlatformToolset> + <PlatformToolset>@TOOLSET@</PlatformToolset> <CharacterSet>Unicode</CharacterSet> <GenerateManifest>No</GenerateManifest> <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</OutDir> @@ -49,7 +49,7 @@ <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration"> <LinkIncremental>true</LinkIncremental> <UseDebugLibraries>true</UseDebugLibraries> - <PlatformToolset>v142</PlatformToolset> + <PlatformToolset>@TOOLSET@</PlatformToolset> <CharacterSet>Unicode</CharacterSet> <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</OutDir> <IntDir>$(Platform)\$(Configuration)\$(ProjectName)\</IntDir> diff --git a/build_msvc/msvc-autogen.py b/build_msvc/msvc-autogen.py index 2a70cd9332..819fe1b7ae 100755 --- a/build_msvc/msvc-autogen.py +++ b/build_msvc/msvc-autogen.py @@ -50,13 +50,6 @@ def parse_makefile(makefile): lib_sources[current_lib] = [] break -def set_common_properties(toolset): - with open(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), 'r', encoding='utf-8') as rfile: - s = rfile.read() - s = re.sub('<PlatformToolset>.*?</PlatformToolset>', '<PlatformToolset>'+toolset+'</PlatformToolset>', s) - with open(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), 'w', encoding='utf-8',newline='\n') as wfile: - wfile.write(s) - def parse_config_into_btc_config(): def find_between( s, first, last ): try: @@ -92,13 +85,18 @@ def parse_config_into_btc_config(): with open(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), "w", encoding="utf8") as btc_config: btc_config.writelines(template) +def set_properties(vcxproj_filename, placeholder, content): + with open(vcxproj_filename + '.in', 'r', encoding='utf-8') as vcxproj_in_file: + with open(vcxproj_filename, 'w', encoding='utf-8') as vcxproj_file: + vcxproj_file.write(vcxproj_in_file.read().replace(placeholder, content)) + def main(): parser = argparse.ArgumentParser(description='Bitcoin-core msbuild configuration initialiser.') - parser.add_argument('-toolset', nargs='?',help='Optionally sets the msbuild platform toolset, e.g. v142 for Visual Studio 2019.' + parser.add_argument('-toolset', nargs='?', default=DEFAULT_PLATFORM_TOOLSET, + help='Optionally sets the msbuild platform toolset, e.g. v142 for Visual Studio 2019.' ' default is %s.'%DEFAULT_PLATFORM_TOOLSET) args = parser.parse_args() - if args.toolset: - set_common_properties(args.toolset) + set_properties(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), '@TOOLSET@', args.toolset) for makefile_name in os.listdir(SOURCE_DIR): if 'Makefile' in makefile_name: @@ -110,10 +108,7 @@ def main(): content += ' <ClCompile Include="..\\..\\src\\' + source_filename + '">\n' content += ' <ObjectFileName>$(IntDir)' + object_filename + '</ObjectFileName>\n' content += ' </ClCompile>\n' - with open(vcxproj_filename + '.in', 'r', encoding='utf-8') as vcxproj_in_file: - with open(vcxproj_filename, 'w', encoding='utf-8') as vcxproj_file: - vcxproj_file.write(vcxproj_in_file.read().replace( - '@SOURCE_FILES@\n', content)) + set_properties(vcxproj_filename, '@SOURCE_FILES@\n', content) parse_config_into_btc_config() copyfile(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), os.path.join(SOURCE_DIR, 'config/bitcoin-config.h')) copyfile(os.path.join(SOURCE_DIR,'../build_msvc/libsecp256k1_config.h'), os.path.join(SOURCE_DIR, 'secp256k1/src/libsecp256k1-config.h')) diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index e806683128..5a150d5f80 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -37,6 +37,7 @@ export USE_BUSY_BOX=${USE_BUSY_BOX:-false} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} +export RUN_TIDY=${RUN_TIDY:-false} export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} # By how much to scale the test_runner timeouts (option --timeout-factor). # This is needed because some ci machines have slow CPU or disk, so sanitizers diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh new file mode 100755 index 0000000000..7d1f763938 --- /dev/null +++ b/ci/test/00_setup_env_native_tidy.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export DOCKER_NAME_TAG="ubuntu:22.04" +export CONTAINER_NAME=ci_native_tidy +export PACKAGES="clang llvm clang-tidy bear libevent-dev libboost-dev" +export NO_DEPENDS=1 +export RUN_UNIT_TESTS=false +export RUN_FUNCTIONAL_TESTS=false +export RUN_FUZZ_TESTS=false +export RUN_TIDY=true +export GOAL="install" +export BITCOIN_CONFIG="CC=clang CXX=clang++ --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0'" +export CCACHE_SIZE=200M diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index 7158b69b4e..4742cb02ea 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -48,7 +48,12 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then CI_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' fi -CI_EXEC make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false ) +if [[ "${RUN_TIDY}" == "true" ]]; then + MAYBE_BEAR="bear" + MAYBE_TOKEN="--" +fi + +CI_EXEC "${MAYBE_BEAR}" "${MAYBE_TOKEN}" make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false ) CI_EXEC "ccache --version | head -n 1 && ccache --show-stats" CI_EXEC du -sh "${DEPENDS_DIR}"/*/ diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index e70d811d5a..30788f1543 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -34,6 +34,11 @@ if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast fi +if [ "${RUN_TIDY}" = "true" ]; then + export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/src/" + CI_EXEC run-clang-tidy "${MAKEJOBS}" +fi + if [ "$RUN_SECURITY_TESTS" = "true" ]; then CI_EXEC make test-security-check fi diff --git a/configure.ac b/configure.ac index c345d590cb..2025064cbd 100644 --- a/configure.ac +++ b/configure.ac @@ -1937,6 +1937,7 @@ AC_CONFIG_LINKS([contrib/devtools/test-security-check.py:contrib/devtools/test-s AC_CONFIG_LINKS([contrib/devtools/test-symbol-check.py:contrib/devtools/test-symbol-check.py]) AC_CONFIG_LINKS([contrib/filter-lcov.py:contrib/filter-lcov.py]) AC_CONFIG_LINKS([contrib/macdeploy/background.tiff:contrib/macdeploy/background.tiff]) +AC_CONFIG_LINKS([src/.clang-tidy:src/.clang-tidy]) AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) AC_CONFIG_LINKS([test/fuzz/test_runner.py:test/fuzz/test_runner.py]) AC_CONFIG_LINKS([test/util/test_runner.py:test/util/test_runner.py]) diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 1f0a07a284..c359725faf 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -47,18 +47,24 @@ The HTTP request and response are both handled entirely in-memory. With the /notxdetails/ option JSON response will only contain the transaction hash instead of the complete transaction details. The option only affects the JSON response. #### Blockheaders -`GET /rest/headers/<COUNT>/<BLOCK-HASH>.<bin|hex|json>` +`GET /rest/headers/<BLOCK-HASH>.<bin|hex|json>?count=<COUNT=5>` Given a block hash: returns <COUNT> amount of blockheaders in upward direction. Returns empty if the block doesn't exist or it isn't in the active chain. +*Deprecated (but not removed) since 24.0:* +`GET /rest/headers/<COUNT>/<BLOCK-HASH>.<bin|hex|json>` + #### Blockfilter Headers -`GET /rest/blockfilterheaders/<FILTERTYPE>/<COUNT>/<BLOCK-HASH>.<bin|hex|json>` +`GET /rest/blockfilterheaders/<FILTERTYPE>/<BLOCK-HASH>.<bin|hex|json>?count=<COUNT=5>` Given a block hash: returns <COUNT> amount of blockfilter headers in upward direction for the filter type <FILTERTYPE>. Returns empty if the block doesn't exist or it isn't in the active chain. +*Deprecated (but not removed) since 24.0:* +`GET /rest/blockfilterheaders/<FILTERTYPE>/<COUNT>/<BLOCK-HASH>.<bin|hex|json>` + #### Blockfilters `GET /rest/blockfilter/<FILTERTYPE>/<BLOCK-HASH>.<bin|hex|json>` diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 275b7ce124..1d8da1143e 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -1,123 +1,133 @@ -OpenBSD build guide -====================== -(updated for OpenBSD 6.9) +# OpenBSD Build Guide -This guide describes how to build bitcoind, bitcoin-qt, and command-line utilities on OpenBSD. +**Updated for OpenBSD [7.0](https://www.openbsd.org/70.html)** -Preparation -------------- +This guide describes how to build bitcoind, command-line utilities, and GUI on OpenBSD. -Run the following as root to install the base dependencies for building: +## Preparation + +### 1. Install Required Dependencies +Run the following as root to install the base dependencies for building. ```bash -pkg_add git gmake libevent libtool boost -pkg_add qt5 # (optional for enabling the GUI) -pkg_add autoconf # (select highest version, e.g. 2.69) -pkg_add automake # (select highest version, e.g. 1.16) -pkg_add python # (select highest version, e.g. 3.8) -pkg_add bash +pkg_add bash git gmake libevent libtool boost +# Select the newest version of the follow packages: +pkg_add autoconf automake python +``` +See [dependencies.md](dependencies.md) for a complete overview. + +### 2. Clone Bitcoin Repo +Clone the Bitcoin Core repository to a directory. All build scripts and commands will run from this directory. +``` bash git clone https://github.com/bitcoin/bitcoin.git ``` -See [dependencies.md](dependencies.md) for a complete overview. +### 3. Install Optional Dependencies -**Important**: From OpenBSD 6.2 onwards a C++11-supporting clang compiler is -part of the base image, and while building it is necessary to make sure that -this compiler is used and not ancient g++ 4.2.1. This is done by appending -`CC=cc CXX=c++` to configuration commands. Mixing different compilers within -the same executable will result in errors. +#### Wallet Dependencies -### Building BerkeleyDB +It is not necessary to build wallet functionality to run either `bitcoind` or `bitcoin-qt`. + +###### Descriptor Wallet Support + +`sqlite3` is required to support [descriptor wallets](descriptors.md). + +``` bash +pkg_add install sqlite3 +``` -BerkeleyDB is only necessary for the wallet functionality. To skip this, pass -`--disable-wallet` to `./configure` and skip to the next section. +###### Legacy Wallet Support +BerkeleyDB is only required to support legacy wallets. It is recommended to use Berkeley DB 4.8. You cannot use the BerkeleyDB library -from ports, for the same reason as boost above (g++/libstd++ incompatibility). -If you have to build it yourself, you can use [the installation script included -in contrib/](/contrib/install_db4.sh) like so: +from ports. However you can build it yourself, [using the installation script included in contrib/](/contrib/install_db4.sh), like so, from the root of the repository. ```bash -./contrib/install_db4.sh `pwd` CC=cc CXX=c++ +./contrib/install_db4.sh `pwd` ``` -from the root of the repository. Then set `BDB_PREFIX` for the next section: +Then set `BDB_PREFIX`: ```bash export BDB_PREFIX="$PWD/db4" ``` -### Building Bitcoin Core +#### GUI Dependencies +###### Qt5 + +Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, Qt 5 is required. + +```bash +pkg_add qt5 +``` + +## Building Bitcoin Core **Important**: Use `gmake` (the non-GNU `make` will exit with an error). Preparation: ```bash -# Replace this with the autoconf version that you installed. Include only -# the major and minor parts of the version: use "2.69" for "autoconf-2.69p2". -export AUTOCONF_VERSION=2.69 - -# Replace this with the automake version that you installed. Include only -# the major and minor parts of the version: use "1.16" for "automake-1.16.1". +# Adapt the following for the version you installed (major.minor only): +export AUTOCONF_VERSION=2.71 export AUTOMAKE_VERSION=1.16 ./autogen.sh ``` -Make sure `BDB_PREFIX` is set to the appropriate path from the above steps. + +### 1. Configuration Note that building with external signer support currently fails on OpenBSD, hence you have to explicitly disable it by passing the parameter -`--disable-external-signer` to the configure script. -(Background: the feature requires the header-only library boost::process, which -is available on OpenBSD 6.9 via Boost 1.72.0, but contains certain system calls -and preprocessor defines like `waitid()` and `WEXITED` that are not available.) +`--disable-external-signer` to the configure script. The feature requires the +header-only library boost::process, which is available on OpenBSD, but contains +certain system calls and preprocessor defines like `waitid()` and `WEXITED` that +are not available. -To configure with wallet: -```bash -./configure --with-gui=no --disable-external-signer CC=cc CXX=c++ \ - BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ - BDB_CFLAGS="-I${BDB_PREFIX}/include" \ - MAKE=gmake -``` +There are many ways to configure Bitcoin Core, here are a few common examples: + +##### Descriptor Wallet and GUI: +This enables the GUI and descriptor wallet support, assuming `sqlite` and `qt5` are installed. -To configure without wallet: ```bash -./configure --disable-wallet --with-gui=no --disable-external-signer CC=cc CXX=c++ MAKE=gmake +./configure --disable-external-signer MAKE=gmake ``` -To configure with GUI: +##### Descriptor & Legacy Wallet. No GUI: +This enables support for both wallet types and disables the GUI: + ```bash -./configure --with-gui=yes --disable-external-signer CC=cc CXX=c++ \ +./configure --disable-external-signer --with-gui=no \ BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ BDB_CFLAGS="-I${BDB_PREFIX}/include" \ MAKE=gmake ``` -Build and run the tests: +### 2. Compile +**Important**: Use `gmake` (the non-GNU `make` will exit with an error). + ```bash -gmake # use "-j N" here for N parallel jobs -gmake check +gmake # use "-j N" for N parallel jobs +gmake check # Run tests if Python 3 is available ``` -Resource limits -------------------- +## Resource limits If the build runs into out-of-memory errors, the instructions in this section might help. The standard ulimit restrictions in OpenBSD are very strict: - - data(kbytes) 1572864 +```bash +data(kbytes) 1572864 +``` This is, unfortunately, in some cases not enough to compile some `.cpp` files in the project, (see issue [#6658](https://github.com/bitcoin/bitcoin/issues/6658)). If your user is in the `staff` group the limit can be raised with: - - ulimit -d 3000000 - +```bash +ulimit -d 3000000 +``` The change will only affect the current shell and processes spawned by it. To make the change system-wide, change `datasize-cur` and `datasize-max` in `/etc/login.conf`, and reboot. - diff --git a/doc/cjdns.md b/doc/cjdns.md index 5b2bcaf874..b69564729f 100644 --- a/doc/cjdns.md +++ b/doc/cjdns.md @@ -10,7 +10,8 @@ CJDNS is like a distributed, shared VPN with multiple entry points where every participant can reach any other participant. All participants use addresses from the `fc00::/8` network (reserved IPv6 range). Installation and configuration is done outside of Bitcoin Core, similarly to a VPN (either in the host/OS or on -the network router). +the network router). See https://github.com/cjdelisle/cjdns#readme and +https://github.com/hyperboria/docs#hyperboriadocs for more information. Compared to IPv4/IPv6, CJDNS provides end-to-end encryption and protects nodes from traffic analysis and filtering. @@ -23,17 +24,37 @@ somewhat centralized. I2P connections have a source address and I2P is slow. CJDNS is fast but does not hide the sender and the recipient from intermediate routers. -## Installing CJDNS and connecting to the network +## Installing CJDNS and finding a peer to connect to the network To install and set up CJDNS, follow the instructions at -https://github.com/cjdelisle/cjdns#cjdns. +https://github.com/cjdelisle/cjdns#how-to-install-cjdns. -Don't skip steps +You need to initiate an outbound connection to a peer on the CJDNS network +before it will work with your Bitcoin Core node. This is described in steps ["2. Find a friend"](https://github.com/cjdelisle/cjdns#2-find-a-friend) and ["3. Connect your node to your friend's -node"](https://github.com/cjdelisle/cjdns#3-connect-your-node-to-your-friends-node). -You need to be connected to the CJDNS network before it will work with your -Bitcoin Core node. +node"](https://github.com/cjdelisle/cjdns#3-connect-your-node-to-your-friends-node) +in the CJDNS documentation. + +One quick way to accomplish these two steps is to query for available public +peers on [Hyperboria](https://github.com/hyperboria) by running the following: + +``` +git clone https://github.com/hyperboria/peers hyperboria-peers +cd hyperboria-peers +./testAvailable.py +``` + +For each peer, the `./testAvailable.py` script prints the filename of the peer's +credentials followed by the ping result. + +Choose one or several peers, copy their credentials from their respective files, +paste them into the relevant IPv4 or IPv6 "connectTo" JSON object in the +`cjdroute.conf` file you created in step ["1. Generate a new configuration +file"](https://github.com/cjdelisle/cjdns#1-generate-a-new-configuration-file), +and save the file. + +## Launching CJDNS Typically, CJDNS might be launched from its directory with `sudo ./cjdroute < cjdroute.conf` and it sheds permissions after setting up the diff --git a/doc/release-notes-24098.md b/doc/release-notes-24098.md new file mode 100644 index 0000000000..79e047e9a5 --- /dev/null +++ b/doc/release-notes-24098.md @@ -0,0 +1,22 @@ +Notable changes +=============== + +Updated REST APIs +----------------- + +- The `/headers/` and `/blockfilterheaders/` endpoints have been updated to use + a query parameter instead of path parameter to specify the result count. The + count parameter is now optional, and defaults to 5 for both endpoints. The old + endpoints are still functional, and have no documented behaviour change. + + For `/headers`, use + `GET /rest/headers/<BLOCK-HASH>.<bin|hex|json>?count=<COUNT=5>` + instead of + `GET /rest/headers/<COUNT>/<BLOCK-HASH>.<bin|hex|json>` (deprecated) + + For `/blockfilterheaders/`, use + `GET /rest/blockfilterheaders/<FILTERTYPE>/<BLOCK-HASH>.<bin|hex|json>?count=<COUNT=5>` + instead of + `GET /rest/blockfilterheaders/<FILTERTYPE>/<COUNT>/<BLOCK-HASH>.<bin|hex|json>` (deprecated) + + (#24098) diff --git a/src/Makefile.am b/src/Makefile.am index 12e4c7d8b7..c089bed0c9 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -206,6 +206,7 @@ BITCOIN_CORE_H = \ psbt.h \ random.h \ randomenv.h \ + rest.h \ reverse_iterator.h \ rpc/blockchain.h \ rpc/client.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 9ae7886a6e..96a9a74802 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -94,6 +94,7 @@ BITCOIN_TESTS =\ test/fs_tests.cpp \ test/getarg_tests.cpp \ test/hash_tests.cpp \ + test/httpserver_tests.cpp \ test/i2p_tests.cpp \ test/interfaces_tests.cpp \ test/key_io_tests.cpp \ @@ -116,6 +117,7 @@ BITCOIN_TESTS =\ test/prevector_tests.cpp \ test/raii_event_tests.cpp \ test/random_tests.cpp \ + test/rest_tests.cpp \ test/reverselock_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ diff --git a/src/httpserver.cpp b/src/httpserver.cpp index e00c68585e..2212097754 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -23,6 +23,7 @@ #include <deque> #include <memory> +#include <optional> #include <stdio.h> #include <stdlib.h> #include <string> @@ -30,11 +31,12 @@ #include <sys/types.h> #include <sys/stat.h> -#include <event2/thread.h> #include <event2/buffer.h> #include <event2/bufferevent.h> -#include <event2/util.h> +#include <event2/http.h> #include <event2/keyvalq_struct.h> +#include <event2/thread.h> +#include <event2/util.h> #include <support/events.h> @@ -639,6 +641,37 @@ HTTPRequest::RequestMethod HTTPRequest::GetRequestMethod() const } } +std::optional<std::string> HTTPRequest::GetQueryParameter(const std::string& key) const +{ + const char* uri{evhttp_request_get_uri(req)}; + + return GetQueryParameterFromUri(uri, key); +} + +std::optional<std::string> GetQueryParameterFromUri(const char* uri, const std::string& key) +{ + evhttp_uri* uri_parsed{evhttp_uri_parse(uri)}; + const char* query{evhttp_uri_get_query(uri_parsed)}; + std::optional<std::string> result; + + if (query) { + // Parse the query string into a key-value queue and iterate over it + struct evkeyvalq params_q; + evhttp_parse_query_str(query, ¶ms_q); + + for (struct evkeyval* param{params_q.tqh_first}; param != nullptr; param = param->next.tqe_next) { + if (param->key == key) { + result = param->value; + break; + } + } + evhttp_clear_headers(¶ms_q); + } + evhttp_uri_free(uri_parsed); + + return result; +} + void RegisterHTTPHandler(const std::string &prefix, bool exactMatch, const HTTPRequestHandler &handler) { LogPrint(BCLog::HTTP, "Registering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); diff --git a/src/httpserver.h b/src/httpserver.h index 97cd63778a..4b60e74e19 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -5,8 +5,9 @@ #ifndef BITCOIN_HTTPSERVER_H #define BITCOIN_HTTPSERVER_H -#include <string> #include <functional> +#include <optional> +#include <string> static const int DEFAULT_HTTP_THREADS=4; static const int DEFAULT_HTTP_WORKQUEUE=16; @@ -83,6 +84,17 @@ public: */ RequestMethod GetRequestMethod() const; + /** Get the query parameter value from request uri for a specified key, or std::nullopt if the + * key is not found. + * + * If the query string contains duplicate keys, the first value is returned. Many web frameworks + * would instead parse this as an array of values, but this is not (yet) implemented as it is + * currently not needed in any of the endpoints. + * + * @param[in] key represents the query parameter of which the value is returned + */ + std::optional<std::string> GetQueryParameter(const std::string& key) const; + /** * Get the request header specified by hdr, or an empty string. * Return a pair (isPresent,string). @@ -115,6 +127,20 @@ public: void WriteReply(int nStatus, const std::string& strReply = ""); }; +/** Get the query parameter value from request uri for a specified key, or std::nullopt if the key + * is not found. + * + * If the query string contains duplicate keys, the first value is returned. Many web frameworks + * would instead parse this as an array of values, but this is not (yet) implemented as it is + * currently not needed in any of the endpoints. + * + * Helper function for HTTPRequest::GetQueryParameter. + * + * @param[in] uri is the entire request uri + * @param[in] key represents the query parameter of which the value is returned + */ +std::optional<std::string> GetQueryParameterFromUri(const char* uri, const std::string& key); + /** Event handler closure. */ class HTTPClosure diff --git a/src/init.cpp b/src/init.cpp index 83937c6925..86e6ec4451 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -792,7 +792,7 @@ bool AppInitBasicSetup(const ArgsManager& args) return true; } -bool AppInitParameterInteraction(const ArgsManager& args) +bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandbox) { const CChainParams& chainparams = Params(); // ********************************************************* Step 2: parameter interactions @@ -1058,6 +1058,9 @@ bool AppInitParameterInteraction(const ArgsManager& args) if (!SetupSyscallSandbox(log_syscall_violation_before_terminating)) { return InitError(Untranslated("Installation of the syscall sandbox failed.")); } + if (use_syscall_sandbox) { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION); + } LogPrintf("Experimental syscall sandbox enabled (-sandbox=%s): bitcoind will terminate if an unexpected (not allowlisted) syscall is invoked.\n", sandbox_arg); } #endif // USE_SYSCALL_SANDBOX diff --git a/src/init.h b/src/init.h index ddd439f619..2250ae20a0 100644 --- a/src/init.h +++ b/src/init.h @@ -41,7 +41,7 @@ bool AppInitBasicSetup(const ArgsManager& args); * @note This can be done before daemonization. Do not call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read, AppInitBasicSetup should have been called. */ -bool AppInitParameterInteraction(const ArgsManager& args); +bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandbox = true); /** * Initialization sanity checks: ecc init, sanity checks, dir lock. * @note This can be done before daemonization. Do not call Shutdown() if this function fails. diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index d71455bc37..73d15652b1 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -90,7 +90,7 @@ public: uint32_t getLogCategories() override { return LogInstance().GetCategoryMask(); } bool baseInitialize() override { - return AppInitBasicSetup(gArgs) && AppInitParameterInteraction(gArgs) && AppInitSanityChecks() && + return AppInitBasicSetup(gArgs) && AppInitParameterInteraction(gArgs, /*use_syscall_sandbox=*/false) && AppInitSanityChecks() && AppInitLockDataDirectory() && AppInitInterfaces(*m_context); } bool appInitMain(interfaces::BlockAndHeaderTipInfo* tip_info) override diff --git a/src/node/miner.cpp b/src/node/miner.cpp index 917df91933..be5d58527b 100644 --- a/src/node/miner.cpp +++ b/src/node/miner.cpp @@ -430,22 +430,4 @@ void BlockAssembler::addPackageTxs(int& nPackagesSelected, int& nDescendantsUpda nDescendantsUpdated += UpdatePackagesForAdded(ancestors, mapModifiedTx); } } - -void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned int& nExtraNonce) -{ - // Update nExtraNonce - static uint256 hashPrevBlock; - if (hashPrevBlock != pblock->hashPrevBlock) { - nExtraNonce = 0; - hashPrevBlock = pblock->hashPrevBlock; - } - ++nExtraNonce; - unsigned int nHeight = pindexPrev->nHeight + 1; // Height first in coinbase required for block.version=2 - CMutableTransaction txCoinbase(*pblock->vtx[0]); - txCoinbase.vin[0].scriptSig = (CScript() << nHeight << CScriptNum(nExtraNonce)); - assert(txCoinbase.vin[0].scriptSig.size() <= 100); - - pblock->vtx[0] = MakeTransactionRef(std::move(txCoinbase)); - pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); -} } // namespace node diff --git a/src/node/miner.h b/src/node/miner.h index 5fd9abc280..c8093ec883 100644 --- a/src/node/miner.h +++ b/src/node/miner.h @@ -200,8 +200,6 @@ private: int UpdatePackagesForAdded(const CTxMemPool::setEntries& alreadyAdded, indexed_modified_transaction_set& mapModifiedTx) EXCLUSIVE_LOCKS_REQUIRED(m_mempool.cs); }; -/** Modify the extranonce in a block */ -void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned int& nExtraNonce); int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev); /** Update an old GenerateCoinbaseCommitment from CreateNewBlock after the block txs have changed */ diff --git a/src/rest.cpp b/src/rest.cpp index d59b6d1c13..956c7d97d0 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -3,6 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <rest.h> + #include <blockfilter.h> #include <chain.h> #include <chainparams.h> @@ -28,6 +30,7 @@ #include <version.h> #include <any> +#include <string> #include <boost/algorithm/string.hpp> @@ -41,21 +44,14 @@ using node::ReadBlockFromDisk; static const size_t MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once static constexpr unsigned int MAX_REST_HEADERS_RESULTS = 2000; -enum class RetFormat { - UNDEF, - BINARY, - HEX, - JSON, -}; - static const struct { - RetFormat rf; + RESTResponseFormat rf; const char* name; } rf_names[] = { - {RetFormat::UNDEF, ""}, - {RetFormat::BINARY, "bin"}, - {RetFormat::HEX, "hex"}, - {RetFormat::JSON, "json"}, + {RESTResponseFormat::UNDEF, ""}, + {RESTResponseFormat::BINARY, "bin"}, + {RESTResponseFormat::HEX, "hex"}, + {RESTResponseFormat::JSON, "json"}, }; struct CCoin { @@ -138,25 +134,28 @@ static ChainstateManager* GetChainman(const std::any& context, HTTPRequest* req) return node_context->chainman.get(); } -static RetFormat ParseDataFormat(std::string& param, const std::string& strReq) +RESTResponseFormat ParseDataFormat(std::string& param, const std::string& strReq) { - const std::string::size_type pos = strReq.rfind('.'); - if (pos == std::string::npos) - { - param = strReq; + // Remove query string (if any, separated with '?') as it should not interfere with + // parsing param and data format + param = strReq.substr(0, strReq.rfind('?')); + const std::string::size_type pos_format{param.rfind('.')}; + + // No format string is found + if (pos_format == std::string::npos) { return rf_names[0].rf; } - param = strReq.substr(0, pos); - const std::string suff(strReq, pos + 1); - + // Match format string to available formats + const std::string suffix(param, pos_format + 1); for (const auto& rf_name : rf_names) { - if (suff == rf_name.name) + if (suffix == rf_name.name) { + param.erase(pos_format); return rf_name.rf; + } } - /* If no suffix is found, return original string. */ - param = strReq; + // If no suffix is found, return RESTResponseFormat::UNDEF and original string without query string return rf_names[0].rf; } @@ -192,19 +191,29 @@ static bool rest_headers(const std::any& context, if (!CheckWarmup(req)) return false; std::string param; - const RetFormat rf = ParseDataFormat(param, strURIPart); + const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); std::vector<std::string> path; boost::split(path, param, boost::is_any_of("/")); - if (path.size() != 2) - return RESTERR(req, HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers/<count>/<hash>.<ext>."); - - const auto parsed_count{ToIntegral<size_t>(path[0])}; + std::string raw_count; + std::string hashStr; + if (path.size() == 2) { + // deprecated path: /rest/headers/<count>/<hash> + hashStr = path[1]; + raw_count = path[0]; + } else if (path.size() == 1) { + // new path with query parameter: /rest/headers/<hash>?count=<count> + hashStr = path[0]; + raw_count = req->GetQueryParameter("count").value_or("5"); + } else { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/headers/<hash>.<ext>?count=<count>"); + } + + const auto parsed_count{ToIntegral<size_t>(raw_count)}; if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > MAX_REST_HEADERS_RESULTS) { - return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count is invalid or out of acceptable range (1-%u): %s", MAX_REST_HEADERS_RESULTS, path[0])); + return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count is invalid or out of acceptable range (1-%u): %s", MAX_REST_HEADERS_RESULTS, raw_count)); } - std::string hashStr = path[1]; uint256 hash; if (!ParseHashStr(hashStr, hash)) return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); @@ -230,7 +239,7 @@ static bool rest_headers(const std::any& context, } switch (rf) { - case RetFormat::BINARY: { + case RESTResponseFormat::BINARY: { CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION); for (const CBlockIndex *pindex : headers) { ssHeader << pindex->GetBlockHeader(); @@ -242,7 +251,7 @@ static bool rest_headers(const std::any& context, return true; } - case RetFormat::HEX: { + case RESTResponseFormat::HEX: { CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION); for (const CBlockIndex *pindex : headers) { ssHeader << pindex->GetBlockHeader(); @@ -253,7 +262,7 @@ static bool rest_headers(const std::any& context, req->WriteReply(HTTP_OK, strHex); return true; } - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { UniValue jsonHeaders(UniValue::VARR); for (const CBlockIndex *pindex : headers) { jsonHeaders.push_back(blockheaderToJSON(tip, pindex)); @@ -277,7 +286,7 @@ static bool rest_block(const std::any& context, if (!CheckWarmup(req)) return false; std::string hashStr; - const RetFormat rf = ParseDataFormat(hashStr, strURIPart); + const RESTResponseFormat rf = ParseDataFormat(hashStr, strURIPart); uint256 hash; if (!ParseHashStr(hashStr, hash)) @@ -305,7 +314,7 @@ static bool rest_block(const std::any& context, } switch (rf) { - case RetFormat::BINARY: { + case RESTResponseFormat::BINARY: { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; std::string binaryBlock = ssBlock.str(); @@ -314,7 +323,7 @@ static bool rest_block(const std::any& context, return true; } - case RetFormat::HEX: { + case RESTResponseFormat::HEX: { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; std::string strHex = HexStr(ssBlock) + "\n"; @@ -323,7 +332,7 @@ static bool rest_block(const std::any& context, return true; } - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { UniValue objBlock = blockToJSON(block, tip, pblockindex, tx_verbosity); std::string strJSON = objBlock.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); @@ -352,17 +361,32 @@ static bool rest_filter_header(const std::any& context, HTTPRequest* req, const if (!CheckWarmup(req)) return false; std::string param; - const RetFormat rf = ParseDataFormat(param, strURIPart); + const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); std::vector<std::string> uri_parts; boost::split(uri_parts, param, boost::is_any_of("/")); - if (uri_parts.size() != 3) { - return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/blockfilterheaders/<filtertype>/<count>/<blockhash>"); + std::string raw_count; + std::string raw_blockhash; + if (uri_parts.size() == 3) { + // deprecated path: /rest/blockfilterheaders/<filtertype>/<count>/<blockhash> + raw_blockhash = uri_parts[2]; + raw_count = uri_parts[1]; + } else if (uri_parts.size() == 2) { + // new path with query parameter: /rest/blockfilterheaders/<filtertype>/<blockhash>?count=<count> + raw_blockhash = uri_parts[1]; + raw_count = req->GetQueryParameter("count").value_or("5"); + } else { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/blockfilterheaders/<filtertype>/<blockhash>.<ext>?count=<count>"); + } + + const auto parsed_count{ToIntegral<size_t>(raw_count)}; + if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > MAX_REST_HEADERS_RESULTS) { + return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count is invalid or out of acceptable range (1-%u): %s", MAX_REST_HEADERS_RESULTS, raw_count)); } uint256 block_hash; - if (!ParseHashStr(uri_parts[2], block_hash)) { - return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + uri_parts[2]); + if (!ParseHashStr(raw_blockhash, block_hash)) { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + raw_blockhash); } BlockFilterType filtertype; @@ -375,11 +399,6 @@ static bool rest_filter_header(const std::any& context, HTTPRequest* req, const return RESTERR(req, HTTP_BAD_REQUEST, "Index is not enabled for filtertype " + uri_parts[0]); } - const auto parsed_count{ToIntegral<size_t>(uri_parts[1])}; - if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > MAX_REST_HEADERS_RESULTS) { - return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count is invalid or out of acceptable range (1-%u): %s", MAX_REST_HEADERS_RESULTS, uri_parts[1])); - } - std::vector<const CBlockIndex*> headers; headers.reserve(*parsed_count); { @@ -418,7 +437,7 @@ static bool rest_filter_header(const std::any& context, HTTPRequest* req, const } switch (rf) { - case RetFormat::BINARY: { + case RESTResponseFormat::BINARY: { CDataStream ssHeader{SER_NETWORK, PROTOCOL_VERSION}; for (const uint256& header : filter_headers) { ssHeader << header; @@ -429,7 +448,7 @@ static bool rest_filter_header(const std::any& context, HTTPRequest* req, const req->WriteReply(HTTP_OK, binaryHeader); return true; } - case RetFormat::HEX: { + case RESTResponseFormat::HEX: { CDataStream ssHeader{SER_NETWORK, PROTOCOL_VERSION}; for (const uint256& header : filter_headers) { ssHeader << header; @@ -440,7 +459,7 @@ static bool rest_filter_header(const std::any& context, HTTPRequest* req, const req->WriteReply(HTTP_OK, strHex); return true; } - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { UniValue jsonHeaders(UniValue::VARR); for (const uint256& header : filter_headers) { jsonHeaders.push_back(header.GetHex()); @@ -462,7 +481,7 @@ static bool rest_block_filter(const std::any& context, HTTPRequest* req, const s if (!CheckWarmup(req)) return false; std::string param; - const RetFormat rf = ParseDataFormat(param, strURIPart); + const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); // request is sent over URI scheme /rest/blockfilter/filtertype/blockhash std::vector<std::string> uri_parts; @@ -518,7 +537,7 @@ static bool rest_block_filter(const std::any& context, HTTPRequest* req, const s } switch (rf) { - case RetFormat::BINARY: { + case RESTResponseFormat::BINARY: { CDataStream ssResp{SER_NETWORK, PROTOCOL_VERSION}; ssResp << filter; @@ -527,7 +546,7 @@ static bool rest_block_filter(const std::any& context, HTTPRequest* req, const s req->WriteReply(HTTP_OK, binaryResp); return true; } - case RetFormat::HEX: { + case RESTResponseFormat::HEX: { CDataStream ssResp{SER_NETWORK, PROTOCOL_VERSION}; ssResp << filter; @@ -536,7 +555,7 @@ static bool rest_block_filter(const std::any& context, HTTPRequest* req, const s req->WriteReply(HTTP_OK, strHex); return true; } - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { UniValue ret(UniValue::VOBJ); ret.pushKV("filter", HexStr(filter.GetEncodedFilter())); std::string strJSON = ret.write() + "\n"; @@ -558,10 +577,10 @@ static bool rest_chaininfo(const std::any& context, HTTPRequest* req, const std: if (!CheckWarmup(req)) return false; std::string param; - const RetFormat rf = ParseDataFormat(param, strURIPart); + const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); switch (rf) { - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { JSONRPCRequest jsonRequest; jsonRequest.context = context; jsonRequest.params = UniValue(UniValue::VARR); @@ -584,10 +603,10 @@ static bool rest_mempool_info(const std::any& context, HTTPRequest* req, const s const CTxMemPool* mempool = GetMemPool(context, req); if (!mempool) return false; std::string param; - const RetFormat rf = ParseDataFormat(param, strURIPart); + const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); switch (rf) { - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { UniValue mempoolInfoObject = MempoolInfoToJSON(*mempool); std::string strJSON = mempoolInfoObject.write() + "\n"; @@ -607,10 +626,10 @@ static bool rest_mempool_contents(const std::any& context, HTTPRequest* req, con const CTxMemPool* mempool = GetMemPool(context, req); if (!mempool) return false; std::string param; - const RetFormat rf = ParseDataFormat(param, strURIPart); + const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); switch (rf) { - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { UniValue mempoolObject = MempoolToJSON(*mempool, true); std::string strJSON = mempoolObject.write() + "\n"; @@ -629,7 +648,7 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string if (!CheckWarmup(req)) return false; std::string hashStr; - const RetFormat rf = ParseDataFormat(hashStr, strURIPart); + const RESTResponseFormat rf = ParseDataFormat(hashStr, strURIPart); uint256 hash; if (!ParseHashStr(hashStr, hash)) @@ -648,7 +667,7 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string } switch (rf) { - case RetFormat::BINARY: { + case RESTResponseFormat::BINARY: { CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssTx << tx; @@ -658,7 +677,7 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string return true; } - case RetFormat::HEX: { + case RESTResponseFormat::HEX: { CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssTx << tx; @@ -668,7 +687,7 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string return true; } - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { UniValue objTx(UniValue::VOBJ); TxToUniv(*tx, /*block_hash=*/hashBlock, /*entry=*/ objTx); std::string strJSON = objTx.write() + "\n"; @@ -688,7 +707,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: if (!CheckWarmup(req)) return false; std::string param; - const RetFormat rf = ParseDataFormat(param, strURIPart); + const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); std::vector<std::string> uriParts; if (param.length() > 1) @@ -735,14 +754,14 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: } switch (rf) { - case RetFormat::HEX: { + case RESTResponseFormat::HEX: { // convert hex to bin, continue then with bin part std::vector<unsigned char> strRequestV = ParseHex(strRequestMutable); strRequestMutable.assign(strRequestV.begin(), strRequestV.end()); [[fallthrough]]; } - case RetFormat::BINARY: { + case RESTResponseFormat::BINARY: { try { //deserialize only if user sent a request if (strRequestMutable.size() > 0) @@ -762,7 +781,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: break; } - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { if (!fInputParsed) return RESTERR(req, HTTP_BAD_REQUEST, "Error: empty request"); break; @@ -816,7 +835,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: } switch (rf) { - case RetFormat::BINARY: { + case RESTResponseFormat::BINARY: { // serialize data // use exact same output as mentioned in Bip64 CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); @@ -828,7 +847,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: return true; } - case RetFormat::HEX: { + case RESTResponseFormat::HEX: { CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); ssGetUTXOResponse << chainman.ActiveChain().Height() << chainman.ActiveChain().Tip()->GetBlockHash() << bitmap << outs; std::string strHex = HexStr(ssGetUTXOResponse) + "\n"; @@ -838,7 +857,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: return true; } - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { UniValue objGetUTXOResponse(UniValue::VOBJ); // pack in some essentials @@ -878,7 +897,7 @@ static bool rest_blockhash_by_height(const std::any& context, HTTPRequest* req, { if (!CheckWarmup(req)) return false; std::string height_str; - const RetFormat rf = ParseDataFormat(height_str, str_uri_part); + const RESTResponseFormat rf = ParseDataFormat(height_str, str_uri_part); int32_t blockheight = -1; // Initialization done only to prevent valgrind false positive, see https://github.com/bitcoin/bitcoin/pull/18785 if (!ParseInt32(height_str, &blockheight) || blockheight < 0) { @@ -898,19 +917,19 @@ static bool rest_blockhash_by_height(const std::any& context, HTTPRequest* req, pblockindex = active_chain[blockheight]; } switch (rf) { - case RetFormat::BINARY: { + case RESTResponseFormat::BINARY: { CDataStream ss_blockhash(SER_NETWORK, PROTOCOL_VERSION); ss_blockhash << pblockindex->GetBlockHash(); req->WriteHeader("Content-Type", "application/octet-stream"); req->WriteReply(HTTP_OK, ss_blockhash.str()); return true; } - case RetFormat::HEX: { + case RESTResponseFormat::HEX: { req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, pblockindex->GetBlockHash().GetHex() + "\n"); return true; } - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { req->WriteHeader("Content-Type", "application/json"); UniValue resp = UniValue(UniValue::VOBJ); resp.pushKV("blockhash", pblockindex->GetBlockHash().GetHex()); diff --git a/src/rest.h b/src/rest.h new file mode 100644 index 0000000000..49b1c333d0 --- /dev/null +++ b/src/rest.h @@ -0,0 +1,28 @@ +// Copyright (c) 2015-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_REST_H +#define BITCOIN_REST_H + +#include <string> + +enum class RESTResponseFormat { + UNDEF, + BINARY, + HEX, + JSON, +}; + +/** + * Parse a URI to get the data format and URI without data format + * and query string. + * + * @param[out] param The strReq without the data format string and + * without the query string (if any). + * @param[in] strReq The URI to be parsed. + * @return RESTResponseFormat that was parsed from the URI. + */ +RESTResponseFormat ParseDataFormat(std::string& param, const std::string& strReq); + +#endif // BITCOIN_REST_H diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 1d1ae92c58..211026c8d9 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -7,6 +7,7 @@ #include <chainparams.h> #include <consensus/amount.h> #include <consensus/consensus.h> +#include <consensus/merkle.h> #include <consensus/params.h> #include <consensus/validation.h> #include <core_io.h> @@ -43,7 +44,6 @@ using node::BlockAssembler; using node::CBlockTemplate; -using node::IncrementExtraNonce; using node::NodeContext; using node::RegenerateCommitments; using node::UpdateTime; @@ -116,14 +116,10 @@ static RPCHelpMan getnetworkhashps() }; } -static bool GenerateBlock(ChainstateManager& chainman, CBlock& block, uint64_t& max_tries, unsigned int& extra_nonce, uint256& block_hash) +static bool GenerateBlock(ChainstateManager& chainman, CBlock& block, uint64_t& max_tries, uint256& block_hash) { block_hash.SetNull(); - - { - LOCK(cs_main); - IncrementExtraNonce(&block, chainman.ActiveChain().Tip(), extra_nonce); - } + block.hashMerkleRoot = BlockMerkleRoot(block); CChainParams chainparams(Params()); @@ -149,30 +145,20 @@ static bool GenerateBlock(ChainstateManager& chainman, CBlock& block, uint64_t& static UniValue generateBlocks(ChainstateManager& chainman, const CTxMemPool& mempool, const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) { - int nHeightEnd = 0; - int nHeight = 0; - - { // Don't keep cs_main locked - LOCK(cs_main); - nHeight = chainman.ActiveChain().Height(); - nHeightEnd = nHeight+nGenerate; - } - unsigned int nExtraNonce = 0; UniValue blockHashes(UniValue::VARR); - while (nHeight < nHeightEnd && !ShutdownRequested()) - { + while (nGenerate > 0 && !ShutdownRequested()) { std::unique_ptr<CBlockTemplate> pblocktemplate(BlockAssembler(chainman.ActiveChainstate(), mempool, Params()).CreateNewBlock(coinbase_script)); if (!pblocktemplate.get()) throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); CBlock *pblock = &pblocktemplate->block; uint256 block_hash; - if (!GenerateBlock(chainman, *pblock, nMaxTries, nExtraNonce, block_hash)) { + if (!GenerateBlock(chainman, *pblock, nMaxTries, block_hash)) { break; } if (!block_hash.IsNull()) { - ++nHeight; + --nGenerate; blockHashes.push_back(block_hash.GetHex()); } } @@ -397,9 +383,8 @@ static RPCHelpMan generateblock() uint256 block_hash; uint64_t max_tries{DEFAULT_MAX_TRIES}; - unsigned int extra_nonce{0}; - if (!GenerateBlock(chainman, block, max_tries, extra_nonce, block_hash) || block_hash.IsNull()) { + if (!GenerateBlock(chainman, block, max_tries, block_hash) || block_hash.IsNull()) { throw JSONRPCError(RPC_MISC_ERROR, "Failed to make block."); } diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 7c502349b3..82b9617384 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -4,6 +4,7 @@ #include <blockfilter.h> #include <chainparams.h> +#include <consensus/merkle.h> #include <consensus/validation.h> #include <index/blockfilterindex.h> #include <node/miner.h> @@ -18,7 +19,6 @@ using node::BlockAssembler; using node::CBlockTemplate; -using node::IncrementExtraNonce; BOOST_AUTO_TEST_SUITE(blockfilter_index_tests) @@ -76,9 +76,12 @@ CBlock BuildChainTestingSetup::CreateBlock(const CBlockIndex* prev, for (const CMutableTransaction& tx : txns) { block.vtx.push_back(MakeTransactionRef(tx)); } - // IncrementExtraNonce creates a valid coinbase and merkleRoot - unsigned int extraNonce = 0; - IncrementExtraNonce(&block, prev, extraNonce); + { + CMutableTransaction tx_coinbase{*block.vtx.at(0)}; + tx_coinbase.vin.at(0).scriptSig = CScript{} << prev->nHeight + 1; + block.vtx.at(0) = MakeTransactionRef(std::move(tx_coinbase)); + block.hashMerkleRoot = BlockMerkleRoot(block); + } while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; diff --git a/src/test/httpserver_tests.cpp b/src/test/httpserver_tests.cpp new file mode 100644 index 0000000000..ee59ec6967 --- /dev/null +++ b/src/test/httpserver_tests.cpp @@ -0,0 +1,38 @@ +// Copyright (c) 2012-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <httpserver.h> +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(httpserver_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(test_query_parameters) +{ + std::string uri {}; + + // No parameters + uri = "localhost:8080/rest/headers/someresource.json"; + BOOST_CHECK(!GetQueryParameterFromUri(uri.c_str(), "p1").has_value()); + + // Single parameter + uri = "localhost:8080/rest/endpoint/someresource.json?p1=v1"; + BOOST_CHECK_EQUAL(GetQueryParameterFromUri(uri.c_str(), "p1").value(), "v1"); + BOOST_CHECK(!GetQueryParameterFromUri(uri.c_str(), "p2").has_value()); + + // Multiple parameters + uri = "/rest/endpoint/someresource.json?p1=v1&p2=v2"; + BOOST_CHECK_EQUAL(GetQueryParameterFromUri(uri.c_str(), "p1").value(), "v1"); + BOOST_CHECK_EQUAL(GetQueryParameterFromUri(uri.c_str(), "p2").value(), "v2"); + + // If the query string contains duplicate keys, the first value is returned + uri = "/rest/endpoint/someresource.json?p1=v1&p1=v2"; + BOOST_CHECK_EQUAL(GetQueryParameterFromUri(uri.c_str(), "p1").value(), "v1"); + + // Invalid query string syntax is the same as not having parameters + uri = "/rest/endpoint/someresource.json&p1=v1&p2=v2"; + BOOST_CHECK(!GetQueryParameterFromUri(uri.c_str(), "p1").has_value()); +} +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/rest_tests.cpp b/src/test/rest_tests.cpp new file mode 100644 index 0000000000..20dfe4b41a --- /dev/null +++ b/src/test/rest_tests.cpp @@ -0,0 +1,48 @@ +// Copyright (c) 2012-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <rest.h> +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +#include <string> + +BOOST_FIXTURE_TEST_SUITE(rest_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(test_query_string) +{ + std::string param; + RESTResponseFormat rf; + // No query string + rf = ParseDataFormat(param, "/rest/endpoint/someresource.json"); + BOOST_CHECK_EQUAL(param, "/rest/endpoint/someresource"); + BOOST_CHECK_EQUAL(rf, RESTResponseFormat::JSON); + + // Query string with single parameter + rf = ParseDataFormat(param, "/rest/endpoint/someresource.bin?p1=v1"); + BOOST_CHECK_EQUAL(param, "/rest/endpoint/someresource"); + BOOST_CHECK_EQUAL(rf, RESTResponseFormat::BINARY); + + // Query string with multiple parameters + rf = ParseDataFormat(param, "/rest/endpoint/someresource.hex?p1=v1&p2=v2"); + BOOST_CHECK_EQUAL(param, "/rest/endpoint/someresource"); + BOOST_CHECK_EQUAL(rf, RESTResponseFormat::HEX); + + // Incorrectly formed query string will not be handled + rf = ParseDataFormat(param, "/rest/endpoint/someresource.json&p1=v1"); + BOOST_CHECK_EQUAL(param, "/rest/endpoint/someresource.json&p1=v1"); + BOOST_CHECK_EQUAL(rf, RESTResponseFormat::UNDEF); + + // Omitted data format with query string should return UNDEF and hide query string + rf = ParseDataFormat(param, "/rest/endpoint/someresource?p1=v1"); + BOOST_CHECK_EQUAL(param, "/rest/endpoint/someresource"); + BOOST_CHECK_EQUAL(rf, RESTResponseFormat::UNDEF); + + // Data format specified after query string + rf = ParseDataFormat(param, "/rest/endpoint/someresource?p1=v1.json"); + BOOST_CHECK_EQUAL(param, "/rest/endpoint/someresource"); + BOOST_CHECK_EQUAL(rf, RESTResponseFormat::UNDEF); +} +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/util/syscall_sandbox.cpp b/src/util/syscall_sandbox.cpp index a05efac602..a69f815ce4 100644 --- a/src/util/syscall_sandbox.cpp +++ b/src/util/syscall_sandbox.cpp @@ -592,8 +592,6 @@ public: allowed_syscalls.insert(__NR_getcwd); // get current working directory allowed_syscalls.insert(__NR_getdents); // get directory entries allowed_syscalls.insert(__NR_getdents64); // get directory entries - allowed_syscalls.insert(__NR_inotify_rm_watch);// remove an existing watch from an inotify instance - allowed_syscalls.insert(__NR_linkat); // create relative to a directory file descriptor allowed_syscalls.insert(__NR_lstat); // get file status allowed_syscalls.insert(__NR_mkdir); // create a directory allowed_syscalls.insert(__NR_newfstatat); // get file status @@ -823,7 +821,6 @@ bool SetupSyscallSandbox(bool log_syscall_violation_before_terminating) return false; } } - SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION); return true; } diff --git a/src/util/syscall_sandbox.h b/src/util/syscall_sandbox.h index f7a1cbdb55..dc02ce29e9 100644 --- a/src/util/syscall_sandbox.h +++ b/src/util/syscall_sandbox.h @@ -45,9 +45,6 @@ void SetSyscallSandboxPolicy(SyscallSandboxPolicy syscall_policy); #if defined(USE_SYSCALL_SANDBOX) //! Setup and enable the experimental syscall sandbox for the running process. -//! -//! SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION) is called as part of -//! SetupSyscallSandbox(...). [[nodiscard]] bool SetupSyscallSandbox(bool log_syscall_violation_before_terminating); //! Invoke a disallowed syscall. Use for testing purposes. diff --git a/src/validation.cpp b/src/validation.cpp index 6db13f1f70..de52610def 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1896,8 +1896,9 @@ static unsigned int GetBlockScriptFlags(const CBlockIndex& block_index, const Co static int64_t nTimeCheck = 0; static int64_t nTimeForks = 0; -static int64_t nTimeVerify = 0; static int64_t nTimeConnect = 0; +static int64_t nTimeVerify = 0; +static int64_t nTimeUndo = 0; static int64_t nTimeIndex = 0; static int64_t nTimeTotal = 0; static int64_t nBlocksTotal = 0; @@ -2191,6 +2192,9 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, return false; } + int64_t nTime5 = GetTimeMicros(); nTimeUndo += nTime5 - nTime4; + LogPrint(BCLog::BENCH, " - Write undo data: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime5 - nTime4), nTimeUndo * MICRO, nTimeUndo * MILLI / nBlocksTotal); + if (!pindex->IsValid(BLOCK_VALID_SCRIPTS)) { pindex->RaiseValidity(BLOCK_VALID_SCRIPTS); m_blockman.m_dirty_blockindex.insert(pindex); @@ -2200,8 +2204,8 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); - int64_t nTime5 = GetTimeMicros(); nTimeIndex += nTime5 - nTime4; - LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime5 - nTime4), nTimeIndex * MICRO, nTimeIndex * MILLI / nBlocksTotal); + int64_t nTime6 = GetTimeMicros(); nTimeIndex += nTime6 - nTime5; + LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime6 - nTime5), nTimeIndex * MICRO, nTimeIndex * MILLI / nBlocksTotal); TRACE6(validation, block_connected, block_hash.data(), @@ -2540,7 +2544,7 @@ bool CChainState::DisconnectTip(BlockValidationState& state, DisconnectedBlockTr return true; } -static int64_t nTimeReadFromDisk = 0; +static int64_t nTimeReadFromDiskTotal = 0; static int64_t nTimeConnectTotal = 0; static int64_t nTimeFlush = 0; static int64_t nTimeChainState = 0; @@ -2608,13 +2612,14 @@ bool CChainState::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew } pthisBlock = pblockNew; } else { + LogPrint(BCLog::BENCH, " - Using cached block\n"); pthisBlock = pblock; } const CBlock& blockConnecting = *pthisBlock; // Apply the block atomically to the chain state. - int64_t nTime2 = GetTimeMicros(); nTimeReadFromDisk += nTime2 - nTime1; + int64_t nTime2 = GetTimeMicros(); nTimeReadFromDiskTotal += nTime2 - nTime1; int64_t nTime3; - LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * MILLI, nTimeReadFromDisk * MICRO); + LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime2 - nTime1) * MILLI, nTimeReadFromDiskTotal * MICRO, nTimeReadFromDiskTotal * MILLI / nBlocksTotal); { CCoinsViewCache view(&CoinsTip()); bool rv = ConnectBlock(blockConnecting, state, pindexNew, view); diff --git a/test/README.md b/test/README.md index 84264277f7..7ff2d6d9f2 100644 --- a/test/README.md +++ b/test/README.md @@ -308,8 +308,9 @@ Use the `-v` option for verbose output. | [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) | [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy) | [`lint-python.sh`](lint/lint-python.sh) | [pyzmq](https://github.com/zeromq/pyzmq) +| [`lint-python-dead-code.py`](lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture) | [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) -| [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) +| [`lint-spelling.py`](lint/lint-spelling.py) | [codespell](https://github.com/codespell-project/codespell) In use versions and install instructions are available in the [CI setup](../ci/lint/04_install.sh). @@ -320,7 +321,7 @@ Please be aware that on Linux distributions all dependencies are usually availab Individual tests can be run by directly calling the test script, e.g.: ``` -test/lint/lint-files.sh +test/lint/lint-files.py ``` You can run all the shell-based lint tests by running: diff --git a/test/functional/feature_blockfilterindex_prune.py b/test/functional/feature_blockfilterindex_prune.py index 2451988135..c983ceda6f 100755 --- a/test/functional/feature_blockfilterindex_prune.py +++ b/test/functional/feature_blockfilterindex_prune.py @@ -31,7 +31,7 @@ class FeatureBlockfilterindexPruneTest(BitcoinTestFramework): pruneheight = self.nodes[0].pruneblockchain(400) # the prune heights used here and below are magic numbers that are determined by the # thresholds at which block files wrap, so they depend on disk serialization and default block file size. - assert_equal(pruneheight, 248) + assert_equal(pruneheight, 249) self.log.info("check if we can access the tips blockfilter when we have pruned some blocks") assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0) @@ -40,19 +40,19 @@ class FeatureBlockfilterindexPruneTest(BitcoinTestFramework): assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getblockhash(2))['filter']), 0) # mine and sync index up to a height that will later be the pruneheight - self.generate(self.nodes[0], 298) - self.sync_index(height=998) + self.generate(self.nodes[0], 51) + self.sync_index(height=751) self.log.info("start node without blockfilterindex") self.restart_node(0, extra_args=["-fastprune", "-prune=1"]) self.log.info("make sure accessing the blockfilters throws an error") assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic", self.nodes[0].getblockfilter, self.nodes[0].getblockhash(2)) - self.generate(self.nodes[0], 502) + self.generate(self.nodes[0], 749) self.log.info("prune exactly up to the blockfilterindexes best block while blockfilters are disabled") pruneheight_2 = self.nodes[0].pruneblockchain(1000) - assert_equal(pruneheight_2, 998) + assert_equal(pruneheight_2, 751) self.restart_node(0, extra_args=["-fastprune", "-prune=1", "-blockfilterindex=1"]) self.log.info("make sure that we can continue with the partially synced index after having pruned up to the index height") self.sync_index(height=1500) diff --git a/test/functional/feature_utxo_set_hash.py b/test/functional/feature_utxo_set_hash.py index 75180e62a2..4d486bc6f4 100755 --- a/test/functional/feature_utxo_set_hash.py +++ b/test/functional/feature_utxo_set_hash.py @@ -69,8 +69,8 @@ class UTXOSetHashTest(BitcoinTestFramework): assert_equal(finalized[::-1].hex(), node_muhash) self.log.info("Test deterministic UTXO set hash results") - assert_equal(node.gettxoutsetinfo()['hash_serialized_2'], "3a570529b4c32e77268de1f81b903c75cc2da53c48df0d125c1e697ba7c8c7b7") - assert_equal(node.gettxoutsetinfo("muhash")['muhash'], "a13e0e70eb8acc786549596e3bc154623f1a5a622ba2f70715f6773ec745f435") + assert_equal(node.gettxoutsetinfo()['hash_serialized_2'], "f9aa4fb5ffd10489b9a6994e70ccf1de8a8bfa2d5f201d9857332e9954b0855d") + assert_equal(node.gettxoutsetinfo("muhash")['muhash'], "d1725b2fe3ef43e55aa4907480aea98d406fc9e0bf8f60169e2305f1fbf5961b") def run_test(self): self.test_muhash_implementation() diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index a3d949c6a8..4f8676ec53 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -10,6 +10,7 @@ import http.client from io import BytesIO import json from struct import pack, unpack +import typing import urllib.parse @@ -57,14 +58,21 @@ class RESTTest (BitcoinTestFramework): args.append("-whitelist=noban@127.0.0.1") self.supports_cli = False - def test_rest_request(self, uri, http_method='GET', req_type=ReqType.JSON, body='', status=200, ret_type=RetType.JSON): + def test_rest_request( + self, + uri: str, + http_method: str = 'GET', + req_type: ReqType = ReqType.JSON, + body: str = '', + status: int = 200, + ret_type: RetType = RetType.JSON, + query_params: typing.Dict[str, typing.Any] = None, + ) -> typing.Union[http.client.HTTPResponse, bytes, str, None]: rest_uri = '/rest' + uri - if req_type == ReqType.JSON: - rest_uri += '.json' - elif req_type == ReqType.BIN: - rest_uri += '.bin' - elif req_type == ReqType.HEX: - rest_uri += '.hex' + if req_type in ReqType: + rest_uri += f'.{req_type.name.lower()}' + if query_params: + rest_uri += f'?{urllib.parse.urlencode(query_params)}' conn = http.client.HTTPConnection(self.url.hostname, self.url.port) self.log.debug(f'{http_method} {rest_uri} {body}') @@ -83,6 +91,8 @@ class RESTTest (BitcoinTestFramework): elif ret_type == RetType.JSON: return json.loads(resp.read().decode('utf-8'), parse_float=Decimal) + return None + def run_test(self): self.url = urllib.parse.urlparse(self.nodes[0].url) self.wallet = MiniWallet(self.nodes[0]) @@ -213,12 +223,12 @@ class RESTTest (BitcoinTestFramework): bb_hash = self.nodes[0].getbestblockhash() # Check result if block does not exists - assert_equal(self.test_rest_request(f"/headers/1/{UNKNOWN_PARAM}"), []) + assert_equal(self.test_rest_request(f"/headers/{UNKNOWN_PARAM}", query_params={"count": 1}), []) self.test_rest_request(f"/block/{UNKNOWN_PARAM}", status=404, ret_type=RetType.OBJ) # Check result if block is not in the active chain self.nodes[0].invalidateblock(bb_hash) - assert_equal(self.test_rest_request(f'/headers/1/{bb_hash}'), []) + assert_equal(self.test_rest_request(f'/headers/{bb_hash}', query_params={'count': 1}), []) self.test_rest_request(f'/block/{bb_hash}') self.nodes[0].reconsiderblock(bb_hash) @@ -228,7 +238,7 @@ class RESTTest (BitcoinTestFramework): response_bytes = response.read() # Compare with block header - response_header = self.test_rest_request(f"/headers/1/{bb_hash}", req_type=ReqType.BIN, ret_type=RetType.OBJ) + response_header = self.test_rest_request(f"/headers/{bb_hash}", req_type=ReqType.BIN, ret_type=RetType.OBJ, query_params={"count": 1}) assert_equal(int(response_header.getheader('content-length')), BLOCK_HEADER_SIZE) response_header_bytes = response_header.read() assert_equal(response_bytes[:BLOCK_HEADER_SIZE], response_header_bytes) @@ -240,7 +250,7 @@ class RESTTest (BitcoinTestFramework): assert_equal(response_bytes.hex().encode(), response_hex_bytes) # Compare with hex block header - response_header_hex = self.test_rest_request(f"/headers/1/{bb_hash}", req_type=ReqType.HEX, ret_type=RetType.OBJ) + response_header_hex = self.test_rest_request(f"/headers/{bb_hash}", req_type=ReqType.HEX, ret_type=RetType.OBJ, query_params={"count": 1}) assert_greater_than(int(response_header_hex.getheader('content-length')), BLOCK_HEADER_SIZE*2) response_header_hex_bytes = response_header_hex.read(BLOCK_HEADER_SIZE*2) assert_equal(response_bytes[:BLOCK_HEADER_SIZE].hex().encode(), response_header_hex_bytes) @@ -267,7 +277,7 @@ class RESTTest (BitcoinTestFramework): self.test_rest_request("/blockhashbyheight/", ret_type=RetType.OBJ, status=400) # Compare with json block header - json_obj = self.test_rest_request(f"/headers/1/{bb_hash}") + json_obj = self.test_rest_request(f"/headers/{bb_hash}", query_params={"count": 1}) assert_equal(len(json_obj), 1) # ensure that there is one header in the json response assert_equal(json_obj[0]['hash'], bb_hash) # request/response hash should be the same @@ -278,9 +288,9 @@ class RESTTest (BitcoinTestFramework): # See if we can get 5 headers in one response self.generate(self.nodes[1], 5) - json_obj = self.test_rest_request(f"/headers/5/{bb_hash}") + json_obj = self.test_rest_request(f"/headers/{bb_hash}", query_params={"count": 5}) assert_equal(len(json_obj), 5) # now we should have 5 header objects - json_obj = self.test_rest_request(f"/blockfilterheaders/basic/5/{bb_hash}") + json_obj = self.test_rest_request(f"/blockfilterheaders/basic/{bb_hash}", query_params={"count": 5}) first_filter_header = json_obj[0] assert_equal(len(json_obj), 5) # now we should have 5 filter header objects json_obj = self.test_rest_request(f"/blockfilter/basic/{bb_hash}") @@ -294,7 +304,7 @@ class RESTTest (BitcoinTestFramework): for num in ['5a', '-5', '0', '2001', '99999999999999999999999999999999999']: assert_equal( bytes(f'Header count is invalid or out of acceptable range (1-2000): {num}\r\n', 'ascii'), - self.test_rest_request(f"/headers/{num}/{bb_hash}", ret_type=RetType.BYTES, status=400), + self.test_rest_request(f"/headers/{bb_hash}", ret_type=RetType.BYTES, status=400, query_params={"count": num}), ) self.log.info("Test tx inclusion in the /mempool and /block URIs") @@ -351,6 +361,11 @@ class RESTTest (BitcoinTestFramework): json_obj = self.test_rest_request("/chaininfo") assert_equal(json_obj['bestblockhash'], bb_hash) + # Test compatibility of deprecated and newer endpoints + self.log.info("Test compatibility of deprecated and newer endpoints") + assert_equal(self.test_rest_request(f"/headers/{bb_hash}", query_params={"count": 1}), self.test_rest_request(f"/headers/1/{bb_hash}")) + assert_equal(self.test_rest_request(f"/blockfilterheaders/basic/{bb_hash}", query_params={"count": 1}), self.test_rest_request(f"/blockfilterheaders/basic/5/{bb_hash}")) + if __name__ == '__main__': RESTTest().main() diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py index 1721b6ffe8..4ca84748b2 100755 --- a/test/functional/rpc_dumptxoutset.py +++ b/test/functional/rpc_dumptxoutset.py @@ -37,16 +37,16 @@ class DumptxoutsetTest(BitcoinTestFramework): # Blockhash should be deterministic based on mocked time. assert_equal( out['base_hash'], - '6fd417acba2a8738b06fee43330c50d58e6a725046c3d843c8dd7e51d46d1ed6') + '09abf0e7b510f61ca6cf33bab104e9ee99b3528b371d27a2d4b39abb800fba7e') with open(str(expected_path), 'rb') as f: digest = hashlib.sha256(f.read()).hexdigest() # UTXO snapshot hash should be deterministic based on mocked time. assert_equal( - digest, '7ae82c986fa5445678d2a21453bb1c86d39e47af13da137640c2b1cf8093691c') + digest, 'b1bacb602eacf5fbc9a7c2ef6eeb0d229c04e98bdf0c2ea5929012cd0eae3830') assert_equal( - out['txoutset_hash'], 'd4b614f476b99a6e569973bf1c0120d88b1a168076f8ce25691fb41dd1cef149') + out['txoutset_hash'], '1f7e3befd45dc13ae198dfbb22869a9c5c4196f8e9ef9735831af1288033f890') assert_equal(out['nchaintx'], 101) # Specifying a path to an existing file will fail. diff --git a/test/lint/lint-python-dead-code.py b/test/lint/lint-python-dead-code.py new file mode 100755 index 0000000000..b3f9394788 --- /dev/null +++ b/test/lint/lint-python-dead-code.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Find dead Python code. +""" + +from subprocess import check_output, STDOUT, CalledProcessError + +FILES_ARGS = ['git', 'ls-files', '--', '*.py'] + + +def check_vulture_install(): + try: + check_output(["vulture", "--version"]) + except FileNotFoundError: + print("Skipping Python dead code linting since vulture is not installed. Install by running \"pip3 install vulture\"") + exit(0) + + +def main(): + check_vulture_install() + + files = check_output(FILES_ARGS).decode("utf-8").splitlines() + # --min-confidence 100 will only report code that is guaranteed to be unused within the analyzed files. + # Any value below 100 introduces the risk of false positives, which would create an unacceptable maintenance burden. + vulture_args = ['vulture', '--min-confidence=100'] + files + + try: + check_output(vulture_args, stderr=STDOUT) + except CalledProcessError as e: + print(e.output.decode("utf-8"), end="") + print("Python dead code detection found some issues") + exit(1) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python-dead-code.sh b/test/lint/lint-python-dead-code.sh deleted file mode 100755 index 247bfb310a..0000000000 --- a/test/lint/lint-python-dead-code.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Find dead Python code. - -export LC_ALL=C - -if ! command -v vulture > /dev/null; then - echo "Skipping Python dead code linting since vulture is not installed. Install by running \"pip3 install vulture\"" - exit 0 -fi - -# --min-confidence 100 will only report code that is guaranteed to be unused within the analyzed files. -# Any value below 100 introduces the risk of false positives, which would create an unacceptable maintenance burden. -mapfile -t FILES < <(git ls-files -- "*.py") -if ! vulture --min-confidence 100 "${FILES[@]}"; then - echo "Python dead code detection found some issues" - exit 1 -fi diff --git a/test/lint/lint-spelling.py b/test/lint/lint-spelling.py new file mode 100755 index 0000000000..5da1b243f7 --- /dev/null +++ b/test/lint/lint-spelling.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Warn in case of spelling errors. +Note: Will exit successfully regardless of spelling errors. +""" + +from subprocess import check_output, STDOUT, CalledProcessError + +IGNORE_WORDS_FILE = 'test/lint/spelling.ignore-words.txt' +FILES_ARGS = ['git', 'ls-files', '--', ":(exclude)build-aux/m4/", ":(exclude)contrib/seeds/*.txt", ":(exclude)depends/", ":(exclude)doc/release-notes/", ":(exclude)src/leveldb/", ":(exclude)src/crc32c/", ":(exclude)src/qt/locale/", ":(exclude)src/qt/*.qrc", ":(exclude)src/secp256k1/", ":(exclude)src/minisketch/", ":(exclude)src/univalue/", ":(exclude)contrib/builder-keys/keys.txt", ":(exclude)contrib/guix/patches"] + + +def check_codespell_install(): + try: + check_output(["codespell", "--version"]) + except FileNotFoundError: + print("Skipping spell check linting since codespell is not installed.") + exit(0) + + +def main(): + check_codespell_install() + + files = check_output(FILES_ARGS).decode("utf-8").splitlines() + codespell_args = ['codespell', '--check-filenames', '--disable-colors', '--quiet-level=7', '--ignore-words={}'.format(IGNORE_WORDS_FILE)] + files + + try: + check_output(codespell_args, stderr=STDOUT) + except CalledProcessError as e: + print(e.output.decode("utf-8"), end="") + print('^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in {}'.format(IGNORE_WORDS_FILE)) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-spelling.sh b/test/lint/lint-spelling.sh deleted file mode 100755 index 8808fbc3c6..0000000000 --- a/test/lint/lint-spelling.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Warn in case of spelling errors. -# Note: Will exit successfully regardless of spelling errors. - -export LC_ALL=C - -if ! command -v codespell > /dev/null; then - echo "Skipping spell check linting since codespell is not installed." - exit 0 -fi - -IGNORE_WORDS_FILE="test/lint/spelling.ignore-words.txt" -mapfile -t FILES < <(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)contrib/builder-keys/keys.txt" ":(exclude)contrib/guix/patches") -if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} "${FILES[@]}"; then - echo "^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in ${IGNORE_WORDS_FILE}" -fi |