diff options
83 files changed, 2310 insertions, 1018 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index cf68110ec1..29116c9940 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -30,6 +30,7 @@ persistent_worker_template_env: &PERSISTENT_WORKER_TEMPLATE_ENV # # The following specific types should exist, with the following requirements: # - lunar: For a machine running the Linux kernel shipped with Ubuntu Lunar 23.04. The machine is recommended to have 4 CPUs and 16 GB of memory. +# - arm64: For an aarch64 machine, recommended to have 2 CPUs and 8 GB of memory. persistent_worker_template: &PERSISTENT_WORKER_TEMPLATE persistent_worker: {} # Only use this if the task does not care about the type at all @@ -206,13 +207,11 @@ task: task: name: 'ARM [unit tests, no functional tests] [bullseye]' << : *GLOBAL_TASK_TEMPLATE - container: - docker_arguments: - CI_IMAGE_NAME_TAG: debian:bullseye - FILE_ENV: "./ci/test/00_setup_env_arm.sh" - << : *CREDITS_TEMPLATE + persistent_worker: + labels: + type: arm64 # Use arm64 worker to sidestep qemu and avoid a slow CI: https://github.com/bitcoin/bitcoin/pull/28087#issuecomment-1649399453 env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_arm.sh" task: name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [jammy]' @@ -230,7 +229,7 @@ task: << : *GLOBAL_TASK_TEMPLATE container: docker_arguments: - CI_IMAGE_NAME_TAG: quay.io/centos/centos:stream9 + CI_IMAGE_NAME_TAG: "quay.io/centos/amd64:stream9" FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh" # For faster CI feedback, immediately schedule one task that runs all tests << : *CREDITS_TEMPLATE @@ -304,7 +303,7 @@ task: cpu: 4 memory: 16G # The default memory is too small, so double everything docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:focal + CI_IMAGE_NAME_TAG: "docker.io/amd64/ubuntu:focal" FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV @@ -322,15 +321,10 @@ task: task: name: 'macOS 11.0 [gui, no tests] [jammy]' - << : *CONTAINER_DEPENDS_TEMPLATE + << : *GLOBAL_TASK_TEMPLATE container: docker_arguments: CI_IMAGE_NAME_TAG: ubuntu:jammy FILE_ENV: "./ci/test/00_setup_env_mac.sh" - macos_sdk_cache: - folder: "depends/SDKs/$MACOS_SDK" - fingerprint_key: "$MACOS_SDK" - << : *MAIN_TEMPLATE env: - MACOS_SDK: "Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers" << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 722a877890..3014714a44 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -8,10 +8,18 @@ export LC_ALL=C.UTF-8 set -ex -# The root dir. +# The source root dir, usually from git, usually read-only. # The ci system copies this folder. -BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) -export BASE_ROOT_DIR +BASE_READ_ONLY_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) +export BASE_READ_ONLY_DIR +# The destination root dir inside the container. +if [ -z "${DANGER_RUN_CI_ON_HOST}" ] ; then + # This folder only exists on the ci guest and will be a copy of BASE_READ_ONLY_DIR + export BASE_ROOT_DIR="/ci_container_base" +else + # This folder is equal to BASE_READ_ONLY_DIR and is read-write + export BASE_ROOT_DIR="${BASE_READ_ONLY_DIR}" +fi # The depends dir. # This folder exists only on the ci guest, and on the ci host as a volume. export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} @@ -32,10 +40,6 @@ fi echo "Fallback to default values in env (if not yet set)" # The number of parallel jobs to pass down to make and test_runner.py export MAKEJOBS=${MAKEJOBS:--j4} -# What host to compile for. See also ./depends/README.md -# Tests that need cross-compilation export the appropriate HOST. -# Tests that run natively guess the host -export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} # Whether to prefer BusyBox over GNU utilities export USE_BUSY_BOX=${USE_BUSY_BOX:-false} @@ -61,13 +65,12 @@ export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} # This folder exists only on the ci guest, and on the ci host as a volume. export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache} # Folder where the build result is put (bin and lib). -export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} +export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out} # Folder where the build is done (dist and out-of-tree build). export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build} # The folder for previous release binaries. # This folder exists only on the ci guest, and on the ci host as a volume. -export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST} -export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu" +export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/prev_releases} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison} export GOAL=${GOAL:-install} diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index ac0c0be96a..65d37f01d9 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -7,18 +7,10 @@ export LC_ALL=C.UTF-8 export HOST=arm-linux-gnueabihf -# The host arch is unknown, so we run the tests through qemu. -# If the host is arm and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string. -if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-arm -L /usr/arm-linux-gnueabihf/"}"; fi export DPKG_ADD_ARCH="armhf" export PACKAGES="python3-zmq g++-arm-linux-gnueabihf busybox libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf" -if [ -n "$QEMU_USER_CMD" ]; then - # Likely cross-compiling, so install the needed gcc and qemu-user - export PACKAGES="$PACKAGES qemu-user" -fi export CONTAINER_NAME=ci_arm_linux -# Use debian to avoid 404 apt errors when cross compiling -export CI_IMAGE_NAME_TAG="debian:bullseye" +export CI_IMAGE_NAME_TAG="docker.io/arm64v8/debian:bookworm" export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index 606c28e252..a8bc0d0ca0 100755 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_centos -export CI_IMAGE_NAME_TAG="quay.io/centos/centos:stream9" +export CI_IMAGE_NAME_TAG="quay.io/centos/amd64:stream9" export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison util-linux" export PIP_PACKAGES="pyzmq" export GOAL="install" diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index 7911c1912f..b11a387660 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_multiprocess -export CI_IMAGE_NAME_TAG=ubuntu:20.04 +export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:20.04" export PACKAGES="cmake llvm clang g++-multilib" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 89006bf95e..007bb4bedf 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="ubuntu:22.04" -LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/" +LIBCXX_DIR="/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index a96ed30dd3..1d39ef9c48 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="ubuntu:22.04" -LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/" +LIBCXX_DIR="/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 523e81c94a..a7516d6b4e 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -7,18 +7,9 @@ export LC_ALL=C.UTF-8 export HOST=s390x-linux-gnu -# The host arch is unknown, so we run the tests through qemu. -# If the host is s390x and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string. -if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-s390x"}"; fi export PACKAGES="python3-zmq" -if [ -n "$QEMU_USER_CMD" ]; then - # Likely cross-compiling, so install the needed gcc and qemu-user - export DPKG_ADD_ARCH="s390x" - export PACKAGES="$PACKAGES g++-s390x-linux-gnu qemu-user libc6:s390x libstdc++6:s390x" -fi -# Use debian to avoid 404 apt errors export CONTAINER_NAME=ci_s390x -export CI_IMAGE_NAME_TAG="debian:bookworm" +export CI_IMAGE_NAME_TAG="docker.io/s390x/debian:bookworm" export TEST_RUNNER_ENV="LC_ALL=C" export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export RUN_FUNCTIONAL_TESTS=true diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 76cde42161..c9a496e6ab 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -42,39 +42,41 @@ if [ -n "$PIP_PACKAGES" ]; then fi if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.6 "${BASE_SCRATCH_DIR}"/msan/llvm-project - - cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/clang_build/ -DLLVM_ENABLE_PROJECTS="clang" \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_TARGETS_TO_BUILD=Native \ - -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \ - -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/llvm - - ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ "$MAKEJOBS" - ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ install-runtimes - - update-alternatives --install /usr/bin/clang++ clang++ "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang++ 100 - update-alternatives --install /usr/bin/clang clang "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang 100 - update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/llvm-symbolizer 100 - - cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/cxx_build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_USE_SANITIZER=MemoryWithOrigins \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DLLVM_TARGETS_TO_BUILD=Native \ - -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ - -DLIBCXX_ENABLE_DEBUG_MODE=ON \ - -DLIBCXX_ENABLE_ASSERTIONS=ON \ - -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes - - ninja -C "${BASE_SCRATCH_DIR}"/msan/cxx_build/ "$MAKEJOBS" + git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.6 /msan/llvm-project + + cmake -G Ninja -B /msan/clang_build/ \ + -DLLVM_ENABLE_PROJECTS="clang" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \ + -S /msan/llvm-project/llvm + + ninja -C /msan/clang_build/ "$MAKEJOBS" + ninja -C /msan/clang_build/ install-runtimes + + update-alternatives --install /usr/bin/clang++ clang++ /msan/clang_build/bin/clang++ 100 + update-alternatives --install /usr/bin/clang clang /msan/clang_build/bin/clang 100 + update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /msan/clang_build/bin/llvm-symbolizer 100 + + cmake -G Ninja -B /msan/cxx_build/ \ + -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_USE_SANITIZER=MemoryWithOrigins \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ + -DLIBCXX_ENABLE_DEBUG_MODE=ON \ + -DLIBCXX_ENABLE_ASSERTIONS=ON \ + -S /msan/llvm-project/runtimes + + ninja -C /msan/cxx_build/ "$MAKEJOBS" fi if [[ "${RUN_TIDY}" == "true" ]]; then - git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_16 "${DIR_IWYU}"/include-what-you-use - cmake -B "${DIR_IWYU}"/build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-16 -S "${DIR_IWYU}"/include-what-you-use - make -C "${DIR_IWYU}"/build/ install "$MAKEJOBS" + git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_16 /include-what-you-use + cmake -B /iwyu-build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-16 -S /include-what-you-use + make -C /iwyu-build/ install "$MAKEJOBS" fi mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 205c79328a..99e16bcb98 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -6,14 +6,6 @@ export LC_ALL=C.UTF-8 -if [[ $QEMU_USER_CMD == qemu-s390* ]]; then - export LC_ALL=C -fi - -# Create folders that are mounted into the docker -mkdir -p "${CCACHE_DIR}" -mkdir -p "${PREVIOUS_RELEASES_DIR}" - export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1" export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" @@ -27,11 +19,11 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then docker run --rm "${CI_IMAGE_NAME_TAG}" bash -c "env | grep --extended-regexp '^(HOME|PATH|USER)='" | tee --append /tmp/env echo "Creating $CI_IMAGE_NAME_TAG container to run in" DOCKER_BUILDKIT=1 docker build \ - --file "${BASE_ROOT_DIR}/ci/test_imagefile" \ + --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \ --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \ --build-arg "FILE_ENV=${FILE_ENV}" \ --tag="${CONTAINER_NAME}" \ - "${BASE_ROOT_DIR}" + "${BASE_READ_ONLY_DIR}" docker volume create "${CONTAINER_NAME}_ccache" || true docker volume create "${CONTAINER_NAME}_depends" || true docker volume create "${CONTAINER_NAME}_previous_releases" || true @@ -45,7 +37,7 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then # shellcheck disable=SC2086 CI_CONTAINER_ID=$(docker run $CI_CONTAINER_CAP --rm --interactive --detach --tty \ - --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ + --mount type=bind,src=$BASE_READ_ONLY_DIR,dst=/ro_base,readonly \ --mount "type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" \ --mount "type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR" \ --mount "type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" \ @@ -56,6 +48,9 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}" else echo "Running on host system without docker wrapper" + echo "Create missing folders" + mkdir -p "${CCACHE_DIR}" + mkdir -p "${PREVIOUS_RELEASES_DIR}" fi CI_EXEC () { diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 6e921d3377..eee4b8d2b3 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -19,6 +19,11 @@ fi echo "Free disk space:" df -h +# What host to compile for. See also ./depends/README.md +# Tests that need cross-compilation export the appropriate HOST. +# Tests that run natively guess the host +export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} + if [ "$RUN_FUZZ_TESTS" = "true" ]; then export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ if [ ! -d "$DIR_FUZZ_IN" ]; then @@ -124,13 +129,6 @@ if [[ $HOST = *-mingw32 ]]; then "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" fi -if [ -n "$QEMU_USER_CMD" ]; then - # Generate all binaries, so that they can be wrapped - make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 - make "$MAKEJOBS" -C src minisketch/test VERBOSE=1 - "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" -fi - if [ -n "$USE_VALGRIND" ]; then "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh" fi @@ -148,9 +146,13 @@ if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then fi if [ "${RUN_TIDY}" = "true" ]; then + cmake -B /tidy-build -DLLVM_DIR=/usr/lib/llvm-16/cmake -DCMAKE_BUILD_TYPE=Release -S "${BASE_ROOT_DIR}"/contrib/devtools/bitcoin-tidy + cmake --build /tidy-build "$MAKEJOBS" + cmake --build /tidy-build --target bitcoin-tidy-tests "$MAKEJOBS" + set -eo pipefail cd "${BASE_BUILD_DIR}/bitcoin-$HOST/src/" - ( run-clang-tidy-16 -quiet "${MAKEJOBS}" ) | grep -C5 "error" + ( run-clang-tidy-16 -quiet -load="/tidy-build/libbitcoin-tidy.so" "${MAKEJOBS}" ) | grep -C5 "error" # Filter out files by regex here, because regex may not be # accepted in src/.bear-tidy-config # Filter out: @@ -158,13 +160,13 @@ if [ "${RUN_TIDY}" = "true" ]; then jq 'map(select(.file | test("src/qt/qrc_.*\\.cpp$|/moc_.*\\.cpp$") | not))' ../compile_commands.json > tmp.json mv tmp.json ../compile_commands.json cd "${BASE_BUILD_DIR}/bitcoin-$HOST/" - python3 "${DIR_IWYU}/include-what-you-use/iwyu_tool.py" \ + python3 "/include-what-you-use/iwyu_tool.py" \ -p . "${MAKEJOBS}" \ -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" \ -Xiwyu --max_line_length=160 \ 2>&1 | tee /tmp/iwyu_ci.out cd "${BASE_ROOT_DIR}/src" - python3 "${DIR_IWYU}/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out + python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out git --no-pager diff fi diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh deleted file mode 100755 index e028ede378..0000000000 --- a/ci/test/wrap-qemu.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{test_json,unitester,object}}; do - # shellcheck disable=SC2044 - for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do - echo "Wrap $b ..." - mv "$b" "${b}_orig" - echo '#!/usr/bin/env bash' > "$b" - echo "$QEMU_USER_CMD \"${b}_orig\" \"\$@\"" >> "$b" - chmod +x "$b" - done -done diff --git a/contrib/devtools/bitcoin-tidy/CMakeLists.txt b/contrib/devtools/bitcoin-tidy/CMakeLists.txt new file mode 100644 index 0000000000..35e60d1d87 --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/CMakeLists.txt @@ -0,0 +1,56 @@ +cmake_minimum_required(VERSION 3.9) + +project(bitcoin-tidy VERSION 1.0.0 DESCRIPTION "clang-tidy checks for Bitcoin Core") + +include(GNUInstallDirs) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) +set(CMAKE_CXX_EXTENSIONS False) + +# TODO: Figure out how to avoid the terminfo check +find_package(LLVM REQUIRED CONFIG) +find_program(CLANG_TIDY_EXE NAMES "clang-tidy-${LLVM_VERSION_MAJOR}" "clang-tidy" HINTS ${LLVM_TOOLS_BINARY_DIR}) +message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") +message(STATUS "Found clang-tidy: ${CLANG_TIDY_EXE}") + +add_library(bitcoin-tidy MODULE bitcoin-tidy.cpp logprintf.cpp) +target_include_directories(bitcoin-tidy SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) + +# Disable RTTI and exceptions as necessary +if (MSVC) + target_compile_options(bitcoin-tidy PRIVATE /GR-) +else() + target_compile_options(bitcoin-tidy PRIVATE -fno-rtti) + target_compile_options(bitcoin-tidy PRIVATE -fno-exceptions) +endif() + +if(CMAKE_HOST_APPLE) + # ld64 expects no undefined symbols by default + target_link_options(bitcoin-tidy PRIVATE -Wl,-flat_namespace) + target_link_options(bitcoin-tidy PRIVATE -Wl,-undefined -Wl,suppress) +endif() + +# Add warnings +if (MSVC) + target_compile_options(bitcoin-tidy PRIVATE /W4) +else() + target_compile_options(bitcoin-tidy PRIVATE -Wall) + target_compile_options(bitcoin-tidy PRIVATE -Wextra) +endif() + +if(CMAKE_VERSION VERSION_LESS 3.27) + set(CLANG_TIDY_COMMAND "${CLANG_TIDY_EXE}" "--load=${CMAKE_BINARY_DIR}/${CMAKE_SHARED_MODULE_PREFIX}bitcoin-tidy${CMAKE_SHARED_MODULE_SUFFIX}" "-checks=-*,bitcoin-*") +else() + # CLANG_TIDY_COMMAND supports generator expressions as of 3.27 + set(CLANG_TIDY_COMMAND "${CLANG_TIDY_EXE}" "--load=$<TARGET_FILE:bitcoin-tidy>" "-checks=-*,bitcoin-*") +endif() + +# Create a dummy library that runs clang-tidy tests as a side-effect of building +add_library(bitcoin-tidy-tests OBJECT EXCLUDE_FROM_ALL example_logprintf.cpp) +add_dependencies(bitcoin-tidy-tests bitcoin-tidy) + +set_target_properties(bitcoin-tidy-tests PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_COMMAND}") + + +install(TARGETS bitcoin-tidy LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/contrib/devtools/bitcoin-tidy/README b/contrib/devtools/bitcoin-tidy/README new file mode 100644 index 0000000000..c15e07c4ed --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/README @@ -0,0 +1,11 @@ +# Bitcoin Tidy + +Example Usage: + +```bash +cmake -S . -B build -DLLVM_DIR=$(llvm-config --cmakedir) -DCMAKE_BUILD_TYPE=Release + +cmake --build build -j$(nproc) + +cmake --build build --target bitcoin-tidy-tests -j$(nproc) +``` diff --git a/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp b/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp new file mode 100644 index 0000000000..0f34d37793 --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp @@ -0,0 +1,22 @@ +// Copyright (c) 2023 Bitcoin Developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "logprintf.h" + +#include <clang-tidy/ClangTidyModule.h> +#include <clang-tidy/ClangTidyModuleRegistry.h> + +class BitcoinModule final : public clang::tidy::ClangTidyModule +{ +public: + void addCheckFactories(clang::tidy::ClangTidyCheckFactories& CheckFactories) override + { + CheckFactories.registerCheck<bitcoin::LogPrintfCheck>("bitcoin-unterminated-logprintf"); + } +}; + +static clang::tidy::ClangTidyModuleRegistry::Add<BitcoinModule> + X("bitcoin-module", "Adds bitcoin checks."); + +volatile int BitcoinModuleAnchorSource = 0; diff --git a/contrib/devtools/bitcoin-tidy/example_logprintf.cpp b/contrib/devtools/bitcoin-tidy/example_logprintf.cpp new file mode 100644 index 0000000000..a3d2768964 --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/example_logprintf.cpp @@ -0,0 +1,91 @@ +// Copyright (c) 2023 Bitcoin Developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +// Warn about any use of LogPrintf that does not end with a newline. +#include <string> + +enum LogFlags { + NONE +}; + +enum Level { + None +}; + +template <typename... Args> +static inline void LogPrintf_(const std::string& logging_function, const std::string& source_file, const int source_line, const LogFlags flag, const Level level, const char* fmt, const Args&... args) +{ +} + +#define LogPrintLevel_(category, level, ...) LogPrintf_(__func__, __FILE__, __LINE__, category, level, __VA_ARGS__) +#define LogPrintf(...) LogPrintLevel_(LogFlags::NONE, Level::None, __VA_ARGS__) + +// Use a macro instead of a function for conditional logging to prevent +// evaluating arguments when logging for the category is not enabled. +#define LogPrint(category, ...) \ + do { \ + LogPrintf(__VA_ARGS__); \ + } while (0) + + +class CWallet +{ + std::string GetDisplayName() const + { + return "default wallet"; + } + +public: + template <typename... Params> + void WalletLogPrintf(std::string fmt, Params... parameters) const + { + LogPrintf(("%s " + fmt).c_str(), GetDisplayName(), parameters...); + }; +}; + +void good_func() +{ + LogPrintf("hello world!\n"); +} +void good_func2() +{ + CWallet wallet; + wallet.WalletLogPrintf("hi\n"); + + const CWallet& walletref = wallet; + walletref.WalletLogPrintf("hi\n"); + + auto* walletptr = new CWallet(); + walletptr->WalletLogPrintf("hi\n"); + delete walletptr; +} +void bad_func() +{ + LogPrintf("hello world!"); +} +void bad_func2() +{ + LogPrintf(""); +} +void bad_func3() +{ + // Ending in "..." has no special meaning. + LogPrintf("hello world!..."); +} +void bad_func4_ignored() +{ + LogPrintf("hello world!"); // NOLINT(bitcoin-unterminated-logprintf) +} +void bad_func5() +{ + CWallet wallet; + wallet.WalletLogPrintf("hi"); + + const CWallet& walletref = wallet; + walletref.WalletLogPrintf("hi"); + + auto* walletptr = new CWallet(); + walletptr->WalletLogPrintf("hi"); + delete walletptr; +} diff --git a/contrib/devtools/bitcoin-tidy/logprintf.cpp b/contrib/devtools/bitcoin-tidy/logprintf.cpp new file mode 100644 index 0000000000..1690c8fde0 --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/logprintf.cpp @@ -0,0 +1,62 @@ +// Copyright (c) 2023 Bitcoin Developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "logprintf.h" + +#include <clang/AST/ASTContext.h> +#include <clang/ASTMatchers/ASTMatchFinder.h> + + +namespace { +AST_MATCHER(clang::StringLiteral, unterminated) +{ + size_t len = Node.getLength(); + if (len > 0 && Node.getCodeUnit(len - 1) == '\n') { + return false; + } + return true; +} +} // namespace + +namespace bitcoin { + +void LogPrintfCheck::registerMatchers(clang::ast_matchers::MatchFinder* finder) +{ + using namespace clang::ast_matchers; + + /* + Logprintf(..., ..., ..., ..., ..., "foo", ...) + */ + + finder->addMatcher( + callExpr( + callee(functionDecl(hasName("LogPrintf_"))), + hasArgument(5, stringLiteral(unterminated()).bind("logstring"))), + this); + + /* + CWallet wallet; + auto walletptr = &wallet; + wallet.WalletLogPrintf("foo"); + wallet->WalletLogPrintf("foo"); + */ + finder->addMatcher( + cxxMemberCallExpr( + thisPointerType(qualType(hasDeclaration(cxxRecordDecl(hasName("CWallet"))))), + callee(cxxMethodDecl(hasName("WalletLogPrintf"))), + hasArgument(0, stringLiteral(unterminated()).bind("logstring"))), + this); +} + +void LogPrintfCheck::check(const clang::ast_matchers::MatchFinder::MatchResult& Result) +{ + if (const clang::StringLiteral* lit = Result.Nodes.getNodeAs<clang::StringLiteral>("logstring")) { + const clang::ASTContext& ctx = *Result.Context; + const auto user_diag = diag(lit->getEndLoc(), "Unterminated format string used with LogPrintf"); + const auto& loc = lit->getLocationOfByte(lit->getByteLength(), *Result.SourceManager, ctx.getLangOpts(), ctx.getTargetInfo()); + user_diag << clang::FixItHint::CreateInsertion(loc, "\\n"); + } +} + +} // namespace bitcoin diff --git a/contrib/devtools/bitcoin-tidy/logprintf.h b/contrib/devtools/bitcoin-tidy/logprintf.h new file mode 100644 index 0000000000..466849ef01 --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/logprintf.h @@ -0,0 +1,28 @@ +// Copyright (c) 2023 Bitcoin Developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef LOGPRINTF_CHECK_H +#define LOGPRINTF_CHECK_H + +#include <clang-tidy/ClangTidyCheck.h> + +namespace bitcoin { + +class LogPrintfCheck final : public clang::tidy::ClangTidyCheck +{ +public: + LogPrintfCheck(clang::StringRef Name, clang::tidy::ClangTidyContext* Context) + : clang::tidy::ClangTidyCheck(Name, Context) {} + + bool isLanguageVersionSupported(const clang::LangOptions& LangOpts) const override + { + return LangOpts.CPlusPlus; + } + void registerMatchers(clang::ast_matchers::MatchFinder* Finder) override; + void check(const clang::ast_matchers::MatchFinder::MatchResult& Result) override; +}; + +} // namespace bitcoin + +#endif // LOGPRINTF_CHECK_H diff --git a/doc/build-unix.md b/doc/build-unix.md index 848dd5f71a..bf367fc421 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -55,7 +55,7 @@ SQLite is required for the descriptor wallet: sudo apt install libsqlite3-dev Berkeley DB is only required for the legacy wallet. Ubuntu and Debian have their own `libdb-dev` and `libdb++-dev` packages, -but these will install Berkeley DB 5.1 or later. This will break binary wallet compatibility with the distributed +but these will install Berkeley DB 5.3 or later. This will break binary wallet compatibility with the distributed executables, which are based on BerkeleyDB 4.8. If you do not care about wallet compatibility, pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley DB [yourself](#berkeley-db). @@ -111,11 +111,7 @@ SQLite is required for the descriptor wallet: sudo dnf install sqlite-devel -Berkeley DB is required for the legacy wallet: - - sudo dnf install libdb4-devel libdb4-cxx-devel - -Berkeley DB is only required for the legacy wallet. Newer Fedora releases have only `libdb-devel` and `libdb-cxx-devel` packages, but these will install +Berkeley DB is only required for the legacy wallet. Fedora releases have only `libdb-devel` and `libdb-cxx-devel` packages, but these will install Berkeley DB 5.3 or later. This will break binary wallet compatibility with the distributed executables, which are based on Berkeley DB 4.8. If you do not care about wallet compatibility, pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley DB [yourself](#berkeley-db). diff --git a/doc/release-notes-27213.md b/doc/release-notes-27213.md new file mode 100644 index 0000000000..3b478f11b7 --- /dev/null +++ b/doc/release-notes-27213.md @@ -0,0 +1,8 @@ +P2P and network changes +------ + +- Nodes with multiple reachable networks will actively try to have at least one + outbound connection to each network. This improves individual resistance to + eclipse attacks and network level resistance to partition attacks. Users no + longer need to perform active measures to ensure being connected to multiple + enabled networks. diff --git a/src/.clang-tidy b/src/.clang-tidy index 84c9d5fb3a..b4d50135dd 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -1,5 +1,6 @@ Checks: ' -*, +bitcoin-unterminated-logprintf, bugprone-argument-comment, bugprone-use-after-move, misc-unused-using-decls, diff --git a/src/Makefile.am b/src/Makefile.am index b48d723bc9..06c156a8c0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -24,7 +24,7 @@ check_PROGRAMS = TESTS = BENCHMARKS = -BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(LEVELDB_CPPFLAGS) +BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) LIBBITCOIN_NODE=libbitcoin_node.a LIBBITCOIN_COMMON=libbitcoin_common.a @@ -124,6 +124,7 @@ BITCOIN_CORE_H = \ banman.h \ base58.h \ bech32.h \ + bip324.h \ blockencodings.h \ blockfilter.h \ chain.h \ @@ -370,12 +371,13 @@ obj/build.h: FORCE libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h # node # -libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) +libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(LEVELDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_node_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_node_a_SOURCES = \ addrdb.cpp \ addrman.cpp \ banman.cpp \ + bip324.cpp \ blockencodings.cpp \ blockfilter.cpp \ chain.cpp \ @@ -546,10 +548,10 @@ crypto_libbitcoin_crypto_base_la_LDFLAGS = $(AM_LDFLAGS) -static crypto_libbitcoin_crypto_base_la_SOURCES = \ crypto/aes.cpp \ crypto/aes.h \ - crypto/chacha_poly_aead.h \ - crypto/chacha_poly_aead.cpp \ crypto/chacha20.h \ crypto/chacha20.cpp \ + crypto/chacha20poly1305.h \ + crypto/chacha20poly1305.cpp \ crypto/common.h \ crypto/hkdf_sha256_32.cpp \ crypto/hkdf_sha256_32.h \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 51bfb1e459..934e9a1fae 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -22,7 +22,6 @@ bench_bench_bitcoin_SOURCES = \ bench/block_assemble.cpp \ bench/ccoins_caching.cpp \ bench/chacha20.cpp \ - bench/chacha_poly_aead.cpp \ bench/checkblock.cpp \ bench/checkqueue.cpp \ bench/crypto_hash.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index f2a82ce73b..5dc20d4fab 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -74,6 +74,7 @@ BITCOIN_TESTS =\ test/base64_tests.cpp \ test/bech32_tests.cpp \ test/bip32_tests.cpp \ + test/bip324_tests.cpp \ test/blockchain_tests.cpp \ test/blockencodings_tests.cpp \ test/blockfilter_index_tests.cpp \ @@ -246,6 +247,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/banman.cpp \ test/fuzz/base_encode_decode.cpp \ test/fuzz/bech32.cpp \ + test/fuzz/bip324.cpp \ test/fuzz/bitdeque.cpp \ test/fuzz/block.cpp \ test/fuzz/block_header.cpp \ @@ -261,7 +263,6 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/crypto_aes256.cpp \ test/fuzz/crypto_aes256cbc.cpp \ test/fuzz/crypto_chacha20.cpp \ - test/fuzz/crypto_chacha20_poly1305_aead.cpp \ test/fuzz/crypto_common.cpp \ test/fuzz/crypto_diff_fuzz_chacha20.cpp \ test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp \ diff --git a/src/bench/chacha20.cpp b/src/bench/chacha20.cpp index 3b57e29f39..d8bebf9319 100644 --- a/src/bench/chacha20.cpp +++ b/src/bench/chacha20.cpp @@ -5,6 +5,7 @@ #include <bench/bench.h> #include <crypto/chacha20.h> +#include <crypto/chacha20poly1305.h> /* Number of bytes to process per iteration */ static const uint64_t BUFFER_SIZE_TINY = 64; @@ -23,6 +24,18 @@ static void CHACHA20(benchmark::Bench& bench, size_t buffersize) }); } +static void FSCHACHA20POLY1305(benchmark::Bench& bench, size_t buffersize) +{ + std::vector<std::byte> key(32); + FSChaCha20Poly1305 ctx(key, 224); + std::vector<std::byte> in(buffersize); + std::vector<std::byte> aad; + std::vector<std::byte> out(buffersize + FSChaCha20Poly1305::EXPANSION); + bench.batch(in.size()).unit("byte").run([&] { + ctx.Encrypt(in, aad, out); + }); +} + static void CHACHA20_64BYTES(benchmark::Bench& bench) { CHACHA20(bench, BUFFER_SIZE_TINY); @@ -38,6 +51,24 @@ static void CHACHA20_1MB(benchmark::Bench& bench) CHACHA20(bench, BUFFER_SIZE_LARGE); } +static void FSCHACHA20POLY1305_64BYTES(benchmark::Bench& bench) +{ + FSCHACHA20POLY1305(bench, BUFFER_SIZE_TINY); +} + +static void FSCHACHA20POLY1305_256BYTES(benchmark::Bench& bench) +{ + FSCHACHA20POLY1305(bench, BUFFER_SIZE_SMALL); +} + +static void FSCHACHA20POLY1305_1MB(benchmark::Bench& bench) +{ + FSCHACHA20POLY1305(bench, BUFFER_SIZE_LARGE); +} + BENCHMARK(CHACHA20_64BYTES, benchmark::PriorityLevel::HIGH); BENCHMARK(CHACHA20_256BYTES, benchmark::PriorityLevel::HIGH); BENCHMARK(CHACHA20_1MB, benchmark::PriorityLevel::HIGH); +BENCHMARK(FSCHACHA20POLY1305_64BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(FSCHACHA20POLY1305_256BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(FSCHACHA20POLY1305_1MB, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/chacha_poly_aead.cpp b/src/bench/chacha_poly_aead.cpp deleted file mode 100644 index 9149eb683a..0000000000 --- a/src/bench/chacha_poly_aead.cpp +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (c) 2019-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - - -#include <bench/bench.h> -#include <crypto/chacha_poly_aead.h> -#include <crypto/poly1305.h> // for the POLY1305_TAGLEN constant -#include <hash.h> - -#include <assert.h> -#include <limits> - -/* Number of bytes to process per iteration */ -static constexpr uint64_t BUFFER_SIZE_TINY = 64; -static constexpr uint64_t BUFFER_SIZE_SMALL = 256; -static constexpr uint64_t BUFFER_SIZE_LARGE = 1024 * 1024; - -static const unsigned char k1[32] = {0}; -static const unsigned char k2[32] = {0}; - -static ChaCha20Poly1305AEAD aead(k1, 32, k2, 32); - -static void CHACHA20_POLY1305_AEAD(benchmark::Bench& bench, size_t buffersize, bool include_decryption) -{ - std::vector<unsigned char> in(buffersize + CHACHA20_POLY1305_AEAD_AAD_LEN + Poly1305::TAGLEN, 0); - std::vector<unsigned char> out(buffersize + CHACHA20_POLY1305_AEAD_AAD_LEN + Poly1305::TAGLEN, 0); - uint64_t seqnr_payload = 0; - uint64_t seqnr_aad = 0; - int aad_pos = 0; - uint32_t len = 0; - bench.batch(buffersize).unit("byte").run([&] { - // encrypt or decrypt the buffer with a static key - const bool crypt_ok_1 = aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffersize, true); - assert(crypt_ok_1); - - if (include_decryption) { - // if we decrypt, include the GetLength - const bool get_length_ok = aead.GetLength(&len, seqnr_aad, aad_pos, in.data()); - assert(get_length_ok); - const bool crypt_ok_2 = aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffersize, true); - assert(crypt_ok_2); - } - - // increase main sequence number - seqnr_payload++; - // increase aad position (position in AAD keystream) - aad_pos += CHACHA20_POLY1305_AEAD_AAD_LEN; - if (aad_pos + CHACHA20_POLY1305_AEAD_AAD_LEN > CHACHA20_ROUND_OUTPUT) { - aad_pos = 0; - seqnr_aad++; - } - if (seqnr_payload + 1 == std::numeric_limits<uint64_t>::max()) { - // reuse of nonce+key is okay while benchmarking. - seqnr_payload = 0; - seqnr_aad = 0; - aad_pos = 0; - } - }); -} - -static void CHACHA20_POLY1305_AEAD_64BYTES_ONLY_ENCRYPT(benchmark::Bench& bench) -{ - CHACHA20_POLY1305_AEAD(bench, BUFFER_SIZE_TINY, false); -} - -static void CHACHA20_POLY1305_AEAD_256BYTES_ONLY_ENCRYPT(benchmark::Bench& bench) -{ - CHACHA20_POLY1305_AEAD(bench, BUFFER_SIZE_SMALL, false); -} - -static void CHACHA20_POLY1305_AEAD_1MB_ONLY_ENCRYPT(benchmark::Bench& bench) -{ - CHACHA20_POLY1305_AEAD(bench, BUFFER_SIZE_LARGE, false); -} - -static void CHACHA20_POLY1305_AEAD_64BYTES_ENCRYPT_DECRYPT(benchmark::Bench& bench) -{ - CHACHA20_POLY1305_AEAD(bench, BUFFER_SIZE_TINY, true); -} - -static void CHACHA20_POLY1305_AEAD_256BYTES_ENCRYPT_DECRYPT(benchmark::Bench& bench) -{ - CHACHA20_POLY1305_AEAD(bench, BUFFER_SIZE_SMALL, true); -} - -static void CHACHA20_POLY1305_AEAD_1MB_ENCRYPT_DECRYPT(benchmark::Bench& bench) -{ - CHACHA20_POLY1305_AEAD(bench, BUFFER_SIZE_LARGE, true); -} - -// Add Hash() (dbl-sha256) bench for comparison - -static void HASH(benchmark::Bench& bench, size_t buffersize) -{ - uint8_t hash[CHash256::OUTPUT_SIZE]; - std::vector<uint8_t> in(buffersize,0); - bench.batch(in.size()).unit("byte").run([&] { - CHash256().Write(in).Finalize(hash); - }); -} - -static void HASH_64BYTES(benchmark::Bench& bench) -{ - HASH(bench, BUFFER_SIZE_TINY); -} - -static void HASH_256BYTES(benchmark::Bench& bench) -{ - HASH(bench, BUFFER_SIZE_SMALL); -} - -static void HASH_1MB(benchmark::Bench& bench) -{ - HASH(bench, BUFFER_SIZE_LARGE); -} - -BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); -BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); -BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); -BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); -BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); -BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); -BENCHMARK(HASH_64BYTES, benchmark::PriorityLevel::HIGH); -BENCHMARK(HASH_256BYTES, benchmark::PriorityLevel::HIGH); -BENCHMARK(HASH_1MB, benchmark::PriorityLevel::HIGH); diff --git a/src/bip324.cpp b/src/bip324.cpp new file mode 100644 index 0000000000..314e756829 --- /dev/null +++ b/src/bip324.cpp @@ -0,0 +1,116 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bip324.h> + +#include <chainparams.h> +#include <crypto/chacha20.h> +#include <crypto/chacha20poly1305.h> +#include <crypto/hkdf_sha256_32.h> +#include <key.h> +#include <pubkey.h> +#include <random.h> +#include <span.h> +#include <support/cleanse.h> +#include <uint256.h> + +#include <algorithm> +#include <assert.h> +#include <cstdint> +#include <cstddef> +#include <iterator> +#include <string> + +BIP324Cipher::BIP324Cipher() noexcept +{ + m_key.MakeNewKey(true); + uint256 entropy = GetRandHash(); + m_our_pubkey = m_key.EllSwiftCreate(MakeByteSpan(entropy)); +} + +BIP324Cipher::BIP324Cipher(const CKey& key, Span<const std::byte> ent32) noexcept : + m_key(key) +{ + m_our_pubkey = m_key.EllSwiftCreate(ent32); +} + +BIP324Cipher::BIP324Cipher(const CKey& key, const EllSwiftPubKey& pubkey) noexcept : + m_key(key), m_our_pubkey(pubkey) {} + +void BIP324Cipher::Initialize(const EllSwiftPubKey& their_pubkey, bool initiator, bool self_decrypt) noexcept +{ + // Determine salt (fixed string + network magic bytes) + const auto& message_header = Params().MessageStart(); + std::string salt = std::string{"bitcoin_v2_shared_secret"} + std::string(std::begin(message_header), std::end(message_header)); + + // Perform ECDH to derive shared secret. + ECDHSecret ecdh_secret = m_key.ComputeBIP324ECDHSecret(their_pubkey, m_our_pubkey, initiator); + + // Derive encryption keys from shared secret, and initialize stream ciphers and AEADs. + bool side = (initiator != self_decrypt); + CHKDF_HMAC_SHA256_L32 hkdf(UCharCast(ecdh_secret.data()), ecdh_secret.size(), salt); + std::array<std::byte, 32> hkdf_32_okm; + hkdf.Expand32("initiator_L", UCharCast(hkdf_32_okm.data())); + (side ? m_send_l_cipher : m_recv_l_cipher).emplace(hkdf_32_okm, REKEY_INTERVAL); + hkdf.Expand32("initiator_P", UCharCast(hkdf_32_okm.data())); + (side ? m_send_p_cipher : m_recv_p_cipher).emplace(hkdf_32_okm, REKEY_INTERVAL); + hkdf.Expand32("responder_L", UCharCast(hkdf_32_okm.data())); + (side ? m_recv_l_cipher : m_send_l_cipher).emplace(hkdf_32_okm, REKEY_INTERVAL); + hkdf.Expand32("responder_P", UCharCast(hkdf_32_okm.data())); + (side ? m_recv_p_cipher : m_send_p_cipher).emplace(hkdf_32_okm, REKEY_INTERVAL); + + // Derive garbage terminators from shared secret. + hkdf.Expand32("garbage_terminators", UCharCast(hkdf_32_okm.data())); + std::copy(std::begin(hkdf_32_okm), std::begin(hkdf_32_okm) + GARBAGE_TERMINATOR_LEN, + (initiator ? m_send_garbage_terminator : m_recv_garbage_terminator).begin()); + std::copy(std::end(hkdf_32_okm) - GARBAGE_TERMINATOR_LEN, std::end(hkdf_32_okm), + (initiator ? m_recv_garbage_terminator : m_send_garbage_terminator).begin()); + + // Derive session id from shared secret. + hkdf.Expand32("session_id", UCharCast(m_session_id.data())); + + // Wipe all variables that contain information which could be used to re-derive encryption keys. + memory_cleanse(ecdh_secret.data(), ecdh_secret.size()); + memory_cleanse(hkdf_32_okm.data(), sizeof(hkdf_32_okm)); + memory_cleanse(&hkdf, sizeof(hkdf)); + m_key = CKey(); +} + +void BIP324Cipher::Encrypt(Span<const std::byte> contents, Span<const std::byte> aad, bool ignore, Span<std::byte> output) noexcept +{ + assert(output.size() == contents.size() + EXPANSION); + + // Encrypt length. + std::byte len[LENGTH_LEN]; + len[0] = std::byte{(uint8_t)(contents.size() & 0xFF)}; + len[1] = std::byte{(uint8_t)((contents.size() >> 8) & 0xFF)}; + len[2] = std::byte{(uint8_t)((contents.size() >> 16) & 0xFF)}; + m_send_l_cipher->Crypt(len, output.first(LENGTH_LEN)); + + // Encrypt plaintext. + std::byte header[HEADER_LEN] = {ignore ? IGNORE_BIT : std::byte{0}}; + m_send_p_cipher->Encrypt(header, contents, aad, output.subspan(LENGTH_LEN)); +} + +uint32_t BIP324Cipher::DecryptLength(Span<const std::byte> input) noexcept +{ + assert(input.size() == LENGTH_LEN); + + std::byte buf[LENGTH_LEN]; + // Decrypt length + m_recv_l_cipher->Crypt(input, buf); + // Convert to number. + return uint32_t(buf[0]) + (uint32_t(buf[1]) << 8) + (uint32_t(buf[2]) << 16); +} + +bool BIP324Cipher::Decrypt(Span<const std::byte> input, Span<const std::byte> aad, bool& ignore, Span<std::byte> contents) noexcept +{ + assert(input.size() + LENGTH_LEN == contents.size() + EXPANSION); + + std::byte header[HEADER_LEN]; + if (!m_recv_p_cipher->Decrypt(input, aad, header, contents)) return false; + + ignore = (header[0] & IGNORE_BIT) == IGNORE_BIT; + return true; +} diff --git a/src/bip324.h b/src/bip324.h new file mode 100644 index 0000000000..0238c479c0 --- /dev/null +++ b/src/bip324.h @@ -0,0 +1,96 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_BIP324_H +#define BITCOIN_BIP324_H + +#include <array> +#include <cstddef> +#include <optional> + +#include <crypto/chacha20.h> +#include <crypto/chacha20poly1305.h> +#include <key.h> +#include <pubkey.h> +#include <span.h> + +/** The BIP324 packet cipher, encapsulating its key derivation, stream cipher, and AEAD. */ +class BIP324Cipher +{ +public: + static constexpr unsigned SESSION_ID_LEN{32}; + static constexpr unsigned GARBAGE_TERMINATOR_LEN{16}; + static constexpr unsigned REKEY_INTERVAL{224}; + static constexpr unsigned LENGTH_LEN{3}; + static constexpr unsigned HEADER_LEN{1}; + static constexpr unsigned EXPANSION = LENGTH_LEN + HEADER_LEN + FSChaCha20Poly1305::EXPANSION; + static constexpr std::byte IGNORE_BIT{0x80}; + +private: + std::optional<FSChaCha20> m_send_l_cipher; + std::optional<FSChaCha20> m_recv_l_cipher; + std::optional<FSChaCha20Poly1305> m_send_p_cipher; + std::optional<FSChaCha20Poly1305> m_recv_p_cipher; + + CKey m_key; + EllSwiftPubKey m_our_pubkey; + + std::array<std::byte, SESSION_ID_LEN> m_session_id; + std::array<std::byte, GARBAGE_TERMINATOR_LEN> m_send_garbage_terminator; + std::array<std::byte, GARBAGE_TERMINATOR_LEN> m_recv_garbage_terminator; + +public: + /** Initialize a BIP324 cipher with securely generated random keys. */ + BIP324Cipher() noexcept; + + /** Initialize a BIP324 cipher with specified key and encoding entropy (testing only). */ + BIP324Cipher(const CKey& key, Span<const std::byte> ent32) noexcept; + + /** Initialize a BIP324 cipher with specified key (testing only). */ + BIP324Cipher(const CKey& key, const EllSwiftPubKey& pubkey) noexcept; + + /** Retrieve our public key. */ + const EllSwiftPubKey& GetOurPubKey() const noexcept { return m_our_pubkey; } + + /** Initialize when the other side's public key is received. Can only be called once. + * + * initiator is set to true if we are the initiator establishing the v2 P2P connection. + * self_decrypt is only for testing, and swaps encryption/decryption keys, so that encryption + * and decryption can be tested without knowing the other side's private key. + */ + void Initialize(const EllSwiftPubKey& their_pubkey, bool initiator, bool self_decrypt = false) noexcept; + + /** Determine whether this cipher is fully initialized. */ + explicit operator bool() const noexcept { return m_send_l_cipher.has_value(); } + + /** Encrypt a packet. Only after Initialize(). + * + * It must hold that output.size() == contents.size() + EXPANSION. + */ + void Encrypt(Span<const std::byte> contents, Span<const std::byte> aad, bool ignore, Span<std::byte> output) noexcept; + + /** Decrypt the length of a packet. Only after Initialize(). + * + * It must hold that input.size() == LENGTH_LEN. + */ + unsigned DecryptLength(Span<const std::byte> input) noexcept; + + /** Decrypt a packet. Only after Initialize(). + * + * It must hold that input.size() + LENGTH_LEN == contents.size() + EXPANSION. + * Contents.size() must equal the length returned by DecryptLength. + */ + bool Decrypt(Span<const std::byte> input, Span<const std::byte> aad, bool& ignore, Span<std::byte> contents) noexcept; + + /** Get the Session ID. Only after Initialize(). */ + Span<const std::byte> GetSessionID() const noexcept { return m_session_id; } + + /** Get the Garbage Terminator to send. Only after Initialize(). */ + Span<const std::byte> GetSendGarbageTerminator() const noexcept { return m_send_garbage_terminator; } + + /** Get the expected Garbage Terminator to receive. Only after Initialize(). */ + Span<const std::byte> GetReceiveGarbageTerminator() const noexcept { return m_recv_garbage_terminator; } +}; + +#endif // BITCOIN_BIP324_H diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 9aa0a6ba20..211b4740be 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -9,6 +9,7 @@ #include <consensus/validation.h> #include <crypto/sha256.h> #include <crypto/siphash.h> +#include <logging.h> #include <random.h> #include <streams.h> #include <txmempool.h> diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp index fafd783ab1..469b280494 100644 --- a/src/crypto/chacha20.cpp +++ b/src/crypto/chacha20.cpp @@ -7,6 +7,7 @@ #include <crypto/common.h> #include <crypto/chacha20.h> +#include <support/cleanse.h> #include <algorithm> #include <string.h> @@ -42,6 +43,11 @@ ChaCha20Aligned::ChaCha20Aligned() memset(input, 0, sizeof(input)); } +ChaCha20Aligned::~ChaCha20Aligned() +{ + memory_cleanse(input, sizeof(input)); +} + ChaCha20Aligned::ChaCha20Aligned(const unsigned char* key32) { SetKey32(key32); @@ -318,3 +324,38 @@ void ChaCha20::Crypt(const unsigned char* m, unsigned char* c, size_t bytes) m_bufleft = 64 - bytes; } } + +ChaCha20::~ChaCha20() +{ + memory_cleanse(m_buffer, sizeof(m_buffer)); +} + +FSChaCha20::FSChaCha20(Span<const std::byte> key, uint32_t rekey_interval) noexcept : + m_chacha20(UCharCast(key.data())), m_rekey_interval(rekey_interval) +{ + assert(key.size() == KEYLEN); +} + +void FSChaCha20::Crypt(Span<const std::byte> input, Span<std::byte> output) noexcept +{ + assert(input.size() == output.size()); + + // Invoke internal stream cipher for actual encryption/decryption. + m_chacha20.Crypt(UCharCast(input.data()), UCharCast(output.data()), input.size()); + + // Rekey after m_rekey_interval encryptions/decryptions. + if (++m_chunk_counter == m_rekey_interval) { + // Get new key from the stream cipher. + std::byte new_key[KEYLEN]; + m_chacha20.Keystream(UCharCast(new_key), sizeof(new_key)); + // Update its key. + m_chacha20.SetKey32(UCharCast(new_key)); + // Wipe the key (a copy remains inside m_chacha20, where it'll be wiped on the next rekey + // or on destruction). + memory_cleanse(new_key, sizeof(new_key)); + // Set the nonce for the new section of output. + m_chacha20.Seek64({0, ++m_rekey_counter}, 0); + // Reset the chunk counter. + m_chunk_counter = 0; + } +} diff --git a/src/crypto/chacha20.h b/src/crypto/chacha20.h index f2ec21d82e..d1b2094e7e 100644 --- a/src/crypto/chacha20.h +++ b/src/crypto/chacha20.h @@ -5,6 +5,10 @@ #ifndef BITCOIN_CRYPTO_CHACHA20_H #define BITCOIN_CRYPTO_CHACHA20_H +#include <span.h> + +#include <array> +#include <cstddef> #include <cstdlib> #include <stdint.h> #include <utility> @@ -29,6 +33,9 @@ public: /** Initialize a cipher with specified 32-byte key. */ ChaCha20Aligned(const unsigned char* key32); + /** Destructor to clean up private memory. */ + ~ChaCha20Aligned(); + /** set 32-byte key. */ void SetKey32(const unsigned char* key32); @@ -72,6 +79,9 @@ public: /** Initialize a cipher with specified 32-byte key. */ ChaCha20(const unsigned char* key32) : m_aligned(key32) {} + /** Destructor to clean up private memory. */ + ~ChaCha20(); + /** set 32-byte key. */ void SetKey32(const unsigned char* key32) { @@ -98,4 +108,43 @@ public: void Crypt(const unsigned char* input, unsigned char* output, size_t bytes); }; +/** Forward-secure ChaCha20 + * + * This implements a stream cipher that automatically transitions to a new stream with a new key + * and new nonce after a predefined number of encryptions or decryptions. + * + * See BIP324 for details. + */ +class FSChaCha20 +{ +private: + /** Internal stream cipher. */ + ChaCha20 m_chacha20; + + /** The number of encryptions/decryptions before a rekey happens. */ + const uint32_t m_rekey_interval; + + /** The number of encryptions/decryptions since the last rekey. */ + uint32_t m_chunk_counter{0}; + + /** The number of rekey operations that have happened. */ + uint64_t m_rekey_counter{0}; + +public: + /** Length of keys expected by the constructor. */ + static constexpr unsigned KEYLEN = 32; + + // No copy or move to protect the secret. + FSChaCha20(const FSChaCha20&) = delete; + FSChaCha20(FSChaCha20&&) = delete; + FSChaCha20& operator=(const FSChaCha20&) = delete; + FSChaCha20& operator=(FSChaCha20&&) = delete; + + /** Construct an FSChaCha20 cipher that rekeys every rekey_interval Crypt() calls. */ + FSChaCha20(Span<const std::byte> key, uint32_t rekey_interval) noexcept; + + /** Encrypt or decrypt a chunk. */ + void Crypt(Span<const std::byte> input, Span<std::byte> output) noexcept; +}; + #endif // BITCOIN_CRYPTO_CHACHA20_H diff --git a/src/crypto/chacha20poly1305.cpp b/src/crypto/chacha20poly1305.cpp new file mode 100644 index 0000000000..26161641bb --- /dev/null +++ b/src/crypto/chacha20poly1305.cpp @@ -0,0 +1,140 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/chacha20poly1305.h> + +#include <crypto/common.h> +#include <crypto/chacha20.h> +#include <crypto/poly1305.h> +#include <span.h> +#include <support/cleanse.h> + +#include <assert.h> +#include <cstddef> + +AEADChaCha20Poly1305::AEADChaCha20Poly1305(Span<const std::byte> key) noexcept : m_chacha20(UCharCast(key.data())) +{ + assert(key.size() == KEYLEN); +} + +void AEADChaCha20Poly1305::SetKey(Span<const std::byte> key) noexcept +{ + assert(key.size() == KEYLEN); + m_chacha20.SetKey32(UCharCast(key.data())); +} + +namespace { + +#ifndef HAVE_TIMINGSAFE_BCMP +#define HAVE_TIMINGSAFE_BCMP + +int timingsafe_bcmp(const unsigned char* b1, const unsigned char* b2, size_t n) noexcept +{ + const unsigned char *p1 = b1, *p2 = b2; + int ret = 0; + for (; n > 0; n--) + ret |= *p1++ ^ *p2++; + return (ret != 0); +} + +#endif + +/** Compute poly1305 tag. chacha20 must be set to the right nonce, block 0. Will be at block 1 after. */ +void ComputeTag(ChaCha20& chacha20, Span<const std::byte> aad, Span<const std::byte> cipher, Span<std::byte> tag) noexcept +{ + static const std::byte PADDING[16] = {{}}; + + // Get block of keystream (use a full 64 byte buffer to avoid the need for chacha20's own buffering). + std::byte first_block[64]; + chacha20.Keystream(UCharCast(first_block), sizeof(first_block)); + + // Use the first 32 bytes of the first keystream block as poly1305 key. + Poly1305 poly1305{Span{first_block}.first(Poly1305::KEYLEN)}; + + // Compute tag: + // - Process the padded AAD with Poly1305. + const unsigned aad_padding_length = (16 - (aad.size() % 16)) % 16; + poly1305.Update(aad).Update(Span{PADDING}.first(aad_padding_length)); + // - Process the padded ciphertext with Poly1305. + const unsigned cipher_padding_length = (16 - (cipher.size() % 16)) % 16; + poly1305.Update(cipher).Update(Span{PADDING}.first(cipher_padding_length)); + // - Process the AAD and plaintext length with Poly1305. + std::byte length_desc[Poly1305::TAGLEN]; + WriteLE64(UCharCast(length_desc), aad.size()); + WriteLE64(UCharCast(length_desc + 8), cipher.size()); + poly1305.Update(length_desc); + + // Output tag. + poly1305.Finalize(tag); +} + +} // namespace + +void AEADChaCha20Poly1305::Encrypt(Span<const std::byte> plain1, Span<const std::byte> plain2, Span<const std::byte> aad, Nonce96 nonce, Span<std::byte> cipher) noexcept +{ + assert(cipher.size() == plain1.size() + plain2.size() + EXPANSION); + + // Encrypt using ChaCha20 (starting at block 1). + m_chacha20.Seek64(nonce, 1); + m_chacha20.Crypt(UCharCast(plain1.data()), UCharCast(cipher.data()), plain1.size()); + m_chacha20.Crypt(UCharCast(plain2.data()), UCharCast(cipher.data() + plain1.size()), plain2.size()); + + // Seek to block 0, and compute tag using key drawn from there. + m_chacha20.Seek64(nonce, 0); + ComputeTag(m_chacha20, aad, cipher.first(cipher.size() - EXPANSION), cipher.last(EXPANSION)); +} + +bool AEADChaCha20Poly1305::Decrypt(Span<const std::byte> cipher, Span<const std::byte> aad, Nonce96 nonce, Span<std::byte> plain1, Span<std::byte> plain2) noexcept +{ + assert(cipher.size() == plain1.size() + plain2.size() + EXPANSION); + + // Verify tag (using key drawn from block 0). + m_chacha20.Seek64(nonce, 0); + std::byte expected_tag[EXPANSION]; + ComputeTag(m_chacha20, aad, cipher.first(cipher.size() - EXPANSION), expected_tag); + if (timingsafe_bcmp(UCharCast(expected_tag), UCharCast(cipher.last(EXPANSION).data()), EXPANSION)) return false; + + // Decrypt (starting at block 1). + m_chacha20.Crypt(UCharCast(cipher.data()), UCharCast(plain1.data()), plain1.size()); + m_chacha20.Crypt(UCharCast(cipher.data() + plain1.size()), UCharCast(plain2.data()), plain2.size()); + return true; +} + +void AEADChaCha20Poly1305::Keystream(Nonce96 nonce, Span<std::byte> keystream) noexcept +{ + // Skip the first output block, as it's used for generating the poly1305 key. + m_chacha20.Seek64(nonce, 1); + m_chacha20.Keystream(UCharCast(keystream.data()), keystream.size()); +} + +void FSChaCha20Poly1305::NextPacket() noexcept +{ + if (++m_packet_counter == m_rekey_interval) { + // Generate a full block of keystream, to avoid needing the ChaCha20 buffer, even though + // we only need KEYLEN (32) bytes. + std::byte one_block[64]; + m_aead.Keystream({0xFFFFFFFF, m_rekey_counter}, one_block); + // Switch keys. + m_aead.SetKey(Span{one_block}.first(KEYLEN)); + // Wipe the generated keystream (a copy remains inside m_aead, which will be cleaned up + // once it cycles again, or is destroyed). + memory_cleanse(one_block, sizeof(one_block)); + // Update counters. + m_packet_counter = 0; + ++m_rekey_counter; + } +} + +void FSChaCha20Poly1305::Encrypt(Span<const std::byte> plain1, Span<const std::byte> plain2, Span<const std::byte> aad, Span<std::byte> cipher) noexcept +{ + m_aead.Encrypt(plain1, plain2, aad, {m_packet_counter, m_rekey_counter}, cipher); + NextPacket(); +} + +bool FSChaCha20Poly1305::Decrypt(Span<const std::byte> cipher, Span<const std::byte> aad, Span<std::byte> plain1, Span<std::byte> plain2) noexcept +{ + bool ret = m_aead.Decrypt(cipher, aad, {m_packet_counter, m_rekey_counter}, plain1, plain2); + NextPacket(); + return ret; +} diff --git a/src/crypto/chacha20poly1305.h b/src/crypto/chacha20poly1305.h new file mode 100644 index 0000000000..a847c258ef --- /dev/null +++ b/src/crypto/chacha20poly1305.h @@ -0,0 +1,148 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_CHACHA20POLY1305_H +#define BITCOIN_CRYPTO_CHACHA20POLY1305_H + +#include <cstddef> +#include <stdint.h> + +#include <crypto/chacha20.h> +#include <crypto/poly1305.h> +#include <span.h> + +/** The AEAD_CHACHA20_POLY1305 authenticated encryption algorithm from RFC8439 section 2.8. */ +class AEADChaCha20Poly1305 +{ + /** Internal stream cipher. */ + ChaCha20 m_chacha20; + +public: + /** Expected size of key argument in constructor. */ + static constexpr unsigned KEYLEN = 32; + + /** Expansion when encrypting. */ + static constexpr unsigned EXPANSION = Poly1305::TAGLEN; + + /** Initialize an AEAD instance with a specified 32-byte key. */ + AEADChaCha20Poly1305(Span<const std::byte> key) noexcept; + + /** Switch to another 32-byte key. */ + void SetKey(Span<const std::byte> key) noexcept; + + /** 96-bit nonce type. */ + using Nonce96 = ChaCha20::Nonce96; + + /** Encrypt a message with a specified 96-bit nonce and aad. + * + * Requires cipher.size() = plain.size() + EXPANSION. + */ + void Encrypt(Span<const std::byte> plain, Span<const std::byte> aad, Nonce96 nonce, Span<std::byte> cipher) noexcept + { + Encrypt(plain, {}, aad, nonce, cipher); + } + + /** Encrypt a message (given split into plain1 + plain2) with a specified 96-bit nonce and aad. + * + * Requires cipher.size() = plain1.size() + plain2.size() + EXPANSION. + */ + void Encrypt(Span<const std::byte> plain1, Span<const std::byte> plain2, Span<const std::byte> aad, Nonce96 nonce, Span<std::byte> cipher) noexcept; + + /** Decrypt a message with a specified 96-bit nonce and aad. Returns true if valid. + * + * Requires cipher.size() = plain.size() + EXPANSION. + */ + bool Decrypt(Span<const std::byte> cipher, Span<const std::byte> aad, Nonce96 nonce, Span<std::byte> plain) noexcept + { + return Decrypt(cipher, aad, nonce, plain, {}); + } + + /** Decrypt a message with a specified 96-bit nonce and aad and split the result. Returns true if valid. + * + * Requires cipher.size() = plain1.size() + plain2.size() + EXPANSION. + */ + bool Decrypt(Span<const std::byte> cipher, Span<const std::byte> aad, Nonce96 nonce, Span<std::byte> plain1, Span<std::byte> plain2) noexcept; + + /** Get a number of keystream bytes from the underlying stream cipher. + * + * This is equivalent to Encrypt() with plain set to that many zero bytes, and dropping the + * last EXPANSION bytes off the result. + */ + void Keystream(Nonce96 nonce, Span<std::byte> keystream) noexcept; +}; + +/** Forward-secure wrapper around AEADChaCha20Poly1305. + * + * This implements an AEAD which automatically increments the nonce on every encryption or + * decryption, and cycles keys after a predetermined number of encryptions or decryptions. + * + * See BIP324 for details. + */ +class FSChaCha20Poly1305 +{ +private: + /** Internal AEAD. */ + AEADChaCha20Poly1305 m_aead; + + /** Every how many iterations this cipher rekeys. */ + const uint32_t m_rekey_interval; + + /** The number of encryptions/decryptions since the last rekey. */ + uint32_t m_packet_counter{0}; + + /** The number of rekeys performed so far. */ + uint64_t m_rekey_counter{0}; + + /** Update counters (and if necessary, key) to transition to the next message. */ + void NextPacket() noexcept; + +public: + /** Length of keys expected by the constructor. */ + static constexpr auto KEYLEN = AEADChaCha20Poly1305::KEYLEN; + + /** Expansion when encrypting. */ + static constexpr auto EXPANSION = AEADChaCha20Poly1305::EXPANSION; + + // No copy or move to protect the secret. + FSChaCha20Poly1305(const FSChaCha20Poly1305&) = delete; + FSChaCha20Poly1305(FSChaCha20Poly1305&&) = delete; + FSChaCha20Poly1305& operator=(const FSChaCha20Poly1305&) = delete; + FSChaCha20Poly1305& operator=(FSChaCha20Poly1305&&) = delete; + + /** Construct an FSChaCha20Poly1305 cipher that rekeys every rekey_interval operations. */ + FSChaCha20Poly1305(Span<const std::byte> key, uint32_t rekey_interval) noexcept : + m_aead(key), m_rekey_interval(rekey_interval) {} + + /** Encrypt a message with a specified aad. + * + * Requires cipher.size() = plain.size() + EXPANSION. + */ + void Encrypt(Span<const std::byte> plain, Span<const std::byte> aad, Span<std::byte> cipher) noexcept + { + Encrypt(plain, {}, aad, cipher); + } + + /** Encrypt a message (given split into plain1 + plain2) with a specified aad. + * + * Requires cipher.size() = plain.size() + EXPANSION. + */ + void Encrypt(Span<const std::byte> plain1, Span<const std::byte> plain2, Span<const std::byte> aad, Span<std::byte> cipher) noexcept; + + /** Decrypt a message with a specified aad. Returns true if valid. + * + * Requires cipher.size() = plain.size() + EXPANSION. + */ + bool Decrypt(Span<const std::byte> cipher, Span<const std::byte> aad, Span<std::byte> plain) noexcept + { + return Decrypt(cipher, aad, plain, {}); + } + + /** Decrypt a message with a specified aad and split the result. Returns true if valid. + * + * Requires cipher.size() = plain1.size() + plain2.size() + EXPANSION. + */ + bool Decrypt(Span<const std::byte> cipher, Span<const std::byte> aad, Span<std::byte> plain1, Span<std::byte> plain2) noexcept; +}; + +#endif // BITCOIN_CRYPTO_CHACHA20POLY1305_H diff --git a/src/crypto/chacha_poly_aead.cpp b/src/crypto/chacha_poly_aead.cpp deleted file mode 100644 index 0d82cf3d74..0000000000 --- a/src/crypto/chacha_poly_aead.cpp +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright (c) 2019-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - -#include <crypto/chacha_poly_aead.h> - -#include <crypto/poly1305.h> -#include <support/cleanse.h> - -#include <assert.h> -#include <string.h> - -#include <cstdio> -#include <limits> - -#ifndef HAVE_TIMINGSAFE_BCMP - -int timingsafe_bcmp(const unsigned char* b1, const unsigned char* b2, size_t n) -{ - const unsigned char *p1 = b1, *p2 = b2; - int ret = 0; - - for (; n > 0; n--) - ret |= *p1++ ^ *p2++; - return (ret != 0); -} - -#endif // TIMINGSAFE_BCMP - -ChaCha20Poly1305AEAD::ChaCha20Poly1305AEAD(const unsigned char* K_1, size_t K_1_len, const unsigned char* K_2, size_t K_2_len) -{ - assert(K_1_len == CHACHA20_POLY1305_AEAD_KEY_LEN); - assert(K_2_len == CHACHA20_POLY1305_AEAD_KEY_LEN); - - static_assert(CHACHA20_POLY1305_AEAD_KEY_LEN == 32); - m_chacha_header.SetKey32(K_1); - m_chacha_main.SetKey32(K_2); - - // set the cached sequence number to uint64 max which hints for an unset cache. - // we can't hit uint64 max since the rekey rule (which resets the sequence number) is 1GB - m_cached_aad_seqnr = std::numeric_limits<uint64_t>::max(); -} - -bool ChaCha20Poly1305AEAD::Crypt(uint64_t seqnr_payload, uint64_t seqnr_aad, int aad_pos, unsigned char* dest, size_t dest_len /* length of the output buffer for sanity checks */, const unsigned char* src, size_t src_len, bool is_encrypt) -{ - // check buffer boundaries - if ( - // if we encrypt, make sure the source contains at least the expected AAD and the destination has at least space for the source + MAC - (is_encrypt && (src_len < CHACHA20_POLY1305_AEAD_AAD_LEN || dest_len < src_len + Poly1305::TAGLEN)) || - // if we decrypt, make sure the source contains at least the expected AAD+MAC and the destination has at least space for the source - MAC - (!is_encrypt && (src_len < CHACHA20_POLY1305_AEAD_AAD_LEN + Poly1305::TAGLEN || dest_len < src_len - Poly1305::TAGLEN))) { - return false; - } - - unsigned char expected_tag[Poly1305::TAGLEN], poly_key[Poly1305::KEYLEN]; - memset(poly_key, 0, sizeof(poly_key)); - - // block counter 0 for the poly1305 key - // use lower 32bytes for the poly1305 key - // (throws away 32 unused bytes (upper 32) from this ChaCha20 round) - m_chacha_main.Seek64({0, seqnr_payload}, 0); - m_chacha_main.Crypt(poly_key, poly_key, sizeof(poly_key)); - - // if decrypting, verify the tag prior to decryption - if (!is_encrypt) { - const unsigned char* tag = src + src_len - Poly1305::TAGLEN; - Poly1305{MakeByteSpan(poly_key)} - .Update(AsBytes(Span{src, src_len - Poly1305::TAGLEN})) - .Finalize(MakeWritableByteSpan(expected_tag)); - - // constant time compare the calculated MAC with the provided MAC - if (timingsafe_bcmp(expected_tag, tag, Poly1305::TAGLEN) != 0) { - memory_cleanse(expected_tag, sizeof(expected_tag)); - memory_cleanse(poly_key, sizeof(poly_key)); - return false; - } - memory_cleanse(expected_tag, sizeof(expected_tag)); - // MAC has been successfully verified, make sure we don't convert it in decryption - src_len -= Poly1305::TAGLEN; - } - - // calculate and cache the next 64byte keystream block if requested sequence number is not yet the cache - if (m_cached_aad_seqnr != seqnr_aad) { - m_cached_aad_seqnr = seqnr_aad; - m_chacha_header.Seek64({0, seqnr_aad}, 0); - m_chacha_header.Keystream(m_aad_keystream_buffer, CHACHA20_ROUND_OUTPUT); - } - // crypt the AAD (3 bytes message length) with given position in AAD cipher instance keystream - dest[0] = src[0] ^ m_aad_keystream_buffer[aad_pos]; - dest[1] = src[1] ^ m_aad_keystream_buffer[aad_pos + 1]; - dest[2] = src[2] ^ m_aad_keystream_buffer[aad_pos + 2]; - - // Set the playload ChaCha instance block counter to 1 and crypt the payload - m_chacha_main.Seek64({0, seqnr_payload}, 1); - m_chacha_main.Crypt(src + CHACHA20_POLY1305_AEAD_AAD_LEN, dest + CHACHA20_POLY1305_AEAD_AAD_LEN, src_len - CHACHA20_POLY1305_AEAD_AAD_LEN); - - // If encrypting, calculate and append tag - if (is_encrypt) { - // the poly1305 tag expands over the AAD (3 bytes length) & encrypted payload - Poly1305{MakeByteSpan(poly_key)} - .Update(AsBytes(Span{dest, src_len})) - .Finalize(AsWritableBytes(Span{dest + src_len, Poly1305::TAGLEN})); - } - - // cleanse no longer required MAC and polykey - memory_cleanse(poly_key, sizeof(poly_key)); - return true; -} - -bool ChaCha20Poly1305AEAD::GetLength(uint32_t* len24_out, uint64_t seqnr_aad, int aad_pos, const uint8_t* ciphertext) -{ - // enforce valid aad position to avoid accessing outside of the 64byte keystream cache - // (there is space for 21 times 3 bytes) - assert(aad_pos >= 0 && aad_pos < CHACHA20_ROUND_OUTPUT - CHACHA20_POLY1305_AEAD_AAD_LEN); - if (m_cached_aad_seqnr != seqnr_aad) { - // we need to calculate the 64 keystream bytes since we reached a new aad sequence number - m_cached_aad_seqnr = seqnr_aad; - m_chacha_header.Seek64({0, seqnr_aad}, 0); // use LE for the nonce - m_chacha_header.Keystream(m_aad_keystream_buffer, CHACHA20_ROUND_OUTPUT); // write keystream to the cache - } - - // decrypt the ciphertext length by XORing the right position of the 64byte keystream cache with the ciphertext - *len24_out = (ciphertext[0] ^ m_aad_keystream_buffer[aad_pos + 0]) | - (ciphertext[1] ^ m_aad_keystream_buffer[aad_pos + 1]) << 8 | - (ciphertext[2] ^ m_aad_keystream_buffer[aad_pos + 2]) << 16; - - return true; -} diff --git a/src/crypto/chacha_poly_aead.h b/src/crypto/chacha_poly_aead.h deleted file mode 100644 index 5d57b5a5e2..0000000000 --- a/src/crypto/chacha_poly_aead.h +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright (c) 2019-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_CRYPTO_CHACHA_POLY_AEAD_H -#define BITCOIN_CRYPTO_CHACHA_POLY_AEAD_H - -#include <crypto/chacha20.h> - -#include <cmath> - -static constexpr int CHACHA20_POLY1305_AEAD_KEY_LEN = 32; -static constexpr int CHACHA20_POLY1305_AEAD_AAD_LEN = 3; /* 3 bytes length */ -static constexpr int CHACHA20_ROUND_OUTPUT = 64; /* 64 bytes per round */ -static constexpr int AAD_PACKAGES_PER_ROUND = 21; /* 64 / 3 round down*/ - -/* A AEAD class for ChaCha20-Poly1305@bitcoin. - * - * ChaCha20 is a stream cipher designed by Daniel Bernstein and described in - * <ref>[https://cr.yp.to/chacha/chacha-20080128.pdf ChaCha20]</ref>. It operates - * by permuting 128 fixed bits, 128 or 256 bits of key, a 64 bit nonce and a 64 - * bit counter into 64 bytes of output. This output is used as a keystream, with - * any unused bytes simply discarded. - * - * Poly1305 <ref>[https://cr.yp.to/mac/poly1305-20050329.pdf Poly1305]</ref>, also - * by Daniel Bernstein, is a one-time Carter-Wegman MAC that computes a 128 bit - * integrity tag given a message and a single-use 256 bit secret key. - * - * The chacha20-poly1305@bitcoin combines these two primitives into an - * authenticated encryption mode. The construction used is based on that proposed - * for TLS by Adam Langley in - * <ref>[http://tools.ietf.org/html/draft-agl-tls-chacha20poly1305-03 "ChaCha20 - * and Poly1305 based Cipher Suites for TLS", Adam Langley]</ref>, but differs in - * the layout of data passed to the MAC and in the addition of encryption of the - * packet lengths. - * - * ==== Detailed Construction ==== - * - * The chacha20-poly1305@bitcoin cipher requires two 256 bits of key material as - * output from the key exchange. Each key (K_1 and K_2) are used by two separate - * instances of chacha20. - * - * The instance keyed by K_1 is a stream cipher that is used only to encrypt the 3 - * byte packet length field and has its own sequence number. The second instance, - * keyed by K_2, is used in conjunction with poly1305 to build an AEAD - * (Authenticated Encryption with Associated Data) that is used to encrypt and - * authenticate the entire packet. - * - * Two separate cipher instances are used here so as to keep the packet lengths - * confidential but not create an oracle for the packet payload cipher by - * decrypting and using the packet length prior to checking the MAC. By using an - * independently-keyed cipher instance to encrypt the length, an active attacker - * seeking to exploit the packet input handling as a decryption oracle can learn - * nothing about the payload contents or its MAC (assuming key derivation, - * ChaCha20 and Poly1305 are secure). - * - * The AEAD is constructed as follows: for each packet, generate a Poly1305 key by - * taking the first 256 bits of ChaCha20 stream output generated using K_2, an IV - * consisting of the packet sequence number encoded as an LE uint64 and a ChaCha20 - * block counter of zero. The K_2 ChaCha20 block counter is then set to the - * little-endian encoding of 1 (i.e. {1, 0, 0, 0, 0, 0, 0, 0}) and this instance - * is used for encryption of the packet payload. - * - * ==== Packet Handling ==== - * - * When receiving a packet, the length must be decrypted first. When 3 bytes of - * ciphertext length have been received, they may be decrypted. - * - * A ChaCha20 round always calculates 64bytes which is sufficient to crypt 21 - * times a 3 bytes length field (21*3 = 63). The length field sequence number can - * thus be used 21 times (keystream caching). - * - * The length field must be enc-/decrypted with the ChaCha20 keystream keyed with - * K_1 defined by block counter 0, the length field sequence number in little - * endian and a keystream position from 0 to 60. - * - * Once the entire packet has been received, the MAC MUST be checked before - * decryption. A per-packet Poly1305 key is generated as described above and the - * MAC tag calculated using Poly1305 with this key over the ciphertext of the - * packet length and the payload together. The calculated MAC is then compared in - * constant time with the one appended to the packet and the packet decrypted - * using ChaCha20 as described above (with K_2, the packet sequence number as - * nonce and a starting block counter of 1). - * - * Detection of an invalid MAC MUST lead to immediate connection termination. - * - * To send a packet, first encode the 3 byte length and encrypt it using K_1 as - * described above. Encrypt the packet payload (using K_2) and append it to the - * encrypted length. Finally, calculate a MAC tag and append it. - * - * The initiating peer MUST use <code>K_1_A, K_2_A</code> to encrypt messages on - * the send channel, <code>K_1_B, K_2_B</code> MUST be used to decrypt messages on - * the receive channel. - * - * The responding peer MUST use <code>K_1_A, K_2_A</code> to decrypt messages on - * the receive channel, <code>K_1_B, K_2_B</code> MUST be used to encrypt messages - * on the send channel. - * - * Optimized implementations of ChaCha20-Poly1305@bitcoin are relatively fast in - * general, therefore it is very likely that encrypted messages require not more - * CPU cycles per bytes then the current unencrypted p2p message format - * (ChaCha20/Poly1305 versus double SHA256). - * - * The initial packet sequence numbers are 0. - * - * K_2 ChaCha20 cipher instance (payload) must never reuse a {key, nonce} for - * encryption nor may it be used to encrypt more than 2^70 bytes under the same - * {key, nonce}. - * - * K_1 ChaCha20 cipher instance (length field/AAD) must never reuse a {key, nonce, - * position-in-keystream} for encryption nor may it be used to encrypt more than - * 2^70 bytes under the same {key, nonce}. - * - * We use message sequence numbers for both communication directions. - */ - -class ChaCha20Poly1305AEAD -{ -private: - ChaCha20 m_chacha_header; // AAD cipher instance (encrypted length) and poly1305 key-derivation cipher instance - ChaCha20 m_chacha_main; // payload - unsigned char m_aad_keystream_buffer[CHACHA20_ROUND_OUTPUT]; // aad keystream cache - uint64_t m_cached_aad_seqnr; // aad keystream cache hint - -public: - ChaCha20Poly1305AEAD(const unsigned char* K_1, size_t K_1_len, const unsigned char* K_2, size_t K_2_len); - - explicit ChaCha20Poly1305AEAD(const ChaCha20Poly1305AEAD&) = delete; - - /** Encrypts/decrypts a packet - seqnr_payload, the message sequence number - seqnr_aad, the messages AAD sequence number which allows reuse of the AAD keystream - aad_pos, position to use in the AAD keystream to encrypt the AAD - dest, output buffer, must be of a size equal or larger then CHACHA20_POLY1305_AEAD_AAD_LEN + payload (+ POLY1305_TAG_LEN in encryption) bytes - destlen, length of the destination buffer - src, the AAD+payload to encrypt or the AAD+payload+MAC to decrypt - src_len, the length of the source buffer - is_encrypt, set to true if we encrypt (creates and appends the MAC instead of verifying it) - */ - bool Crypt(uint64_t seqnr_payload, uint64_t seqnr_aad, int aad_pos, unsigned char* dest, size_t dest_len, const unsigned char* src, size_t src_len, bool is_encrypt); - - /** decrypts the 3 bytes AAD data and decodes it into a uint32_t field */ - bool GetLength(uint32_t* len24_out, uint64_t seqnr_aad, int aad_pos, const uint8_t* ciphertext); -}; - -#endif // BITCOIN_CRYPTO_CHACHA_POLY_AEAD_H diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 2aade14ef4..c95937ba75 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -4,9 +4,12 @@ #include <dbwrapper.h> +#include <clientversion.h> #include <logging.h> #include <random.h> -#include <tinyformat.h> +#include <serialize.h> +#include <span.h> +#include <streams.h> #include <util/fs.h> #include <util/fs_helpers.h> #include <util/strencodings.h> @@ -23,9 +26,31 @@ #include <leveldb/helpers/memenv/memenv.h> #include <leveldb/iterator.h> #include <leveldb/options.h> +#include <leveldb/slice.h> #include <leveldb/status.h> +#include <leveldb/write_batch.h> #include <memory> #include <optional> +#include <utility> + +static auto CharCast(const std::byte* data) { return reinterpret_cast<const char*>(data); } + +bool DestroyDB(const std::string& path_str) +{ + return leveldb::DestroyDB(path_str, {}).ok(); +} + +/** Handle database error by throwing dbwrapper_error exception. + */ +static void HandleError(const leveldb::Status& status) +{ + if (status.ok()) + return; + const std::string errmsg = "Fatal LevelDB error: " + status.ToString(); + LogPrintf("%s\n", errmsg); + LogPrintf("You can use -debug=leveldb to get more complete diagnostic messages\n"); + throw dbwrapper_error(errmsg); +} class CBitcoinLevelDBLogger : public leveldb::Logger { public: @@ -76,7 +101,7 @@ public: assert(p <= limit); base[std::min(bufsize - 1, (int)(p - base))] = '\0'; - LogPrintLevel(BCLog::LEVELDB, BCLog::Level::Debug, "%s", base); /* Continued */ + LogPrintLevel(BCLog::LEVELDB, BCLog::Level::Debug, "%s", base); // NOLINT(bitcoin-unterminated-logprintf) if (base != buffer) { delete[] base; } @@ -127,24 +152,91 @@ static leveldb::Options GetOptions(size_t nCacheSize) return options; } +struct CDBBatch::WriteBatchImpl { + leveldb::WriteBatch batch; +}; + +CDBBatch::CDBBatch(const CDBWrapper& _parent) : parent(_parent), + m_impl_batch{std::make_unique<CDBBatch::WriteBatchImpl>()}, + ssValue(SER_DISK, CLIENT_VERSION){}; + +CDBBatch::~CDBBatch() = default; + +void CDBBatch::Clear() +{ + m_impl_batch->batch.Clear(); + size_estimate = 0; +} + +void CDBBatch::WriteImpl(Span<const std::byte> key, CDataStream& ssValue) +{ + leveldb::Slice slKey(CharCast(key.data()), key.size()); + ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); + leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size()); + m_impl_batch->batch.Put(slKey, slValue); + // LevelDB serializes writes as: + // - byte: header + // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...) + // - byte[]: key + // - varint: value length + // - byte[]: value + // The formula below assumes the key and value are both less than 16k. + size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size(); +} + +void CDBBatch::EraseImpl(Span<const std::byte> key) +{ + leveldb::Slice slKey(CharCast(key.data()), key.size()); + m_impl_batch->batch.Delete(slKey); + // LevelDB serializes erases as: + // - byte: header + // - varint: key length + // - byte[]: key + // The formula below assumes the key is less than 16kB. + size_estimate += 2 + (slKey.size() > 127) + slKey.size(); +} + +struct LevelDBContext { + //! custom environment this database is using (may be nullptr in case of default environment) + leveldb::Env* penv; + + //! database options used + leveldb::Options options; + + //! options used when reading from the database + leveldb::ReadOptions readoptions; + + //! options used when iterating over values of the database + leveldb::ReadOptions iteroptions; + + //! options used when writing to the database + leveldb::WriteOptions writeoptions; + + //! options used when sync writing to the database + leveldb::WriteOptions syncoptions; + + //! the database itself + leveldb::DB* pdb; +}; + CDBWrapper::CDBWrapper(const DBParams& params) - : m_name{fs::PathToString(params.path.stem())}, m_path{params.path}, m_is_memory{params.memory_only} + : m_db_context{std::make_unique<LevelDBContext>()}, m_name{fs::PathToString(params.path.stem())}, m_path{params.path}, m_is_memory{params.memory_only} { - penv = nullptr; - readoptions.verify_checksums = true; - iteroptions.verify_checksums = true; - iteroptions.fill_cache = false; - syncoptions.sync = true; - options = GetOptions(params.cache_bytes); - options.create_if_missing = true; + DBContext().penv = nullptr; + DBContext().readoptions.verify_checksums = true; + DBContext().iteroptions.verify_checksums = true; + DBContext().iteroptions.fill_cache = false; + DBContext().syncoptions.sync = true; + DBContext().options = GetOptions(params.cache_bytes); + DBContext().options.create_if_missing = true; if (params.memory_only) { - penv = leveldb::NewMemEnv(leveldb::Env::Default()); - options.env = penv; + DBContext().penv = leveldb::NewMemEnv(leveldb::Env::Default()); + DBContext().options.env = DBContext().penv; } else { if (params.wipe_data) { LogPrintf("Wiping LevelDB in %s\n", fs::PathToString(params.path)); - leveldb::Status result = leveldb::DestroyDB(fs::PathToString(params.path), options); - dbwrapper_private::HandleError(result); + leveldb::Status result = leveldb::DestroyDB(fs::PathToString(params.path), DBContext().options); + HandleError(result); } TryCreateDirectories(params.path); LogPrintf("Opening LevelDB in %s\n", fs::PathToString(params.path)); @@ -153,13 +245,13 @@ CDBWrapper::CDBWrapper(const DBParams& params) // because on POSIX leveldb passes the byte string directly to ::open(), and // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW // (see env_posix.cc and env_windows.cc). - leveldb::Status status = leveldb::DB::Open(options, fs::PathToString(params.path), &pdb); - dbwrapper_private::HandleError(status); + leveldb::Status status = leveldb::DB::Open(DBContext().options, fs::PathToString(params.path), &DBContext().pdb); + HandleError(status); LogPrintf("Opened LevelDB successfully\n"); if (params.options.force_compact) { LogPrintf("Starting database compaction of %s\n", fs::PathToString(params.path)); - pdb->CompactRange(nullptr, nullptr); + DBContext().pdb->CompactRange(nullptr, nullptr); LogPrintf("Finished database compaction of %s\n", fs::PathToString(params.path)); } @@ -185,16 +277,16 @@ CDBWrapper::CDBWrapper(const DBParams& params) CDBWrapper::~CDBWrapper() { - delete pdb; - pdb = nullptr; - delete options.filter_policy; - options.filter_policy = nullptr; - delete options.info_log; - options.info_log = nullptr; - delete options.block_cache; - options.block_cache = nullptr; - delete penv; - options.env = nullptr; + delete DBContext().pdb; + DBContext().pdb = nullptr; + delete DBContext().options.filter_policy; + DBContext().options.filter_policy = nullptr; + delete DBContext().options.info_log; + DBContext().options.info_log = nullptr; + delete DBContext().options.block_cache; + DBContext().options.block_cache = nullptr; + delete DBContext().penv; + DBContext().options.env = nullptr; } bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) @@ -204,8 +296,8 @@ bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) if (log_memory) { mem_before = DynamicMemoryUsage() / 1024.0 / 1024; } - leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch); - dbwrapper_private::HandleError(status); + leveldb::Status status = DBContext().pdb->Write(fSync ? DBContext().syncoptions : DBContext().writeoptions, &batch.m_impl_batch->batch); + HandleError(status); if (log_memory) { double mem_after = DynamicMemoryUsage() / 1024.0 / 1024; LogPrint(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n", @@ -218,7 +310,7 @@ size_t CDBWrapper::DynamicMemoryUsage() const { std::string memory; std::optional<size_t> parsed; - if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral<size_t>(memory))) { + if (!DBContext().pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral<size_t>(memory))) { LogPrint(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n"); return 0; } @@ -244,6 +336,45 @@ std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const return ret; } +std::optional<std::string> CDBWrapper::ReadImpl(Span<const std::byte> key) const +{ + leveldb::Slice slKey(CharCast(key.data()), key.size()); + std::string strValue; + leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue); + if (!status.ok()) { + if (status.IsNotFound()) + return std::nullopt; + LogPrintf("LevelDB read failure: %s\n", status.ToString()); + HandleError(status); + } + return strValue; +} + +bool CDBWrapper::ExistsImpl(Span<const std::byte> key) const +{ + leveldb::Slice slKey(CharCast(key.data()), key.size()); + + std::string strValue; + leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue); + if (!status.ok()) { + if (status.IsNotFound()) + return false; + LogPrintf("LevelDB read failure: %s\n", status.ToString()); + HandleError(status); + } + return true; +} + +size_t CDBWrapper::EstimateSizeImpl(Span<const std::byte> key1, Span<const std::byte> key2) const +{ + leveldb::Slice slKey1(CharCast(key1.data()), key1.size()); + leveldb::Slice slKey2(CharCast(key2.data()), key2.size()); + uint64_t size = 0; + leveldb::Range range(slKey1, slKey2); + DBContext().pdb->GetApproximateSizes(&range, 1, &size); + return size; +} + bool CDBWrapper::IsEmpty() { std::unique_ptr<CDBIterator> it(NewIterator()); @@ -251,23 +382,43 @@ bool CDBWrapper::IsEmpty() return !(it->Valid()); } -CDBIterator::~CDBIterator() { delete piter; } -bool CDBIterator::Valid() const { return piter->Valid(); } -void CDBIterator::SeekToFirst() { piter->SeekToFirst(); } -void CDBIterator::Next() { piter->Next(); } +struct CDBIterator::IteratorImpl { + const std::unique_ptr<leveldb::Iterator> iter; -namespace dbwrapper_private { + explicit IteratorImpl(leveldb::Iterator* _iter) : iter{_iter} {} +}; -void HandleError(const leveldb::Status& status) +CDBIterator::CDBIterator(const CDBWrapper& _parent, std::unique_ptr<IteratorImpl> _piter) : parent(_parent), + m_impl_iter(std::move(_piter)) {} + +CDBIterator* CDBWrapper::NewIterator() { - if (status.ok()) - return; - const std::string errmsg = "Fatal LevelDB error: " + status.ToString(); - LogPrintf("%s\n", errmsg); - LogPrintf("You can use -debug=leveldb to get more complete diagnostic messages\n"); - throw dbwrapper_error(errmsg); + return new CDBIterator{*this, std::make_unique<CDBIterator::IteratorImpl>(DBContext().pdb->NewIterator(DBContext().iteroptions))}; } +void CDBIterator::SeekImpl(Span<const std::byte> key) +{ + leveldb::Slice slKey(CharCast(key.data()), key.size()); + m_impl_iter->iter->Seek(slKey); +} + +Span<const std::byte> CDBIterator::GetKeyImpl() const +{ + return MakeByteSpan(m_impl_iter->iter->key()); +} + +Span<const std::byte> CDBIterator::GetValueImpl() const +{ + return MakeByteSpan(m_impl_iter->iter->value()); +} + +CDBIterator::~CDBIterator() = default; +bool CDBIterator::Valid() const { return m_impl_iter->iter->Valid(); } +void CDBIterator::SeekToFirst() { m_impl_iter->iter->SeekToFirst(); } +void CDBIterator::Next() { m_impl_iter->iter->Next(); } + +namespace dbwrapper_private { + const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w) { return w.obfuscate_key; diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 4ae2106211..eac9594aa1 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -5,28 +5,21 @@ #ifndef BITCOIN_DBWRAPPER_H #define BITCOIN_DBWRAPPER_H +#include <attributes.h> #include <clientversion.h> -#include <logging.h> #include <serialize.h> #include <span.h> #include <streams.h> +#include <util/check.h> #include <util/fs.h> #include <cstddef> -#include <cstdint> #include <exception> -#include <leveldb/db.h> -#include <leveldb/iterator.h> -#include <leveldb/options.h> -#include <leveldb/slice.h> -#include <leveldb/status.h> -#include <leveldb/write_batch.h> +#include <memory> +#include <optional> #include <stdexcept> #include <string> #include <vector> -namespace leveldb { -class Env; -} static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64; static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024; @@ -54,8 +47,6 @@ struct DBParams { DBOptions options{}; }; -inline auto CharCast(const std::byte* data) { return reinterpret_cast<const char*>(data); } - class dbwrapper_error : public std::runtime_error { public: @@ -64,25 +55,19 @@ public: class CDBWrapper; -namespace dbwrapper { - using leveldb::DestroyDB; -} - /** These should be considered an implementation detail of the specific database. */ namespace dbwrapper_private { -/** Handle database error by throwing dbwrapper_error exception. - */ -void HandleError(const leveldb::Status& status); - /** Work around circular dependency, as well as for testing in dbwrapper_tests. * Database obfuscation should be considered an implementation detail of the * specific database. */ const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w); -}; +}; // namespace dbwrapper_private + +bool DestroyDB(const std::string& path_str); /** Batch of changes queued to be written to a CDBWrapper */ class CDBBatch @@ -91,46 +76,34 @@ class CDBBatch private: const CDBWrapper &parent; - leveldb::WriteBatch batch; + + struct WriteBatchImpl; + const std::unique_ptr<WriteBatchImpl> m_impl_batch; DataStream ssKey{}; CDataStream ssValue; size_t size_estimate{0}; + void WriteImpl(Span<const std::byte> key, CDataStream& ssValue); + void EraseImpl(Span<const std::byte> key); + public: /** * @param[in] _parent CDBWrapper that this batch is to be submitted to */ - explicit CDBBatch(const CDBWrapper& _parent) : parent(_parent), ssValue(SER_DISK, CLIENT_VERSION){}; - - void Clear() - { - batch.Clear(); - size_estimate = 0; - } + explicit CDBBatch(const CDBWrapper& _parent); + ~CDBBatch(); + void Clear(); template <typename K, typename V> void Write(const K& key, const V& value) { ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); - ssKey << key; - leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); - ssValue.reserve(DBWRAPPER_PREALLOC_VALUE_SIZE); + ssKey << key; ssValue << value; - ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); - leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size()); - - batch.Put(slKey, slValue); - // LevelDB serializes writes as: - // - byte: header - // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...) - // - byte[]: key - // - varint: value length - // - byte[]: value - // The formula below assumes the key and value are both less than 16k. - size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size(); + WriteImpl(ssKey, ssValue); ssKey.clear(); ssValue.clear(); } @@ -140,15 +113,7 @@ public: { ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); - - batch.Delete(slKey); - // LevelDB serializes erases as: - // - byte: header - // - varint: key length - // - byte[]: key - // The formula below assumes the key is less than 16kB. - size_estimate += 2 + (slKey.size() > 127) + slKey.size(); + EraseImpl(ssKey); ssKey.clear(); } @@ -157,9 +122,16 @@ public: class CDBIterator { +public: + struct IteratorImpl; + private: const CDBWrapper &parent; - leveldb::Iterator *piter; + const std::unique_ptr<IteratorImpl> m_impl_iter; + + void SeekImpl(Span<const std::byte> key); + Span<const std::byte> GetKeyImpl() const; + Span<const std::byte> GetValueImpl() const; public: @@ -167,8 +139,7 @@ public: * @param[in] _parent Parent CDBWrapper instance. * @param[in] _piter The original leveldb iterator. */ - CDBIterator(const CDBWrapper &_parent, leveldb::Iterator *_piter) : - parent(_parent), piter(_piter) { }; + CDBIterator(const CDBWrapper& _parent, std::unique_ptr<IteratorImpl> _piter); ~CDBIterator(); bool Valid() const; @@ -179,16 +150,14 @@ public: DataStream ssKey{}; ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); - piter->Seek(slKey); + SeekImpl(ssKey); } void Next(); template<typename K> bool GetKey(K& key) { - leveldb::Slice slKey = piter->key(); try { - DataStream ssKey{MakeByteSpan(slKey)}; + DataStream ssKey{GetKeyImpl()}; ssKey >> key; } catch (const std::exception&) { return false; @@ -197,9 +166,8 @@ public: } template<typename V> bool GetValue(V& value) { - leveldb::Slice slValue = piter->value(); try { - CDataStream ssValue{MakeByteSpan(slValue), SER_DISK, CLIENT_VERSION}; + CDataStream ssValue{GetValueImpl(), SER_DISK, CLIENT_VERSION}; ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); ssValue >> value; } catch (const std::exception&) { @@ -209,30 +177,14 @@ public: } }; +struct LevelDBContext; + class CDBWrapper { friend const std::vector<unsigned char>& dbwrapper_private::GetObfuscateKey(const CDBWrapper &w); private: - //! custom environment this database is using (may be nullptr in case of default environment) - leveldb::Env* penv; - - //! database options used - leveldb::Options options; - - //! options used when reading from the database - leveldb::ReadOptions readoptions; - - //! options used when iterating over values of the database - leveldb::ReadOptions iteroptions; - - //! options used when writing to the database - leveldb::WriteOptions writeoptions; - - //! options used when sync writing to the database - leveldb::WriteOptions syncoptions; - - //! the database itself - leveldb::DB* pdb; + //! holds all leveldb-specific fields of this class + std::unique_ptr<LevelDBContext> m_db_context; //! the name of this database std::string m_name; @@ -254,6 +206,11 @@ private: //! whether or not the database resides in memory bool m_is_memory; + std::optional<std::string> ReadImpl(Span<const std::byte> key) const; + bool ExistsImpl(Span<const std::byte> key) const; + size_t EstimateSizeImpl(Span<const std::byte> key1, Span<const std::byte> key2) const; + auto& DBContext() const LIFETIMEBOUND { return *Assert(m_db_context); } + public: CDBWrapper(const DBParams& params); ~CDBWrapper(); @@ -267,18 +224,12 @@ public: DataStream ssKey{}; ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); - - std::string strValue; - leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); - if (!status.ok()) { - if (status.IsNotFound()) - return false; - LogPrintf("LevelDB read failure: %s\n", status.ToString()); - dbwrapper_private::HandleError(status); + std::optional<std::string> strValue{ReadImpl(ssKey)}; + if (!strValue) { + return false; } try { - CDataStream ssValue{MakeByteSpan(strValue), SER_DISK, CLIENT_VERSION}; + CDataStream ssValue{MakeByteSpan(*strValue), SER_DISK, CLIENT_VERSION}; ssValue.Xor(obfuscate_key); ssValue >> value; } catch (const std::exception&) { @@ -309,17 +260,7 @@ public: DataStream ssKey{}; ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); - - std::string strValue; - leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); - if (!status.ok()) { - if (status.IsNotFound()) - return false; - LogPrintf("LevelDB read failure: %s\n", status.ToString()); - dbwrapper_private::HandleError(status); - } - return true; + return ExistsImpl(ssKey); } template <typename K> @@ -335,10 +276,7 @@ public: // Get an estimate of LevelDB memory usage (in bytes). size_t DynamicMemoryUsage() const; - CDBIterator *NewIterator() - { - return new CDBIterator(*this, pdb->NewIterator(iteroptions)); - } + CDBIterator* NewIterator(); /** * Return true if the database managed by this class contains no entries. @@ -353,12 +291,7 @@ public: ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey1 << key_begin; ssKey2 << key_end; - leveldb::Slice slKey1(CharCast(ssKey1.data()), ssKey1.size()); - leveldb::Slice slKey2(CharCast(ssKey2.data()), ssKey2.size()); - uint64_t size = 0; - leveldb::Range range(slKey1, slKey2); - pdb->GetApproximateSizes(&range, 1, &size); - return size; + return EstimateSizeImpl(ssKey1, ssKey2); } }; diff --git a/src/index/base.cpp b/src/index/base.cpp index 55fb154d99..f18205a76f 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -270,7 +270,7 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const // in the ValidationInterface queue backlog even after the sync thread has caught up to the // new chain tip. In this unlikely event, log a warning and let the queue clear. if (best_block_index->GetAncestor(pindex->nHeight - 1) != pindex->pprev) { - LogPrintf("%s: WARNING: Block %s does not connect to an ancestor of " /* Continued */ + LogPrintf("%s: WARNING: Block %s does not connect to an ancestor of " "known best chain (tip=%s); not updating index\n", __func__, pindex->GetBlockHash().ToString(), best_block_index->GetBlockHash().ToString()); @@ -322,7 +322,7 @@ void BaseIndex::ChainStateFlushed(const CBlockLocator& locator) // event, log a warning and let the queue clear. const CBlockIndex* best_block_index = m_best_block_index.load(); if (best_block_index->GetAncestor(locator_tip_index->nHeight) != locator_tip_index) { - LogPrintf("%s: WARNING: Locator contains block (hash=%s) not on known best " /* Continued */ + LogPrintf("%s: WARNING: Locator contains block (hash=%s) not on known best " "chain (tip=%s); not writing index locator\n", __func__, locator_tip_hash.ToString(), best_block_index->GetBlockHash().ToString()); diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index a860b3a94d..cc7d6687b8 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -8,6 +8,7 @@ #include <dbwrapper.h> #include <hash.h> #include <index/blockfilterindex.h> +#include <logging.h> #include <node/blockstorage.h> #include <util/fs_helpers.h> #include <validation.h> diff --git a/src/init.cpp b/src/init.cpp index 4d526bd0de..c2c4dbe459 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -33,6 +33,7 @@ #include <interfaces/chain.h> #include <interfaces/init.h> #include <interfaces/node.h> +#include <logging.h> #include <mapport.h> #include <net.h> #include <net_permissions.h> @@ -115,6 +116,7 @@ #endif using kernel::DumpMempool; +using kernel::LoadMempool; using kernel::ValidationCacheSizes; using node::ApplyArgsManOptions; @@ -1082,7 +1084,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // Warn about relative -datadir path. if (args.IsArgSet("-datadir") && !args.GetPathArg("-datadir").is_absolute()) { - LogPrintf("Warning: relative datadir option '%s' specified, which will be interpreted relative to the " /* Continued */ + LogPrintf("Warning: relative datadir option '%s' specified, which will be interpreted relative to the " "current working directory '%s'. This is fragile, because if bitcoin is started in the future " "from a different location, it will be unable to locate the current data files. There could " "also be data loss if bitcoin is started while in a temporary directory.\n", @@ -1675,7 +1677,10 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return; } // Load mempool from disk - chainman.ActiveChainstate().LoadMempool(ShouldPersistMempool(args) ? MempoolPath(args) : fs::path{}); + if (auto* pool{chainman.ActiveChainstate().GetMempool()}) { + LoadMempool(*pool, ShouldPersistMempool(args) ? MempoolPath(args) : fs::path{}, chainman.ActiveChainstate(), {}); + pool->SetLoadTried(!chainman.m_interrupt); + } }); // Wait for genesis block to be processed diff --git a/src/kernel/mempool_persist.cpp b/src/kernel/mempool_persist.cpp index d060e45af3..6be07da222 100644 --- a/src/kernel/mempool_persist.cpp +++ b/src/kernel/mempool_persist.cpp @@ -19,7 +19,6 @@ #include <util/time.h> #include <validation.h> -#include <chrono> #include <cstdint> #include <cstdio> #include <exception> @@ -37,11 +36,11 @@ namespace kernel { static const uint64_t MEMPOOL_DUMP_VERSION = 1; -bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, FopenFn mockable_fopen_function) +bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, ImportMempoolOptions&& opts) { if (load_path.empty()) return false; - FILE* filestr{mockable_fopen_function(load_path, "rb")}; + FILE* filestr{opts.mockable_fopen_function(load_path, "rb")}; CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); if (file.IsNull()) { LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); @@ -53,7 +52,7 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active int64_t failed = 0; int64_t already_there = 0; int64_t unbroadcast = 0; - auto now = NodeClock::now(); + const auto now{NodeClock::now()}; try { uint64_t version; @@ -72,8 +71,12 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active file >> nTime; file >> nFeeDelta; + if (opts.use_current_time) { + nTime = TicksSinceEpoch<std::chrono::seconds>(now); + } + CAmount amountdelta = nFeeDelta; - if (amountdelta) { + if (amountdelta && opts.apply_fee_delta_priority) { pool.PrioritiseTransaction(tx->GetHash(), amountdelta); } if (nTime > TicksSinceEpoch<std::chrono::seconds>(now - pool.m_expiry)) { @@ -101,17 +104,21 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active std::map<uint256, CAmount> mapDeltas; file >> mapDeltas; - for (const auto& i : mapDeltas) { - pool.PrioritiseTransaction(i.first, i.second); + if (opts.apply_fee_delta_priority) { + for (const auto& i : mapDeltas) { + pool.PrioritiseTransaction(i.first, i.second); + } } std::set<uint256> unbroadcast_txids; file >> unbroadcast_txids; - unbroadcast = unbroadcast_txids.size(); - for (const auto& txid : unbroadcast_txids) { - // Ensure transactions were accepted to mempool then add to - // unbroadcast set. - if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid); + if (opts.apply_unbroadcast_set) { + unbroadcast = unbroadcast_txids.size(); + for (const auto& txid : unbroadcast_txids) { + // Ensure transactions were accepted to mempool then add to + // unbroadcast set. + if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid); + } } } catch (const std::exception& e) { LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); diff --git a/src/kernel/mempool_persist.h b/src/kernel/mempool_persist.h index cb09119e4a..e124a8eadf 100644 --- a/src/kernel/mempool_persist.h +++ b/src/kernel/mempool_persist.h @@ -12,15 +12,21 @@ class CTxMemPool; namespace kernel { -/** Dump the mempool to disk. */ +/** Dump the mempool to a file. */ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen, bool skip_file_commit = false); -/** Load the mempool from disk. */ +struct ImportMempoolOptions { + fsbridge::FopenFn mockable_fopen_function{fsbridge::fopen}; + bool use_current_time{false}; + bool apply_fee_delta_priority{true}; + bool apply_unbroadcast_set{true}; +}; +/** Import the file and attempt to add its contents to the mempool. */ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, - fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen); + ImportMempoolOptions&& opts); } // namespace kernel diff --git a/src/net.cpp b/src/net.cpp index 1eda51e9df..b51043ba27 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -83,6 +83,9 @@ static constexpr std::chrono::seconds MAX_UPLOAD_TIMEFRAME{60 * 60 * 24}; // A random time period (0 to 1 seconds) is added to feeler connections to prevent synchronization. static constexpr auto FEELER_SLEEP_WINDOW{1s}; +/** Frequency to attempt extra connections to reachable networks we're not connected to yet **/ +static constexpr auto EXTRA_NETWORK_PEER_INTERVAL{5min}; + /** Used to pass flags to the Bind() function */ enum BindFlags { BF_NONE = 0, @@ -1138,6 +1141,9 @@ void CConnman::DisconnectNodes() // close socket and cleanup pnode->CloseSocketDisconnect(); + // update connection count by network + if (pnode->IsManualOrFullOutboundConn()) --m_network_conn_counts[pnode->addr.GetNetwork()]; + // hold in disconnected pool until all refs are released pnode->Release(); m_nodes_disconnected.push_back(pnode); @@ -1605,6 +1611,28 @@ std::unordered_set<Network> CConnman::GetReachableEmptyNetworks() const return networks; } +bool CConnman::MultipleManualOrFullOutboundConns(Network net) const +{ + AssertLockHeld(m_nodes_mutex); + return m_network_conn_counts[net] > 1; +} + +bool CConnman::MaybePickPreferredNetwork(std::optional<Network>& network) +{ + std::array<Network, 5> nets{NET_IPV4, NET_IPV6, NET_ONION, NET_I2P, NET_CJDNS}; + Shuffle(nets.begin(), nets.end(), FastRandomContext()); + + LOCK(m_nodes_mutex); + for (const auto net : nets) { + if (IsReachable(net) && m_network_conn_counts[net] == 0 && addrman.Size(net) != 0) { + network = net; + return true; + } + } + + return false; +} + void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) { AssertLockNotHeld(m_unused_i2p_sessions_mutex); @@ -1635,6 +1663,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // Minimum time before next feeler connection (in microseconds). auto next_feeler = GetExponentialRand(start, FEELER_INTERVAL); auto next_extra_block_relay = GetExponentialRand(start, EXTRA_BLOCK_RELAY_ONLY_PEER_INTERVAL); + auto next_extra_network_peer{GetExponentialRand(start, EXTRA_NETWORK_PEER_INTERVAL)}; const bool dnsseed = gArgs.GetBoolArg("-dnsseed", DEFAULT_DNSSEED); bool add_fixed_seeds = gArgs.GetBoolArg("-fixedseeds", DEFAULT_FIXEDSEEDS); const bool use_seednodes{gArgs.IsArgSet("-seednode")}; @@ -1747,6 +1776,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) auto now = GetTime<std::chrono::microseconds>(); bool anchor = false; bool fFeeler = false; + std::optional<Network> preferred_net; // Determine what type of connection to open. Opening // BLOCK_RELAY connections to addresses from anchors.dat gets the highest @@ -1796,6 +1826,17 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) next_feeler = GetExponentialRand(now, FEELER_INTERVAL); conn_type = ConnectionType::FEELER; fFeeler = true; + } else if (nOutboundFullRelay == m_max_outbound_full_relay && + m_max_outbound_full_relay == MAX_OUTBOUND_FULL_RELAY_CONNECTIONS && + now > next_extra_network_peer && + MaybePickPreferredNetwork(preferred_net)) { + // Full outbound connection management: Attempt to get at least one + // outbound peer from each reachable network by making extra connections + // and then protecting "only" peers from a network during outbound eviction. + // This is not attempted if the user changed -maxconnections to a value + // so low that less than MAX_OUTBOUND_FULL_RELAY_CONNECTIONS are made, + // to prevent interactions with otherwise protected outbound peers. + next_extra_network_peer = GetExponentialRand(now, EXTRA_NETWORK_PEER_INTERVAL); } else { // skip to next iteration of while loop continue; @@ -1849,7 +1890,10 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) } } else { // Not a feeler - std::tie(addr, addr_last_try) = addrman.Select(); + // If preferred_net has a value set, pick an extra outbound + // peer from that network. The eviction logic in net_processing + // ensures that a peer from another network will be evicted. + std::tie(addr, addr_last_try) = addrman.Select(false, preferred_net); } // Require outbound IPv4/IPv6 connections, other than feelers, to be to distinct network groups @@ -1896,6 +1940,9 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) } LogPrint(BCLog::NET, "Making feeler connection to %s\n", addrConnect.ToStringAddrPort()); } + + if (preferred_net != std::nullopt) LogPrint(BCLog::NET, "Making network specific connection to %s on %s.\n", addrConnect.ToStringAddrPort(), GetNetworkName(preferred_net.value())); + // Record addrman failure attempts when node has at least 2 persistent outbound connections to peers with // different netgroups in ipv4/ipv6 networks + all peers in Tor/I2P/CJDNS networks. // Don't record addrman failure attempts when node is offline. This can be identified since all local @@ -2035,6 +2082,9 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai { LOCK(m_nodes_mutex); m_nodes.push_back(pnode); + + // update connection count by network + if (pnode->IsManualOrFullOutboundConn()) ++m_network_conn_counts[pnode->addr.GetNetwork()]; } } @@ -465,6 +465,22 @@ public: return m_conn_type == ConnectionType::MANUAL; } + bool IsManualOrFullOutboundConn() const + { + switch (m_conn_type) { + case ConnectionType::INBOUND: + case ConnectionType::FEELER: + case ConnectionType::BLOCK_RELAY: + case ConnectionType::ADDR_FETCH: + return false; + case ConnectionType::OUTBOUND_FULL_RELAY: + case ConnectionType::MANUAL: + return true; + } // no default case, so the compiler can warn about missing cases + + assert(false); + } + bool IsBlockOnlyConn() const { return m_conn_type == ConnectionType::BLOCK_RELAY; } @@ -777,6 +793,9 @@ public: void OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant* grantOutbound, const char* strDest, ConnectionType conn_type) EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex); bool CheckIncomingNonce(uint64_t nonce); + // alias for thread safety annotations only, not defined + RecursiveMutex& GetNodesMutex() const LOCK_RETURNED(m_nodes_mutex); + bool ForNode(NodeId id, std::function<bool(CNode* pnode)> func); void PushMessage(CNode* pnode, CSerializedNetMsg&& msg) EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); @@ -893,6 +912,8 @@ public: /** Return true if we should disconnect the peer for failing an inactivity check. */ bool ShouldRunInactivityChecks(const CNode& node, std::chrono::seconds now) const; + bool MultipleManualOrFullOutboundConns(Network net) const EXCLUSIVE_LOCKS_REQUIRED(m_nodes_mutex); + private: struct ListenSocket { public: @@ -1010,6 +1031,18 @@ private: */ std::vector<CAddress> GetCurrentBlockRelayOnlyConns() const; + /** + * Search for a "preferred" network, a reachable network to which we + * currently don't have any OUTBOUND_FULL_RELAY or MANUAL connections. + * There needs to be at least one address in AddrMan for a preferred + * network to be picked. + * + * @param[out] network Preferred network, if found. + * + * @return bool Whether a preferred network was found. + */ + bool MaybePickPreferredNetwork(std::optional<Network>& network); + // Whether the node should be passed out in ForEach* callbacks static bool NodeFullyConnected(const CNode* pnode); @@ -1048,6 +1081,9 @@ private: std::atomic<NodeId> nLastNodeId{0}; unsigned int nPrevNodeCount{0}; + // Stores number of full-tx connections (outbound and manual) per network + std::array<unsigned int, Network::NET_MAX> m_network_conn_counts GUARDED_BY(m_nodes_mutex) = {}; + /** * Cache responses to addr requests to minimize privacy leak. * Attack example: scraping addrs in real-time may allow an attacker diff --git a/src/net_processing.cpp b/src/net_processing.cpp index be6777d14b..e2bbfe3308 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -17,6 +17,7 @@ #include <headerssync.h> #include <index/blockfilterindex.h> #include <kernel/mempool_entry.h> +#include <logging.h> #include <merkleblock.h> #include <netbase.h> #include <netmessagemaker.h> @@ -2212,7 +2213,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& // Fast-path: in this case it is possible to serve the block directly from disk, // as the network format matches the format on disk std::vector<uint8_t> block_data; - if (!m_chainman.m_blockman.ReadRawBlockFromDisk(block_data, pindex->GetBlockPos(), m_chainparams.MessageStart())) { + if (!m_chainman.m_blockman.ReadRawBlockFromDisk(block_data, pindex->GetBlockPos())) { assert(!"cannot load block from disk"); } m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, Span{block_data})); @@ -3048,7 +3049,7 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& node, Peer& peer, uint32_t stop_height = stop_index->nHeight; if (start_height > stop_height) { - LogPrint(BCLog::NET, "peer %d sent invalid getcfilters/getcfheaders with " /* Continued */ + LogPrint(BCLog::NET, "peer %d sent invalid getcfilters/getcfheaders with " "start height %d and stop height %d\n", node.GetId(), start_height, stop_height); node.fDisconnect = true; @@ -5142,10 +5143,12 @@ void PeerManagerImpl::EvictExtraOutboundPeers(std::chrono::seconds now) // Pick the outbound-full-relay peer that least recently announced // us a new block, with ties broken by choosing the more recent // connection (higher node id) + // Protect peers from eviction if we don't have another connection + // to their network, counting both outbound-full-relay and manual peers. NodeId worst_peer = -1; int64_t oldest_block_announcement = std::numeric_limits<int64_t>::max(); - m_connman.ForEachNode([&](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + m_connman.ForEachNode([&](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_connman.GetNodesMutex()) { AssertLockHeld(::cs_main); // Only consider outbound-full-relay peers that are not already @@ -5155,6 +5158,9 @@ void PeerManagerImpl::EvictExtraOutboundPeers(std::chrono::seconds now) if (state == nullptr) return; // shouldn't be possible, but just in case // Don't evict our protected peers if (state->m_chain_sync.m_protect) return; + // If this is the only connection on a particular network that is + // OUTBOUND_FULL_RELAY or MANUAL, protect it. + if (!m_connman.MultipleManualOrFullOutboundConns(pnode->addr.GetNetwork())) return; if (state->m_last_block_announcement < oldest_block_announcement || (state->m_last_block_announcement == oldest_block_announcement && pnode->GetId() > worst_peer)) { worst_peer = pnode->GetId(); oldest_block_announcement = state->m_last_block_announcement; diff --git a/src/net_processing.h b/src/net_processing.h index a0cbe92289..837e308617 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -17,9 +17,10 @@ class ChainstateManager; /** Whether transaction reconciliation protocol should be enabled by default. */ static constexpr bool DEFAULT_TXRECONCILIATION_ENABLE{false}; /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ -static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; -/** Default number of orphan+recently-replaced txn to keep around for block reconstruction */ -static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; +static const uint32_t DEFAULT_MAX_ORPHAN_TRANSACTIONS{100}; +/** Default number of non-mempool transactions to keep around for block reconstruction. Includes + orphan, replaced, and rejected transactions. */ +static const uint32_t DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN{100}; static const bool DEFAULT_PEERBLOOMFILTERS = false; static const bool DEFAULT_PEERBLOCKFILTERS = false; /** Threshold for marking a node to be discouraged, e.g. disconnected and added to the discouragement filter. */ @@ -46,11 +47,16 @@ class PeerManager : public CValidationInterface, public NetEventsInterface { public: struct Options { - /** Whether this node is running in -blocksonly mode */ + //! Whether this node is running in -blocksonly mode bool ignore_incoming_txs{DEFAULT_BLOCKSONLY}; + //! Whether transaction reconciliation protocol is enabled bool reconcile_txs{DEFAULT_TXRECONCILIATION_ENABLE}; + //! Maximum number of orphan transactions kept in memory uint32_t max_orphan_txs{DEFAULT_MAX_ORPHAN_TRANSACTIONS}; - size_t max_extra_txs{DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN}; + //! Number of non-mempool transactions to keep around for block reconstruction. Includes + //! orphan, replaced, and rejected transactions. + uint32_t max_extra_txs{DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN}; + //! Whether all P2P messages are captured to disk bool capture_messages{false}; }; diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 0d25c798ce..01b4c36a8f 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -476,7 +476,7 @@ CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n) return &m_blockfile_info.at(n); } -bool BlockManager::UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) const +bool BlockManager::UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const { // Open history file to append AutoFile fileout{OpenUndoFile(pos)}; @@ -486,7 +486,7 @@ bool BlockManager::UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos // Write index header unsigned int nSize = GetSerializeSize(blockundo, CLIENT_VERSION); - fileout << messageStart << nSize; + fileout << GetParams().MessageStart() << nSize; // Write undo data long fileOutPos = ftell(fileout.Get()); @@ -708,7 +708,7 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP return true; } -bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) const +bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const { // Open history file to append CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); @@ -718,7 +718,7 @@ bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const // Write index header unsigned int nSize = GetSerializeSize(block, fileout.GetVersion()); - fileout << messageStart << nSize; + fileout << GetParams().MessageStart() << nSize; // Write block long fileOutPos = ftell(fileout.Get()); @@ -740,7 +740,7 @@ bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValid if (!FindUndoPos(state, block.nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) { return error("ConnectBlock(): FindUndoPos failed"); } - if (!UndoWriteToDisk(blockundo, _pos, block.pprev->GetBlockHash(), GetParams().MessageStart())) { + if (!UndoWriteToDisk(blockundo, _pos, block.pprev->GetBlockHash())) { return FatalError(m_opts.notifications, state, "Failed to write undo data"); } // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order) @@ -806,7 +806,7 @@ bool BlockManager::ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) co return true; } -bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) const +bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos) const { FlatFilePos hpos = pos; hpos.nPos -= 8; // Seek back 8 bytes for meta header @@ -821,10 +821,10 @@ bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatF filein >> blk_start >> blk_size; - if (memcmp(blk_start, message_start, CMessageHeader::MESSAGE_START_SIZE)) { + if (memcmp(blk_start, GetParams().MessageStart(), CMessageHeader::MESSAGE_START_SIZE)) { return error("%s: Block magic mismatch for %s: %s versus expected %s", __func__, pos.ToString(), HexStr(blk_start), - HexStr(message_start)); + HexStr(GetParams().MessageStart())); } if (blk_size > MAX_SIZE) { @@ -859,7 +859,7 @@ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, cons return FlatFilePos(); } if (!position_known) { - if (!WriteBlockToDisk(block, blockPos, GetParams().MessageStart())) { + if (!WriteBlockToDisk(block, blockPos)) { m_opts.notifications.fatalError("Failed to write block"); return FlatFilePos(); } diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index eb40d45aba..0180124a79 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -101,8 +101,8 @@ private: FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const; - bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) const; - bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) const; + bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const; + bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const; /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */ void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight, int chain_tip_height); @@ -264,7 +264,7 @@ public: /** Functions for disk access for blocks */ bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const; bool ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) const; - bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) const; + bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos) const; bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& index) const; diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 9c98e4cf0c..42e021fcc9 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -18,6 +18,7 @@ #include <interfaces/wallet.h> #include <kernel/chain.h> #include <kernel/mempool_entry.h> +#include <logging.h> #include <mapport.h> #include <net.h> #include <net_processing.h> diff --git a/src/node/miner.cpp b/src/node/miner.cpp index aa1a9a155c..caa2991819 100644 --- a/src/node/miner.cpp +++ b/src/node/miner.cpp @@ -15,6 +15,7 @@ #include <consensus/tx_verify.h> #include <consensus/validation.h> #include <deploymentstatus.h> +#include <logging.h> #include <policy/feerate.h> #include <policy/policy.h> #include <pow.h> diff --git a/src/node/peerman_args.cpp b/src/node/peerman_args.cpp index e0dcf21c33..b0ae266365 100644 --- a/src/node/peerman_args.cpp +++ b/src/node/peerman_args.cpp @@ -3,6 +3,9 @@ #include <common/args.h> #include <net_processing.h> +#include <algorithm> +#include <limits> + namespace node { void ApplyArgsManOptions(const ArgsManager& argsman, PeerManager::Options& options) @@ -10,11 +13,11 @@ void ApplyArgsManOptions(const ArgsManager& argsman, PeerManager::Options& optio if (auto value{argsman.GetBoolArg("-txreconciliation")}) options.reconcile_txs = *value; if (auto value{argsman.GetIntArg("-maxorphantx")}) { - options.max_orphan_txs = uint32_t(std::max(int64_t{0}, *value)); + options.max_orphan_txs = uint32_t((std::clamp<int64_t>(*value, 0, std::numeric_limits<uint32_t>::max()))); } if (auto value{argsman.GetIntArg("-blockreconstructionextratxn")}) { - options.max_extra_txs = size_t(std::max(int64_t{0}, *value)); + options.max_extra_txs = uint32_t((std::clamp<int64_t>(*value, 0, std::numeric_limits<uint32_t>::max()))); } if (auto value{argsman.GetBoolArg("-capturemessages")}) options.capture_messages = *value; diff --git a/src/node/utxo_snapshot.cpp b/src/node/utxo_snapshot.cpp index 036a25d0a5..976421e455 100644 --- a/src/node/utxo_snapshot.cpp +++ b/src/node/utxo_snapshot.cpp @@ -49,7 +49,7 @@ bool WriteSnapshotBaseBlockhash(Chainstate& snapshot_chainstate) std::optional<uint256> ReadSnapshotBaseBlockhash(fs::path chaindir) { if (!fs::exists(chaindir)) { - LogPrintf("[snapshot] cannot read base blockhash: no chainstate dir " /* Continued */ + LogPrintf("[snapshot] cannot read base blockhash: no chainstate dir " "exists at path %s\n", fs::PathToString(chaindir)); return std::nullopt; } @@ -57,7 +57,7 @@ std::optional<uint256> ReadSnapshotBaseBlockhash(fs::path chaindir) const std::string read_from_str = fs::PathToString(read_from); if (!fs::exists(read_from)) { - LogPrintf("[snapshot] snapshot chainstate dir is malformed! no base blockhash file " /* Continued */ + LogPrintf("[snapshot] snapshot chainstate dir is malformed! no base blockhash file " "exists at path %s. Try deleting %s and calling loadtxoutset again?\n", fs::PathToString(chaindir), read_from_str); return std::nullopt; diff --git a/src/pubkey.h b/src/pubkey.h index 7d37504b01..00defa25a0 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -299,6 +299,9 @@ private: std::array<std::byte, SIZE> m_pubkey; public: + /** Default constructor creates all-zero pubkey (which is valid). */ + EllSwiftPubKey() noexcept = default; + /** Construct a new ellswift public key from a given serialization. */ EllSwiftPubKey(const std::array<std::byte, SIZE>& ellswift) : m_pubkey(ellswift) {} diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 8f45af9485..865871a6d4 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -9,7 +9,6 @@ #include <qt/bitcoin.h> #include <chainparams.h> -#include <node/context.h> #include <common/args.h> #include <common/init.h> #include <common/system.h> @@ -17,6 +16,8 @@ #include <interfaces/handler.h> #include <interfaces/init.h> #include <interfaces/node.h> +#include <logging.h> +#include <node/context.h> #include <node/interface_ui.h> #include <noui.h> #include <qt/bitcoingui.h> diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index fb8029cb65..b05009965f 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -6,6 +6,7 @@ #include <chainparams.h> #include <key.h> +#include <logging.h> #include <qt/bitcoin.h> #include <qt/bitcoingui.h> #include <qt/networkstyle.h> diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index fa110cfbc9..dae6a2dea9 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -18,6 +18,7 @@ #include <interfaces/node.h> #include <interfaces/wallet.h> #include <key_io.h> +#include <logging.h> #include <policy/policy.h> #include <validation.h> #include <wallet/types.h> diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index d289a9240e..2908c37c1f 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -229,6 +229,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "importaddress", 2, "rescan" }, { "importaddress", 3, "p2sh" }, { "importpubkey", 2, "rescan" }, + { "importmempool", 1, "options" }, + { "importmempool", 1, "apply_fee_delta_priority" }, + { "importmempool", 1, "use_current_time" }, + { "importmempool", 1, "apply_unbroadcast_set" }, { "importmulti", 0, "requests" }, { "importmulti", 1, "options" }, { "importmulti", 1, "rescan" }, diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 11d2874961..90ee2a48af 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -696,7 +696,7 @@ static RPCHelpMan getmempoolinfo() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::BOOL, "loaded", "True if the mempool is fully loaded"}, + {RPCResult::Type::BOOL, "loaded", "True if the initial load attempt of the persisted mempool finished"}, {RPCResult::Type::NUM, "size", "Current tx count"}, {RPCResult::Type::NUM, "bytes", "Sum of all virtual transaction sizes as defined in BIP 141. Differs from actual serialized size because witness data is discounted"}, {RPCResult::Type::NUM, "usage", "Total memory usage for the mempool"}, @@ -719,6 +719,66 @@ static RPCHelpMan getmempoolinfo() }; } +static RPCHelpMan importmempool() +{ + return RPCHelpMan{ + "importmempool", + "Import a mempool.dat file and attempt to add its contents to the mempool.\n" + "Warning: Importing untrusted files is dangerous, especially if metadata from the file is taken over.", + { + {"filepath", RPCArg::Type::STR, RPCArg::Optional::NO, "The mempool file"}, + {"options", + RPCArg::Type::OBJ_NAMED_PARAMS, + RPCArg::Optional::OMITTED, + "", + { + {"use_current_time", RPCArg::Type::BOOL, RPCArg::Default{true}, + "Whether to use the current system time or use the entry time metadata from the mempool file.\n" + "Warning: Importing untrusted metadata may lead to unexpected issues and undesirable behavior."}, + {"apply_fee_delta_priority", RPCArg::Type::BOOL, RPCArg::Default{false}, + "Whether to apply the fee delta metadata from the mempool file.\n" + "It will be added to any existing fee deltas.\n" + "The fee delta can be set by the prioritisetransaction RPC.\n" + "Warning: Importing untrusted metadata may lead to unexpected issues and undesirable behavior.\n" + "Only set this bool if you understand what it does."}, + {"apply_unbroadcast_set", RPCArg::Type::BOOL, RPCArg::Default{false}, + "Whether to apply the unbroadcast set metadata from the mempool file.\n" + "Warning: Importing untrusted metadata may lead to unexpected issues and undesirable behavior."}, + }, + RPCArgOptions{.oneline_description = "\"options\""}}, + }, + RPCResult{RPCResult::Type::OBJ, "", "", std::vector<RPCResult>{}}, + RPCExamples{HelpExampleCli("importmempool", "/path/to/mempool.dat") + HelpExampleRpc("importmempool", "/path/to/mempool.dat")}, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + const NodeContext& node{EnsureAnyNodeContext(request.context)}; + + CTxMemPool& mempool{EnsureMemPool(node)}; + Chainstate& chainstate = EnsureChainman(node).ActiveChainstate(); + + if (chainstate.IsInitialBlockDownload()) { + throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Can only import the mempool after the block download and sync is done."); + } + + const fs::path load_path{fs::u8path(request.params[0].get_str())}; + const UniValue& use_current_time{request.params[1]["use_current_time"]}; + const UniValue& apply_fee_delta{request.params[1]["apply_fee_delta_priority"]}; + const UniValue& apply_unbroadcast{request.params[1]["apply_unbroadcast_set"]}; + kernel::ImportMempoolOptions opts{ + .use_current_time = use_current_time.isNull() ? true : use_current_time.get_bool(), + .apply_fee_delta_priority = apply_fee_delta.isNull() ? false : apply_fee_delta.get_bool(), + .apply_unbroadcast_set = apply_unbroadcast.isNull() ? false : apply_unbroadcast.get_bool(), + }; + + if (!kernel::LoadMempool(mempool, load_path, chainstate, std::move(opts))) { + throw JSONRPCError(RPC_MISC_ERROR, "Unable to import mempool file, see debug.log for details."); + } + + UniValue ret{UniValue::VOBJ}; + return ret; + }, + }; +} + static RPCHelpMan savemempool() { return RPCHelpMan{"savemempool", @@ -921,6 +981,7 @@ void RegisterMempoolRPCCommands(CRPCTable& t) {"blockchain", &gettxspendingprevout}, {"blockchain", &getmempoolinfo}, {"blockchain", &getrawmempool}, + {"blockchain", &importmempool}, {"blockchain", &savemempool}, {"hidden", &submitpackage}, }; diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index dc5ecb9cbc..6b3662996c 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -13,6 +13,7 @@ #include <interfaces/init.h> #include <interfaces/ipc.h> #include <kernel/cs_main.h> +#include <logging.h> #include <node/context.h> #include <rpc/server.h> #include <rpc/server_util.h> diff --git a/src/test/bip324_tests.cpp b/src/test/bip324_tests.cpp new file mode 100644 index 0000000000..04472611ec --- /dev/null +++ b/src/test/bip324_tests.cpp @@ -0,0 +1,306 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bip324.h> +#include <chainparams.h> +#include <key.h> +#include <pubkey.h> +#include <span.h> +#include <test/util/random.h> +#include <test/util/setup_common.h> +#include <util/strencodings.h> + +#include <array> +#include <cstddef> +#include <cstdint> +#include <vector> + +#include <boost/test/unit_test.hpp> + +namespace { + +void TestBIP324PacketVector( + uint32_t in_idx, + const std::string& in_priv_ours_hex, + const std::string& in_ellswift_ours_hex, + const std::string& in_ellswift_theirs_hex, + bool in_initiating, + const std::string& in_contents_hex, + uint32_t in_multiply, + const std::string& in_aad_hex, + bool in_ignore, + const std::string& mid_send_garbage_hex, + const std::string& mid_recv_garbage_hex, + const std::string& out_session_id_hex, + const std::string& out_ciphertext_hex, + const std::string& out_ciphertext_endswith_hex) +{ + // Convert input from hex to char/byte vectors/arrays. + const auto in_priv_ours = ParseHex(in_priv_ours_hex); + const auto in_ellswift_ours_vec = ParseHex<std::byte>(in_ellswift_ours_hex); + assert(in_ellswift_ours_vec.size() == 64); + std::array<std::byte, 64> in_ellswift_ours; + std::copy(in_ellswift_ours_vec.begin(), in_ellswift_ours_vec.end(), in_ellswift_ours.begin()); + const auto in_ellswift_theirs_vec = ParseHex<std::byte>(in_ellswift_theirs_hex); + assert(in_ellswift_theirs_vec.size() == 64); + std::array<std::byte, 64> in_ellswift_theirs; + std::copy(in_ellswift_theirs_vec.begin(), in_ellswift_theirs_vec.end(), in_ellswift_theirs.begin()); + const auto in_contents = ParseHex<std::byte>(in_contents_hex); + const auto in_aad = ParseHex<std::byte>(in_aad_hex); + const auto mid_send_garbage = ParseHex<std::byte>(mid_send_garbage_hex); + const auto mid_recv_garbage = ParseHex<std::byte>(mid_recv_garbage_hex); + const auto out_session_id = ParseHex<std::byte>(out_session_id_hex); + const auto out_ciphertext = ParseHex<std::byte>(out_ciphertext_hex); + const auto out_ciphertext_endswith = ParseHex<std::byte>(out_ciphertext_endswith_hex); + + // Load keys + CKey key; + key.Set(in_priv_ours.begin(), in_priv_ours.end(), true); + EllSwiftPubKey ellswift_ours(in_ellswift_ours); + EllSwiftPubKey ellswift_theirs(in_ellswift_theirs); + + // Instantiate encryption BIP324 cipher. + BIP324Cipher cipher(key, ellswift_ours); + BOOST_CHECK(!cipher); + BOOST_CHECK(cipher.GetOurPubKey() == ellswift_ours); + cipher.Initialize(ellswift_theirs, in_initiating); + BOOST_CHECK(cipher); + + // Compare session variables. + BOOST_CHECK(Span{out_session_id} == cipher.GetSessionID()); + BOOST_CHECK(Span{mid_send_garbage} == cipher.GetSendGarbageTerminator()); + BOOST_CHECK(Span{mid_recv_garbage} == cipher.GetReceiveGarbageTerminator()); + + // Vector of encrypted empty messages, encrypted in order to seek to the right position. + std::vector<std::vector<std::byte>> dummies(in_idx); + + // Seek to the numbered packet. + for (uint32_t i = 0; i < in_idx; ++i) { + dummies[i].resize(cipher.EXPANSION); + cipher.Encrypt({}, {}, true, dummies[i]); + } + + // Construct contents and encrypt it. + std::vector<std::byte> contents; + for (uint32_t i = 0; i < in_multiply; ++i) { + contents.insert(contents.end(), in_contents.begin(), in_contents.end()); + } + std::vector<std::byte> ciphertext(contents.size() + cipher.EXPANSION); + cipher.Encrypt(contents, in_aad, in_ignore, ciphertext); + + // Verify ciphertext. Note that the test vectors specify either out_ciphertext (for short + // messages) or out_ciphertext_endswith (for long messages), so only check the relevant one. + if (!out_ciphertext.empty()) { + BOOST_CHECK(out_ciphertext == ciphertext); + } else { + BOOST_CHECK(ciphertext.size() >= out_ciphertext_endswith.size()); + BOOST_CHECK(Span{out_ciphertext_endswith} == Span{ciphertext}.last(out_ciphertext_endswith.size())); + } + + for (unsigned error = 0; error <= 12; ++error) { + // error selects a type of error introduced: + // - error=0: no errors, decryption should be successful + // - error=1: wrong side + // - error=2..9: bit error in ciphertext + // - error=10: bit error in aad + // - error=11: extra 0x00 at end of aad + // - error=12: message index wrong + + // Instantiate self-decrypting BIP324 cipher. + BIP324Cipher dec_cipher(key, ellswift_ours); + BOOST_CHECK(!dec_cipher); + BOOST_CHECK(dec_cipher.GetOurPubKey() == ellswift_ours); + dec_cipher.Initialize(ellswift_theirs, (error == 1) ^ in_initiating, /*self_decrypt=*/true); + BOOST_CHECK(dec_cipher); + + // Compare session variables. + BOOST_CHECK((Span{out_session_id} == dec_cipher.GetSessionID()) == (error != 1)); + BOOST_CHECK((Span{mid_send_garbage} == dec_cipher.GetSendGarbageTerminator()) == (error != 1)); + BOOST_CHECK((Span{mid_recv_garbage} == dec_cipher.GetReceiveGarbageTerminator()) == (error != 1)); + + // Seek to the numbered packet. + if (in_idx == 0 && error == 12) continue; + uint32_t dec_idx = in_idx ^ (error == 12 ? (1U << InsecureRandRange(16)) : 0); + for (uint32_t i = 0; i < dec_idx; ++i) { + unsigned use_idx = i < in_idx ? i : 0; + bool dec_ignore{false}; + dec_cipher.DecryptLength(Span{dummies[use_idx]}.first(cipher.LENGTH_LEN)); + dec_cipher.Decrypt(Span{dummies[use_idx]}.subspan(cipher.LENGTH_LEN), {}, dec_ignore, {}); + } + + // Construct copied (and possibly damaged) copy of ciphertext. + // Decrypt length + auto to_decrypt = ciphertext; + if (error >= 2 && error <= 9) { + to_decrypt[InsecureRandRange(to_decrypt.size())] ^= std::byte(1U << (error - 2)); + } + + // Decrypt length and resize ciphertext to accommodate. + uint32_t dec_len = dec_cipher.DecryptLength(MakeByteSpan(to_decrypt).first(cipher.LENGTH_LEN)); + to_decrypt.resize(dec_len + cipher.EXPANSION); + + // Construct copied (and possibly damaged) copy of aad. + auto dec_aad = in_aad; + if (error == 10) { + if (in_aad.size() == 0) continue; + dec_aad[InsecureRandRange(dec_aad.size())] ^= std::byte(1U << InsecureRandRange(8)); + } + if (error == 11) dec_aad.push_back({}); + + // Decrypt contents. + std::vector<std::byte> decrypted(dec_len); + bool dec_ignore{false}; + bool dec_ok = dec_cipher.Decrypt(Span{to_decrypt}.subspan(cipher.LENGTH_LEN), dec_aad, dec_ignore, decrypted); + + // Verify result. + BOOST_CHECK(dec_ok == !error); + if (dec_ok) { + BOOST_CHECK(decrypted == contents); + BOOST_CHECK(dec_ignore == in_ignore); + } + } +} + +} // namespace + +BOOST_FIXTURE_TEST_SUITE(bip324_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(packet_test_vectors) { + // BIP324 key derivation uses network magic in the HKDF process. We use mainnet params here + // as that is what the test vectors are written for. + SelectParams(ChainType::MAIN); + + // The test vectors are converted using the following Python code in the BIP bip-0324/ directory: + // + // import sys + // import csv + // with open('packet_encoding_test_vectors.csv', newline='', encoding='utf-8') as csvfile: + // reader = csv.DictReader(csvfile) + // quote = lambda x: "\"" + x + "\"" + // for row in reader: + // args = [ + // row['in_idx'], + // quote(row['in_priv_ours']), + // quote(row['in_ellswift_ours']), + // quote(row['in_ellswift_theirs']), + // "true" if int(row['in_initiating']) else "false", + // quote(row['in_contents']), + // row['in_multiply'], + // quote(row['in_aad']), + // "true" if int(row['in_ignore']) else "false", + // quote(row['mid_send_garbage_terminator']), + // quote(row['mid_recv_garbage_terminator']), + // quote(row['out_session_id']), + // quote(row['out_ciphertext']), + // quote(row['out_ciphertext_endswith']) + // ] + // print(" TestBIP324PacketVector(\n " + ",\n ".join(args) + ");") + TestBIP324PacketVector( + 1, + "61062ea5071d800bbfd59e2e8b53d47d194b095ae5a4df04936b49772ef0d4d7", + "ec0adff257bbfe500c188c80b4fdd640f6b45a482bbc15fc7cef5931deff0aa186f6eb9bba7b85dc4dcc28b28722de1e3d9108b985e2967045668f66098e475b", + "a4a94dfce69b4a2a0a099313d10f9f7e7d649d60501c9e1d274c300e0d89aafaffffffffffffffffffffffffffffffffffffffffffffffffffffffff8faf88d5", + true, + "8e", + 1, + "", + false, + "faef555dfcdb936425d84aba524758f3", + "02cb8ff24307a6e27de3b4e7ea3fa65b", + "ce72dffb015da62b0d0f5474cab8bc72605225b0cee3f62312ec680ec5f41ba5", + "7530d2a18720162ac09c25329a60d75adf36eda3c3", + ""); + TestBIP324PacketVector( + 999, + "1f9c581b35231838f0f17cf0c979835baccb7f3abbbb96ffcc318ab71e6e126f", + "a1855e10e94e00baa23041d916e259f7044e491da6171269694763f018c7e63693d29575dcb464ac816baa1be353ba12e3876cba7628bd0bd8e755e721eb0140", + "fffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2f0000000000000000000000000000000000000000000000000000000000000000", + false, + "3eb1d4e98035cfd8eeb29bac969ed3824a", + 1, + "", + false, + "efb64fd80acd3825ac9bc2a67216535a", + "b3cb553453bceb002897e751ff7588bf", + "9267c54560607de73f18c563b76a2442718879c52dd39852885d4a3c9912c9ea", + "1da1bcf589f9b61872f45b7fa5371dd3f8bdf5d515b0c5f9fe9f0044afb8dc0aa1cd39a8c4", + ""); + TestBIP324PacketVector( + 0, + "0286c41cd30913db0fdff7a64ebda5c8e3e7cef10f2aebc00a7650443cf4c60d", + "d1ee8a93a01130cbf299249a258f94feb5f469e7d0f2f28f69ee5e9aa8f9b54a60f2c3ff2d023634ec7f4127a96cc11662e402894cf1f694fb9a7eaa5f1d9244", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff22d5e441524d571a52b3def126189d3f416890a99d4da6ede2b0cde1760ce2c3f98457ae", + true, + "054290a6c6ba8d80478172e89d32bf690913ae9835de6dcf206ff1f4d652286fe0ddf74deba41d55de3edc77c42a32af79bbea2c00bae7492264c60866ae5a", + 1, + "84932a55aac22b51e7b128d31d9f0550da28e6a3f394224707d878603386b2f9d0c6bcd8046679bfed7b68c517e7431e75d9dd34605727d2ef1c2babbf680ecc8d68d2c4886e9953a4034abde6da4189cd47c6bb3192242cf714d502ca6103ee84e08bc2ca4fd370d5ad4e7d06c7fbf496c6c7cc7eb19c40c61fb33df2a9ba48497a96c98d7b10c1f91098a6b7b16b4bab9687f27585ade1491ae0dba6a79e1e2d85dd9d9d45c5135ca5fca3f0f99a60ea39edbc9efc7923111c937913f225d67788d5f7e8852b697e26b92ec7bfcaa334a1665511c2b4c0a42d06f7ab98a9719516c8fd17f73804555ee84ab3b7d1762f6096b778d3cb9c799cbd49a9e4a325197b4e6cc4a5c4651f8b41ff88a92ec428354531f970263b467c77ed11312e2617d0d53fe9a8707f51f9f57a77bfb49afe3d89d85ec05ee17b9186f360c94ab8bb2926b65ca99dae1d6ee1af96cad09de70b6767e949023e4b380e66669914a741ed0fa420a48dbc7bfae5ef2019af36d1022283dd90655f25eec7151d471265d22a6d3f91dc700ba749bb67c0fe4bc0888593fbaf59d3c6fff1bf756a125910a63b9682b597c20f560ecb99c11a92c8c8c3f7fbfaa103146083a0ccaecf7a5f5e735a784a8820155914a289d57d8141870ffcaf588882332e0bcd8779efa931aa108dab6c3cce76691e345df4a91a03b71074d66333fd3591bff071ea099360f787bbe43b7b3dff2a59c41c7642eb79870222ad1c6f2e5a191ed5acea51134679587c9cf71c7d8ee290be6bf465c4ee47897a125708704ad610d8d00252d01959209d7cd04d5ecbbb1419a7e84037a55fefa13dee464b48a35c96bcb9a53e7ed461c3a1607ee00c3c302fd47cd73fda7493e947c9834a92d63dcfbd65aa7c38c3e3a2748bb5d9a58e7495d243d6b741078c8f7ee9c8813e473a323375702702b0afae1550c8341eedf5247627343a95240cb02e3e17d5dca16f8d8d3b2228e19c06399f8ec5c5e9dbe4caef6a0ea3ffb1d3c7eac03ae030e791fa12e537c80d56b55b764cadf27a8701052df1282ba8b5e3eb62b5dc7973ac40160e00722fa958d95102fc25c549d8c0e84bed95b7acb61ba65700c4de4feebf78d13b9682c52e937d23026fb4c6193e6644e2d3c99f91f4f39a8b9fc6d013f89c3793ef703987954dc0412b550652c01d922f525704d32d70d6d4079bc3551b563fb29577b3aecdc9505011701dddfd94830431e7a4918927ee44fb3831ce8c4513839e2deea1287f3fa1ab9b61a256c09637dbc7b4f0f8fbb783840f9c24526da883b0df0c473cf231656bd7bc1aaba7f321fec0971c8c2c3444bff2f55e1df7fea66ec3e440a612db9aa87bb505163a59e06b96d46f50d8120b92814ac5ab146bc78dbbf91065af26107815678ce6e33812e6bf3285d4ef3b7b04b076f21e7820dcbfdb4ad5218cf4ff6a65812d8fcb98ecc1e95e2fa58e3efe4ce26cd0bd400d6036ab2ad4f6c713082b5e3f1e04eb9e3b6c8f63f57953894b9e220e0130308e1fd91f72d398c1e7962ca2c31be83f31d6157633581a0a6910496de8d55d3d07090b6aa087159e388b7e7dec60f5d8a60d93ca2ae91296bd484d916bfaaa17c8f45ea4b1a91b37c82821199a2b7596672c37156d8701e7352aa48671d3b1bbbd2bd5f0a2268894a25b0cb2514af39c8743f8cce8ab4b523053739fd8a522222a09acf51ac704489cf17e4b7125455cb8f125b4d31af1eba1f8cf7f81a5a100a141a7ee72e8083e065616649c241f233645c5fc865d17f0285f5c52d9f45312c979bfb3ce5f2a1b951deddf280ffb3f370410cffd1583bfa90077835aa201a0712d1dcd1293ee177738b14e6b5e2a496d05220c3253bb6578d6aff774be91946a614dd7e879fb3dcf7451e0b9adb6a8c44f53c2c464bcc0019e9fad89cac7791a0a3f2974f759a9856351d4d2d7c5612c17cfc50f8479945df57716767b120a590f4bf656f4645029a525694d8a238446c5f5c2c1c995c09c1405b8b1eb9e0352ffdf766cc964f8dcf9f8f043dfab6d102cf4b298021abd78f1d9025fa1f8e1d710b38d9d1652f2d88d1305874ec41609b6617b65c5adb19b6295dc5c5da5fdf69f28144ea12f17c3c6fcce6b9b5157b3dfc969d6725fa5b098a4d9b1d31547ed4c9187452d281d0a5d456008caf1aa251fac8f950ca561982dc2dc908d3691ee3b6ad3ae3d22d002577264ca8e49c523bd51c4846be0d198ad9407bf6f7b82c79893eb2c05fe9981f687a97a4f01fe45ff8c8b7ecc551135cd960a0d6001ad35020be07ffb53cb9e731522ca8ae9364628914b9b8e8cc2f37f03393263603cc2b45295767eb0aac29b0930390eb89587ab2779d2e3decb8042acece725ba42eda650863f418f8d0d50d104e44fbbe5aa7389a4a144a8cecf00f45fb14c39112f9bfb56c0acbd44fa3ff261f5ce4acaa5134c2c1d0cca447040820c81ab1bcdc16aa075b7c68b10d06bbb7ce08b5b805e0238f24402cf24a4b4e00701935a0c68add3de090903f9b85b153cb179a582f57113bfc21c2093803f0cfa4d9d4672c2b05a24f7e4c34a8e9101b70303a7378b9c50b6cddd46814ef7fd73ef6923feceab8fc5aa8b0d185f2e83c7a99dcb1077c0ab5c1f5d5f01ba2f0420443f75c4417db9ebf1665efbb33dca224989920a64b44dc26f682cc77b4632c8454d49135e52503da855bc0f6ff8edc1145451a9772c06891f41064036b66c3119a0fc6e80dffeb65dc456108b7ca0296f4175fff3ed2b0f842cd46bd7e86f4c62dfaf1ddbf836263c00b34803de164983d0811cebfac86e7720c726d3048934c36c23189b02386a722ca9f0fe00233ab50db928d3bccea355cc681144b8b7edcaae4884d5a8f04425c0890ae2c74326e138066d8c05f4c82b29df99b034ea727afde590a1f2177ace3af99cfb1729d6539ce7f7f7314b046aab74497e63dd399e1f7d5f16517c23bd830d1fdee810f3c3b77573dd69c4b97d80d71fb5a632e00acdfa4f8e829faf3580d6a72c40b28a82172f8dcd4627663ebf6069736f21735fd84a226f427cd06bb055f94e7c92f31c48075a2955d82a5b9d2d0198ce0d4e131a112570a8ee40fb80462a81436a58e7db4e34b6e2c422e82f934ecda9949893da5730fc5c23c7c920f363f85ab28cc6a4206713c3152669b47efa8238fa826735f17b4e78750276162024ec85458cd5808e06f40dd9fd43775a456a3ff6cae90550d76d8b2899e0762ad9a371482b3e38083b1274708301d6346c22fea9bb4b73db490ff3ab05b2f7f9e187adef139a7794454b7300b8cc64d3ad76c0e4bc54e08833a4419251550655380d675bc91855aeb82585220bb97f03e976579c08f321b5f8f70988d3061f41465517d53ac571dbf1b24b94443d2e9a8e8a79b392b3d6a4ecdd7f626925c365ef6221305105ce9b5f5b6ecc5bed3d702bd4b7f5008aa8eb8c7aa3ade8ecf6251516fbefeea4e1082aa0e1848eddb31ffe44b04792d296054402826e4bd054e671f223e5557e4c94f89ca01c25c44f1a2ff2c05a70b43408250705e1b858bf0670679fdcd379203e36be3500dd981b1a6422c3cf15224f7fefdef0a5f225c5a09d15767598ecd9e262460bb33a4b5d09a64591efabc57c923d3be406979032ae0bc0997b65336a06dd75b253332ad6a8b63ef043f780a1b3fb6d0b6cad98b1ef4a02535eb39e14a866cfc5fc3a9c5deb2261300d71280ebe66a0776a151469551c3c5fa308757f956655278ec6330ae9e3625468c5f87e02cd9a6489910d4143c1f4ee13aa21a6859d907b788e28572fecee273d44e4a900fa0aa668dd861a60fb6b6b12c2c5ef3c8df1bd7ef5d4b0d1cdb8c15fffbb365b9784bd94abd001c6966216b9b67554ad7cb7f958b70092514f7800fc40244003e0fd1133a9b850fb17f4fcafde07fc87b07fb510670654a5d2d6fc9876ac74728ea41593beef003d6858786a52d3a40af7529596767c17000bfaf8dc52e871359f4ad8bf6e7b2853e5229bdf39657e213580294a5317c5df172865e1e17fe37093b585e04613f5f078f761b2b1752eb32983afda24b523af8851df9a02b37e77f543f18888a782a994a50563334282bf9cdfccc183fdf4fcd75ad86ee0d94f91ee2300a5befbccd14e03a77fc031a8cfe4f01e4c5290f5ac1da0d58ea054bd4837cfd93e5e34fc0eb16e48044ba76131f228d16cde9b0bb978ca7cdcd10653c358bdb26fdb723a530232c32ae0a4cecc06082f46e1c1d596bfe60621ad1e354e01e07b040cc7347c016653f44d926d13ca74e6cbc9d4ab4c99f4491c95c76fff5076b3936eb9d0a286b97c035ca88a3c6309f5febfd4cdaac869e4f58ed409b1e9eb4192fb2f9c2f12176d460fd98286c9d6df84598f260119fd29c63f800c07d8df83d5cc95f8c2fea2812e7890e8a0718bb1e031ecbebc0436dcf3e3b9a58bcc06b4c17f711f80fe1dffc3326a6eb6e00283055c6dabe20d311bfd5019591b7954f8163c9afad9ef8390a38f3582e0a79cdf0353de8eeb6b5f9f27b16ffdef7dd62869b4840ee226ccdce95e02c4545eb981b60571cd83f03dc5eaf8c97a0829a4318a9b3dc06c0e003db700b2260ff1fa8fee66890e637b109abb03ec901b05ca599775f48af50154c0e67d82bf0f558d7d3e0778dc38bea1eb5f74dc8d7f90abdf5511a424be66bf8b6a3cacb477d2e7ef4db68d2eba4d5289122d851f9501ba7e9c4957d8eba3be3fc8e785c4265a1d65c46f2809b70846c693864b169c9dcb78be26ea14b8613f145b01887222979a9e67aee5f800caa6f5c4229bdeefc901232ace6143c9865e4d9c07f51aa200afaf7e48a7d1d8faf366023beab12906ffcb3eaf72c0eb68075e4daf3c080e0c31911befc16f0cc4a09908bb7c1e26abab38bd7b788e1a09c0edf1a35a38d2ff1d3ed47fcdaae2f0934224694f5b56705b9409b6d3d64f3833b686f7576ec64bbdd6ff174e56c2d1edac0011f904681a73face26573fbba4e34652f7ae84acfb2fa5a5b3046f98178cd0831df7477de70e06a4c00e305f31aafc026ef064dd68fd3e4252b1b91d617b26c6d09b6891a00df68f105b5962e7f9d82da101dd595d286da721443b72b2aba2377f6e7772e33b3a5e3753da9c2578c5d1daab80187f55518c72a64ee150a7cb5649823c08c9f62cd7d020b45ec2cba8310db1a7785a46ab24785b4d54ff1660b5ca78e05a9a55edba9c60bf044737bc468101c4e8bd1480d749be5024adefca1d998abe33eaeb6b11fbb39da5d905fdd3f611b2e51517ccee4b8af72c2d948573505590d61a6783ab7278fc43fe55b1fcc0e7216444d3c8039bb8145ef1ce01c50e95a3f3feab0aee883fdb94cc13ee4d21c542aa795e18932228981690f4d4c57ca4db6eb5c092e29d8a05139d509a8aeb48baa1eb97a76e597a32b280b5e9d6c36859064c98ff96ef5126130264fa8d2f49213870d9fb036cff95da51f270311d9976208554e48ffd486470d0ecdb4e619ccbd8226147204baf8e235f54d8b1cba8fa34a9a4d055de515cdf180d2bb6739a175183c472e30b5c914d09eeb1b7dafd6872b38b48c6afc146101200e6e6a44fe5684e220adc11f5c403ddb15df8051e6bdef09117a3a5349938513776286473a3cf1d2788bb875052a2e6459fa7926da33380149c7f98d7700528a60c954e6f5ecb65842fde69d614be69eaa2040a4819ae6e756accf936e14c1e894489744a79c1f2c1eb295d13e2d767c09964b61f9cfe497649f712", + false, + "d4e3f18ac2e2095edb5c3b94236118ad", + "4faa6c4233d9fd53d170ede4172142a8", + "23f154ac43cfc59c4243e9fc68aeec8f19ad3942d74108e833b36f0dd3dcd357", + "8da7de6ea7bf2a81a396a42880ba1f5756734c4821309ac9aeffa2a26ce86873b9dc4935a772de6ec5162c6d075b14536800fb174841153511bfb597e992e2fe8a450c4bce102cc550bb37fd564c4d60bf884e", + ""); + TestBIP324PacketVector( + 223, + "6c77432d1fda31e9f942f8af44607e10f3ad38a65f8a4bddae823e5eff90dc38", + "d2685070c1e6376e633e825296634fd461fa9e5bdf2109bcebd735e5a91f3e587c5cb782abb797fbf6bb5074fd1542a474f2a45b673763ec2db7fb99b737bbb9", + "56bd0c06f10352c3a1a9f4b4c92f6fa2b26df124b57878353c1fc691c51abea77c8817daeeb9fa546b77c8daf79d89b22b0e1b87574ece42371f00237aa9d83a", + false, + "7e0e78eb6990b059e6cf0ded66ea93ef82e72aa2f18ac24f2fc6ebab561ae557420729da103f64cecfa20527e15f9fb669a49bbbf274ef0389b3e43c8c44e5f60bf2ac38e2b55e7ec4273dba15ba41d21f8f5b3ee1688b3c29951218caf847a97fb50d75a86515d445699497d968164bf740012679b8962de573be941c62b7ef", + 1, + "", + true, + "cf2e25f23501399f30738d7eee652b90", + "225a477a28a54ea7671d2b217a9c29db", + "7ec02fea8c1484e3d0875f978c5f36d63545e2e4acf56311394422f4b66af612", + "", + "729847a3e9eba7a5bff454b5de3b393431ee360736b6c030d7a5bd01d1203d2e98f528543fd2bf886ccaa1ada5e215a730a36b3f4abfc4e252c89eb01d9512f94916dae8a76bf16e4da28986ffe159090fe5267ee3394300b7ccf4dfad389a26321b3a3423e4594a82ccfbad16d6561ecb8772b0cb040280ff999a29e3d9d4fd"); + TestBIP324PacketVector( + 448, + "a6ec25127ca1aa4cf16b20084ba1e6516baae4d32422288e9b36d8bddd2de35a", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff053d7ecca53e33e185a8b9be4e7699a97c6ff4c795522e5918ab7cd6b6884f67e683f3dc", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffa7730be30000000000000000000000000000000000000000000000000000000000000000", + true, + "00cf68f8f7ac49ffaa02c4864fdf6dfe7bbf2c740b88d98c50ebafe32c92f3427f57601ffcb21a3435979287db8fee6c302926741f9d5e464c647eeb9b7acaeda46e00abd7506fc9a719847e9a7328215801e96198dac141a15c7c2f68e0690dd1176292a0dded04d1f548aad88f1aebdc0a8f87da4bb22df32dd7c160c225b843e83f6525d6d484f502f16d923124fc538794e21da2eb689d18d87406ecced5b9f92137239ed1d37bcfa7836641a83cf5e0a1cf63f51b06f158e499a459ede41c", + 1, + "", + false, + "fead69be77825a23daec377c362aa560", + "511d4980526c5e64aa7187462faeafdd", + "acb8f084ea763ddd1b92ac4ed23bf44de20b84ab677d4e4e6666a6090d40353d", + "", + "77b4656934a82de1a593d8481f020194ddafd8cac441f9d72aeb8721e6a14f49698ca6d9b2b6d59d07a01aa552fd4d5b68d0d1617574c77dea10bfadbaa31b83885b7ceac2fd45e3e4a331c51a74e7b1698d81b64c87c73c5b9258b4d83297f9debc2e9aa07f8572ff434dc792b83ecf07b3197de8dc9cf7be56acb59c66cff5"); + TestBIP324PacketVector( + 673, + "0af952659ed76f80f585966b95ab6e6fd68654672827878684c8b547b1b94f5a", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffc81017fd92fd31637c26c906b42092e11cc0d3afae8d9019d2578af22735ce7bc469c72d", + "9652d78baefc028cd37a6a92625b8b8f85fde1e4c944ad3f20e198bef8c02f19fffffffffffffffffffffffffffffffffffffffffffffffffffffffff2e91870", + false, + "5c6272ee55da855bbbf7b1246d9885aa7aa601a715ab86fa46c50da533badf82b97597c968293ae04e", + 97561, + "", + false, + "5e2375ac629b8df1e4ff3617c6255a70", + "70bcbffcb62e4d29d2605d30bceef137", + "7332e92a3f9d2792c4d444fac5ed888c39a073043a65eefb626318fd649328f8", + "", + "657a4a19711ce593c3844cb391b224f60124aba7e04266233bc50cafb971e26c7716b76e98376448f7d214dd11e629ef9a974d60e3770a695810a61c4ba66d78b936ee7892b98f0b48ddae9fcd8b599dca1c9b43e9b95e0226cf8d4459b8a7c2c4e6db80f1d58c7b20dd7208fa5c1057fb78734223ee801dbd851db601fee61e"); + TestBIP324PacketVector( + 1024, + "f90e080c64b05824c5a24b2501d5aeaf08af3872ee860aa80bdcd430f7b63494", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff115173765dc202cf029ad3f15479735d57697af12b0131dd21430d5772e4ef11474d58b9", + "12a50f3fafea7c1eeada4cf8d33777704b77361453afc83bda91eef349ae044d20126c6200547ea5a6911776c05dee2a7f1a9ba7dfbabbbd273c3ef29ef46e46", + true, + "5f67d15d22ca9b2804eeab0a66f7f8e3a10fa5de5809a046084348cbc5304e843ef96f59a59c7d7fdfe5946489f3ea297d941bac326225df316a25fc90f0e65b0d31a9c497e960fdbf8c482516bc8a9c1c77b7f6d0e1143810c737f76f9224e6f2c9af5186b4f7259c7e8d165b6e4fe3d38a60bdbdd4d06ecdcaaf62086070dbb68686b802d53dfd7db14b18743832605f5461ad81e2af4b7e8ff0eff0867a25b93cec7becf15c43131895fed09a83bf1ee4a87d44dd0f02a837bf5a1232e201cb882734eb9643dc2dc4d4e8b5690840766212c7ac8f38ad8a9ec47c7a9b3e022ae3eb6a32522128b518bd0d0085dd81c5", + 69615, + "", + true, + "b709dea25e0be287c50e3603482c2e98", + "1f677e9d7392ebe3633fd82c9efb0f16", + "889f339285564fd868401fac8380bb9887925122ec8f31c8ae51ce067def103b", + "", + "7c4b9e1e6c1ce69da7b01513cdc4588fd93b04dafefaf87f31561763d906c672bac3dfceb751ebd126728ac017d4d580e931b8e5c7d5dfe0123be4dc9b2d2238b655c8a7fadaf8082c31e310909b5b731efc12f0a56e849eae6bfeedcc86dd27ef9b91d159256aa8e8d2b71a311f73350863d70f18d0d7302cf551e4303c7733"); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 8332f54591..6fbe74a680 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -4,7 +4,7 @@ #include <crypto/aes.h> #include <crypto/chacha20.h> -#include <crypto/chacha_poly_aead.h> +#include <crypto/chacha20poly1305.h> #include <crypto/hkdf_sha256_32.h> #include <crypto/hmac_sha256.h> #include <crypto/hmac_sha512.h> @@ -182,6 +182,46 @@ static void TestChaCha20(const std::string &hex_message, const std::string &hexk } } +static void TestFSChaCha20(const std::string& hex_plaintext, const std::string& hexkey, uint32_t rekey_interval, const std::string& ciphertext_after_rotation) +{ + auto key = ParseHex<std::byte>(hexkey); + BOOST_CHECK_EQUAL(FSChaCha20::KEYLEN, key.size()); + + auto plaintext = ParseHex<std::byte>(hex_plaintext); + + auto fsc20 = FSChaCha20{key, rekey_interval}; + auto c20 = ChaCha20{UCharCast(key.data())}; + + std::vector<std::byte> fsc20_output; + fsc20_output.resize(plaintext.size()); + + std::vector<std::byte> c20_output; + c20_output.resize(plaintext.size()); + + for (size_t i = 0; i < rekey_interval; i++) { + fsc20.Crypt(plaintext, fsc20_output); + c20.Crypt(UCharCast(plaintext.data()), UCharCast(c20_output.data()), plaintext.size()); + BOOST_CHECK(c20_output == fsc20_output); + } + + // At the rotation interval, the outputs will no longer match + fsc20.Crypt(plaintext, fsc20_output); + auto c20_copy = c20; + c20.Crypt(UCharCast(plaintext.data()), UCharCast(c20_output.data()), plaintext.size()); + BOOST_CHECK(c20_output != fsc20_output); + + std::byte new_key[FSChaCha20::KEYLEN]; + c20_copy.Keystream(UCharCast(new_key), sizeof(new_key)); + c20.SetKey32(UCharCast(new_key)); + c20.Seek64({0, 1}, 0); + + // Outputs should match again after simulating key rotation + c20.Crypt(UCharCast(plaintext.data()), UCharCast(c20_output.data()), plaintext.size()); + BOOST_CHECK(c20_output == fsc20_output); + + BOOST_CHECK_EQUAL(HexStr(fsc20_output), ciphertext_after_rotation); +} + static void TestPoly1305(const std::string &hexmessage, const std::string &hexkey, const std::string& hextag) { auto key = ParseHex<std::byte>(hexkey); @@ -208,6 +248,93 @@ static void TestPoly1305(const std::string &hexmessage, const std::string &hexke } } +static void TestChaCha20Poly1305(const std::string& plain_hex, const std::string& aad_hex, const std::string& key_hex, ChaCha20::Nonce96 nonce, const std::string& cipher_hex) +{ + auto plain = ParseHex<std::byte>(plain_hex); + auto aad = ParseHex<std::byte>(aad_hex); + auto key = ParseHex<std::byte>(key_hex); + auto expected_cipher = ParseHex<std::byte>(cipher_hex); + + for (int i = 0; i < 10; ++i) { + // During i=0, use single-plain Encrypt/Decrypt; others use a split at prefix. + size_t prefix = i ? InsecureRandRange(plain.size() + 1) : plain.size(); + // Encrypt. + std::vector<std::byte> cipher(plain.size() + AEADChaCha20Poly1305::EXPANSION); + AEADChaCha20Poly1305 aead{key}; + if (i == 0) { + aead.Encrypt(plain, aad, nonce, cipher); + } else { + aead.Encrypt(Span{plain}.first(prefix), Span{plain}.subspan(prefix), aad, nonce, cipher); + } + BOOST_CHECK(cipher == expected_cipher); + + // Decrypt. + std::vector<std::byte> decipher(cipher.size() - AEADChaCha20Poly1305::EXPANSION); + bool ret{false}; + if (i == 0) { + ret = aead.Decrypt(cipher, aad, nonce, decipher); + } else { + ret = aead.Decrypt(cipher, aad, nonce, Span{decipher}.first(prefix), Span{decipher}.subspan(prefix)); + } + BOOST_CHECK(ret); + BOOST_CHECK(decipher == plain); + } + + // Test Keystream output. + std::vector<std::byte> keystream(plain.size()); + AEADChaCha20Poly1305 aead{key}; + aead.Keystream(nonce, keystream); + for (size_t i = 0; i < plain.size(); ++i) { + BOOST_CHECK_EQUAL(plain[i] ^ keystream[i], expected_cipher[i]); + } +} + +static void TestFSChaCha20Poly1305(const std::string& plain_hex, const std::string& aad_hex, const std::string& key_hex, uint64_t msg_idx, const std::string& cipher_hex) +{ + auto plain = ParseHex<std::byte>(plain_hex); + auto aad = ParseHex<std::byte>(aad_hex); + auto key = ParseHex<std::byte>(key_hex); + auto expected_cipher = ParseHex<std::byte>(cipher_hex); + std::vector<std::byte> cipher(plain.size() + FSChaCha20Poly1305::EXPANSION); + + for (int it = 0; it < 10; ++it) { + // During it==0 we use the single-plain Encrypt/Decrypt; others use a split at prefix. + size_t prefix = it ? InsecureRandRange(plain.size() + 1) : plain.size(); + std::byte dummy_tag[FSChaCha20Poly1305::EXPANSION] = {{}}; + + // Do msg_idx dummy encryptions to seek to the correct packet. + FSChaCha20Poly1305 enc_aead{key, 224}; + for (uint64_t i = 0; i < msg_idx; ++i) { + enc_aead.Encrypt(Span{dummy_tag}.first(0), Span{dummy_tag}.first(0), dummy_tag); + } + + // Invoke single-plain or plain1/plain2 Encrypt. + if (it == 0) { + enc_aead.Encrypt(plain, aad, cipher); + } else { + enc_aead.Encrypt(Span{plain}.first(prefix), Span{plain}.subspan(prefix), aad, cipher); + } + BOOST_CHECK(cipher == expected_cipher); + + // Do msg_idx dummy decryptions to seek to the correct packet. + FSChaCha20Poly1305 dec_aead{key, 224}; + for (uint64_t i = 0; i < msg_idx; ++i) { + dec_aead.Decrypt(dummy_tag, Span{dummy_tag}.first(0), Span{dummy_tag}.first(0)); + } + + // Invoke single-plain or plain1/plain2 Decrypt. + std::vector<std::byte> decipher(cipher.size() - AEADChaCha20Poly1305::EXPANSION); + bool ret{false}; + if (it == 0) { + ret = dec_aead.Decrypt(cipher, aad, decipher); + } else { + ret = dec_aead.Decrypt(cipher, aad, Span{decipher}.first(prefix), Span{decipher}.subspan(prefix)); + } + BOOST_CHECK(ret); + BOOST_CHECK(decipher == plain); + } +} + static void TestHKDF_SHA256_32(const std::string &ikm_hex, const std::string &salt_hex, const std::string &info_hex, const std::string &okm_check_hex) { std::vector<unsigned char> initial_key_material = ParseHex(ikm_hex); std::vector<unsigned char> salt = ParseHex(salt_hex); @@ -678,6 +805,20 @@ BOOST_AUTO_TEST_CASE(chacha20_testvector) "fd565dea5addbdb914208fde7950f23e0385f9a727143f6a6ac51d84b1c0fb3e" "2e3b00b63d6841a1cc6d1538b1d3a74bef1eb2f54c7b7281e36e484dba89b351" "c8f572617e61e342879f211b0e4c515df50ea9d0771518fad96cd0baee62deb6"); + + // Forward secure ChaCha20 + TestFSChaCha20("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "0000000000000000000000000000000000000000000000000000000000000000", + 256, + "a93df4ef03011f3db95f60d996e1785df5de38fc39bfcb663a47bb5561928349"); + TestFSChaCha20("01", + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + 5, + "ea"); + TestFSChaCha20("e93fdb5c762804b9a706816aca31e35b11d2aa3080108ef46a5b1f1508819c0a", + "8ec4c3ccdaea336bdeb245636970be01266509b33f3d2642504eaf412206207a", + 4096, + "8bfaa4eacff308fdb4a94a5ff25bd9d0c1f84b77f81239f67ff39d6e1ac280c9"); } BOOST_AUTO_TEST_CASE(chacha20_midblock) @@ -686,7 +827,7 @@ BOOST_AUTO_TEST_CASE(chacha20_midblock) ChaCha20 c20{key.data()}; // get one block of keystream unsigned char block[64]; - c20.Keystream(block, CHACHA20_ROUND_OUTPUT); + c20.Keystream(block, sizeof(block)); unsigned char b1[5], b2[7], b3[52]; c20 = ChaCha20{key.data()}; c20.Keystream(b1, 5); @@ -819,6 +960,87 @@ BOOST_AUTO_TEST_CASE(poly1305_testvector) "0e410fa9d7a40ac582e77546be9a72bb"); } +BOOST_AUTO_TEST_CASE(chacha20poly1305_testvectors) +{ + // Note that in our implementation, the authentication is suffixed to the ciphertext. + // The RFC test vectors specify them separately. + + // RFC 8439 Example from section 2.8.2 + TestChaCha20Poly1305("4c616469657320616e642047656e746c656d656e206f662074686520636c6173" + "73206f66202739393a204966204920636f756c64206f6666657220796f75206f" + "6e6c79206f6e652074697020666f7220746865206675747572652c2073756e73" + "637265656e20776f756c642062652069742e", + "50515253c0c1c2c3c4c5c6c7", + "808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9f", + {7, 0x4746454443424140}, + "d31a8d34648e60db7b86afbc53ef7ec2a4aded51296e08fea9e2b5a736ee62d6" + "3dbea45e8ca9671282fafb69da92728b1a71de0a9e060b2905d6a5b67ecd3b36" + "92ddbd7f2d778b8c9803aee328091b58fab324e4fad675945585808b4831d7bc" + "3ff4def08e4b7a9de576d26586cec64b61161ae10b594f09e26a7e902ecbd060" + "0691"); + + // RFC 8439 Test vector A.5 + TestChaCha20Poly1305("496e7465726e65742d4472616674732061726520647261667420646f63756d65" + "6e74732076616c696420666f722061206d6178696d756d206f6620736978206d" + "6f6e74687320616e64206d617920626520757064617465642c207265706c6163" + "65642c206f72206f62736f6c65746564206279206f7468657220646f63756d65" + "6e747320617420616e792074696d652e20497420697320696e617070726f7072" + "6961746520746f2075736520496e7465726e65742d4472616674732061732072" + "65666572656e6365206d6174657269616c206f7220746f206369746520746865" + "6d206f74686572207468616e206173202fe2809c776f726b20696e2070726f67" + "726573732e2fe2809d", + "f33388860000000000004e91", + "1c9240a5eb55d38af333888604f6b5f0473917c1402b80099dca5cbc207075c0", + {0, 0x0807060504030201}, + "64a0861575861af460f062c79be643bd5e805cfd345cf389f108670ac76c8cb2" + "4c6cfc18755d43eea09ee94e382d26b0bdb7b73c321b0100d4f03b7f355894cf" + "332f830e710b97ce98c8a84abd0b948114ad176e008d33bd60f982b1ff37c855" + "9797a06ef4f0ef61c186324e2b3506383606907b6a7c02b0f9f6157b53c867e4" + "b9166c767b804d46a59b5216cde7a4e99040c5a40433225ee282a1b0a06c523e" + "af4534d7f83fa1155b0047718cbc546a0d072b04b3564eea1b422273f548271a" + "0bb2316053fa76991955ebd63159434ecebb4e466dae5a1073a6727627097a10" + "49e617d91d361094fa68f0ff77987130305beaba2eda04df997b714d6c6f2c29" + "a6ad5cb4022b02709beead9d67890cbb22392336fea1851f38"); + + // Test vectors exercising aad and plaintext which are multiples of 16 bytes. + TestChaCha20Poly1305("8d2d6a8befd9716fab35819eaac83b33269afb9f1a00fddf66095a6c0cd91951" + "a6b7ad3db580be0674c3f0b55f618e34", + "", + "72ddc73f07101282bbbcf853b9012a9f9695fc5d36b303a97fd0845d0314e0c3", + {0x3432b75f, 0xb3585537eb7f4024}, + "f760b8224fb2a317b1b07875092606131232a5b86ae142df5df1c846a7f6341a" + "f2564483dd77f836be45e6230808ffe402a6f0a3e8be074b3d1f4ea8a7b09451"); + TestChaCha20Poly1305("", + "36970d8a704c065de16250c18033de5a400520ac1b5842b24551e5823a3314f3" + "946285171e04a81ebfbe3566e312e74ab80e94c7dd2ff4e10de0098a58d0f503", + "77adda51d6730b9ad6c995658cbd49f581b2547e7c0c08fcc24ceec797461021", + {0x1f90da88, 0x75dafa3ef84471a4}, + "aaae5bb81e8407c94b2ae86ae0c7efbe"); + + // FSChaCha20Poly1305 tests. + TestFSChaCha20Poly1305("d6a4cb04ef0f7c09c1866ed29dc24d820e75b0491032a51b4c3366f9ca35c19e" + "a3047ec6be9d45f9637b63e1cf9eb4c2523a5aab7b851ebeba87199db0e839cf" + "0d5c25e50168306377aedbe9089fd2463ded88b83211cf51b73b150608cc7a60" + "0d0f11b9a742948482e1b109d8faf15b450aa7322e892fa2208c6691e3fecf4c" + "711191b14d75a72147", + "786cb9b6ebf44288974cf0", + "5c9e1c3951a74fba66708bf9d2c217571684556b6a6a3573bff2847d38612654", + 500, + "9dcebbd3281ea3dd8e9a1ef7d55a97abd6743e56ebc0c190cb2c4e14160b385e" + "0bf508dddf754bd02c7c208447c131ce23e47a4a14dfaf5dd8bc601323950f75" + "4e05d46e9232f83fc5120fbbef6f5347a826ec79a93820718d4ec7a2b7cfaaa4" + "4b21e16d726448b62f803811aff4f6d827ed78e738ce8a507b81a8ae13131192" + "8039213de18a5120dc9b7370baca878f50ff254418de3da50c"); + TestFSChaCha20Poly1305("8349b7a2690b63d01204800c288ff1138a1d473c832c90ea8b3fc102d0bb3adc" + "44261b247c7c3d6760bfbe979d061c305f46d94c0582ac3099f0bf249f8cb234", + "", + "3bd2093fcbcb0d034d8c569583c5425c1a53171ea299f8cc3bbf9ae3530adfce", + 60000, + "30a6757ff8439b975363f166a0fa0e36722ab35936abd704297948f45083f4d4" + "99433137ce931f7fca28a0acd3bc30f57b550acbc21cbd45bbef0739d9caf30c" + "14b94829deb27f0b1923a2af704ae5d6"); +} + BOOST_AUTO_TEST_CASE(hkdf_hmac_sha256_l32_tests) { // Use rfc5869 test vectors but truncated to 32 bytes (our implementation only support length 32) @@ -839,129 +1061,6 @@ BOOST_AUTO_TEST_CASE(hkdf_hmac_sha256_l32_tests) "8da4e775a563c18f715f802a063c5a31b8a11f5c5ee1879ec3454e5f3c738d2d"); } -static void TestChaCha20Poly1305AEAD(bool must_succeed, unsigned int expected_aad_length, const std::string& hex_m, const std::string& hex_k1, const std::string& hex_k2, const std::string& hex_aad_keystream, const std::string& hex_encrypted_message, const std::string& hex_encrypted_message_seq_999) -{ - // we need two sequence numbers, one for the payload cipher instance... - uint32_t seqnr_payload = 0; - // ... and one for the AAD (length) cipher instance - uint32_t seqnr_aad = 0; - // we need to keep track of the position in the AAD cipher instance - // keystream since we use the same 64byte output 21 times - // (21 times 3 bytes length < 64) - int aad_pos = 0; - - std::vector<unsigned char> aead_K_1 = ParseHex(hex_k1); - std::vector<unsigned char> aead_K_2 = ParseHex(hex_k2); - std::vector<unsigned char> plaintext_buf = ParseHex(hex_m); - std::vector<unsigned char> expected_aad_keystream = ParseHex(hex_aad_keystream); - std::vector<unsigned char> expected_ciphertext_and_mac = ParseHex(hex_encrypted_message); - std::vector<unsigned char> expected_ciphertext_and_mac_sequence999 = ParseHex(hex_encrypted_message_seq_999); - - std::vector<unsigned char> ciphertext_buf(plaintext_buf.size() + Poly1305::TAGLEN, 0); - std::vector<unsigned char> plaintext_buf_new(plaintext_buf.size(), 0); - std::vector<unsigned char> cmp_ctx_buffer(64); - uint32_t out_len = 0; - - // create the AEAD instance - ChaCha20Poly1305AEAD aead(aead_K_1.data(), aead_K_1.size(), aead_K_2.data(), aead_K_2.size()); - - // create a chacha20 instance to compare against - ChaCha20 cmp_ctx(aead_K_1.data()); - - // encipher - bool res = aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, ciphertext_buf.data(), ciphertext_buf.size(), plaintext_buf.data(), plaintext_buf.size(), true); - // make sure the operation succeeded if expected to succeed - BOOST_CHECK_EQUAL(res, must_succeed); - if (!res) return; - - // verify ciphertext & mac against the test vector - BOOST_CHECK_EQUAL(expected_ciphertext_and_mac.size(), ciphertext_buf.size()); - BOOST_CHECK(memcmp(ciphertext_buf.data(), expected_ciphertext_and_mac.data(), ciphertext_buf.size()) == 0); - - // manually construct the AAD keystream - cmp_ctx.Seek64({0, seqnr_aad}, 0); - cmp_ctx.Keystream(cmp_ctx_buffer.data(), 64); - BOOST_CHECK(memcmp(expected_aad_keystream.data(), cmp_ctx_buffer.data(), expected_aad_keystream.size()) == 0); - // crypt the 3 length bytes and compare the length - uint32_t len_cmp = 0; - len_cmp = (ciphertext_buf[0] ^ cmp_ctx_buffer[aad_pos + 0]) | - (ciphertext_buf[1] ^ cmp_ctx_buffer[aad_pos + 1]) << 8 | - (ciphertext_buf[2] ^ cmp_ctx_buffer[aad_pos + 2]) << 16; - BOOST_CHECK_EQUAL(len_cmp, expected_aad_length); - - // encrypt / decrypt 1000 packets - for (size_t i = 0; i < 1000; ++i) { - res = aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, ciphertext_buf.data(), ciphertext_buf.size(), plaintext_buf.data(), plaintext_buf.size(), true); - BOOST_CHECK(res); - BOOST_CHECK(aead.GetLength(&out_len, seqnr_aad, aad_pos, ciphertext_buf.data())); - BOOST_CHECK_EQUAL(out_len, expected_aad_length); - res = aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, plaintext_buf_new.data(), plaintext_buf_new.size(), ciphertext_buf.data(), ciphertext_buf.size(), false); - BOOST_CHECK(res); - - // make sure we repetitive get the same plaintext - BOOST_CHECK(memcmp(plaintext_buf.data(), plaintext_buf_new.data(), plaintext_buf.size()) == 0); - - // compare sequence number 999 against the test vector - if (seqnr_payload == 999) { - BOOST_CHECK(memcmp(ciphertext_buf.data(), expected_ciphertext_and_mac_sequence999.data(), expected_ciphertext_and_mac_sequence999.size()) == 0); - } - // set nonce and block counter, output the keystream - cmp_ctx.Seek64({0, seqnr_aad}, 0); - cmp_ctx.Keystream(cmp_ctx_buffer.data(), 64); - - // crypt the 3 length bytes and compare the length - len_cmp = 0; - len_cmp = (ciphertext_buf[0] ^ cmp_ctx_buffer[aad_pos + 0]) | - (ciphertext_buf[1] ^ cmp_ctx_buffer[aad_pos + 1]) << 8 | - (ciphertext_buf[2] ^ cmp_ctx_buffer[aad_pos + 2]) << 16; - BOOST_CHECK_EQUAL(len_cmp, expected_aad_length); - - // increment the sequence number(s) - // always increment the payload sequence number - // increment the AAD keystream position by its size (3) - // increment the AAD sequence number if we would hit the 64 byte limit - seqnr_payload++; - aad_pos += CHACHA20_POLY1305_AEAD_AAD_LEN; - if (aad_pos + CHACHA20_POLY1305_AEAD_AAD_LEN > CHACHA20_ROUND_OUTPUT) { - aad_pos = 0; - seqnr_aad++; - } - } -} - -BOOST_AUTO_TEST_CASE(chacha20_poly1305_aead_testvector) -{ - /* test chacha20poly1305@bitcoin AEAD */ - - // must fail with no message - TestChaCha20Poly1305AEAD(false, 0, - "", - "0000000000000000000000000000000000000000000000000000000000000000", - "0000000000000000000000000000000000000000000000000000000000000000", "", "", ""); - - TestChaCha20Poly1305AEAD(true, 0, - /* m */ "0000000000000000000000000000000000000000000000000000000000000000", - /* k1 (AAD) */ "0000000000000000000000000000000000000000000000000000000000000000", - /* k2 (payload) */ "0000000000000000000000000000000000000000000000000000000000000000", - /* AAD keystream */ "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586", - /* encrypted message & MAC */ "76b8e09f07e7be5551387a98ba977c732d080dcb0f29a048e3656912c6533e32d2fc11829c1b6c1df1f551cd6131ff08", - /* encrypted message & MAC at sequence 999 */ "b0a03d5bd2855d60699e7d3a3133fa47be740fe4e4c1f967555e2d9271f31c3aaa7aa16ec62c5e24f040c08bb20c3598"); - TestChaCha20Poly1305AEAD(true, 1, - "0100000000000000000000000000000000000000000000000000000000000000", - "0000000000000000000000000000000000000000000000000000000000000000", - "0000000000000000000000000000000000000000000000000000000000000000", - "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586", - "77b8e09f07e7be5551387a98ba977c732d080dcb0f29a048e3656912c6533e32baf0c85b6dff8602b06cf52a6aefc62e", - "b1a03d5bd2855d60699e7d3a3133fa47be740fe4e4c1f967555e2d9271f31c3a8bd94d54b5ecabbc41ffbb0c90924080"); - TestChaCha20Poly1305AEAD(true, 255, - "ff0000f195e66982105ffb640bb7757f579da31602fc93ec01ac56f85ac3c134a4547b733b46413042c9440049176905d3be59ea1c53f15916155c2be8241a38008b9a26bc35941e2444177c8ade6689de95264986d95889fb60e84629c9bd9a5acb1cc118be563eb9b3a4a472f82e09a7e778492b562ef7130e88dfe031c79db9d4f7c7a899151b9a475032b63fc385245fe054e3dd5a97a5f576fe064025d3ce042c566ab2c507b138db853e3d6959660996546cc9c4a6eafdc777c040d70eaf46f76dad3979e5c5360c3317166a1c894c94a371876a94df7628fe4eaaf2ccb27d5aaae0ad7ad0f9d4b6ad3b54098746d4524d38407a6deb3ab78fab78c9", - "ff0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", - "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", - "c640c1711e3ee904ac35c57ab9791c8a1c408603a90b77a83b54f6c844cb4b06d94e7fc6c800e165acd66147e80ec45a567f6ce66d05ec0cae679dceeb890017", - "3940c1e92da4582ff6f92a776aeb14d014d384eeb30f660dacf70a14a23fd31e91212701334e2ce1acf5199dc84f4d61ddbe6571bca5af874b4c9226c26e650995d157644e1848b96ed6c2102d5489a050e71d29a5a66ece11de5fb5c9558d54da28fe45b0bc4db4e5b88030bfc4a352b4b7068eccf656bae7ad6a35615315fc7c49d4200388d5eca67c2e822e069336c69b40db67e0f3c81209c50f3216a4b89fb3ae1b984b7851a2ec6f68ab12b101ab120e1ea7313bb93b5a0f71185c7fea017ddb92769861c29dba4fbc432280d5dff21b36d1c4c790128b22699950bb18bf74c448cdfe547d8ed4f657d8005fdc0cd7a050c2d46050a44c4376355858981fbe8b184288276e7a93eabc899c4a", - "f039c6689eaeef0456685200feaab9d54bbd9acde4410a3b6f4321296f4a8ca2604b49727d8892c57e005d799b2a38e85e809f20146e08eec75169691c8d4f54a0d51a1e1c7b381e0474eb02f994be9415ef3ffcbd2343f0601e1f3b172a1d494f838824e4df570f8e3b0c04e27966e36c82abd352d07054ef7bd36b84c63f9369afe7ed79b94f953873006b920c3fa251a771de1b63da927058ade119aa898b8c97e42a606b2f6df1e2d957c22f7593c1e2002f4252f4c9ae4bf773499e5cfcfe14dfc1ede26508953f88553bf4a76a802f6a0068d59295b01503fd9a600067624203e880fdf53933b96e1f4d9eb3f4e363dd8165a278ff667a41ee42b9892b077cefff92b93441f7be74cf10e6cd"); -} - BOOST_AUTO_TEST_CASE(countbits_tests) { FastRandomContext ctx; diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 9193d9a8b3..b740a51574 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -107,9 +107,19 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) peerman.FinalizeNode(dummyNode1); } -static void AddRandomOutboundPeer(NodeId& id, std::vector<CNode*>& vNodes, PeerManager& peerLogic, ConnmanTestMsg& connman, ConnectionType connType) +static void AddRandomOutboundPeer(NodeId& id, std::vector<CNode*>& vNodes, PeerManager& peerLogic, ConnmanTestMsg& connman, ConnectionType connType, bool onion_peer = false) { - CAddress addr(ip(g_insecure_rand_ctx.randbits(32)), NODE_NONE); + CAddress addr; + + if (onion_peer) { + auto tor_addr{g_insecure_rand_ctx.randbytes(ADDR_TORV3_SIZE)}; + BOOST_REQUIRE(addr.SetSpecial(OnionToString(tor_addr))); + } + + while (!addr.IsRoutable()) { + addr = CAddress(ip(g_insecure_rand_ctx.randbits(32)), NODE_NONE); + } + vNodes.emplace_back(new CNode{id++, /*sock=*/nullptr, addr, @@ -197,6 +207,30 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) BOOST_CHECK(vNodes[max_outbound_full_relay-1]->fDisconnect == true); BOOST_CHECK(vNodes.back()->fDisconnect == false); + vNodes[max_outbound_full_relay - 1]->fDisconnect = false; + + // Add an onion peer, that will be protected because it is the only one for + // its network, so another peer gets disconnected instead. + SetMockTime(time_init); + AddRandomOutboundPeer(id, vNodes, *peerLogic, *connman, ConnectionType::OUTBOUND_FULL_RELAY, /*onion_peer=*/true); + SetMockTime(time_later); + peerLogic->CheckForStaleTipAndEvictPeers(); + + for (int i = 0; i < max_outbound_full_relay - 2; ++i) { + BOOST_CHECK(vNodes[i]->fDisconnect == false); + } + BOOST_CHECK(vNodes[max_outbound_full_relay - 2]->fDisconnect == false); + BOOST_CHECK(vNodes[max_outbound_full_relay - 1]->fDisconnect == true); + BOOST_CHECK(vNodes[max_outbound_full_relay]->fDisconnect == false); + + // Add a second onion peer which won't be protected + SetMockTime(time_init); + AddRandomOutboundPeer(id, vNodes, *peerLogic, *connman, ConnectionType::OUTBOUND_FULL_RELAY, /*onion_peer=*/true); + SetMockTime(time_later); + peerLogic->CheckForStaleTipAndEvictPeers(); + + BOOST_CHECK(vNodes.back()->fDisconnect == true); + for (const CNode *node : vNodes) { peerLogic->FinalizeNode(*node); } diff --git a/src/test/fuzz/bip324.cpp b/src/test/fuzz/bip324.cpp new file mode 100644 index 0000000000..98ac10e364 --- /dev/null +++ b/src/test/fuzz/bip324.cpp @@ -0,0 +1,136 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bip324.h> +#include <chainparams.h> +#include <span.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/xoroshiro128plusplus.h> + +#include <cstdint> +#include <vector> + +namespace { + +void Initialize() +{ + ECC_Start(); + SelectParams(ChainType::MAIN); +} + +} // namespace + +FUZZ_TARGET(bip324_cipher_roundtrip, .init=Initialize) +{ + // Test that BIP324Cipher's encryption and decryption agree. + + // Load keys from fuzzer. + FuzzedDataProvider provider(buffer.data(), buffer.size()); + // Initiator key + auto init_key_data = provider.ConsumeBytes<unsigned char>(32); + init_key_data.resize(32); + CKey init_key; + init_key.Set(init_key_data.begin(), init_key_data.end(), true); + if (!init_key.IsValid()) return; + // Initiator entropy + auto init_ent = provider.ConsumeBytes<std::byte>(32); + init_ent.resize(32); + // Responder key + auto resp_key_data = provider.ConsumeBytes<unsigned char>(32); + resp_key_data.resize(32); + CKey resp_key; + resp_key.Set(resp_key_data.begin(), resp_key_data.end(), true); + if (!resp_key.IsValid()) return; + // Responder entropy + auto resp_ent = provider.ConsumeBytes<std::byte>(32); + resp_ent.resize(32); + + // Initialize ciphers by exchanging public keys. + BIP324Cipher initiator(init_key, init_ent); + assert(!initiator); + BIP324Cipher responder(resp_key, resp_ent); + assert(!responder); + initiator.Initialize(responder.GetOurPubKey(), true); + assert(initiator); + responder.Initialize(initiator.GetOurPubKey(), false); + assert(responder); + + // Initialize RNG deterministically, to generate contents and AAD. We assume that there are no + // (potentially buggy) edge cases triggered by specific values of contents/AAD, so we can avoid + // reading the actual data for those from the fuzzer input (which would need large amounts of + // data). + XoRoShiRo128PlusPlus rng(provider.ConsumeIntegral<uint64_t>()); + + // Compare session IDs and garbage terminators. + assert(initiator.GetSessionID() == responder.GetSessionID()); + assert(initiator.GetSendGarbageTerminator() == responder.GetReceiveGarbageTerminator()); + assert(initiator.GetReceiveGarbageTerminator() == responder.GetSendGarbageTerminator()); + + LIMITED_WHILE(provider.remaining_bytes(), 1000) { + // Mode: + // - Bit 0: whether the ignore bit is set in message + // - Bit 1: whether the responder (0) or initiator (1) sends + // - Bit 2: whether this ciphertext will be corrupted (making it the last sent one) + // - Bit 3-4: controls the maximum aad length (max 4095 bytes) + // - Bit 5-7: controls the maximum content length (max 16383 bytes, for performance reasons) + unsigned mode = provider.ConsumeIntegral<uint8_t>(); + bool ignore = mode & 1; + bool from_init = mode & 2; + bool damage = mode & 4; + unsigned aad_length_bits = 4 * ((mode >> 3) & 3); + unsigned aad_length = provider.ConsumeIntegralInRange<unsigned>(0, (1 << aad_length_bits) - 1); + unsigned length_bits = 2 * ((mode >> 5) & 7); + unsigned length = provider.ConsumeIntegralInRange<unsigned>(0, (1 << length_bits) - 1); + // Generate aad and content. + std::vector<std::byte> aad(aad_length); + for (auto& val : aad) val = std::byte{(uint8_t)rng()}; + std::vector<std::byte> contents(length); + for (auto& val : contents) val = std::byte{(uint8_t)rng()}; + + // Pick sides. + auto& sender{from_init ? initiator : responder}; + auto& receiver{from_init ? responder : initiator}; + + // Encrypt + std::vector<std::byte> ciphertext(length + initiator.EXPANSION); + sender.Encrypt(contents, aad, ignore, ciphertext); + + // Optionally damage 1 bit in either the ciphertext (corresponding to a change in transit) + // or the aad (to make sure that decryption will fail if the AAD mismatches). + if (damage) { + unsigned damage_bit = provider.ConsumeIntegralInRange<unsigned>(0, + (ciphertext.size() + aad.size()) * 8U - 1U); + unsigned damage_pos = damage_bit >> 3; + std::byte damage_val{(uint8_t)(1U << (damage_bit & 3))}; + if (damage_pos >= ciphertext.size()) { + aad[damage_pos - ciphertext.size()] ^= damage_val; + } else { + ciphertext[damage_pos] ^= damage_val; + } + } + + // Decrypt length + uint32_t dec_length = receiver.DecryptLength(Span{ciphertext}.first(initiator.LENGTH_LEN)); + if (!damage) { + assert(dec_length == length); + } else { + // For performance reasons, don't try to decode if length got increased too much. + if (dec_length > 16384 + length) break; + // Otherwise, just append zeros if dec_length > length. + ciphertext.resize(dec_length + initiator.EXPANSION); + } + + // Decrypt + std::vector<std::byte> decrypt(dec_length); + bool dec_ignore{false}; + bool ok = receiver.Decrypt(Span{ciphertext}.subspan(initiator.LENGTH_LEN), aad, dec_ignore, decrypt); + // Decryption *must* fail if the packet was damaged, and succeed if it wasn't. + assert(!ok == damage); + if (!ok) break; + assert(ignore == dec_ignore); + assert(decrypt == contents); + } +} diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index 723dc6420f..b088aa0bd7 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -155,14 +155,16 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) } assert((exists_using_access_coin && exists_using_have_coin_in_cache && exists_using_have_coin && exists_using_get_coin) || (!exists_using_access_coin && !exists_using_have_coin_in_cache && !exists_using_have_coin && !exists_using_get_coin)); + // If HaveCoin on the backend is true, it must also be on the cache if the coin wasn't spent. const bool exists_using_have_coin_in_backend = backend_coins_view.HaveCoin(random_out_point); - if (exists_using_have_coin_in_backend) { + if (!coin_using_access_coin.IsSpent() && exists_using_have_coin_in_backend) { assert(exists_using_have_coin); } Coin coin_using_backend_get_coin; if (backend_coins_view.GetCoin(random_out_point, coin_using_backend_get_coin)) { assert(exists_using_have_coin_in_backend); - assert(coin_using_get_coin == coin_using_backend_get_coin); + // Note we can't assert that `coin_using_get_coin == coin_using_backend_get_coin` because the coin in + // the cache may have been modified but not yet flushed. } else { assert(!exists_using_have_coin_in_backend); } diff --git a/src/test/fuzz/crypto_chacha20.cpp b/src/test/fuzz/crypto_chacha20.cpp index 63c7bf3b45..76370b4e57 100644 --- a/src/test/fuzz/crypto_chacha20.cpp +++ b/src/test/fuzz/crypto_chacha20.cpp @@ -8,6 +8,8 @@ #include <test/fuzz/util.h> #include <test/util/xoroshiro128plusplus.h> +#include <array> +#include <cstddef> #include <cstdint> #include <vector> @@ -151,3 +153,21 @@ FUZZ_TARGET(chacha20_split_keystream) FuzzedDataProvider provider{buffer.data(), buffer.size()}; ChaCha20SplitFuzz<false>(provider); } + +FUZZ_TARGET(crypto_fschacha20) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + + auto key = fuzzed_data_provider.ConsumeBytes<std::byte>(FSChaCha20::KEYLEN); + key.resize(FSChaCha20::KEYLEN); + + auto fsc20 = FSChaCha20{key, fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(1, 1024)}; + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + { + auto input = fuzzed_data_provider.ConsumeBytes<std::byte>(fuzzed_data_provider.ConsumeIntegralInRange(0, 4096)); + std::vector<std::byte> output; + output.resize(input.size()); + fsc20.Crypt(input, output); + } +} diff --git a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp deleted file mode 100644 index 84ac65dd88..0000000000 --- a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2020-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <crypto/chacha_poly_aead.h> -#include <crypto/poly1305.h> -#include <test/fuzz/FuzzedDataProvider.h> -#include <test/fuzz/fuzz.h> -#include <test/fuzz/util.h> -#include <util/overflow.h> - -#include <cassert> -#include <cstdint> -#include <limits> -#include <vector> - -FUZZ_TARGET(crypto_chacha20_poly1305_aead) -{ - FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - - const std::vector<uint8_t> k1 = ConsumeFixedLengthByteVector(fuzzed_data_provider, CHACHA20_POLY1305_AEAD_KEY_LEN); - const std::vector<uint8_t> k2 = ConsumeFixedLengthByteVector(fuzzed_data_provider, CHACHA20_POLY1305_AEAD_KEY_LEN); - - ChaCha20Poly1305AEAD aead(k1.data(), k1.size(), k2.data(), k2.size()); - uint64_t seqnr_payload = 0; - uint64_t seqnr_aad = 0; - int aad_pos = 0; - size_t buffer_size = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096); - std::vector<uint8_t> in(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + Poly1305::TAGLEN, 0); - std::vector<uint8_t> out(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + Poly1305::TAGLEN, 0); - bool is_encrypt = fuzzed_data_provider.ConsumeBool(); - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { - CallOneOf( - fuzzed_data_provider, - [&] { - buffer_size = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(64, 4096); - in = std::vector<uint8_t>(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + Poly1305::TAGLEN, 0); - out = std::vector<uint8_t>(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + Poly1305::TAGLEN, 0); - }, - [&] { - (void)aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffer_size, is_encrypt); - }, - [&] { - uint32_t len = 0; - const bool ok = aead.GetLength(&len, seqnr_aad, aad_pos, in.data()); - assert(ok); - }, - [&] { - if (AdditionOverflow(seqnr_payload, static_cast<uint64_t>(1))) { - return; - } - seqnr_payload += 1; - aad_pos += CHACHA20_POLY1305_AEAD_AAD_LEN; - if (aad_pos + CHACHA20_POLY1305_AEAD_AAD_LEN > CHACHA20_ROUND_OUTPUT) { - aad_pos = 0; - if (AdditionOverflow(seqnr_aad, static_cast<uint64_t>(1))) { - return; - } - seqnr_aad += 1; - } - }, - [&] { - seqnr_payload = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); - }, - [&] { - seqnr_aad = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); - }, - [&] { - is_encrypt = fuzzed_data_provider.ConsumeBool(); - }); - } -} diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 2782888dc3..9e76c7be3e 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -78,6 +78,7 @@ const std::vector<std::string> RPC_COMMANDS_NOT_SAFE_FOR_FUZZING{ "generatetoaddress", // avoid prohibitively slow execution (when `num_blocks` is large) "generatetodescriptor", // avoid prohibitively slow execution (when `nblocks` is large) "gettxoutproof", // avoid prohibitively slow execution + "importmempool", // avoid reading from disk "importwallet", // avoid reading from disk "loadwallet", // avoid reading from disk "savemempool", // disabled as a precautionary measure: may take a file path argument in the future diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index c203dd4e39..5d020b4d59 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -20,6 +20,7 @@ #include <vector> using kernel::DumpMempool; +using kernel::LoadMempool; using node::MempoolPath; @@ -47,6 +48,10 @@ FUZZ_TARGET(validation_load_mempool, .init = initialize_validation_load_mempool) auto fuzzed_fopen = [&](const fs::path&, const char*) { return fuzzed_file_provider.open(); }; - (void)chainstate.LoadMempool(MempoolPath(g_setup->m_args), fuzzed_fopen); + (void)LoadMempool(pool, MempoolPath(g_setup->m_args), chainstate, + { + .mockable_fopen_function = fuzzed_fopen, + }); + pool.SetLoadTried(true); (void)DumpMempool(pool, MempoolPath(g_setup->m_args), fuzzed_fopen, true); } diff --git a/src/test/logging_tests.cpp b/src/test/logging_tests.cpp index 2699d316da..e448805e69 100644 --- a/src/test/logging_tests.cpp +++ b/src/test/logging_tests.cpp @@ -83,10 +83,10 @@ BOOST_AUTO_TEST_CASE(logging_timer) BOOST_FIXTURE_TEST_CASE(logging_LogPrintf_, LogSetup) { LogInstance().m_log_sourcelocations = true; - LogPrintf_("fn1", "src1", 1, BCLog::LogFlags::NET, BCLog::Level::Debug, "foo1: %s", "bar1\n"); - LogPrintf_("fn2", "src2", 2, BCLog::LogFlags::NET, BCLog::Level::None, "foo2: %s", "bar2\n"); - LogPrintf_("fn3", "src3", 3, BCLog::LogFlags::NONE, BCLog::Level::Debug, "foo3: %s", "bar3\n"); - LogPrintf_("fn4", "src4", 4, BCLog::LogFlags::NONE, BCLog::Level::None, "foo4: %s", "bar4\n"); + LogPrintf_("fn1", "src1", 1, BCLog::LogFlags::NET, BCLog::Level::Debug, "foo1: %s\n", "bar1"); + LogPrintf_("fn2", "src2", 2, BCLog::LogFlags::NET, BCLog::Level::None, "foo2: %s\n", "bar2"); + LogPrintf_("fn3", "src3", 3, BCLog::LogFlags::NONE, BCLog::Level::Debug, "foo3: %s\n", "bar3"); + LogPrintf_("fn4", "src4", 4, BCLog::LogFlags::NONE, BCLog::Level::None, "foo4: %s\n", "bar4"); std::ifstream file{tmp_log_path}; std::vector<std::string> log_lines; for (std::string log; std::getline(file, log);) { diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index 9ff2c08807..7f55916870 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -50,7 +50,7 @@ CreateAndActivateUTXOSnapshot( UniValue result = CreateUTXOSnapshot( node, node.chainman->ActiveChainstate(), auto_outfile, snapshot_path, snapshot_path); LogPrintf( - "Wrote UTXO snapshot to %s: %s", fs::PathToString(snapshot_path.make_preferred()), result.write()); + "Wrote UTXO snapshot to %s: %s\n", fs::PathToString(snapshot_path.make_preferred()), result.write()); // Read the written snapshot in and then activate it. // diff --git a/src/test/util/net.h b/src/test/util/net.h index e6506b0d08..b2f6ebb163 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -29,7 +29,10 @@ struct ConnmanTestMsg : public CConnman { { LOCK(m_nodes_mutex); m_nodes.push_back(&node); + + if (node.IsManualOrFullOutboundConn()) ++m_network_conn_counts[node.addr.GetNetwork()]; } + void ClearTestNodes() { LOCK(m_nodes_mutex); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 65c657da96..08ef890ec4 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -19,6 +19,7 @@ #include <init/common.h> #include <interfaces/chain.h> #include <kernel/mempool_entry.h> +#include <logging.h> #include <net.h> #include <net_processing.h> #include <node/blockstorage.h> diff --git a/src/txmempool.h b/src/txmempool.h index a1867eb895..fa1dbbf4b5 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -663,13 +663,13 @@ public: void GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants, size_t* ancestorsize = nullptr, CAmount* ancestorfees = nullptr) const; /** - * @returns true if we've made an attempt to load the mempool regardless of + * @returns true if an initial attempt to load the persisted mempool was made, regardless of * whether the attempt was successful or not */ bool GetLoadTried() const; /** - * Set whether or not we've made an attempt to load the mempool (regardless + * Set whether or not an initial attempt to load the persisted mempool was made (regardless * of whether the attempt was successful or not) */ void SetLoadTried(bool load_tried); diff --git a/src/util/trace.h b/src/util/trace.h index 7a63f39c83..051921a0d2 100644 --- a/src/util/trace.h +++ b/src/util/trace.h @@ -9,19 +9,28 @@ #include <sys/sdt.h> -#define TRACE(context, event) DTRACE_PROBE(context, event) -#define TRACE1(context, event, a) DTRACE_PROBE1(context, event, a) -#define TRACE2(context, event, a, b) DTRACE_PROBE2(context, event, a, b) -#define TRACE3(context, event, a, b, c) DTRACE_PROBE3(context, event, a, b, c) -#define TRACE4(context, event, a, b, c, d) DTRACE_PROBE4(context, event, a, b, c, d) -#define TRACE5(context, event, a, b, c, d, e) DTRACE_PROBE5(context, event, a, b, c, d, e) -#define TRACE6(context, event, a, b, c, d, e, f) DTRACE_PROBE6(context, event, a, b, c, d, e, f) -#define TRACE7(context, event, a, b, c, d, e, f, g) DTRACE_PROBE7(context, event, a, b, c, d, e, f, g) -#define TRACE8(context, event, a, b, c, d, e, f, g, h) DTRACE_PROBE8(context, event, a, b, c, d, e, f, g, h) -#define TRACE9(context, event, a, b, c, d, e, f, g, h, i) DTRACE_PROBE9(context, event, a, b, c, d, e, f, g, h, i) -#define TRACE10(context, event, a, b, c, d, e, f, g, h, i, j) DTRACE_PROBE10(context, event, a, b, c, d, e, f, g, h, i, j) -#define TRACE11(context, event, a, b, c, d, e, f, g, h, i, j, k) DTRACE_PROBE11(context, event, a, b, c, d, e, f, g, h, i, j, k) -#define TRACE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) DTRACE_PROBE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) +// Disabling this warning can be removed once we switch to C++20 +#if defined(__clang__) && __cplusplus < 202002L +#define BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wgnu-zero-variadic-macro-arguments\"") +#define BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP _Pragma("clang diagnostic pop") +#else +#define BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH +#define BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP +#endif + +#define TRACE(context, event) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE(context, event) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP +#define TRACE1(context, event, a) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE1(context, event, a) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP +#define TRACE2(context, event, a, b) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE2(context, event, a, b) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP +#define TRACE3(context, event, a, b, c) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE3(context, event, a, b, c) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP +#define TRACE4(context, event, a, b, c, d) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE4(context, event, a, b, c, d) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP +#define TRACE5(context, event, a, b, c, d, e) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE5(context, event, a, b, c, d, e) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP +#define TRACE6(context, event, a, b, c, d, e, f) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE6(context, event, a, b, c, d, e, f) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP +#define TRACE7(context, event, a, b, c, d, e, f, g) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE7(context, event, a, b, c, d, e, f, g) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP +#define TRACE8(context, event, a, b, c, d, e, f, g, h) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE8(context, event, a, b, c, d, e, f, g, h) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP +#define TRACE9(context, event, a, b, c, d, e, f, g, h, i) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE9(context, event, a, b, c, d, e, f, g, h, i) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP +#define TRACE10(context, event, a, b, c, d, e, f, g, h, i, j) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE10(context, event, a, b, c, d, e, f, g, h, i, j) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP +#define TRACE11(context, event, a, b, c, d, e, f, g, h, i, j, k) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE11(context, event, a, b, c, d, e, f, g, h, i, j, k) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP +#define TRACE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP #else diff --git a/src/validation.cpp b/src/validation.cpp index cd6654abe4..cec6d13181 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -69,7 +69,6 @@ using kernel::CCoinsStats; using kernel::CoinStatsHashType; using kernel::ComputeUTXOStats; -using kernel::LoadMempool; using kernel::Notifications; using fsbridge::FopenFn; @@ -3102,7 +3101,7 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< // Belt-and-suspenders check that we aren't attempting to advance the background // chainstate past the snapshot base block. if (WITH_LOCK(::cs_main, return m_disabled)) { - LogPrintf("m_disabled is set - this chainstate should not be in operation. " /* Continued */ + LogPrintf("m_disabled is set - this chainstate should not be in operation. " "Please report this as a bug. %s\n", PACKAGE_BUGREPORT); return false; } @@ -4126,13 +4125,6 @@ void PruneBlockFilesManual(Chainstate& active_chainstate, int nManualPruneHeight } } -void Chainstate::LoadMempool(const fs::path& load_path, FopenFn mockable_fopen_function) -{ - if (!m_mempool) return; - ::LoadMempool(*m_mempool, load_path, *this, mockable_fopen_function); - m_mempool->SetLoadTried(!m_chainman.m_interrupt); -} - bool Chainstate::LoadChainTip() { AssertLockHeld(cs_main); @@ -5056,7 +5048,7 @@ static bool DeleteCoinsDBFromDisk(const fs::path db_path, bool is_snapshot) // We have to destruct before this call leveldb::DB in order to release the db // lock, otherwise `DestroyDB` will fail. See `leveldb::~DBImpl()`. - const bool destroyed = dbwrapper::DestroyDB(path_str, {}).ok(); + const bool destroyed = DestroyDB(path_str); if (!destroyed) { LogPrintf("error: leveldb DestroyDB call failed on %s\n", path_str); @@ -5226,7 +5218,7 @@ bool ChainstateManager::PopulateAndValidateSnapshot( auto maybe_au_data = ExpectedAssumeutxo(base_height, GetParams()); if (!maybe_au_data) { - LogPrintf("[snapshot] assumeutxo height in snapshot metadata not recognized " /* Continued */ + LogPrintf("[snapshot] assumeutxo height in snapshot metadata not recognized " "(%d) - refusing to load snapshot\n", base_height); return false; } @@ -5473,8 +5465,8 @@ SnapshotCompletionResult ChainstateManager::MaybeCompleteSnapshotValidation() }; if (index_new.GetBlockHash() != snapshot_blockhash) { - LogPrintf("[snapshot] supposed base block %s does not match the " /* Continued */ - "snapshot base block %s (height %d). Snapshot is not valid.", + LogPrintf("[snapshot] supposed base block %s does not match the " + "snapshot base block %s (height %d). Snapshot is not valid.\n", index_new.ToString(), snapshot_blockhash.ToString(), snapshot_base_height); handle_invalid_snapshot(); return SnapshotCompletionResult::BASE_BLOCKHASH_MISMATCH; @@ -5494,7 +5486,7 @@ SnapshotCompletionResult ChainstateManager::MaybeCompleteSnapshotValidation() auto maybe_au_data = ExpectedAssumeutxo(curr_height, m_options.chainparams); if (!maybe_au_data) { - LogPrintf("[snapshot] assumeutxo data not found for height " /* Continued */ + LogPrintf("[snapshot] assumeutxo data not found for height " "(%d) - refusing to validate snapshot\n", curr_height); handle_invalid_snapshot(); return SnapshotCompletionResult::MISSING_CHAINPARAMS; @@ -5502,7 +5494,7 @@ SnapshotCompletionResult ChainstateManager::MaybeCompleteSnapshotValidation() const AssumeutxoData& au_data = *maybe_au_data; std::optional<CCoinsStats> maybe_ibd_stats; - LogPrintf("[snapshot] computing UTXO stats for background chainstate to validate " /* Continued */ + LogPrintf("[snapshot] computing UTXO stats for background chainstate to validate " "snapshot - this could take a few minutes\n"); try { maybe_ibd_stats = ComputeUTXOStats( @@ -5740,7 +5732,7 @@ bool ChainstateManager::ValidatedSnapshotCleanup() // is in-memory, in which case we can't do on-disk cleanup. You'd better be // in a unittest! if (!ibd_chainstate_path_maybe || !snapshot_chainstate_path_maybe) { - LogPrintf("[snapshot] snapshot chainstate cleanup cannot happen with " /* Continued */ + LogPrintf("[snapshot] snapshot chainstate cleanup cannot happen with " "in-memory chainstates. You are testing, right?\n"); return false; } @@ -5783,7 +5775,7 @@ bool ChainstateManager::ValidatedSnapshotCleanup() throw; } - LogPrintf("[snapshot] moving snapshot chainstate (%s) to " /* Continued */ + LogPrintf("[snapshot] moving snapshot chainstate (%s) to " "default chainstate directory (%s)\n", fs::PathToString(snapshot_chainstate_path), fs::PathToString(ibd_chainstate_path)); @@ -5797,7 +5789,7 @@ bool ChainstateManager::ValidatedSnapshotCleanup() if (!DeleteCoinsDBFromDisk(tmp_old, /*is_snapshot=*/false)) { // No need to FatalError because once the unneeded bg chainstate data is // moved, it will not interfere with subsequent initialization. - LogPrintf("Deletion of %s failed. Please remove it manually, as the " /* Continued */ + LogPrintf("Deletion of %s failed. Please remove it manually, as the " "directory is now unnecessary.\n", fs::PathToString(tmp_old)); } else { diff --git a/src/validation.h b/src/validation.h index d7ad86a5e8..4c9f807f5d 100644 --- a/src/validation.h +++ b/src/validation.h @@ -712,9 +712,6 @@ public: /** Find the last common block of this chain and a locator. */ const CBlockIndex* FindForkInGlobalIndex(const CBlockLocator& locator) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** Load the persisted mempool from disk */ - void LoadMempool(const fs::path& load_path, fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen); - /** Update the chain tip based on database information, i.e. CoinsTip()'s best block. */ bool LoadChainTip() EXCLUSIVE_LOCKS_REQUIRED(cs_main); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 8fa93b97d6..6b2755ea53 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2319,7 +2319,7 @@ OutputType CWallet::TransactionChangeType(const std::optional<OutputType>& chang void CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm) { LOCK(cs_wallet); - WalletLogPrintf("CommitTransaction:\n%s", tx->ToString()); /* Continued */ + WalletLogPrintf("CommitTransaction:\n%s", tx->ToString()); // Add tx to wallet, because if it has change it's also ours, // otherwise just for transaction history. diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index a1335ff069..32a927084a 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -46,7 +46,7 @@ from test_framework.util import ( assert_greater_than_or_equal, assert_raises_rpc_error, ) -from test_framework.wallet import MiniWallet +from test_framework.wallet import MiniWallet, COIN class MempoolPersistTest(BitcoinTestFramework): @@ -159,6 +159,16 @@ class MempoolPersistTest(BitcoinTestFramework): assert self.nodes[0].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[0].getrawmempool()), 0) + self.log.debug("Import mempool at runtime to node0.") + assert_equal({}, self.nodes[0].importmempool(mempooldat0)) + assert_equal(len(self.nodes[0].getrawmempool()), 7) + fees = self.nodes[0].getmempoolentry(txid=last_txid)["fees"] + assert_equal(fees["base"], fees["modified"]) + assert_equal({}, self.nodes[0].importmempool(mempooldat0, {"apply_fee_delta_priority": True, "apply_unbroadcast_set": True})) + assert_equal(2, self.nodes[0].getmempoolinfo()["unbroadcastcount"]) + fees = self.nodes[0].getmempoolentry(txid=last_txid)["fees"] + assert_equal(fees["base"] + Decimal("0.00001000"), fees["modified"]) + self.log.debug("Stop-start node0. Verify that it has the transactions in its mempool.") self.stop_nodes() self.start_node(0) @@ -186,6 +196,7 @@ class MempoolPersistTest(BitcoinTestFramework): assert_raises_rpc_error(-1, "Unable to dump mempool to disk", self.nodes[1].savemempool) os.rmdir(mempooldotnew1) + self.test_importmempool_union() self.test_persist_unbroadcast() def test_persist_unbroadcast(self): @@ -210,6 +221,46 @@ class MempoolPersistTest(BitcoinTestFramework): node0.mockscheduler(16 * 60) # 15 min + 1 for buffer self.wait_until(lambda: len(conn.get_invs()) == 1) + def test_importmempool_union(self): + self.log.debug("Submit different transactions to node0 and node1's mempools") + self.start_node(0) + self.start_node(2) + tx_node0 = self.mini_wallet.send_self_transfer(from_node=self.nodes[0]) + tx_node1 = self.mini_wallet.send_self_transfer(from_node=self.nodes[1]) + tx_node01 = self.mini_wallet.create_self_transfer() + tx_node01_secret = self.mini_wallet.create_self_transfer() + self.nodes[0].prioritisetransaction(tx_node01["txid"], 0, COIN) + self.nodes[0].prioritisetransaction(tx_node01_secret["txid"], 0, 2 * COIN) + self.nodes[1].prioritisetransaction(tx_node01_secret["txid"], 0, 3 * COIN) + self.nodes[0].sendrawtransaction(tx_node01["hex"]) + self.nodes[1].sendrawtransaction(tx_node01["hex"]) + assert tx_node0["txid"] in self.nodes[0].getrawmempool() + assert not tx_node0["txid"] in self.nodes[1].getrawmempool() + assert not tx_node1["txid"] in self.nodes[0].getrawmempool() + assert tx_node1["txid"] in self.nodes[1].getrawmempool() + assert tx_node01["txid"] in self.nodes[0].getrawmempool() + assert tx_node01["txid"] in self.nodes[1].getrawmempool() + assert not tx_node01_secret["txid"] in self.nodes[0].getrawmempool() + assert not tx_node01_secret["txid"] in self.nodes[1].getrawmempool() + + self.log.debug("Check that importmempool can add txns without replacing the entire mempool") + mempooldat0 = str(self.nodes[0].chain_path / "mempool.dat") + result0 = self.nodes[0].savemempool() + assert_equal(mempooldat0, result0["filename"]) + assert_equal({}, self.nodes[1].importmempool(mempooldat0, {"apply_fee_delta_priority": True})) + # All transactions should be in node1's mempool now. + assert tx_node0["txid"] in self.nodes[1].getrawmempool() + assert tx_node1["txid"] in self.nodes[1].getrawmempool() + assert not tx_node1["txid"] in self.nodes[0].getrawmempool() + # For transactions that already existed, priority should be changed + entry_node01 = self.nodes[1].getmempoolentry(tx_node01["txid"]) + assert_equal(entry_node01["fees"]["base"] + 1, entry_node01["fees"]["modified"]) + # Deltas for not-yet-submitted transactions should be applied as well (prioritisation is stackable). + self.nodes[1].sendrawtransaction(tx_node01_secret["hex"]) + entry_node01_secret = self.nodes[1].getmempoolentry(tx_node01_secret["txid"]) + assert_equal(entry_node01_secret["fees"]["base"] + 5, entry_node01_secret["fees"]["modified"]) + self.stop_nodes() + if __name__ == "__main__": MempoolPersistTest().main() diff --git a/test/functional/wallet_fundrawtransaction.py b/test/functional/wallet_fundrawtransaction.py index e367daae2c..fa4f009f34 100755 --- a/test/functional/wallet_fundrawtransaction.py +++ b/test/functional/wallet_fundrawtransaction.py @@ -23,6 +23,7 @@ from test_framework.util import ( assert_raises_rpc_error, count_bytes, find_vout_for_address, + get_fee, ) from test_framework.wallet_util import generate_keypair @@ -570,6 +571,8 @@ class RawTransactionsTest(BitcoinTestFramework): df_wallet = self.nodes[1].get_wallet_rpc(self.default_wallet_name) self.nodes[1].createwallet(wallet_name="locked_wallet", descriptors=self.options.descriptors) wallet = self.nodes[1].get_wallet_rpc("locked_wallet") + # This test is not meant to exercise fee estimation. Making sure all txs are sent at a consistent fee rate. + wallet.settxfee(self.min_relay_tx_fee) # Add some balance to the wallet (this will be reverted at the end of the test) df_wallet.sendall(recipients=[wallet.getnewaddress()]) @@ -599,8 +602,11 @@ class RawTransactionsTest(BitcoinTestFramework): # Choose input inputs = wallet.listunspent() - # Deduce fee to produce a changeless transaction - value = inputs[0]["amount"] - Decimal("0.00002200") + + # Deduce exact fee to produce a changeless transaction + tx_size = 110 # Total tx size: 110 vbytes, p2wpkh -> p2wpkh. Input 68 vbytes + rest of tx is 42 vbytes. + value = inputs[0]["amount"] - get_fee(tx_size, self.min_relay_tx_fee) + outputs = {self.nodes[0].getnewaddress():value} rawtx = wallet.createrawtransaction(inputs, outputs) # fund a transaction that does not require a new key for the change output diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index 43addab2f3..5ac5840ecf 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -77,7 +77,7 @@ def main(): matching_files_filtered = [] for matching_file in matching_files: - if not re.search('^src/(leveldb|secp256k1|minisketch|tinyformat|test/fuzz/strprintf.cpp)', matching_file): + if not re.search('^src/(leveldb|secp256k1|minisketch|tinyformat|test/fuzz/strprintf.cpp)|contrib/devtools/bitcoin-tidy/example_logprintf.cpp', matching_file): matching_files_filtered.append(matching_file) matching_files_filtered.sort() diff --git a/test/lint/lint-include-guards.py b/test/lint/lint-include-guards.py index 5867aae028..48b918e9da 100755 --- a/test/lint/lint-include-guards.py +++ b/test/lint/lint-include-guards.py @@ -17,7 +17,8 @@ from typing import List HEADER_ID_PREFIX = 'BITCOIN_' HEADER_ID_SUFFIX = '_H' -EXCLUDE_FILES_WITH_PREFIX = ['src/crypto/ctaes', +EXCLUDE_FILES_WITH_PREFIX = ['contrib/devtools/bitcoin-tidy', + 'src/crypto/ctaes', 'src/leveldb', 'src/crc32c', 'src/secp256k1', diff --git a/test/lint/lint-includes.py b/test/lint/lint-includes.py index b14caa4855..8e79ba5121 100755 --- a/test/lint/lint-includes.py +++ b/test/lint/lint-includes.py @@ -15,7 +15,8 @@ import sys from subprocess import check_output, CalledProcessError -EXCLUDED_DIRS = ["src/leveldb/", +EXCLUDED_DIRS = ["contrib/devtools/bitcoin-tidy/", + "src/leveldb/", "src/crc32c/", "src/secp256k1/", "src/minisketch/", diff --git a/test/lint/lint-logs.py b/test/lint/lint-logs.py deleted file mode 100755 index de04a1aeca..0000000000 --- a/test/lint/lint-logs.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (c) 2018-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check that all logs are terminated with '\n' -# -# Some logs are continued over multiple lines. They should be explicitly -# commented with /* Continued */ - -import re -import sys - -from subprocess import check_output - - -def main(): - logs_list = check_output(["git", "grep", "--extended-regexp", r"(LogPrintLevel|LogPrintfCategory|LogPrintf?)\(", "--", "*.cpp"], text=True, encoding="utf8").splitlines() - - unterminated_logs = [line for line in logs_list if not re.search(r'(\\n"|/\* Continued \*/)', line)] - - if unterminated_logs != []: - print("All calls to LogPrintf(), LogPrintfCategory(), LogPrint(), LogPrintLevel(), and WalletLogPrintf() should be terminated with \"\\n\".") - print("") - - for line in unterminated_logs: - print(line) - - sys.exit(1) - - -if __name__ == "__main__": - main() |