diff options
62 files changed, 1832 insertions, 837 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index c0b85b24a7..d72a2b289d 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -79,8 +79,8 @@ container_depends_template: &CONTAINER_DEPENDS_TEMPLATE cpu: 2 memory: 8G # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers dockerfile: ci/test_imagefile # https://cirrus-ci.org/guide/docker-builder-vm/#dockerfile-as-a-ci-environment - depends_built_cache: - folder: "depends/built" + base_depends_built_cache: + folder: "/ci_container_base/depends/built" fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-parse HEAD:depends) global_task_template: &GLOBAL_TASK_TEMPLATE @@ -333,32 +333,10 @@ task: task: name: 'macOS 11.0 [gui, no tests] [jammy]' - << : *CONTAINER_DEPENDS_TEMPLATE + << : *GLOBAL_TASK_TEMPLATE container: docker_arguments: CI_IMAGE_NAME_TAG: ubuntu:jammy FILE_ENV: "./ci/test/00_setup_env_mac.sh" - macos_sdk_cache: - folder: "depends/SDKs/$MACOS_SDK" - fingerprint_key: "$MACOS_SDK" - << : *MAIN_TEMPLATE - env: - MACOS_SDK: "Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers" - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - -task: - name: 'macOS 13 native arm64 [gui, sqlite only] [no depends]' - macos_instance: - # Use latest image, but hardcode version to avoid silent upgrades (and breaks) - image: ghcr.io/cirruslabs/macos-ventura-xcode:14.3.1 # https://cirrus-ci.org/guide/macOS - << : *BASE_TEMPLATE - check_clang_script: - - clang --version - brew_install_script: - - brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt - << : *MAIN_TEMPLATE env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - CI_USE_APT_INSTALL: "no" - PACKAGE_MANAGER_INSTALL: "echo" # Nothing to do - FILE_ENV: "./ci/test/00_setup_env_mac_native_arm64.sh" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..44ee0b5c59 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,60 @@ +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +name: CI +on: + # See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request. + pull_request: + # See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push. + push: + branches: + - '**' + tags-ignore: + - '**' + +env: + DANGER_RUN_CI_ON_HOST: 1 + TEST_RUNNER_TIMEOUT_FACTOR: 40 + +jobs: + macos-native-x86_64: + name: macOS 13 native, x86_64 [no depends, sqlite only, gui] + # Use latest image, but hardcode version to avoid silent upgrades (and breaks). + # See: https://github.com/actions/runner-images#available-images. + runs-on: macos-13 + + # No need to run on the read-only mirror, unless it is a PR. + if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request' + + timeout-minutes: 120 + + env: + MAKEJOBS: '-j4' + CI_USE_APT_INSTALL: 'no' + PACKAGE_MANAGER_INSTALL: 'echo' # Nothing to do + FILE_ENV: './ci/test/00_setup_env_mac_native.sh' + BASE_ROOT_DIR: ${{ github.workspace }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Clang version + run: clang --version + + - name: Install Homebrew packages + run: brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt + + - name: Set Ccache directory + run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" + + - name: Ccache cache + uses: actions/cache@v3 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ github.job }}-ccache-cache-${{ github.run_id }} + restore-keys: ${{ github.job }}-ccache-cache + + - name: CI script + run: ./ci/test_run_all.sh diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 75b47af0ed..c479a8a3fe 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -8,10 +8,14 @@ export LC_ALL=C.UTF-8 set -ex -# The root dir. +# The source root dir, usually from git, usually read-only. # The ci system copies this folder. -BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) -export BASE_ROOT_DIR +BASE_READ_ONLY_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) +export BASE_READ_ONLY_DIR +# The destination root dir inside the container. +# This folder will also hold any SDKs. +# This folder only exists on the ci guest and will be a copy of BASE_READ_ONLY_DIR +export BASE_ROOT_DIR="${BASE_ROOT_DIR:-/ci_container_base}" # The depends dir. # This folder exists only on the ci guest, and on the ci host as a volume. export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} diff --git a/ci/test/00_setup_env_mac_native_arm64.sh b/ci/test/00_setup_env_mac_native.sh index eee72db435..c9f65bf397 100755 --- a/ci/test/00_setup_env_mac_native_arm64.sh +++ b/ci/test/00_setup_env_mac_native.sh @@ -1,18 +1,18 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export HOST=arm64-apple-darwin +export HOST=x86_64-apple-darwin export PIP_PACKAGES="zmq" export GOAL="install" export BITCOIN_CONFIG="--with-gui --with-miniupnpc --with-natpmp --enable-reduce-exports" export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" -export CCACHE_MAXSIZE=300M +export CCACHE_MAXSIZE=400M export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--exclude banman" # https://github.com/bitcoin/bitcoin/issues/27924 diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 89006bf95e..007bb4bedf 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="ubuntu:22.04" -LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/" +LIBCXX_DIR="/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index a96ed30dd3..1d39ef9c48 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="ubuntu:22.04" -LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/" +LIBCXX_DIR="/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 83213b2856..c9a496e6ab 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -42,33 +42,35 @@ if [ -n "$PIP_PACKAGES" ]; then fi if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.6 "${BASE_SCRATCH_DIR}"/msan/llvm-project - - cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/clang_build/ -DLLVM_ENABLE_PROJECTS="clang" \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_TARGETS_TO_BUILD=Native \ - -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \ - -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/llvm - - ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ "$MAKEJOBS" - ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ install-runtimes - - update-alternatives --install /usr/bin/clang++ clang++ "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang++ 100 - update-alternatives --install /usr/bin/clang clang "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang 100 - update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/llvm-symbolizer 100 - - cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/cxx_build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_USE_SANITIZER=MemoryWithOrigins \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DLLVM_TARGETS_TO_BUILD=Native \ - -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ - -DLIBCXX_ENABLE_DEBUG_MODE=ON \ - -DLIBCXX_ENABLE_ASSERTIONS=ON \ - -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes - - ninja -C "${BASE_SCRATCH_DIR}"/msan/cxx_build/ "$MAKEJOBS" + git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.6 /msan/llvm-project + + cmake -G Ninja -B /msan/clang_build/ \ + -DLLVM_ENABLE_PROJECTS="clang" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \ + -S /msan/llvm-project/llvm + + ninja -C /msan/clang_build/ "$MAKEJOBS" + ninja -C /msan/clang_build/ install-runtimes + + update-alternatives --install /usr/bin/clang++ clang++ /msan/clang_build/bin/clang++ 100 + update-alternatives --install /usr/bin/clang clang /msan/clang_build/bin/clang 100 + update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /msan/clang_build/bin/llvm-symbolizer 100 + + cmake -G Ninja -B /msan/cxx_build/ \ + -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_USE_SANITIZER=MemoryWithOrigins \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ + -DLIBCXX_ENABLE_DEBUG_MODE=ON \ + -DLIBCXX_ENABLE_ASSERTIONS=ON \ + -S /msan/llvm-project/runtimes + + ninja -C /msan/cxx_build/ "$MAKEJOBS" fi if [[ "${RUN_TIDY}" == "true" ]]; then diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 7ecf00097c..a2ea2b7093 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -6,10 +6,6 @@ export LC_ALL=C.UTF-8 -# Create folders that are mounted into the docker -mkdir -p "${CCACHE_DIR}" -mkdir -p "${PREVIOUS_RELEASES_DIR}" - export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1" export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" @@ -23,11 +19,11 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then docker run --rm "${CI_IMAGE_NAME_TAG}" bash -c "env | grep --extended-regexp '^(HOME|PATH|USER)='" | tee --append /tmp/env echo "Creating $CI_IMAGE_NAME_TAG container to run in" DOCKER_BUILDKIT=1 docker build \ - --file "${BASE_ROOT_DIR}/ci/test_imagefile" \ + --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \ --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \ --build-arg "FILE_ENV=${FILE_ENV}" \ --tag="${CONTAINER_NAME}" \ - "${BASE_ROOT_DIR}" + "${BASE_READ_ONLY_DIR}" docker volume create "${CONTAINER_NAME}_ccache" || true docker volume create "${CONTAINER_NAME}_depends" || true docker volume create "${CONTAINER_NAME}_previous_releases" || true @@ -41,7 +37,7 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then # shellcheck disable=SC2086 CI_CONTAINER_ID=$(docker run $CI_CONTAINER_CAP --rm --interactive --detach --tty \ - --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ + --mount type=bind,src=$BASE_READ_ONLY_DIR,dst=/ro_base,readonly \ --mount "type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" \ --mount "type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR" \ --mount "type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" \ @@ -52,6 +48,9 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}" else echo "Running on host system without docker wrapper" + echo "Create missing folders" + mkdir -p "${CCACHE_DIR}" + mkdir -p "${PREVIOUS_RELEASES_DIR}" fi CI_EXEC () { @@ -59,7 +58,8 @@ CI_EXEC () { } export -f CI_EXEC -CI_EXEC rsync --archive --stats --human-readable /ci_base_install/ "${BASE_ROOT_DIR}" || echo "/ci_base_install/ missing" +# Normalize all folders to BASE_ROOT_DIR +CI_EXEC rsync --archive --stats --human-readable "${BASE_READ_ONLY_DIR}/" "${BASE_ROOT_DIR}" || echo "Nothing to copy from ${BASE_READ_ONLY_DIR}/" CI_EXEC "${BASE_ROOT_DIR}/ci/test/01_base_install.sh" CI_EXEC rsync --archive --stats --human-readable /ro_base/ "${BASE_ROOT_DIR}" || echo "Nothing to copy from ro_base" # Fixes permission issues when there is a container UID/GID mismatch with the owner diff --git a/ci/test_imagefile b/ci/test_imagefile index 8dfb5916b1..f8b5eea1c8 100644 --- a/ci/test_imagefile +++ b/ci/test_imagefile @@ -11,6 +11,6 @@ ARG FILE_ENV ENV FILE_ENV=${FILE_ENV} COPY ./ci/retry/retry /usr/bin/retry -COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh /ci_base_install/ci/test/ +COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh /ci_container_base/ci/test/ -RUN ["bash", "-c", "cd /ci_base_install/ && set -o errexit && source ./ci/test/00_setup_env.sh && ./ci/test/01_base_install.sh"] +RUN ["bash", "-c", "cd /ci_container_base/ && set -o errexit && source ./ci/test/00_setup_env.sh && ./ci/test/01_base_install.sh"] diff --git a/contrib/devtools/bitcoin-tidy/CMakeLists.txt b/contrib/devtools/bitcoin-tidy/CMakeLists.txt index 9ed18696d4..35e60d1d87 100644 --- a/contrib/devtools/bitcoin-tidy/CMakeLists.txt +++ b/contrib/devtools/bitcoin-tidy/CMakeLists.txt @@ -25,6 +25,12 @@ else() target_compile_options(bitcoin-tidy PRIVATE -fno-exceptions) endif() +if(CMAKE_HOST_APPLE) + # ld64 expects no undefined symbols by default + target_link_options(bitcoin-tidy PRIVATE -Wl,-flat_namespace) + target_link_options(bitcoin-tidy PRIVATE -Wl,-undefined -Wl,suppress) +endif() + # Add warnings if (MSVC) target_compile_options(bitcoin-tidy PRIVATE /W4) @@ -33,7 +39,12 @@ else() target_compile_options(bitcoin-tidy PRIVATE -Wextra) endif() -set(CLANG_TIDY_COMMAND "${CLANG_TIDY_EXE}" "--load=${CMAKE_BINARY_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}bitcoin-tidy${CMAKE_SHARED_LIBRARY_SUFFIX}" "-checks=-*,bitcoin-*") +if(CMAKE_VERSION VERSION_LESS 3.27) + set(CLANG_TIDY_COMMAND "${CLANG_TIDY_EXE}" "--load=${CMAKE_BINARY_DIR}/${CMAKE_SHARED_MODULE_PREFIX}bitcoin-tidy${CMAKE_SHARED_MODULE_SUFFIX}" "-checks=-*,bitcoin-*") +else() + # CLANG_TIDY_COMMAND supports generator expressions as of 3.27 + set(CLANG_TIDY_COMMAND "${CLANG_TIDY_EXE}" "--load=$<TARGET_FILE:bitcoin-tidy>" "-checks=-*,bitcoin-*") +endif() # Create a dummy library that runs clang-tidy tests as a side-effect of building add_library(bitcoin-tidy-tests OBJECT EXCLUDE_FROM_ALL example_logprintf.cpp) diff --git a/contrib/devtools/bitcoin-tidy/README b/contrib/devtools/bitcoin-tidy/README index 1669fe98f5..c15e07c4ed 100644 --- a/contrib/devtools/bitcoin-tidy/README +++ b/contrib/devtools/bitcoin-tidy/README @@ -3,6 +3,9 @@ Example Usage: ```bash -cmake -S . -B build -DLLVM_DIR=/path/to/lib/cmake/llvm -DCMAKE_BUILD_TYPE=Release -make -C build -j$(nproc) +cmake -S . -B build -DLLVM_DIR=$(llvm-config --cmakedir) -DCMAKE_BUILD_TYPE=Release + +cmake --build build -j$(nproc) + +cmake --build build --target bitcoin-tidy-tests -j$(nproc) ``` diff --git a/doc/release-notes-27213.md b/doc/release-notes-27213.md new file mode 100644 index 0000000000..3b478f11b7 --- /dev/null +++ b/doc/release-notes-27213.md @@ -0,0 +1,8 @@ +P2P and network changes +------ + +- Nodes with multiple reachable networks will actively try to have at least one + outbound connection to each network. This improves individual resistance to + eclipse attacks and network level resistance to partition attacks. Users no + longer need to perform active measures to ensure being connected to multiple + enabled networks. diff --git a/src/Makefile.am b/src/Makefile.am index a108e60c86..06c156a8c0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -124,6 +124,7 @@ BITCOIN_CORE_H = \ banman.h \ base58.h \ bech32.h \ + bip324.h \ blockencodings.h \ blockfilter.h \ chain.h \ @@ -376,6 +377,7 @@ libbitcoin_node_a_SOURCES = \ addrdb.cpp \ addrman.cpp \ banman.cpp \ + bip324.cpp \ blockencodings.cpp \ blockfilter.cpp \ chain.cpp \ @@ -546,10 +548,10 @@ crypto_libbitcoin_crypto_base_la_LDFLAGS = $(AM_LDFLAGS) -static crypto_libbitcoin_crypto_base_la_SOURCES = \ crypto/aes.cpp \ crypto/aes.h \ - crypto/chacha_poly_aead.h \ - crypto/chacha_poly_aead.cpp \ crypto/chacha20.h \ crypto/chacha20.cpp \ + crypto/chacha20poly1305.h \ + crypto/chacha20poly1305.cpp \ crypto/common.h \ crypto/hkdf_sha256_32.cpp \ crypto/hkdf_sha256_32.h \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 51bfb1e459..934e9a1fae 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -22,7 +22,6 @@ bench_bench_bitcoin_SOURCES = \ bench/block_assemble.cpp \ bench/ccoins_caching.cpp \ bench/chacha20.cpp \ - bench/chacha_poly_aead.cpp \ bench/checkblock.cpp \ bench/checkqueue.cpp \ bench/crypto_hash.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index f2a82ce73b..5dc20d4fab 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -74,6 +74,7 @@ BITCOIN_TESTS =\ test/base64_tests.cpp \ test/bech32_tests.cpp \ test/bip32_tests.cpp \ + test/bip324_tests.cpp \ test/blockchain_tests.cpp \ test/blockencodings_tests.cpp \ test/blockfilter_index_tests.cpp \ @@ -246,6 +247,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/banman.cpp \ test/fuzz/base_encode_decode.cpp \ test/fuzz/bech32.cpp \ + test/fuzz/bip324.cpp \ test/fuzz/bitdeque.cpp \ test/fuzz/block.cpp \ test/fuzz/block_header.cpp \ @@ -261,7 +263,6 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/crypto_aes256.cpp \ test/fuzz/crypto_aes256cbc.cpp \ test/fuzz/crypto_chacha20.cpp \ - test/fuzz/crypto_chacha20_poly1305_aead.cpp \ test/fuzz/crypto_common.cpp \ test/fuzz/crypto_diff_fuzz_chacha20.cpp \ test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp \ diff --git a/src/bench/chacha20.cpp b/src/bench/chacha20.cpp index 3b57e29f39..d8bebf9319 100644 --- a/src/bench/chacha20.cpp +++ b/src/bench/chacha20.cpp @@ -5,6 +5,7 @@ #include <bench/bench.h> #include <crypto/chacha20.h> +#include <crypto/chacha20poly1305.h> /* Number of bytes to process per iteration */ static const uint64_t BUFFER_SIZE_TINY = 64; @@ -23,6 +24,18 @@ static void CHACHA20(benchmark::Bench& bench, size_t buffersize) }); } +static void FSCHACHA20POLY1305(benchmark::Bench& bench, size_t buffersize) +{ + std::vector<std::byte> key(32); + FSChaCha20Poly1305 ctx(key, 224); + std::vector<std::byte> in(buffersize); + std::vector<std::byte> aad; + std::vector<std::byte> out(buffersize + FSChaCha20Poly1305::EXPANSION); + bench.batch(in.size()).unit("byte").run([&] { + ctx.Encrypt(in, aad, out); + }); +} + static void CHACHA20_64BYTES(benchmark::Bench& bench) { CHACHA20(bench, BUFFER_SIZE_TINY); @@ -38,6 +51,24 @@ static void CHACHA20_1MB(benchmark::Bench& bench) CHACHA20(bench, BUFFER_SIZE_LARGE); } +static void FSCHACHA20POLY1305_64BYTES(benchmark::Bench& bench) +{ + FSCHACHA20POLY1305(bench, BUFFER_SIZE_TINY); +} + +static void FSCHACHA20POLY1305_256BYTES(benchmark::Bench& bench) +{ + FSCHACHA20POLY1305(bench, BUFFER_SIZE_SMALL); +} + +static void FSCHACHA20POLY1305_1MB(benchmark::Bench& bench) +{ + FSCHACHA20POLY1305(bench, BUFFER_SIZE_LARGE); +} + BENCHMARK(CHACHA20_64BYTES, benchmark::PriorityLevel::HIGH); BENCHMARK(CHACHA20_256BYTES, benchmark::PriorityLevel::HIGH); BENCHMARK(CHACHA20_1MB, benchmark::PriorityLevel::HIGH); +BENCHMARK(FSCHACHA20POLY1305_64BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(FSCHACHA20POLY1305_256BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(FSCHACHA20POLY1305_1MB, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/chacha_poly_aead.cpp b/src/bench/chacha_poly_aead.cpp deleted file mode 100644 index 9149eb683a..0000000000 --- a/src/bench/chacha_poly_aead.cpp +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (c) 2019-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - - -#include <bench/bench.h> -#include <crypto/chacha_poly_aead.h> -#include <crypto/poly1305.h> // for the POLY1305_TAGLEN constant -#include <hash.h> - -#include <assert.h> -#include <limits> - -/* Number of bytes to process per iteration */ -static constexpr uint64_t BUFFER_SIZE_TINY = 64; -static constexpr uint64_t BUFFER_SIZE_SMALL = 256; -static constexpr uint64_t BUFFER_SIZE_LARGE = 1024 * 1024; - -static const unsigned char k1[32] = {0}; -static const unsigned char k2[32] = {0}; - -static ChaCha20Poly1305AEAD aead(k1, 32, k2, 32); - -static void CHACHA20_POLY1305_AEAD(benchmark::Bench& bench, size_t buffersize, bool include_decryption) -{ - std::vector<unsigned char> in(buffersize + CHACHA20_POLY1305_AEAD_AAD_LEN + Poly1305::TAGLEN, 0); - std::vector<unsigned char> out(buffersize + CHACHA20_POLY1305_AEAD_AAD_LEN + Poly1305::TAGLEN, 0); - uint64_t seqnr_payload = 0; - uint64_t seqnr_aad = 0; - int aad_pos = 0; - uint32_t len = 0; - bench.batch(buffersize).unit("byte").run([&] { - // encrypt or decrypt the buffer with a static key - const bool crypt_ok_1 = aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffersize, true); - assert(crypt_ok_1); - - if (include_decryption) { - // if we decrypt, include the GetLength - const bool get_length_ok = aead.GetLength(&len, seqnr_aad, aad_pos, in.data()); - assert(get_length_ok); - const bool crypt_ok_2 = aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffersize, true); - assert(crypt_ok_2); - } - - // increase main sequence number - seqnr_payload++; - // increase aad position (position in AAD keystream) - aad_pos += CHACHA20_POLY1305_AEAD_AAD_LEN; - if (aad_pos + CHACHA20_POLY1305_AEAD_AAD_LEN > CHACHA20_ROUND_OUTPUT) { - aad_pos = 0; - seqnr_aad++; - } - if (seqnr_payload + 1 == std::numeric_limits<uint64_t>::max()) { - // reuse of nonce+key is okay while benchmarking. - seqnr_payload = 0; - seqnr_aad = 0; - aad_pos = 0; - } - }); -} - -static void CHACHA20_POLY1305_AEAD_64BYTES_ONLY_ENCRYPT(benchmark::Bench& bench) -{ - CHACHA20_POLY1305_AEAD(bench, BUFFER_SIZE_TINY, false); -} - -static void CHACHA20_POLY1305_AEAD_256BYTES_ONLY_ENCRYPT(benchmark::Bench& bench) -{ - CHACHA20_POLY1305_AEAD(bench, BUFFER_SIZE_SMALL, false); -} - -static void CHACHA20_POLY1305_AEAD_1MB_ONLY_ENCRYPT(benchmark::Bench& bench) -{ - CHACHA20_POLY1305_AEAD(bench, BUFFER_SIZE_LARGE, false); -} - -static void CHACHA20_POLY1305_AEAD_64BYTES_ENCRYPT_DECRYPT(benchmark::Bench& bench) -{ - CHACHA20_POLY1305_AEAD(bench, BUFFER_SIZE_TINY, true); -} - -static void CHACHA20_POLY1305_AEAD_256BYTES_ENCRYPT_DECRYPT(benchmark::Bench& bench) -{ - CHACHA20_POLY1305_AEAD(bench, BUFFER_SIZE_SMALL, true); -} - -static void CHACHA20_POLY1305_AEAD_1MB_ENCRYPT_DECRYPT(benchmark::Bench& bench) -{ - CHACHA20_POLY1305_AEAD(bench, BUFFER_SIZE_LARGE, true); -} - -// Add Hash() (dbl-sha256) bench for comparison - -static void HASH(benchmark::Bench& bench, size_t buffersize) -{ - uint8_t hash[CHash256::OUTPUT_SIZE]; - std::vector<uint8_t> in(buffersize,0); - bench.batch(in.size()).unit("byte").run([&] { - CHash256().Write(in).Finalize(hash); - }); -} - -static void HASH_64BYTES(benchmark::Bench& bench) -{ - HASH(bench, BUFFER_SIZE_TINY); -} - -static void HASH_256BYTES(benchmark::Bench& bench) -{ - HASH(bench, BUFFER_SIZE_SMALL); -} - -static void HASH_1MB(benchmark::Bench& bench) -{ - HASH(bench, BUFFER_SIZE_LARGE); -} - -BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); -BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); -BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); -BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); -BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); -BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); -BENCHMARK(HASH_64BYTES, benchmark::PriorityLevel::HIGH); -BENCHMARK(HASH_256BYTES, benchmark::PriorityLevel::HIGH); -BENCHMARK(HASH_1MB, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 735dc92dfb..1a9b013277 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -13,11 +13,12 @@ static void AddTx(const CTransactionRef& tx, const CAmount& nFee, CTxMemPool& po { int64_t nTime = 0; unsigned int nHeight = 1; + uint64_t sequence = 0; bool spendsCoinbase = false; unsigned int sigOpCost = 4; LockPoints lp; pool.addUnchecked(CTxMemPoolEntry( - tx, nFee, nTime, nHeight, + tx, nFee, nTime, nHeight, sequence, spendsCoinbase, sigOpCost, lp)); } diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index 826da73800..1f94461d19 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -16,10 +16,11 @@ static void AddTx(const CTransactionRef& tx, CTxMemPool& pool) EXCLUSIVE_LOCKS_R { int64_t nTime = 0; unsigned int nHeight = 1; + uint64_t sequence = 0; bool spendsCoinbase = false; unsigned int sigOpCost = 4; LockPoints lp; - pool.addUnchecked(CTxMemPoolEntry(tx, 1000, nTime, nHeight, spendsCoinbase, sigOpCost, lp)); + pool.addUnchecked(CTxMemPoolEntry(tx, 1000, nTime, nHeight, sequence, spendsCoinbase, sigOpCost, lp)); } struct Available { diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp index 7e274370e0..a55aa0c794 100644 --- a/src/bench/rpc_mempool.cpp +++ b/src/bench/rpc_mempool.cpp @@ -16,7 +16,7 @@ static void AddTx(const CTransactionRef& tx, const CAmount& fee, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) { LockPoints lp; - pool.addUnchecked(CTxMemPoolEntry(tx, fee, /*time=*/0, /*entry_height=*/1, /*spends_coinbase=*/false, /*sigops_cost=*/4, lp)); + pool.addUnchecked(CTxMemPoolEntry(tx, fee, /*time=*/0, /*entry_height=*/1, /*entry_sequence=*/0, /*spends_coinbase=*/false, /*sigops_cost=*/4, lp)); } static void RpcMempool(benchmark::Bench& bench) diff --git a/src/bip324.cpp b/src/bip324.cpp new file mode 100644 index 0000000000..314e756829 --- /dev/null +++ b/src/bip324.cpp @@ -0,0 +1,116 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bip324.h> + +#include <chainparams.h> +#include <crypto/chacha20.h> +#include <crypto/chacha20poly1305.h> +#include <crypto/hkdf_sha256_32.h> +#include <key.h> +#include <pubkey.h> +#include <random.h> +#include <span.h> +#include <support/cleanse.h> +#include <uint256.h> + +#include <algorithm> +#include <assert.h> +#include <cstdint> +#include <cstddef> +#include <iterator> +#include <string> + +BIP324Cipher::BIP324Cipher() noexcept +{ + m_key.MakeNewKey(true); + uint256 entropy = GetRandHash(); + m_our_pubkey = m_key.EllSwiftCreate(MakeByteSpan(entropy)); +} + +BIP324Cipher::BIP324Cipher(const CKey& key, Span<const std::byte> ent32) noexcept : + m_key(key) +{ + m_our_pubkey = m_key.EllSwiftCreate(ent32); +} + +BIP324Cipher::BIP324Cipher(const CKey& key, const EllSwiftPubKey& pubkey) noexcept : + m_key(key), m_our_pubkey(pubkey) {} + +void BIP324Cipher::Initialize(const EllSwiftPubKey& their_pubkey, bool initiator, bool self_decrypt) noexcept +{ + // Determine salt (fixed string + network magic bytes) + const auto& message_header = Params().MessageStart(); + std::string salt = std::string{"bitcoin_v2_shared_secret"} + std::string(std::begin(message_header), std::end(message_header)); + + // Perform ECDH to derive shared secret. + ECDHSecret ecdh_secret = m_key.ComputeBIP324ECDHSecret(their_pubkey, m_our_pubkey, initiator); + + // Derive encryption keys from shared secret, and initialize stream ciphers and AEADs. + bool side = (initiator != self_decrypt); + CHKDF_HMAC_SHA256_L32 hkdf(UCharCast(ecdh_secret.data()), ecdh_secret.size(), salt); + std::array<std::byte, 32> hkdf_32_okm; + hkdf.Expand32("initiator_L", UCharCast(hkdf_32_okm.data())); + (side ? m_send_l_cipher : m_recv_l_cipher).emplace(hkdf_32_okm, REKEY_INTERVAL); + hkdf.Expand32("initiator_P", UCharCast(hkdf_32_okm.data())); + (side ? m_send_p_cipher : m_recv_p_cipher).emplace(hkdf_32_okm, REKEY_INTERVAL); + hkdf.Expand32("responder_L", UCharCast(hkdf_32_okm.data())); + (side ? m_recv_l_cipher : m_send_l_cipher).emplace(hkdf_32_okm, REKEY_INTERVAL); + hkdf.Expand32("responder_P", UCharCast(hkdf_32_okm.data())); + (side ? m_recv_p_cipher : m_send_p_cipher).emplace(hkdf_32_okm, REKEY_INTERVAL); + + // Derive garbage terminators from shared secret. + hkdf.Expand32("garbage_terminators", UCharCast(hkdf_32_okm.data())); + std::copy(std::begin(hkdf_32_okm), std::begin(hkdf_32_okm) + GARBAGE_TERMINATOR_LEN, + (initiator ? m_send_garbage_terminator : m_recv_garbage_terminator).begin()); + std::copy(std::end(hkdf_32_okm) - GARBAGE_TERMINATOR_LEN, std::end(hkdf_32_okm), + (initiator ? m_recv_garbage_terminator : m_send_garbage_terminator).begin()); + + // Derive session id from shared secret. + hkdf.Expand32("session_id", UCharCast(m_session_id.data())); + + // Wipe all variables that contain information which could be used to re-derive encryption keys. + memory_cleanse(ecdh_secret.data(), ecdh_secret.size()); + memory_cleanse(hkdf_32_okm.data(), sizeof(hkdf_32_okm)); + memory_cleanse(&hkdf, sizeof(hkdf)); + m_key = CKey(); +} + +void BIP324Cipher::Encrypt(Span<const std::byte> contents, Span<const std::byte> aad, bool ignore, Span<std::byte> output) noexcept +{ + assert(output.size() == contents.size() + EXPANSION); + + // Encrypt length. + std::byte len[LENGTH_LEN]; + len[0] = std::byte{(uint8_t)(contents.size() & 0xFF)}; + len[1] = std::byte{(uint8_t)((contents.size() >> 8) & 0xFF)}; + len[2] = std::byte{(uint8_t)((contents.size() >> 16) & 0xFF)}; + m_send_l_cipher->Crypt(len, output.first(LENGTH_LEN)); + + // Encrypt plaintext. + std::byte header[HEADER_LEN] = {ignore ? IGNORE_BIT : std::byte{0}}; + m_send_p_cipher->Encrypt(header, contents, aad, output.subspan(LENGTH_LEN)); +} + +uint32_t BIP324Cipher::DecryptLength(Span<const std::byte> input) noexcept +{ + assert(input.size() == LENGTH_LEN); + + std::byte buf[LENGTH_LEN]; + // Decrypt length + m_recv_l_cipher->Crypt(input, buf); + // Convert to number. + return uint32_t(buf[0]) + (uint32_t(buf[1]) << 8) + (uint32_t(buf[2]) << 16); +} + +bool BIP324Cipher::Decrypt(Span<const std::byte> input, Span<const std::byte> aad, bool& ignore, Span<std::byte> contents) noexcept +{ + assert(input.size() + LENGTH_LEN == contents.size() + EXPANSION); + + std::byte header[HEADER_LEN]; + if (!m_recv_p_cipher->Decrypt(input, aad, header, contents)) return false; + + ignore = (header[0] & IGNORE_BIT) == IGNORE_BIT; + return true; +} diff --git a/src/bip324.h b/src/bip324.h new file mode 100644 index 0000000000..0238c479c0 --- /dev/null +++ b/src/bip324.h @@ -0,0 +1,96 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_BIP324_H +#define BITCOIN_BIP324_H + +#include <array> +#include <cstddef> +#include <optional> + +#include <crypto/chacha20.h> +#include <crypto/chacha20poly1305.h> +#include <key.h> +#include <pubkey.h> +#include <span.h> + +/** The BIP324 packet cipher, encapsulating its key derivation, stream cipher, and AEAD. */ +class BIP324Cipher +{ +public: + static constexpr unsigned SESSION_ID_LEN{32}; + static constexpr unsigned GARBAGE_TERMINATOR_LEN{16}; + static constexpr unsigned REKEY_INTERVAL{224}; + static constexpr unsigned LENGTH_LEN{3}; + static constexpr unsigned HEADER_LEN{1}; + static constexpr unsigned EXPANSION = LENGTH_LEN + HEADER_LEN + FSChaCha20Poly1305::EXPANSION; + static constexpr std::byte IGNORE_BIT{0x80}; + +private: + std::optional<FSChaCha20> m_send_l_cipher; + std::optional<FSChaCha20> m_recv_l_cipher; + std::optional<FSChaCha20Poly1305> m_send_p_cipher; + std::optional<FSChaCha20Poly1305> m_recv_p_cipher; + + CKey m_key; + EllSwiftPubKey m_our_pubkey; + + std::array<std::byte, SESSION_ID_LEN> m_session_id; + std::array<std::byte, GARBAGE_TERMINATOR_LEN> m_send_garbage_terminator; + std::array<std::byte, GARBAGE_TERMINATOR_LEN> m_recv_garbage_terminator; + +public: + /** Initialize a BIP324 cipher with securely generated random keys. */ + BIP324Cipher() noexcept; + + /** Initialize a BIP324 cipher with specified key and encoding entropy (testing only). */ + BIP324Cipher(const CKey& key, Span<const std::byte> ent32) noexcept; + + /** Initialize a BIP324 cipher with specified key (testing only). */ + BIP324Cipher(const CKey& key, const EllSwiftPubKey& pubkey) noexcept; + + /** Retrieve our public key. */ + const EllSwiftPubKey& GetOurPubKey() const noexcept { return m_our_pubkey; } + + /** Initialize when the other side's public key is received. Can only be called once. + * + * initiator is set to true if we are the initiator establishing the v2 P2P connection. + * self_decrypt is only for testing, and swaps encryption/decryption keys, so that encryption + * and decryption can be tested without knowing the other side's private key. + */ + void Initialize(const EllSwiftPubKey& their_pubkey, bool initiator, bool self_decrypt = false) noexcept; + + /** Determine whether this cipher is fully initialized. */ + explicit operator bool() const noexcept { return m_send_l_cipher.has_value(); } + + /** Encrypt a packet. Only after Initialize(). + * + * It must hold that output.size() == contents.size() + EXPANSION. + */ + void Encrypt(Span<const std::byte> contents, Span<const std::byte> aad, bool ignore, Span<std::byte> output) noexcept; + + /** Decrypt the length of a packet. Only after Initialize(). + * + * It must hold that input.size() == LENGTH_LEN. + */ + unsigned DecryptLength(Span<const std::byte> input) noexcept; + + /** Decrypt a packet. Only after Initialize(). + * + * It must hold that input.size() + LENGTH_LEN == contents.size() + EXPANSION. + * Contents.size() must equal the length returned by DecryptLength. + */ + bool Decrypt(Span<const std::byte> input, Span<const std::byte> aad, bool& ignore, Span<std::byte> contents) noexcept; + + /** Get the Session ID. Only after Initialize(). */ + Span<const std::byte> GetSessionID() const noexcept { return m_session_id; } + + /** Get the Garbage Terminator to send. Only after Initialize(). */ + Span<const std::byte> GetSendGarbageTerminator() const noexcept { return m_send_garbage_terminator; } + + /** Get the expected Garbage Terminator to receive. Only after Initialize(). */ + Span<const std::byte> GetReceiveGarbageTerminator() const noexcept { return m_recv_garbage_terminator; } +}; + +#endif // BITCOIN_BIP324_H diff --git a/src/compat/compat.h b/src/compat/compat.h index 8195bceaec..435a403552 100644 --- a/src/compat/compat.h +++ b/src/compat/compat.h @@ -22,19 +22,18 @@ #include <ws2tcpip.h> #include <cstdint> #else -#include <fcntl.h> -#include <sys/mman.h> -#include <sys/select.h> -#include <sys/socket.h> -#include <sys/types.h> -#include <net/if.h> -#include <netinet/in.h> -#include <netinet/tcp.h> -#include <arpa/inet.h> -#include <ifaddrs.h> -#include <limits.h> -#include <netdb.h> -#include <unistd.h> +#include <arpa/inet.h> // IWYU pragma: export +#include <fcntl.h> // IWYU pragma: export +#include <ifaddrs.h> // IWYU pragma: export +#include <net/if.h> // IWYU pragma: export +#include <netdb.h> // IWYU pragma: export +#include <netinet/in.h> // IWYU pragma: export +#include <netinet/tcp.h> // IWYU pragma: export +#include <sys/mman.h> // IWYU pragma: export +#include <sys/select.h> // IWYU pragma: export +#include <sys/socket.h> // IWYU pragma: export +#include <sys/types.h> // IWYU pragma: export +#include <unistd.h> // IWYU pragma: export #endif // We map Linux / BSD error functions and codes, to the equivalent diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp index fafd783ab1..469b280494 100644 --- a/src/crypto/chacha20.cpp +++ b/src/crypto/chacha20.cpp @@ -7,6 +7,7 @@ #include <crypto/common.h> #include <crypto/chacha20.h> +#include <support/cleanse.h> #include <algorithm> #include <string.h> @@ -42,6 +43,11 @@ ChaCha20Aligned::ChaCha20Aligned() memset(input, 0, sizeof(input)); } +ChaCha20Aligned::~ChaCha20Aligned() +{ + memory_cleanse(input, sizeof(input)); +} + ChaCha20Aligned::ChaCha20Aligned(const unsigned char* key32) { SetKey32(key32); @@ -318,3 +324,38 @@ void ChaCha20::Crypt(const unsigned char* m, unsigned char* c, size_t bytes) m_bufleft = 64 - bytes; } } + +ChaCha20::~ChaCha20() +{ + memory_cleanse(m_buffer, sizeof(m_buffer)); +} + +FSChaCha20::FSChaCha20(Span<const std::byte> key, uint32_t rekey_interval) noexcept : + m_chacha20(UCharCast(key.data())), m_rekey_interval(rekey_interval) +{ + assert(key.size() == KEYLEN); +} + +void FSChaCha20::Crypt(Span<const std::byte> input, Span<std::byte> output) noexcept +{ + assert(input.size() == output.size()); + + // Invoke internal stream cipher for actual encryption/decryption. + m_chacha20.Crypt(UCharCast(input.data()), UCharCast(output.data()), input.size()); + + // Rekey after m_rekey_interval encryptions/decryptions. + if (++m_chunk_counter == m_rekey_interval) { + // Get new key from the stream cipher. + std::byte new_key[KEYLEN]; + m_chacha20.Keystream(UCharCast(new_key), sizeof(new_key)); + // Update its key. + m_chacha20.SetKey32(UCharCast(new_key)); + // Wipe the key (a copy remains inside m_chacha20, where it'll be wiped on the next rekey + // or on destruction). + memory_cleanse(new_key, sizeof(new_key)); + // Set the nonce for the new section of output. + m_chacha20.Seek64({0, ++m_rekey_counter}, 0); + // Reset the chunk counter. + m_chunk_counter = 0; + } +} diff --git a/src/crypto/chacha20.h b/src/crypto/chacha20.h index f2ec21d82e..d1b2094e7e 100644 --- a/src/crypto/chacha20.h +++ b/src/crypto/chacha20.h @@ -5,6 +5,10 @@ #ifndef BITCOIN_CRYPTO_CHACHA20_H #define BITCOIN_CRYPTO_CHACHA20_H +#include <span.h> + +#include <array> +#include <cstddef> #include <cstdlib> #include <stdint.h> #include <utility> @@ -29,6 +33,9 @@ public: /** Initialize a cipher with specified 32-byte key. */ ChaCha20Aligned(const unsigned char* key32); + /** Destructor to clean up private memory. */ + ~ChaCha20Aligned(); + /** set 32-byte key. */ void SetKey32(const unsigned char* key32); @@ -72,6 +79,9 @@ public: /** Initialize a cipher with specified 32-byte key. */ ChaCha20(const unsigned char* key32) : m_aligned(key32) {} + /** Destructor to clean up private memory. */ + ~ChaCha20(); + /** set 32-byte key. */ void SetKey32(const unsigned char* key32) { @@ -98,4 +108,43 @@ public: void Crypt(const unsigned char* input, unsigned char* output, size_t bytes); }; +/** Forward-secure ChaCha20 + * + * This implements a stream cipher that automatically transitions to a new stream with a new key + * and new nonce after a predefined number of encryptions or decryptions. + * + * See BIP324 for details. + */ +class FSChaCha20 +{ +private: + /** Internal stream cipher. */ + ChaCha20 m_chacha20; + + /** The number of encryptions/decryptions before a rekey happens. */ + const uint32_t m_rekey_interval; + + /** The number of encryptions/decryptions since the last rekey. */ + uint32_t m_chunk_counter{0}; + + /** The number of rekey operations that have happened. */ + uint64_t m_rekey_counter{0}; + +public: + /** Length of keys expected by the constructor. */ + static constexpr unsigned KEYLEN = 32; + + // No copy or move to protect the secret. + FSChaCha20(const FSChaCha20&) = delete; + FSChaCha20(FSChaCha20&&) = delete; + FSChaCha20& operator=(const FSChaCha20&) = delete; + FSChaCha20& operator=(FSChaCha20&&) = delete; + + /** Construct an FSChaCha20 cipher that rekeys every rekey_interval Crypt() calls. */ + FSChaCha20(Span<const std::byte> key, uint32_t rekey_interval) noexcept; + + /** Encrypt or decrypt a chunk. */ + void Crypt(Span<const std::byte> input, Span<std::byte> output) noexcept; +}; + #endif // BITCOIN_CRYPTO_CHACHA20_H diff --git a/src/crypto/chacha20poly1305.cpp b/src/crypto/chacha20poly1305.cpp new file mode 100644 index 0000000000..26161641bb --- /dev/null +++ b/src/crypto/chacha20poly1305.cpp @@ -0,0 +1,140 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/chacha20poly1305.h> + +#include <crypto/common.h> +#include <crypto/chacha20.h> +#include <crypto/poly1305.h> +#include <span.h> +#include <support/cleanse.h> + +#include <assert.h> +#include <cstddef> + +AEADChaCha20Poly1305::AEADChaCha20Poly1305(Span<const std::byte> key) noexcept : m_chacha20(UCharCast(key.data())) +{ + assert(key.size() == KEYLEN); +} + +void AEADChaCha20Poly1305::SetKey(Span<const std::byte> key) noexcept +{ + assert(key.size() == KEYLEN); + m_chacha20.SetKey32(UCharCast(key.data())); +} + +namespace { + +#ifndef HAVE_TIMINGSAFE_BCMP +#define HAVE_TIMINGSAFE_BCMP + +int timingsafe_bcmp(const unsigned char* b1, const unsigned char* b2, size_t n) noexcept +{ + const unsigned char *p1 = b1, *p2 = b2; + int ret = 0; + for (; n > 0; n--) + ret |= *p1++ ^ *p2++; + return (ret != 0); +} + +#endif + +/** Compute poly1305 tag. chacha20 must be set to the right nonce, block 0. Will be at block 1 after. */ +void ComputeTag(ChaCha20& chacha20, Span<const std::byte> aad, Span<const std::byte> cipher, Span<std::byte> tag) noexcept +{ + static const std::byte PADDING[16] = {{}}; + + // Get block of keystream (use a full 64 byte buffer to avoid the need for chacha20's own buffering). + std::byte first_block[64]; + chacha20.Keystream(UCharCast(first_block), sizeof(first_block)); + + // Use the first 32 bytes of the first keystream block as poly1305 key. + Poly1305 poly1305{Span{first_block}.first(Poly1305::KEYLEN)}; + + // Compute tag: + // - Process the padded AAD with Poly1305. + const unsigned aad_padding_length = (16 - (aad.size() % 16)) % 16; + poly1305.Update(aad).Update(Span{PADDING}.first(aad_padding_length)); + // - Process the padded ciphertext with Poly1305. + const unsigned cipher_padding_length = (16 - (cipher.size() % 16)) % 16; + poly1305.Update(cipher).Update(Span{PADDING}.first(cipher_padding_length)); + // - Process the AAD and plaintext length with Poly1305. + std::byte length_desc[Poly1305::TAGLEN]; + WriteLE64(UCharCast(length_desc), aad.size()); + WriteLE64(UCharCast(length_desc + 8), cipher.size()); + poly1305.Update(length_desc); + + // Output tag. + poly1305.Finalize(tag); +} + +} // namespace + +void AEADChaCha20Poly1305::Encrypt(Span<const std::byte> plain1, Span<const std::byte> plain2, Span<const std::byte> aad, Nonce96 nonce, Span<std::byte> cipher) noexcept +{ + assert(cipher.size() == plain1.size() + plain2.size() + EXPANSION); + + // Encrypt using ChaCha20 (starting at block 1). + m_chacha20.Seek64(nonce, 1); + m_chacha20.Crypt(UCharCast(plain1.data()), UCharCast(cipher.data()), plain1.size()); + m_chacha20.Crypt(UCharCast(plain2.data()), UCharCast(cipher.data() + plain1.size()), plain2.size()); + + // Seek to block 0, and compute tag using key drawn from there. + m_chacha20.Seek64(nonce, 0); + ComputeTag(m_chacha20, aad, cipher.first(cipher.size() - EXPANSION), cipher.last(EXPANSION)); +} + +bool AEADChaCha20Poly1305::Decrypt(Span<const std::byte> cipher, Span<const std::byte> aad, Nonce96 nonce, Span<std::byte> plain1, Span<std::byte> plain2) noexcept +{ + assert(cipher.size() == plain1.size() + plain2.size() + EXPANSION); + + // Verify tag (using key drawn from block 0). + m_chacha20.Seek64(nonce, 0); + std::byte expected_tag[EXPANSION]; + ComputeTag(m_chacha20, aad, cipher.first(cipher.size() - EXPANSION), expected_tag); + if (timingsafe_bcmp(UCharCast(expected_tag), UCharCast(cipher.last(EXPANSION).data()), EXPANSION)) return false; + + // Decrypt (starting at block 1). + m_chacha20.Crypt(UCharCast(cipher.data()), UCharCast(plain1.data()), plain1.size()); + m_chacha20.Crypt(UCharCast(cipher.data() + plain1.size()), UCharCast(plain2.data()), plain2.size()); + return true; +} + +void AEADChaCha20Poly1305::Keystream(Nonce96 nonce, Span<std::byte> keystream) noexcept +{ + // Skip the first output block, as it's used for generating the poly1305 key. + m_chacha20.Seek64(nonce, 1); + m_chacha20.Keystream(UCharCast(keystream.data()), keystream.size()); +} + +void FSChaCha20Poly1305::NextPacket() noexcept +{ + if (++m_packet_counter == m_rekey_interval) { + // Generate a full block of keystream, to avoid needing the ChaCha20 buffer, even though + // we only need KEYLEN (32) bytes. + std::byte one_block[64]; + m_aead.Keystream({0xFFFFFFFF, m_rekey_counter}, one_block); + // Switch keys. + m_aead.SetKey(Span{one_block}.first(KEYLEN)); + // Wipe the generated keystream (a copy remains inside m_aead, which will be cleaned up + // once it cycles again, or is destroyed). + memory_cleanse(one_block, sizeof(one_block)); + // Update counters. + m_packet_counter = 0; + ++m_rekey_counter; + } +} + +void FSChaCha20Poly1305::Encrypt(Span<const std::byte> plain1, Span<const std::byte> plain2, Span<const std::byte> aad, Span<std::byte> cipher) noexcept +{ + m_aead.Encrypt(plain1, plain2, aad, {m_packet_counter, m_rekey_counter}, cipher); + NextPacket(); +} + +bool FSChaCha20Poly1305::Decrypt(Span<const std::byte> cipher, Span<const std::byte> aad, Span<std::byte> plain1, Span<std::byte> plain2) noexcept +{ + bool ret = m_aead.Decrypt(cipher, aad, {m_packet_counter, m_rekey_counter}, plain1, plain2); + NextPacket(); + return ret; +} diff --git a/src/crypto/chacha20poly1305.h b/src/crypto/chacha20poly1305.h new file mode 100644 index 0000000000..a847c258ef --- /dev/null +++ b/src/crypto/chacha20poly1305.h @@ -0,0 +1,148 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_CHACHA20POLY1305_H +#define BITCOIN_CRYPTO_CHACHA20POLY1305_H + +#include <cstddef> +#include <stdint.h> + +#include <crypto/chacha20.h> +#include <crypto/poly1305.h> +#include <span.h> + +/** The AEAD_CHACHA20_POLY1305 authenticated encryption algorithm from RFC8439 section 2.8. */ +class AEADChaCha20Poly1305 +{ + /** Internal stream cipher. */ + ChaCha20 m_chacha20; + +public: + /** Expected size of key argument in constructor. */ + static constexpr unsigned KEYLEN = 32; + + /** Expansion when encrypting. */ + static constexpr unsigned EXPANSION = Poly1305::TAGLEN; + + /** Initialize an AEAD instance with a specified 32-byte key. */ + AEADChaCha20Poly1305(Span<const std::byte> key) noexcept; + + /** Switch to another 32-byte key. */ + void SetKey(Span<const std::byte> key) noexcept; + + /** 96-bit nonce type. */ + using Nonce96 = ChaCha20::Nonce96; + + /** Encrypt a message with a specified 96-bit nonce and aad. + * + * Requires cipher.size() = plain.size() + EXPANSION. + */ + void Encrypt(Span<const std::byte> plain, Span<const std::byte> aad, Nonce96 nonce, Span<std::byte> cipher) noexcept + { + Encrypt(plain, {}, aad, nonce, cipher); + } + + /** Encrypt a message (given split into plain1 + plain2) with a specified 96-bit nonce and aad. + * + * Requires cipher.size() = plain1.size() + plain2.size() + EXPANSION. + */ + void Encrypt(Span<const std::byte> plain1, Span<const std::byte> plain2, Span<const std::byte> aad, Nonce96 nonce, Span<std::byte> cipher) noexcept; + + /** Decrypt a message with a specified 96-bit nonce and aad. Returns true if valid. + * + * Requires cipher.size() = plain.size() + EXPANSION. + */ + bool Decrypt(Span<const std::byte> cipher, Span<const std::byte> aad, Nonce96 nonce, Span<std::byte> plain) noexcept + { + return Decrypt(cipher, aad, nonce, plain, {}); + } + + /** Decrypt a message with a specified 96-bit nonce and aad and split the result. Returns true if valid. + * + * Requires cipher.size() = plain1.size() + plain2.size() + EXPANSION. + */ + bool Decrypt(Span<const std::byte> cipher, Span<const std::byte> aad, Nonce96 nonce, Span<std::byte> plain1, Span<std::byte> plain2) noexcept; + + /** Get a number of keystream bytes from the underlying stream cipher. + * + * This is equivalent to Encrypt() with plain set to that many zero bytes, and dropping the + * last EXPANSION bytes off the result. + */ + void Keystream(Nonce96 nonce, Span<std::byte> keystream) noexcept; +}; + +/** Forward-secure wrapper around AEADChaCha20Poly1305. + * + * This implements an AEAD which automatically increments the nonce on every encryption or + * decryption, and cycles keys after a predetermined number of encryptions or decryptions. + * + * See BIP324 for details. + */ +class FSChaCha20Poly1305 +{ +private: + /** Internal AEAD. */ + AEADChaCha20Poly1305 m_aead; + + /** Every how many iterations this cipher rekeys. */ + const uint32_t m_rekey_interval; + + /** The number of encryptions/decryptions since the last rekey. */ + uint32_t m_packet_counter{0}; + + /** The number of rekeys performed so far. */ + uint64_t m_rekey_counter{0}; + + /** Update counters (and if necessary, key) to transition to the next message. */ + void NextPacket() noexcept; + +public: + /** Length of keys expected by the constructor. */ + static constexpr auto KEYLEN = AEADChaCha20Poly1305::KEYLEN; + + /** Expansion when encrypting. */ + static constexpr auto EXPANSION = AEADChaCha20Poly1305::EXPANSION; + + // No copy or move to protect the secret. + FSChaCha20Poly1305(const FSChaCha20Poly1305&) = delete; + FSChaCha20Poly1305(FSChaCha20Poly1305&&) = delete; + FSChaCha20Poly1305& operator=(const FSChaCha20Poly1305&) = delete; + FSChaCha20Poly1305& operator=(FSChaCha20Poly1305&&) = delete; + + /** Construct an FSChaCha20Poly1305 cipher that rekeys every rekey_interval operations. */ + FSChaCha20Poly1305(Span<const std::byte> key, uint32_t rekey_interval) noexcept : + m_aead(key), m_rekey_interval(rekey_interval) {} + + /** Encrypt a message with a specified aad. + * + * Requires cipher.size() = plain.size() + EXPANSION. + */ + void Encrypt(Span<const std::byte> plain, Span<const std::byte> aad, Span<std::byte> cipher) noexcept + { + Encrypt(plain, {}, aad, cipher); + } + + /** Encrypt a message (given split into plain1 + plain2) with a specified aad. + * + * Requires cipher.size() = plain.size() + EXPANSION. + */ + void Encrypt(Span<const std::byte> plain1, Span<const std::byte> plain2, Span<const std::byte> aad, Span<std::byte> cipher) noexcept; + + /** Decrypt a message with a specified aad. Returns true if valid. + * + * Requires cipher.size() = plain.size() + EXPANSION. + */ + bool Decrypt(Span<const std::byte> cipher, Span<const std::byte> aad, Span<std::byte> plain) noexcept + { + return Decrypt(cipher, aad, plain, {}); + } + + /** Decrypt a message with a specified aad and split the result. Returns true if valid. + * + * Requires cipher.size() = plain1.size() + plain2.size() + EXPANSION. + */ + bool Decrypt(Span<const std::byte> cipher, Span<const std::byte> aad, Span<std::byte> plain1, Span<std::byte> plain2) noexcept; +}; + +#endif // BITCOIN_CRYPTO_CHACHA20POLY1305_H diff --git a/src/crypto/chacha_poly_aead.cpp b/src/crypto/chacha_poly_aead.cpp deleted file mode 100644 index 0d82cf3d74..0000000000 --- a/src/crypto/chacha_poly_aead.cpp +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright (c) 2019-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - -#include <crypto/chacha_poly_aead.h> - -#include <crypto/poly1305.h> -#include <support/cleanse.h> - -#include <assert.h> -#include <string.h> - -#include <cstdio> -#include <limits> - -#ifndef HAVE_TIMINGSAFE_BCMP - -int timingsafe_bcmp(const unsigned char* b1, const unsigned char* b2, size_t n) -{ - const unsigned char *p1 = b1, *p2 = b2; - int ret = 0; - - for (; n > 0; n--) - ret |= *p1++ ^ *p2++; - return (ret != 0); -} - -#endif // TIMINGSAFE_BCMP - -ChaCha20Poly1305AEAD::ChaCha20Poly1305AEAD(const unsigned char* K_1, size_t K_1_len, const unsigned char* K_2, size_t K_2_len) -{ - assert(K_1_len == CHACHA20_POLY1305_AEAD_KEY_LEN); - assert(K_2_len == CHACHA20_POLY1305_AEAD_KEY_LEN); - - static_assert(CHACHA20_POLY1305_AEAD_KEY_LEN == 32); - m_chacha_header.SetKey32(K_1); - m_chacha_main.SetKey32(K_2); - - // set the cached sequence number to uint64 max which hints for an unset cache. - // we can't hit uint64 max since the rekey rule (which resets the sequence number) is 1GB - m_cached_aad_seqnr = std::numeric_limits<uint64_t>::max(); -} - -bool ChaCha20Poly1305AEAD::Crypt(uint64_t seqnr_payload, uint64_t seqnr_aad, int aad_pos, unsigned char* dest, size_t dest_len /* length of the output buffer for sanity checks */, const unsigned char* src, size_t src_len, bool is_encrypt) -{ - // check buffer boundaries - if ( - // if we encrypt, make sure the source contains at least the expected AAD and the destination has at least space for the source + MAC - (is_encrypt && (src_len < CHACHA20_POLY1305_AEAD_AAD_LEN || dest_len < src_len + Poly1305::TAGLEN)) || - // if we decrypt, make sure the source contains at least the expected AAD+MAC and the destination has at least space for the source - MAC - (!is_encrypt && (src_len < CHACHA20_POLY1305_AEAD_AAD_LEN + Poly1305::TAGLEN || dest_len < src_len - Poly1305::TAGLEN))) { - return false; - } - - unsigned char expected_tag[Poly1305::TAGLEN], poly_key[Poly1305::KEYLEN]; - memset(poly_key, 0, sizeof(poly_key)); - - // block counter 0 for the poly1305 key - // use lower 32bytes for the poly1305 key - // (throws away 32 unused bytes (upper 32) from this ChaCha20 round) - m_chacha_main.Seek64({0, seqnr_payload}, 0); - m_chacha_main.Crypt(poly_key, poly_key, sizeof(poly_key)); - - // if decrypting, verify the tag prior to decryption - if (!is_encrypt) { - const unsigned char* tag = src + src_len - Poly1305::TAGLEN; - Poly1305{MakeByteSpan(poly_key)} - .Update(AsBytes(Span{src, src_len - Poly1305::TAGLEN})) - .Finalize(MakeWritableByteSpan(expected_tag)); - - // constant time compare the calculated MAC with the provided MAC - if (timingsafe_bcmp(expected_tag, tag, Poly1305::TAGLEN) != 0) { - memory_cleanse(expected_tag, sizeof(expected_tag)); - memory_cleanse(poly_key, sizeof(poly_key)); - return false; - } - memory_cleanse(expected_tag, sizeof(expected_tag)); - // MAC has been successfully verified, make sure we don't convert it in decryption - src_len -= Poly1305::TAGLEN; - } - - // calculate and cache the next 64byte keystream block if requested sequence number is not yet the cache - if (m_cached_aad_seqnr != seqnr_aad) { - m_cached_aad_seqnr = seqnr_aad; - m_chacha_header.Seek64({0, seqnr_aad}, 0); - m_chacha_header.Keystream(m_aad_keystream_buffer, CHACHA20_ROUND_OUTPUT); - } - // crypt the AAD (3 bytes message length) with given position in AAD cipher instance keystream - dest[0] = src[0] ^ m_aad_keystream_buffer[aad_pos]; - dest[1] = src[1] ^ m_aad_keystream_buffer[aad_pos + 1]; - dest[2] = src[2] ^ m_aad_keystream_buffer[aad_pos + 2]; - - // Set the playload ChaCha instance block counter to 1 and crypt the payload - m_chacha_main.Seek64({0, seqnr_payload}, 1); - m_chacha_main.Crypt(src + CHACHA20_POLY1305_AEAD_AAD_LEN, dest + CHACHA20_POLY1305_AEAD_AAD_LEN, src_len - CHACHA20_POLY1305_AEAD_AAD_LEN); - - // If encrypting, calculate and append tag - if (is_encrypt) { - // the poly1305 tag expands over the AAD (3 bytes length) & encrypted payload - Poly1305{MakeByteSpan(poly_key)} - .Update(AsBytes(Span{dest, src_len})) - .Finalize(AsWritableBytes(Span{dest + src_len, Poly1305::TAGLEN})); - } - - // cleanse no longer required MAC and polykey - memory_cleanse(poly_key, sizeof(poly_key)); - return true; -} - -bool ChaCha20Poly1305AEAD::GetLength(uint32_t* len24_out, uint64_t seqnr_aad, int aad_pos, const uint8_t* ciphertext) -{ - // enforce valid aad position to avoid accessing outside of the 64byte keystream cache - // (there is space for 21 times 3 bytes) - assert(aad_pos >= 0 && aad_pos < CHACHA20_ROUND_OUTPUT - CHACHA20_POLY1305_AEAD_AAD_LEN); - if (m_cached_aad_seqnr != seqnr_aad) { - // we need to calculate the 64 keystream bytes since we reached a new aad sequence number - m_cached_aad_seqnr = seqnr_aad; - m_chacha_header.Seek64({0, seqnr_aad}, 0); // use LE for the nonce - m_chacha_header.Keystream(m_aad_keystream_buffer, CHACHA20_ROUND_OUTPUT); // write keystream to the cache - } - - // decrypt the ciphertext length by XORing the right position of the 64byte keystream cache with the ciphertext - *len24_out = (ciphertext[0] ^ m_aad_keystream_buffer[aad_pos + 0]) | - (ciphertext[1] ^ m_aad_keystream_buffer[aad_pos + 1]) << 8 | - (ciphertext[2] ^ m_aad_keystream_buffer[aad_pos + 2]) << 16; - - return true; -} diff --git a/src/crypto/chacha_poly_aead.h b/src/crypto/chacha_poly_aead.h deleted file mode 100644 index 5d57b5a5e2..0000000000 --- a/src/crypto/chacha_poly_aead.h +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright (c) 2019-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_CRYPTO_CHACHA_POLY_AEAD_H -#define BITCOIN_CRYPTO_CHACHA_POLY_AEAD_H - -#include <crypto/chacha20.h> - -#include <cmath> - -static constexpr int CHACHA20_POLY1305_AEAD_KEY_LEN = 32; -static constexpr int CHACHA20_POLY1305_AEAD_AAD_LEN = 3; /* 3 bytes length */ -static constexpr int CHACHA20_ROUND_OUTPUT = 64; /* 64 bytes per round */ -static constexpr int AAD_PACKAGES_PER_ROUND = 21; /* 64 / 3 round down*/ - -/* A AEAD class for ChaCha20-Poly1305@bitcoin. - * - * ChaCha20 is a stream cipher designed by Daniel Bernstein and described in - * <ref>[https://cr.yp.to/chacha/chacha-20080128.pdf ChaCha20]</ref>. It operates - * by permuting 128 fixed bits, 128 or 256 bits of key, a 64 bit nonce and a 64 - * bit counter into 64 bytes of output. This output is used as a keystream, with - * any unused bytes simply discarded. - * - * Poly1305 <ref>[https://cr.yp.to/mac/poly1305-20050329.pdf Poly1305]</ref>, also - * by Daniel Bernstein, is a one-time Carter-Wegman MAC that computes a 128 bit - * integrity tag given a message and a single-use 256 bit secret key. - * - * The chacha20-poly1305@bitcoin combines these two primitives into an - * authenticated encryption mode. The construction used is based on that proposed - * for TLS by Adam Langley in - * <ref>[http://tools.ietf.org/html/draft-agl-tls-chacha20poly1305-03 "ChaCha20 - * and Poly1305 based Cipher Suites for TLS", Adam Langley]</ref>, but differs in - * the layout of data passed to the MAC and in the addition of encryption of the - * packet lengths. - * - * ==== Detailed Construction ==== - * - * The chacha20-poly1305@bitcoin cipher requires two 256 bits of key material as - * output from the key exchange. Each key (K_1 and K_2) are used by two separate - * instances of chacha20. - * - * The instance keyed by K_1 is a stream cipher that is used only to encrypt the 3 - * byte packet length field and has its own sequence number. The second instance, - * keyed by K_2, is used in conjunction with poly1305 to build an AEAD - * (Authenticated Encryption with Associated Data) that is used to encrypt and - * authenticate the entire packet. - * - * Two separate cipher instances are used here so as to keep the packet lengths - * confidential but not create an oracle for the packet payload cipher by - * decrypting and using the packet length prior to checking the MAC. By using an - * independently-keyed cipher instance to encrypt the length, an active attacker - * seeking to exploit the packet input handling as a decryption oracle can learn - * nothing about the payload contents or its MAC (assuming key derivation, - * ChaCha20 and Poly1305 are secure). - * - * The AEAD is constructed as follows: for each packet, generate a Poly1305 key by - * taking the first 256 bits of ChaCha20 stream output generated using K_2, an IV - * consisting of the packet sequence number encoded as an LE uint64 and a ChaCha20 - * block counter of zero. The K_2 ChaCha20 block counter is then set to the - * little-endian encoding of 1 (i.e. {1, 0, 0, 0, 0, 0, 0, 0}) and this instance - * is used for encryption of the packet payload. - * - * ==== Packet Handling ==== - * - * When receiving a packet, the length must be decrypted first. When 3 bytes of - * ciphertext length have been received, they may be decrypted. - * - * A ChaCha20 round always calculates 64bytes which is sufficient to crypt 21 - * times a 3 bytes length field (21*3 = 63). The length field sequence number can - * thus be used 21 times (keystream caching). - * - * The length field must be enc-/decrypted with the ChaCha20 keystream keyed with - * K_1 defined by block counter 0, the length field sequence number in little - * endian and a keystream position from 0 to 60. - * - * Once the entire packet has been received, the MAC MUST be checked before - * decryption. A per-packet Poly1305 key is generated as described above and the - * MAC tag calculated using Poly1305 with this key over the ciphertext of the - * packet length and the payload together. The calculated MAC is then compared in - * constant time with the one appended to the packet and the packet decrypted - * using ChaCha20 as described above (with K_2, the packet sequence number as - * nonce and a starting block counter of 1). - * - * Detection of an invalid MAC MUST lead to immediate connection termination. - * - * To send a packet, first encode the 3 byte length and encrypt it using K_1 as - * described above. Encrypt the packet payload (using K_2) and append it to the - * encrypted length. Finally, calculate a MAC tag and append it. - * - * The initiating peer MUST use <code>K_1_A, K_2_A</code> to encrypt messages on - * the send channel, <code>K_1_B, K_2_B</code> MUST be used to decrypt messages on - * the receive channel. - * - * The responding peer MUST use <code>K_1_A, K_2_A</code> to decrypt messages on - * the receive channel, <code>K_1_B, K_2_B</code> MUST be used to encrypt messages - * on the send channel. - * - * Optimized implementations of ChaCha20-Poly1305@bitcoin are relatively fast in - * general, therefore it is very likely that encrypted messages require not more - * CPU cycles per bytes then the current unencrypted p2p message format - * (ChaCha20/Poly1305 versus double SHA256). - * - * The initial packet sequence numbers are 0. - * - * K_2 ChaCha20 cipher instance (payload) must never reuse a {key, nonce} for - * encryption nor may it be used to encrypt more than 2^70 bytes under the same - * {key, nonce}. - * - * K_1 ChaCha20 cipher instance (length field/AAD) must never reuse a {key, nonce, - * position-in-keystream} for encryption nor may it be used to encrypt more than - * 2^70 bytes under the same {key, nonce}. - * - * We use message sequence numbers for both communication directions. - */ - -class ChaCha20Poly1305AEAD -{ -private: - ChaCha20 m_chacha_header; // AAD cipher instance (encrypted length) and poly1305 key-derivation cipher instance - ChaCha20 m_chacha_main; // payload - unsigned char m_aad_keystream_buffer[CHACHA20_ROUND_OUTPUT]; // aad keystream cache - uint64_t m_cached_aad_seqnr; // aad keystream cache hint - -public: - ChaCha20Poly1305AEAD(const unsigned char* K_1, size_t K_1_len, const unsigned char* K_2, size_t K_2_len); - - explicit ChaCha20Poly1305AEAD(const ChaCha20Poly1305AEAD&) = delete; - - /** Encrypts/decrypts a packet - seqnr_payload, the message sequence number - seqnr_aad, the messages AAD sequence number which allows reuse of the AAD keystream - aad_pos, position to use in the AAD keystream to encrypt the AAD - dest, output buffer, must be of a size equal or larger then CHACHA20_POLY1305_AEAD_AAD_LEN + payload (+ POLY1305_TAG_LEN in encryption) bytes - destlen, length of the destination buffer - src, the AAD+payload to encrypt or the AAD+payload+MAC to decrypt - src_len, the length of the source buffer - is_encrypt, set to true if we encrypt (creates and appends the MAC instead of verifying it) - */ - bool Crypt(uint64_t seqnr_payload, uint64_t seqnr_aad, int aad_pos, unsigned char* dest, size_t dest_len, const unsigned char* src, size_t src_len, bool is_encrypt); - - /** decrypts the 3 bytes AAD data and decodes it into a uint32_t field */ - bool GetLength(uint32_t* len24_out, uint64_t seqnr_aad, int aad_pos, const uint8_t* ciphertext); -}; - -#endif // BITCOIN_CRYPTO_CHACHA_POLY_AEAD_H diff --git a/src/init.cpp b/src/init.cpp index 33389e695c..c2c4dbe459 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -116,6 +116,7 @@ #endif using kernel::DumpMempool; +using kernel::LoadMempool; using kernel::ValidationCacheSizes; using node::ApplyArgsManOptions; @@ -1676,7 +1677,10 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return; } // Load mempool from disk - chainman.ActiveChainstate().LoadMempool(ShouldPersistMempool(args) ? MempoolPath(args) : fs::path{}); + if (auto* pool{chainman.ActiveChainstate().GetMempool()}) { + LoadMempool(*pool, ShouldPersistMempool(args) ? MempoolPath(args) : fs::path{}, chainman.ActiveChainstate(), {}); + pool->SetLoadTried(!chainman.m_interrupt); + } }); // Wait for genesis block to be processed diff --git a/src/kernel/mempool_entry.h b/src/kernel/mempool_entry.h index 886e1e1b3a..1f175a5ccf 100644 --- a/src/kernel/mempool_entry.h +++ b/src/kernel/mempool_entry.h @@ -78,6 +78,7 @@ private: const int32_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) const size_t nUsageSize; //!< ... and total memory usage const int64_t nTime; //!< Local time when entering the mempool + const uint64_t entry_sequence; //!< Sequence number used to determine whether this transaction is too recent for relay const unsigned int entryHeight; //!< Chain height when entering the mempool const bool spendsCoinbase; //!< keep track of transactions that spend a coinbase const int64_t sigOpCost; //!< Total sigop cost @@ -101,7 +102,7 @@ private: public: CTxMemPoolEntry(const CTransactionRef& tx, CAmount fee, - int64_t time, unsigned int entry_height, + int64_t time, unsigned int entry_height, uint64_t entry_sequence, bool spends_coinbase, int64_t sigops_cost, LockPoints lp) : tx{tx}, @@ -109,6 +110,7 @@ public: nTxWeight{GetTransactionWeight(*tx)}, nUsageSize{RecursiveDynamicUsage(tx)}, nTime{time}, + entry_sequence{entry_sequence}, entryHeight{entry_height}, spendsCoinbase{spends_coinbase}, sigOpCost{sigops_cost}, @@ -130,6 +132,7 @@ public: int32_t GetTxWeight() const { return nTxWeight; } std::chrono::seconds GetTime() const { return std::chrono::seconds{nTime}; } unsigned int GetHeight() const { return entryHeight; } + uint64_t GetSequence() const { return entry_sequence; } int64_t GetSigOpCost() const { return sigOpCost; } CAmount GetModifiedFee() const { return m_modified_fee; } size_t DynamicMemoryUsage() const { return nUsageSize; } diff --git a/src/kernel/mempool_persist.cpp b/src/kernel/mempool_persist.cpp index d060e45af3..6be07da222 100644 --- a/src/kernel/mempool_persist.cpp +++ b/src/kernel/mempool_persist.cpp @@ -19,7 +19,6 @@ #include <util/time.h> #include <validation.h> -#include <chrono> #include <cstdint> #include <cstdio> #include <exception> @@ -37,11 +36,11 @@ namespace kernel { static const uint64_t MEMPOOL_DUMP_VERSION = 1; -bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, FopenFn mockable_fopen_function) +bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, ImportMempoolOptions&& opts) { if (load_path.empty()) return false; - FILE* filestr{mockable_fopen_function(load_path, "rb")}; + FILE* filestr{opts.mockable_fopen_function(load_path, "rb")}; CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); if (file.IsNull()) { LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); @@ -53,7 +52,7 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active int64_t failed = 0; int64_t already_there = 0; int64_t unbroadcast = 0; - auto now = NodeClock::now(); + const auto now{NodeClock::now()}; try { uint64_t version; @@ -72,8 +71,12 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active file >> nTime; file >> nFeeDelta; + if (opts.use_current_time) { + nTime = TicksSinceEpoch<std::chrono::seconds>(now); + } + CAmount amountdelta = nFeeDelta; - if (amountdelta) { + if (amountdelta && opts.apply_fee_delta_priority) { pool.PrioritiseTransaction(tx->GetHash(), amountdelta); } if (nTime > TicksSinceEpoch<std::chrono::seconds>(now - pool.m_expiry)) { @@ -101,17 +104,21 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active std::map<uint256, CAmount> mapDeltas; file >> mapDeltas; - for (const auto& i : mapDeltas) { - pool.PrioritiseTransaction(i.first, i.second); + if (opts.apply_fee_delta_priority) { + for (const auto& i : mapDeltas) { + pool.PrioritiseTransaction(i.first, i.second); + } } std::set<uint256> unbroadcast_txids; file >> unbroadcast_txids; - unbroadcast = unbroadcast_txids.size(); - for (const auto& txid : unbroadcast_txids) { - // Ensure transactions were accepted to mempool then add to - // unbroadcast set. - if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid); + if (opts.apply_unbroadcast_set) { + unbroadcast = unbroadcast_txids.size(); + for (const auto& txid : unbroadcast_txids) { + // Ensure transactions were accepted to mempool then add to + // unbroadcast set. + if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid); + } } } catch (const std::exception& e) { LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); diff --git a/src/kernel/mempool_persist.h b/src/kernel/mempool_persist.h index cb09119e4a..e124a8eadf 100644 --- a/src/kernel/mempool_persist.h +++ b/src/kernel/mempool_persist.h @@ -12,15 +12,21 @@ class CTxMemPool; namespace kernel { -/** Dump the mempool to disk. */ +/** Dump the mempool to a file. */ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen, bool skip_file_commit = false); -/** Load the mempool from disk. */ +struct ImportMempoolOptions { + fsbridge::FopenFn mockable_fopen_function{fsbridge::fopen}; + bool use_current_time{false}; + bool apply_fee_delta_priority{true}; + bool apply_unbroadcast_set{true}; +}; +/** Import the file and attempt to add its contents to the mempool. */ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, - fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen); + ImportMempoolOptions&& opts); } // namespace kernel diff --git a/src/net_processing.cpp b/src/net_processing.cpp index e2bbfe3308..6d011c239b 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -51,8 +51,6 @@ #include <optional> #include <typeinfo> -/** How long a transaction has to be in the mempool before it can unconditionally be relayed. */ -static constexpr auto UNCONDITIONAL_RELAY_DELAY = 2min; /** Headers download timeout. * Timeout = base + per_header * (expected number of headers) */ static constexpr auto HEADERS_DOWNLOAD_TIMEOUT_BASE = 15min; @@ -149,15 +147,12 @@ static constexpr auto OUTBOUND_INVENTORY_BROADCAST_INTERVAL{2s}; /** Maximum rate of inventory items to send per second. * Limits the impact of low-fee transaction floods. */ static constexpr unsigned int INVENTORY_BROADCAST_PER_SECOND = 7; +/** Target number of tx inventory items to send per transmission. */ +static constexpr unsigned int INVENTORY_BROADCAST_TARGET = INVENTORY_BROADCAST_PER_SECOND * count_seconds(INBOUND_INVENTORY_BROADCAST_INTERVAL); /** Maximum number of inventory items to send per transmission. */ -static constexpr unsigned int INVENTORY_BROADCAST_MAX = INVENTORY_BROADCAST_PER_SECOND * count_seconds(INBOUND_INVENTORY_BROADCAST_INTERVAL); -/** The number of most recently announced transactions a peer can request. */ -static constexpr unsigned int INVENTORY_MAX_RECENT_RELAY = 3500; -/** Verify that INVENTORY_MAX_RECENT_RELAY is enough to cache everything typically - * relayed before unconditional relay from the mempool kicks in. This is only a - * lower bound, and it should be larger to account for higher inv rate to outbound - * peers, and random variations in the broadcast mechanism. */ -static_assert(INVENTORY_MAX_RECENT_RELAY >= INVENTORY_BROADCAST_PER_SECOND * UNCONDITIONAL_RELAY_DELAY / std::chrono::seconds{1}, "INVENTORY_RELAY_MAX too low"); +static constexpr unsigned int INVENTORY_BROADCAST_MAX = 1000; +static_assert(INVENTORY_BROADCAST_MAX >= INVENTORY_BROADCAST_TARGET, "INVENTORY_BROADCAST_MAX too low"); +static_assert(INVENTORY_BROADCAST_MAX <= MAX_PEER_TX_ANNOUNCEMENTS, "INVENTORY_BROADCAST_MAX too high"); /** Average delay between feefilter broadcasts in seconds. */ static constexpr auto AVG_FEEFILTER_BROADCAST_INTERVAL{10min}; /** Maximum feefilter broadcast delay after significant change. */ @@ -273,13 +268,10 @@ struct Peer { /** A bloom filter for which transactions to announce to the peer. See BIP37. */ std::unique_ptr<CBloomFilter> m_bloom_filter PT_GUARDED_BY(m_bloom_filter_mutex) GUARDED_BY(m_bloom_filter_mutex){nullptr}; - /** A rolling bloom filter of all announced tx CInvs to this peer */ - CRollingBloomFilter m_recently_announced_invs GUARDED_BY(NetEventsInterface::g_msgproc_mutex){INVENTORY_MAX_RECENT_RELAY, 0.000001}; - mutable RecursiveMutex m_tx_inventory_mutex; - /** A filter of all the txids and wtxids that the peer has announced to + /** A filter of all the (w)txids that the peer has announced to * us or we have announced to the peer. We use this to avoid announcing - * the same txid/wtxid to a peer that already has the transaction. */ + * the same (w)txid to a peer that already has the transaction. */ CRollingBloomFilter m_tx_inventory_known_filter GUARDED_BY(m_tx_inventory_mutex){50000, 0.000001}; /** Set of transaction ids we still have to announce (txid for * non-wtxid-relay peers, wtxid for wtxid-relay peers). We use the @@ -290,11 +282,12 @@ struct Peer { * permitted if the peer has NetPermissionFlags::Mempool or we advertise * NODE_BLOOM. See BIP35. */ bool m_send_mempool GUARDED_BY(m_tx_inventory_mutex){false}; - /** The last time a BIP35 `mempool` request was serviced. */ - std::atomic<std::chrono::seconds> m_last_mempool_req{0s}; /** The next time after which we will send an `inv` message containing * transaction announcements to this peer. */ std::chrono::microseconds m_next_inv_send_time GUARDED_BY(m_tx_inventory_mutex){0}; + /** The mempool sequence num at which we sent the last `inv` message to this peer. + * Can relay txs with lower sequence numbers than this (see CTxMempool::info_for_relay). */ + uint64_t m_last_inv_sequence GUARDED_BY(NetEventsInterface::g_msgproc_mutex){1}; /** Minimum fee rate with which to filter transaction announcements to this node. See BIP133. */ std::atomic<CAmount> m_fee_filter_received{0}; @@ -907,7 +900,7 @@ private: std::atomic<std::chrono::seconds> m_last_tip_update{0s}; /** Determine whether or not a peer can request a transaction, and return it (or nullptr if not found or not allowed). */ - CTransactionRef FindTxForGetData(const Peer::TxRelay& tx_relay, const GenTxid& gtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) + CTransactionRef FindTxForGetData(const Peer::TxRelay& tx_relay, const GenTxid& gtxid) EXCLUSIVE_LOCKS_REQUIRED(!m_most_recent_block_mutex, NetEventsInterface::g_msgproc_mutex); void ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic<bool>& interruptMsgProc) @@ -2288,22 +2281,14 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& } } -CTransactionRef PeerManagerImpl::FindTxForGetData(const Peer::TxRelay& tx_relay, const GenTxid& gtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) +CTransactionRef PeerManagerImpl::FindTxForGetData(const Peer::TxRelay& tx_relay, const GenTxid& gtxid) { - auto txinfo = m_mempool.info(gtxid); + // If a tx was in the mempool prior to the last INV for this peer, permit the request. + auto txinfo = m_mempool.info_for_relay(gtxid, tx_relay.m_last_inv_sequence); if (txinfo.tx) { - // If a TX could have been INVed in reply to a MEMPOOL request, - // or is older than UNCONDITIONAL_RELAY_DELAY, permit the request - // unconditionally. - if ((mempool_req.count() && txinfo.m_time <= mempool_req) || txinfo.m_time <= now - UNCONDITIONAL_RELAY_DELAY) { - return std::move(txinfo.tx); - } + return std::move(txinfo.tx); } - // Otherwise, the transaction might have been announced recently. - bool recent = tx_relay.m_recently_announced_invs.contains(gtxid.GetHash()); - if (recent && txinfo.tx) return std::move(txinfo.tx); - // Or it might be from the most recent block { LOCK(m_most_recent_block_mutex); @@ -2326,10 +2311,6 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic std::vector<CInv> vNotFound; const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); - const auto now{GetTime<std::chrono::seconds>()}; - // Get last mempool request time - const auto mempool_req = tx_relay != nullptr ? tx_relay->m_last_mempool_req.load() : std::chrono::seconds::min(); - // Process as many TX items from the front of the getdata queue as // possible, since they're common and it's efficient to batch process // them. @@ -2347,33 +2328,12 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic continue; } - CTransactionRef tx = FindTxForGetData(*tx_relay, ToGenTxid(inv), mempool_req, now); + CTransactionRef tx = FindTxForGetData(*tx_relay, ToGenTxid(inv)); if (tx) { // WTX and WITNESS_TX imply we serialize with witness int nSendFlags = (inv.IsMsgTx() ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); m_connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *tx)); m_mempool.RemoveUnbroadcastTx(tx->GetHash()); - // As we're going to send tx, make sure its unconfirmed parents are made requestable. - std::vector<uint256> parent_ids_to_add; - { - LOCK(m_mempool.cs); - auto tx_iter = m_mempool.GetIter(tx->GetHash()); - if (tx_iter) { - const CTxMemPoolEntry::Parents& parents = (*tx_iter)->GetMemPoolParentsConst(); - parent_ids_to_add.reserve(parents.size()); - for (const CTxMemPoolEntry& parent : parents) { - if (parent.GetTime() > now - UNCONDITIONAL_RELAY_DELAY) { - parent_ids_to_add.push_back(parent.GetTx().GetHash()); - } - } - } - } - for (const uint256& parent_txid : parent_ids_to_add) { - // Relaying a transaction with a recent but unconfirmed parent. - if (WITH_LOCK(tx_relay->m_tx_inventory_mutex, return !tx_relay->m_tx_inventory_known_filter.contains(parent_txid))) { - tx_relay->m_recently_announced_invs.insert(parent_txid); - } - } } else { vNotFound.push_back(inv); } @@ -4131,14 +4091,6 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, const uint256& hash = peer->m_wtxid_relay ? wtxid : txid; AddKnownTx(*peer, hash); - if (peer->m_wtxid_relay && txid != wtxid) { - // Insert txid into m_tx_inventory_known_filter, even for - // wtxidrelay peers. This prevents re-adding of - // unconfirmed parents to the recently_announced - // filter, when a child tx is requested. See - // ProcessGetData(). - AddKnownTx(*peer, txid); - } LOCK(cs_main); @@ -5684,7 +5636,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) std::vector<CInv> vInv; { LOCK(peer->m_block_inv_mutex); - vInv.reserve(std::max<size_t>(peer->m_blocks_for_inv_relay.size(), INVENTORY_BROADCAST_MAX)); + vInv.reserve(std::max<size_t>(peer->m_blocks_for_inv_relay.size(), INVENTORY_BROADCAST_TARGET)); // Add blocks for (const uint256& hash : peer->m_blocks_for_inv_relay) { @@ -5736,14 +5688,12 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (!tx_relay->m_bloom_filter->IsRelevantAndUpdate(*txinfo.tx)) continue; } tx_relay->m_tx_inventory_known_filter.insert(hash); - // Responses to MEMPOOL requests bypass the m_recently_announced_invs filter. vInv.push_back(inv); if (vInv.size() == MAX_INV_SZ) { m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } } - tx_relay->m_last_mempool_req = std::chrono::duration_cast<std::chrono::seconds>(current_time); } // Determine transactions to relay @@ -5763,8 +5713,8 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // especially since we have many peers and some will draw much shorter delays. unsigned int nRelayedTransactions = 0; LOCK(tx_relay->m_bloom_filter_mutex); - size_t broadcast_max{INVENTORY_BROADCAST_MAX + (tx_relay->m_tx_inventory_to_send.size()/1000)*5}; - broadcast_max = std::min<size_t>(1000, broadcast_max); + size_t broadcast_max{INVENTORY_BROADCAST_TARGET + (tx_relay->m_tx_inventory_to_send.size()/1000)*5}; + broadcast_max = std::min<size_t>(INVENTORY_BROADCAST_MAX, broadcast_max); while (!vInvTx.empty() && nRelayedTransactions < broadcast_max) { // Fetch the top element from the heap std::pop_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); @@ -5783,14 +5733,12 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (!txinfo.tx) { continue; } - auto txid = txinfo.tx->GetHash(); // Peer told you to not send transactions at that feerate? Don't bother sending it. if (txinfo.fee < filterrate.GetFee(txinfo.vsize)) { continue; } if (tx_relay->m_bloom_filter && !tx_relay->m_bloom_filter->IsRelevantAndUpdate(*txinfo.tx)) continue; // Send - tx_relay->m_recently_announced_invs.insert(hash); vInv.push_back(inv); nRelayedTransactions++; if (vInv.size() == MAX_INV_SZ) { @@ -5798,15 +5746,11 @@ bool PeerManagerImpl::SendMessages(CNode* pto) vInv.clear(); } tx_relay->m_tx_inventory_known_filter.insert(hash); - if (hash != txid) { - // Insert txid into m_tx_inventory_known_filter, even for - // wtxidrelay peers. This prevents re-adding of - // unconfirmed parents to the recently_announced - // filter, when a child tx is requested. See - // ProcessGetData(). - tx_relay->m_tx_inventory_known_filter.insert(txid); - } } + + // Ensure we'll respond to GETDATA requests for anything we've just announced + LOCK(m_mempool.cs); + tx_relay->m_last_inv_sequence = m_mempool.GetSequence(); } } if (!vInv.empty()) diff --git a/src/net_processing.h b/src/net_processing.h index a0cbe92289..837e308617 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -17,9 +17,10 @@ class ChainstateManager; /** Whether transaction reconciliation protocol should be enabled by default. */ static constexpr bool DEFAULT_TXRECONCILIATION_ENABLE{false}; /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ -static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; -/** Default number of orphan+recently-replaced txn to keep around for block reconstruction */ -static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; +static const uint32_t DEFAULT_MAX_ORPHAN_TRANSACTIONS{100}; +/** Default number of non-mempool transactions to keep around for block reconstruction. Includes + orphan, replaced, and rejected transactions. */ +static const uint32_t DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN{100}; static const bool DEFAULT_PEERBLOOMFILTERS = false; static const bool DEFAULT_PEERBLOCKFILTERS = false; /** Threshold for marking a node to be discouraged, e.g. disconnected and added to the discouragement filter. */ @@ -46,11 +47,16 @@ class PeerManager : public CValidationInterface, public NetEventsInterface { public: struct Options { - /** Whether this node is running in -blocksonly mode */ + //! Whether this node is running in -blocksonly mode bool ignore_incoming_txs{DEFAULT_BLOCKSONLY}; + //! Whether transaction reconciliation protocol is enabled bool reconcile_txs{DEFAULT_TXRECONCILIATION_ENABLE}; + //! Maximum number of orphan transactions kept in memory uint32_t max_orphan_txs{DEFAULT_MAX_ORPHAN_TRANSACTIONS}; - size_t max_extra_txs{DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN}; + //! Number of non-mempool transactions to keep around for block reconstruction. Includes + //! orphan, replaced, and rejected transactions. + uint32_t max_extra_txs{DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN}; + //! Whether all P2P messages are captured to disk bool capture_messages{false}; }; diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 42e021fcc9..14192bd962 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -677,7 +677,7 @@ public: { if (!m_node.mempool) return true; LockPoints lp; - CTxMemPoolEntry entry(tx, 0, 0, 0, false, 0, lp); + CTxMemPoolEntry entry(tx, 0, 0, 0, 0, false, 0, lp); const CTxMemPool::Limits& limits{m_node.mempool->m_limits}; LOCK(m_node.mempool->cs); return m_node.mempool->CalculateMemPoolAncestors(entry, limits).has_value(); diff --git a/src/node/peerman_args.cpp b/src/node/peerman_args.cpp index e0dcf21c33..b0ae266365 100644 --- a/src/node/peerman_args.cpp +++ b/src/node/peerman_args.cpp @@ -3,6 +3,9 @@ #include <common/args.h> #include <net_processing.h> +#include <algorithm> +#include <limits> + namespace node { void ApplyArgsManOptions(const ArgsManager& argsman, PeerManager::Options& options) @@ -10,11 +13,11 @@ void ApplyArgsManOptions(const ArgsManager& argsman, PeerManager::Options& optio if (auto value{argsman.GetBoolArg("-txreconciliation")}) options.reconcile_txs = *value; if (auto value{argsman.GetIntArg("-maxorphantx")}) { - options.max_orphan_txs = uint32_t(std::max(int64_t{0}, *value)); + options.max_orphan_txs = uint32_t((std::clamp<int64_t>(*value, 0, std::numeric_limits<uint32_t>::max()))); } if (auto value{argsman.GetIntArg("-blockreconstructionextratxn")}) { - options.max_extra_txs = size_t(std::max(int64_t{0}, *value)); + options.max_extra_txs = uint32_t((std::clamp<int64_t>(*value, 0, std::numeric_limits<uint32_t>::max()))); } if (auto value{argsman.GetBoolArg("-capturemessages")}) options.capture_messages = *value; diff --git a/src/pubkey.h b/src/pubkey.h index 7d37504b01..00defa25a0 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -299,6 +299,9 @@ private: std::array<std::byte, SIZE> m_pubkey; public: + /** Default constructor creates all-zero pubkey (which is valid). */ + EllSwiftPubKey() noexcept = default; + /** Construct a new ellswift public key from a given serialization. */ EllSwiftPubKey(const std::array<std::byte, SIZE>& ellswift) : m_pubkey(ellswift) {} diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index d289a9240e..2908c37c1f 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -229,6 +229,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "importaddress", 2, "rescan" }, { "importaddress", 3, "p2sh" }, { "importpubkey", 2, "rescan" }, + { "importmempool", 1, "options" }, + { "importmempool", 1, "apply_fee_delta_priority" }, + { "importmempool", 1, "use_current_time" }, + { "importmempool", 1, "apply_unbroadcast_set" }, { "importmulti", 0, "requests" }, { "importmulti", 1, "options" }, { "importmulti", 1, "rescan" }, diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 11d2874961..90ee2a48af 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -696,7 +696,7 @@ static RPCHelpMan getmempoolinfo() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::BOOL, "loaded", "True if the mempool is fully loaded"}, + {RPCResult::Type::BOOL, "loaded", "True if the initial load attempt of the persisted mempool finished"}, {RPCResult::Type::NUM, "size", "Current tx count"}, {RPCResult::Type::NUM, "bytes", "Sum of all virtual transaction sizes as defined in BIP 141. Differs from actual serialized size because witness data is discounted"}, {RPCResult::Type::NUM, "usage", "Total memory usage for the mempool"}, @@ -719,6 +719,66 @@ static RPCHelpMan getmempoolinfo() }; } +static RPCHelpMan importmempool() +{ + return RPCHelpMan{ + "importmempool", + "Import a mempool.dat file and attempt to add its contents to the mempool.\n" + "Warning: Importing untrusted files is dangerous, especially if metadata from the file is taken over.", + { + {"filepath", RPCArg::Type::STR, RPCArg::Optional::NO, "The mempool file"}, + {"options", + RPCArg::Type::OBJ_NAMED_PARAMS, + RPCArg::Optional::OMITTED, + "", + { + {"use_current_time", RPCArg::Type::BOOL, RPCArg::Default{true}, + "Whether to use the current system time or use the entry time metadata from the mempool file.\n" + "Warning: Importing untrusted metadata may lead to unexpected issues and undesirable behavior."}, + {"apply_fee_delta_priority", RPCArg::Type::BOOL, RPCArg::Default{false}, + "Whether to apply the fee delta metadata from the mempool file.\n" + "It will be added to any existing fee deltas.\n" + "The fee delta can be set by the prioritisetransaction RPC.\n" + "Warning: Importing untrusted metadata may lead to unexpected issues and undesirable behavior.\n" + "Only set this bool if you understand what it does."}, + {"apply_unbroadcast_set", RPCArg::Type::BOOL, RPCArg::Default{false}, + "Whether to apply the unbroadcast set metadata from the mempool file.\n" + "Warning: Importing untrusted metadata may lead to unexpected issues and undesirable behavior."}, + }, + RPCArgOptions{.oneline_description = "\"options\""}}, + }, + RPCResult{RPCResult::Type::OBJ, "", "", std::vector<RPCResult>{}}, + RPCExamples{HelpExampleCli("importmempool", "/path/to/mempool.dat") + HelpExampleRpc("importmempool", "/path/to/mempool.dat")}, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + const NodeContext& node{EnsureAnyNodeContext(request.context)}; + + CTxMemPool& mempool{EnsureMemPool(node)}; + Chainstate& chainstate = EnsureChainman(node).ActiveChainstate(); + + if (chainstate.IsInitialBlockDownload()) { + throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Can only import the mempool after the block download and sync is done."); + } + + const fs::path load_path{fs::u8path(request.params[0].get_str())}; + const UniValue& use_current_time{request.params[1]["use_current_time"]}; + const UniValue& apply_fee_delta{request.params[1]["apply_fee_delta_priority"]}; + const UniValue& apply_unbroadcast{request.params[1]["apply_unbroadcast_set"]}; + kernel::ImportMempoolOptions opts{ + .use_current_time = use_current_time.isNull() ? true : use_current_time.get_bool(), + .apply_fee_delta_priority = apply_fee_delta.isNull() ? false : apply_fee_delta.get_bool(), + .apply_unbroadcast_set = apply_unbroadcast.isNull() ? false : apply_unbroadcast.get_bool(), + }; + + if (!kernel::LoadMempool(mempool, load_path, chainstate, std::move(opts))) { + throw JSONRPCError(RPC_MISC_ERROR, "Unable to import mempool file, see debug.log for details."); + } + + UniValue ret{UniValue::VOBJ}; + return ret; + }, + }; +} + static RPCHelpMan savemempool() { return RPCHelpMan{"savemempool", @@ -921,6 +981,7 @@ void RegisterMempoolRPCCommands(CRPCTable& t) {"blockchain", &gettxspendingprevout}, {"blockchain", &getmempoolinfo}, {"blockchain", &getrawmempool}, + {"blockchain", &importmempool}, {"blockchain", &savemempool}, {"hidden", &submitpackage}, }; diff --git a/src/test/bip324_tests.cpp b/src/test/bip324_tests.cpp new file mode 100644 index 0000000000..04472611ec --- /dev/null +++ b/src/test/bip324_tests.cpp @@ -0,0 +1,306 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bip324.h> +#include <chainparams.h> +#include <key.h> +#include <pubkey.h> +#include <span.h> +#include <test/util/random.h> +#include <test/util/setup_common.h> +#include <util/strencodings.h> + +#include <array> +#include <cstddef> +#include <cstdint> +#include <vector> + +#include <boost/test/unit_test.hpp> + +namespace { + +void TestBIP324PacketVector( + uint32_t in_idx, + const std::string& in_priv_ours_hex, + const std::string& in_ellswift_ours_hex, + const std::string& in_ellswift_theirs_hex, + bool in_initiating, + const std::string& in_contents_hex, + uint32_t in_multiply, + const std::string& in_aad_hex, + bool in_ignore, + const std::string& mid_send_garbage_hex, + const std::string& mid_recv_garbage_hex, + const std::string& out_session_id_hex, + const std::string& out_ciphertext_hex, + const std::string& out_ciphertext_endswith_hex) +{ + // Convert input from hex to char/byte vectors/arrays. + const auto in_priv_ours = ParseHex(in_priv_ours_hex); + const auto in_ellswift_ours_vec = ParseHex<std::byte>(in_ellswift_ours_hex); + assert(in_ellswift_ours_vec.size() == 64); + std::array<std::byte, 64> in_ellswift_ours; + std::copy(in_ellswift_ours_vec.begin(), in_ellswift_ours_vec.end(), in_ellswift_ours.begin()); + const auto in_ellswift_theirs_vec = ParseHex<std::byte>(in_ellswift_theirs_hex); + assert(in_ellswift_theirs_vec.size() == 64); + std::array<std::byte, 64> in_ellswift_theirs; + std::copy(in_ellswift_theirs_vec.begin(), in_ellswift_theirs_vec.end(), in_ellswift_theirs.begin()); + const auto in_contents = ParseHex<std::byte>(in_contents_hex); + const auto in_aad = ParseHex<std::byte>(in_aad_hex); + const auto mid_send_garbage = ParseHex<std::byte>(mid_send_garbage_hex); + const auto mid_recv_garbage = ParseHex<std::byte>(mid_recv_garbage_hex); + const auto out_session_id = ParseHex<std::byte>(out_session_id_hex); + const auto out_ciphertext = ParseHex<std::byte>(out_ciphertext_hex); + const auto out_ciphertext_endswith = ParseHex<std::byte>(out_ciphertext_endswith_hex); + + // Load keys + CKey key; + key.Set(in_priv_ours.begin(), in_priv_ours.end(), true); + EllSwiftPubKey ellswift_ours(in_ellswift_ours); + EllSwiftPubKey ellswift_theirs(in_ellswift_theirs); + + // Instantiate encryption BIP324 cipher. + BIP324Cipher cipher(key, ellswift_ours); + BOOST_CHECK(!cipher); + BOOST_CHECK(cipher.GetOurPubKey() == ellswift_ours); + cipher.Initialize(ellswift_theirs, in_initiating); + BOOST_CHECK(cipher); + + // Compare session variables. + BOOST_CHECK(Span{out_session_id} == cipher.GetSessionID()); + BOOST_CHECK(Span{mid_send_garbage} == cipher.GetSendGarbageTerminator()); + BOOST_CHECK(Span{mid_recv_garbage} == cipher.GetReceiveGarbageTerminator()); + + // Vector of encrypted empty messages, encrypted in order to seek to the right position. + std::vector<std::vector<std::byte>> dummies(in_idx); + + // Seek to the numbered packet. + for (uint32_t i = 0; i < in_idx; ++i) { + dummies[i].resize(cipher.EXPANSION); + cipher.Encrypt({}, {}, true, dummies[i]); + } + + // Construct contents and encrypt it. + std::vector<std::byte> contents; + for (uint32_t i = 0; i < in_multiply; ++i) { + contents.insert(contents.end(), in_contents.begin(), in_contents.end()); + } + std::vector<std::byte> ciphertext(contents.size() + cipher.EXPANSION); + cipher.Encrypt(contents, in_aad, in_ignore, ciphertext); + + // Verify ciphertext. Note that the test vectors specify either out_ciphertext (for short + // messages) or out_ciphertext_endswith (for long messages), so only check the relevant one. + if (!out_ciphertext.empty()) { + BOOST_CHECK(out_ciphertext == ciphertext); + } else { + BOOST_CHECK(ciphertext.size() >= out_ciphertext_endswith.size()); + BOOST_CHECK(Span{out_ciphertext_endswith} == Span{ciphertext}.last(out_ciphertext_endswith.size())); + } + + for (unsigned error = 0; error <= 12; ++error) { + // error selects a type of error introduced: + // - error=0: no errors, decryption should be successful + // - error=1: wrong side + // - error=2..9: bit error in ciphertext + // - error=10: bit error in aad + // - error=11: extra 0x00 at end of aad + // - error=12: message index wrong + + // Instantiate self-decrypting BIP324 cipher. + BIP324Cipher dec_cipher(key, ellswift_ours); + BOOST_CHECK(!dec_cipher); + BOOST_CHECK(dec_cipher.GetOurPubKey() == ellswift_ours); + dec_cipher.Initialize(ellswift_theirs, (error == 1) ^ in_initiating, /*self_decrypt=*/true); + BOOST_CHECK(dec_cipher); + + // Compare session variables. + BOOST_CHECK((Span{out_session_id} == dec_cipher.GetSessionID()) == (error != 1)); + BOOST_CHECK((Span{mid_send_garbage} == dec_cipher.GetSendGarbageTerminator()) == (error != 1)); + BOOST_CHECK((Span{mid_recv_garbage} == dec_cipher.GetReceiveGarbageTerminator()) == (error != 1)); + + // Seek to the numbered packet. + if (in_idx == 0 && error == 12) continue; + uint32_t dec_idx = in_idx ^ (error == 12 ? (1U << InsecureRandRange(16)) : 0); + for (uint32_t i = 0; i < dec_idx; ++i) { + unsigned use_idx = i < in_idx ? i : 0; + bool dec_ignore{false}; + dec_cipher.DecryptLength(Span{dummies[use_idx]}.first(cipher.LENGTH_LEN)); + dec_cipher.Decrypt(Span{dummies[use_idx]}.subspan(cipher.LENGTH_LEN), {}, dec_ignore, {}); + } + + // Construct copied (and possibly damaged) copy of ciphertext. + // Decrypt length + auto to_decrypt = ciphertext; + if (error >= 2 && error <= 9) { + to_decrypt[InsecureRandRange(to_decrypt.size())] ^= std::byte(1U << (error - 2)); + } + + // Decrypt length and resize ciphertext to accommodate. + uint32_t dec_len = dec_cipher.DecryptLength(MakeByteSpan(to_decrypt).first(cipher.LENGTH_LEN)); + to_decrypt.resize(dec_len + cipher.EXPANSION); + + // Construct copied (and possibly damaged) copy of aad. + auto dec_aad = in_aad; + if (error == 10) { + if (in_aad.size() == 0) continue; + dec_aad[InsecureRandRange(dec_aad.size())] ^= std::byte(1U << InsecureRandRange(8)); + } + if (error == 11) dec_aad.push_back({}); + + // Decrypt contents. + std::vector<std::byte> decrypted(dec_len); + bool dec_ignore{false}; + bool dec_ok = dec_cipher.Decrypt(Span{to_decrypt}.subspan(cipher.LENGTH_LEN), dec_aad, dec_ignore, decrypted); + + // Verify result. + BOOST_CHECK(dec_ok == !error); + if (dec_ok) { + BOOST_CHECK(decrypted == contents); + BOOST_CHECK(dec_ignore == in_ignore); + } + } +} + +} // namespace + +BOOST_FIXTURE_TEST_SUITE(bip324_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(packet_test_vectors) { + // BIP324 key derivation uses network magic in the HKDF process. We use mainnet params here + // as that is what the test vectors are written for. + SelectParams(ChainType::MAIN); + + // The test vectors are converted using the following Python code in the BIP bip-0324/ directory: + // + // import sys + // import csv + // with open('packet_encoding_test_vectors.csv', newline='', encoding='utf-8') as csvfile: + // reader = csv.DictReader(csvfile) + // quote = lambda x: "\"" + x + "\"" + // for row in reader: + // args = [ + // row['in_idx'], + // quote(row['in_priv_ours']), + // quote(row['in_ellswift_ours']), + // quote(row['in_ellswift_theirs']), + // "true" if int(row['in_initiating']) else "false", + // quote(row['in_contents']), + // row['in_multiply'], + // quote(row['in_aad']), + // "true" if int(row['in_ignore']) else "false", + // quote(row['mid_send_garbage_terminator']), + // quote(row['mid_recv_garbage_terminator']), + // quote(row['out_session_id']), + // quote(row['out_ciphertext']), + // quote(row['out_ciphertext_endswith']) + // ] + // print(" TestBIP324PacketVector(\n " + ",\n ".join(args) + ");") + TestBIP324PacketVector( + 1, + "61062ea5071d800bbfd59e2e8b53d47d194b095ae5a4df04936b49772ef0d4d7", + "ec0adff257bbfe500c188c80b4fdd640f6b45a482bbc15fc7cef5931deff0aa186f6eb9bba7b85dc4dcc28b28722de1e3d9108b985e2967045668f66098e475b", + "a4a94dfce69b4a2a0a099313d10f9f7e7d649d60501c9e1d274c300e0d89aafaffffffffffffffffffffffffffffffffffffffffffffffffffffffff8faf88d5", + true, + "8e", + 1, + "", + false, + "faef555dfcdb936425d84aba524758f3", + "02cb8ff24307a6e27de3b4e7ea3fa65b", + "ce72dffb015da62b0d0f5474cab8bc72605225b0cee3f62312ec680ec5f41ba5", + "7530d2a18720162ac09c25329a60d75adf36eda3c3", + ""); + TestBIP324PacketVector( + 999, + "1f9c581b35231838f0f17cf0c979835baccb7f3abbbb96ffcc318ab71e6e126f", + "a1855e10e94e00baa23041d916e259f7044e491da6171269694763f018c7e63693d29575dcb464ac816baa1be353ba12e3876cba7628bd0bd8e755e721eb0140", + "fffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2f0000000000000000000000000000000000000000000000000000000000000000", + false, + "3eb1d4e98035cfd8eeb29bac969ed3824a", + 1, + "", + false, + "efb64fd80acd3825ac9bc2a67216535a", + "b3cb553453bceb002897e751ff7588bf", + "9267c54560607de73f18c563b76a2442718879c52dd39852885d4a3c9912c9ea", + "1da1bcf589f9b61872f45b7fa5371dd3f8bdf5d515b0c5f9fe9f0044afb8dc0aa1cd39a8c4", + ""); + TestBIP324PacketVector( + 0, + "0286c41cd30913db0fdff7a64ebda5c8e3e7cef10f2aebc00a7650443cf4c60d", + "d1ee8a93a01130cbf299249a258f94feb5f469e7d0f2f28f69ee5e9aa8f9b54a60f2c3ff2d023634ec7f4127a96cc11662e402894cf1f694fb9a7eaa5f1d9244", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff22d5e441524d571a52b3def126189d3f416890a99d4da6ede2b0cde1760ce2c3f98457ae", + true, + "054290a6c6ba8d80478172e89d32bf690913ae9835de6dcf206ff1f4d652286fe0ddf74deba41d55de3edc77c42a32af79bbea2c00bae7492264c60866ae5a", + 1, + "84932a55aac22b51e7b128d31d9f0550da28e6a3f394224707d878603386b2f9d0c6bcd8046679bfed7b68c517e7431e75d9dd34605727d2ef1c2babbf680ecc8d68d2c4886e9953a4034abde6da4189cd47c6bb3192242cf714d502ca6103ee84e08bc2ca4fd370d5ad4e7d06c7fbf496c6c7cc7eb19c40c61fb33df2a9ba48497a96c98d7b10c1f91098a6b7b16b4bab9687f27585ade1491ae0dba6a79e1e2d85dd9d9d45c5135ca5fca3f0f99a60ea39edbc9efc7923111c937913f225d67788d5f7e8852b697e26b92ec7bfcaa334a1665511c2b4c0a42d06f7ab98a9719516c8fd17f73804555ee84ab3b7d1762f6096b778d3cb9c799cbd49a9e4a325197b4e6cc4a5c4651f8b41ff88a92ec428354531f970263b467c77ed11312e2617d0d53fe9a8707f51f9f57a77bfb49afe3d89d85ec05ee17b9186f360c94ab8bb2926b65ca99dae1d6ee1af96cad09de70b6767e949023e4b380e66669914a741ed0fa420a48dbc7bfae5ef2019af36d1022283dd90655f25eec7151d471265d22a6d3f91dc700ba749bb67c0fe4bc0888593fbaf59d3c6fff1bf756a125910a63b9682b597c20f560ecb99c11a92c8c8c3f7fbfaa103146083a0ccaecf7a5f5e735a784a8820155914a289d57d8141870ffcaf588882332e0bcd8779efa931aa108dab6c3cce76691e345df4a91a03b71074d66333fd3591bff071ea099360f787bbe43b7b3dff2a59c41c7642eb79870222ad1c6f2e5a191ed5acea51134679587c9cf71c7d8ee290be6bf465c4ee47897a125708704ad610d8d00252d01959209d7cd04d5ecbbb1419a7e84037a55fefa13dee464b48a35c96bcb9a53e7ed461c3a1607ee00c3c302fd47cd73fda7493e947c9834a92d63dcfbd65aa7c38c3e3a2748bb5d9a58e7495d243d6b741078c8f7ee9c8813e473a323375702702b0afae1550c8341eedf5247627343a95240cb02e3e17d5dca16f8d8d3b2228e19c06399f8ec5c5e9dbe4caef6a0ea3ffb1d3c7eac03ae030e791fa12e537c80d56b55b764cadf27a8701052df1282ba8b5e3eb62b5dc7973ac40160e00722fa958d95102fc25c549d8c0e84bed95b7acb61ba65700c4de4feebf78d13b9682c52e937d23026fb4c6193e6644e2d3c99f91f4f39a8b9fc6d013f89c3793ef703987954dc0412b550652c01d922f525704d32d70d6d4079bc3551b563fb29577b3aecdc9505011701dddfd94830431e7a4918927ee44fb3831ce8c4513839e2deea1287f3fa1ab9b61a256c09637dbc7b4f0f8fbb783840f9c24526da883b0df0c473cf231656bd7bc1aaba7f321fec0971c8c2c3444bff2f55e1df7fea66ec3e440a612db9aa87bb505163a59e06b96d46f50d8120b92814ac5ab146bc78dbbf91065af26107815678ce6e33812e6bf3285d4ef3b7b04b076f21e7820dcbfdb4ad5218cf4ff6a65812d8fcb98ecc1e95e2fa58e3efe4ce26cd0bd400d6036ab2ad4f6c713082b5e3f1e04eb9e3b6c8f63f57953894b9e220e0130308e1fd91f72d398c1e7962ca2c31be83f31d6157633581a0a6910496de8d55d3d07090b6aa087159e388b7e7dec60f5d8a60d93ca2ae91296bd484d916bfaaa17c8f45ea4b1a91b37c82821199a2b7596672c37156d8701e7352aa48671d3b1bbbd2bd5f0a2268894a25b0cb2514af39c8743f8cce8ab4b523053739fd8a522222a09acf51ac704489cf17e4b7125455cb8f125b4d31af1eba1f8cf7f81a5a100a141a7ee72e8083e065616649c241f233645c5fc865d17f0285f5c52d9f45312c979bfb3ce5f2a1b951deddf280ffb3f370410cffd1583bfa90077835aa201a0712d1dcd1293ee177738b14e6b5e2a496d05220c3253bb6578d6aff774be91946a614dd7e879fb3dcf7451e0b9adb6a8c44f53c2c464bcc0019e9fad89cac7791a0a3f2974f759a9856351d4d2d7c5612c17cfc50f8479945df57716767b120a590f4bf656f4645029a525694d8a238446c5f5c2c1c995c09c1405b8b1eb9e0352ffdf766cc964f8dcf9f8f043dfab6d102cf4b298021abd78f1d9025fa1f8e1d710b38d9d1652f2d88d1305874ec41609b6617b65c5adb19b6295dc5c5da5fdf69f28144ea12f17c3c6fcce6b9b5157b3dfc969d6725fa5b098a4d9b1d31547ed4c9187452d281d0a5d456008caf1aa251fac8f950ca561982dc2dc908d3691ee3b6ad3ae3d22d002577264ca8e49c523bd51c4846be0d198ad9407bf6f7b82c79893eb2c05fe9981f687a97a4f01fe45ff8c8b7ecc551135cd960a0d6001ad35020be07ffb53cb9e731522ca8ae9364628914b9b8e8cc2f37f03393263603cc2b45295767eb0aac29b0930390eb89587ab2779d2e3decb8042acece725ba42eda650863f418f8d0d50d104e44fbbe5aa7389a4a144a8cecf00f45fb14c39112f9bfb56c0acbd44fa3ff261f5ce4acaa5134c2c1d0cca447040820c81ab1bcdc16aa075b7c68b10d06bbb7ce08b5b805e0238f24402cf24a4b4e00701935a0c68add3de090903f9b85b153cb179a582f57113bfc21c2093803f0cfa4d9d4672c2b05a24f7e4c34a8e9101b70303a7378b9c50b6cddd46814ef7fd73ef6923feceab8fc5aa8b0d185f2e83c7a99dcb1077c0ab5c1f5d5f01ba2f0420443f75c4417db9ebf1665efbb33dca224989920a64b44dc26f682cc77b4632c8454d49135e52503da855bc0f6ff8edc1145451a9772c06891f41064036b66c3119a0fc6e80dffeb65dc456108b7ca0296f4175fff3ed2b0f842cd46bd7e86f4c62dfaf1ddbf836263c00b34803de164983d0811cebfac86e7720c726d3048934c36c23189b02386a722ca9f0fe00233ab50db928d3bccea355cc681144b8b7edcaae4884d5a8f04425c0890ae2c74326e138066d8c05f4c82b29df99b034ea727afde590a1f2177ace3af99cfb1729d6539ce7f7f7314b046aab74497e63dd399e1f7d5f16517c23bd830d1fdee810f3c3b77573dd69c4b97d80d71fb5a632e00acdfa4f8e829faf3580d6a72c40b28a82172f8dcd4627663ebf6069736f21735fd84a226f427cd06bb055f94e7c92f31c48075a2955d82a5b9d2d0198ce0d4e131a112570a8ee40fb80462a81436a58e7db4e34b6e2c422e82f934ecda9949893da5730fc5c23c7c920f363f85ab28cc6a4206713c3152669b47efa8238fa826735f17b4e78750276162024ec85458cd5808e06f40dd9fd43775a456a3ff6cae90550d76d8b2899e0762ad9a371482b3e38083b1274708301d6346c22fea9bb4b73db490ff3ab05b2f7f9e187adef139a7794454b7300b8cc64d3ad76c0e4bc54e08833a4419251550655380d675bc91855aeb82585220bb97f03e976579c08f321b5f8f70988d3061f41465517d53ac571dbf1b24b94443d2e9a8e8a79b392b3d6a4ecdd7f626925c365ef6221305105ce9b5f5b6ecc5bed3d702bd4b7f5008aa8eb8c7aa3ade8ecf6251516fbefeea4e1082aa0e1848eddb31ffe44b04792d296054402826e4bd054e671f223e5557e4c94f89ca01c25c44f1a2ff2c05a70b43408250705e1b858bf0670679fdcd379203e36be3500dd981b1a6422c3cf15224f7fefdef0a5f225c5a09d15767598ecd9e262460bb33a4b5d09a64591efabc57c923d3be406979032ae0bc0997b65336a06dd75b253332ad6a8b63ef043f780a1b3fb6d0b6cad98b1ef4a02535eb39e14a866cfc5fc3a9c5deb2261300d71280ebe66a0776a151469551c3c5fa308757f956655278ec6330ae9e3625468c5f87e02cd9a6489910d4143c1f4ee13aa21a6859d907b788e28572fecee273d44e4a900fa0aa668dd861a60fb6b6b12c2c5ef3c8df1bd7ef5d4b0d1cdb8c15fffbb365b9784bd94abd001c6966216b9b67554ad7cb7f958b70092514f7800fc40244003e0fd1133a9b850fb17f4fcafde07fc87b07fb510670654a5d2d6fc9876ac74728ea41593beef003d6858786a52d3a40af7529596767c17000bfaf8dc52e871359f4ad8bf6e7b2853e5229bdf39657e213580294a5317c5df172865e1e17fe37093b585e04613f5f078f761b2b1752eb32983afda24b523af8851df9a02b37e77f543f18888a782a994a50563334282bf9cdfccc183fdf4fcd75ad86ee0d94f91ee2300a5befbccd14e03a77fc031a8cfe4f01e4c5290f5ac1da0d58ea054bd4837cfd93e5e34fc0eb16e48044ba76131f228d16cde9b0bb978ca7cdcd10653c358bdb26fdb723a530232c32ae0a4cecc06082f46e1c1d596bfe60621ad1e354e01e07b040cc7347c016653f44d926d13ca74e6cbc9d4ab4c99f4491c95c76fff5076b3936eb9d0a286b97c035ca88a3c6309f5febfd4cdaac869e4f58ed409b1e9eb4192fb2f9c2f12176d460fd98286c9d6df84598f260119fd29c63f800c07d8df83d5cc95f8c2fea2812e7890e8a0718bb1e031ecbebc0436dcf3e3b9a58bcc06b4c17f711f80fe1dffc3326a6eb6e00283055c6dabe20d311bfd5019591b7954f8163c9afad9ef8390a38f3582e0a79cdf0353de8eeb6b5f9f27b16ffdef7dd62869b4840ee226ccdce95e02c4545eb981b60571cd83f03dc5eaf8c97a0829a4318a9b3dc06c0e003db700b2260ff1fa8fee66890e637b109abb03ec901b05ca599775f48af50154c0e67d82bf0f558d7d3e0778dc38bea1eb5f74dc8d7f90abdf5511a424be66bf8b6a3cacb477d2e7ef4db68d2eba4d5289122d851f9501ba7e9c4957d8eba3be3fc8e785c4265a1d65c46f2809b70846c693864b169c9dcb78be26ea14b8613f145b01887222979a9e67aee5f800caa6f5c4229bdeefc901232ace6143c9865e4d9c07f51aa200afaf7e48a7d1d8faf366023beab12906ffcb3eaf72c0eb68075e4daf3c080e0c31911befc16f0cc4a09908bb7c1e26abab38bd7b788e1a09c0edf1a35a38d2ff1d3ed47fcdaae2f0934224694f5b56705b9409b6d3d64f3833b686f7576ec64bbdd6ff174e56c2d1edac0011f904681a73face26573fbba4e34652f7ae84acfb2fa5a5b3046f98178cd0831df7477de70e06a4c00e305f31aafc026ef064dd68fd3e4252b1b91d617b26c6d09b6891a00df68f105b5962e7f9d82da101dd595d286da721443b72b2aba2377f6e7772e33b3a5e3753da9c2578c5d1daab80187f55518c72a64ee150a7cb5649823c08c9f62cd7d020b45ec2cba8310db1a7785a46ab24785b4d54ff1660b5ca78e05a9a55edba9c60bf044737bc468101c4e8bd1480d749be5024adefca1d998abe33eaeb6b11fbb39da5d905fdd3f611b2e51517ccee4b8af72c2d948573505590d61a6783ab7278fc43fe55b1fcc0e7216444d3c8039bb8145ef1ce01c50e95a3f3feab0aee883fdb94cc13ee4d21c542aa795e18932228981690f4d4c57ca4db6eb5c092e29d8a05139d509a8aeb48baa1eb97a76e597a32b280b5e9d6c36859064c98ff96ef5126130264fa8d2f49213870d9fb036cff95da51f270311d9976208554e48ffd486470d0ecdb4e619ccbd8226147204baf8e235f54d8b1cba8fa34a9a4d055de515cdf180d2bb6739a175183c472e30b5c914d09eeb1b7dafd6872b38b48c6afc146101200e6e6a44fe5684e220adc11f5c403ddb15df8051e6bdef09117a3a5349938513776286473a3cf1d2788bb875052a2e6459fa7926da33380149c7f98d7700528a60c954e6f5ecb65842fde69d614be69eaa2040a4819ae6e756accf936e14c1e894489744a79c1f2c1eb295d13e2d767c09964b61f9cfe497649f712", + false, + "d4e3f18ac2e2095edb5c3b94236118ad", + "4faa6c4233d9fd53d170ede4172142a8", + "23f154ac43cfc59c4243e9fc68aeec8f19ad3942d74108e833b36f0dd3dcd357", + "8da7de6ea7bf2a81a396a42880ba1f5756734c4821309ac9aeffa2a26ce86873b9dc4935a772de6ec5162c6d075b14536800fb174841153511bfb597e992e2fe8a450c4bce102cc550bb37fd564c4d60bf884e", + ""); + TestBIP324PacketVector( + 223, + "6c77432d1fda31e9f942f8af44607e10f3ad38a65f8a4bddae823e5eff90dc38", + "d2685070c1e6376e633e825296634fd461fa9e5bdf2109bcebd735e5a91f3e587c5cb782abb797fbf6bb5074fd1542a474f2a45b673763ec2db7fb99b737bbb9", + "56bd0c06f10352c3a1a9f4b4c92f6fa2b26df124b57878353c1fc691c51abea77c8817daeeb9fa546b77c8daf79d89b22b0e1b87574ece42371f00237aa9d83a", + false, + "7e0e78eb6990b059e6cf0ded66ea93ef82e72aa2f18ac24f2fc6ebab561ae557420729da103f64cecfa20527e15f9fb669a49bbbf274ef0389b3e43c8c44e5f60bf2ac38e2b55e7ec4273dba15ba41d21f8f5b3ee1688b3c29951218caf847a97fb50d75a86515d445699497d968164bf740012679b8962de573be941c62b7ef", + 1, + "", + true, + "cf2e25f23501399f30738d7eee652b90", + "225a477a28a54ea7671d2b217a9c29db", + "7ec02fea8c1484e3d0875f978c5f36d63545e2e4acf56311394422f4b66af612", + "", + "729847a3e9eba7a5bff454b5de3b393431ee360736b6c030d7a5bd01d1203d2e98f528543fd2bf886ccaa1ada5e215a730a36b3f4abfc4e252c89eb01d9512f94916dae8a76bf16e4da28986ffe159090fe5267ee3394300b7ccf4dfad389a26321b3a3423e4594a82ccfbad16d6561ecb8772b0cb040280ff999a29e3d9d4fd"); + TestBIP324PacketVector( + 448, + "a6ec25127ca1aa4cf16b20084ba1e6516baae4d32422288e9b36d8bddd2de35a", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff053d7ecca53e33e185a8b9be4e7699a97c6ff4c795522e5918ab7cd6b6884f67e683f3dc", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffa7730be30000000000000000000000000000000000000000000000000000000000000000", + true, + "00cf68f8f7ac49ffaa02c4864fdf6dfe7bbf2c740b88d98c50ebafe32c92f3427f57601ffcb21a3435979287db8fee6c302926741f9d5e464c647eeb9b7acaeda46e00abd7506fc9a719847e9a7328215801e96198dac141a15c7c2f68e0690dd1176292a0dded04d1f548aad88f1aebdc0a8f87da4bb22df32dd7c160c225b843e83f6525d6d484f502f16d923124fc538794e21da2eb689d18d87406ecced5b9f92137239ed1d37bcfa7836641a83cf5e0a1cf63f51b06f158e499a459ede41c", + 1, + "", + false, + "fead69be77825a23daec377c362aa560", + "511d4980526c5e64aa7187462faeafdd", + "acb8f084ea763ddd1b92ac4ed23bf44de20b84ab677d4e4e6666a6090d40353d", + "", + "77b4656934a82de1a593d8481f020194ddafd8cac441f9d72aeb8721e6a14f49698ca6d9b2b6d59d07a01aa552fd4d5b68d0d1617574c77dea10bfadbaa31b83885b7ceac2fd45e3e4a331c51a74e7b1698d81b64c87c73c5b9258b4d83297f9debc2e9aa07f8572ff434dc792b83ecf07b3197de8dc9cf7be56acb59c66cff5"); + TestBIP324PacketVector( + 673, + "0af952659ed76f80f585966b95ab6e6fd68654672827878684c8b547b1b94f5a", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffc81017fd92fd31637c26c906b42092e11cc0d3afae8d9019d2578af22735ce7bc469c72d", + "9652d78baefc028cd37a6a92625b8b8f85fde1e4c944ad3f20e198bef8c02f19fffffffffffffffffffffffffffffffffffffffffffffffffffffffff2e91870", + false, + "5c6272ee55da855bbbf7b1246d9885aa7aa601a715ab86fa46c50da533badf82b97597c968293ae04e", + 97561, + "", + false, + "5e2375ac629b8df1e4ff3617c6255a70", + "70bcbffcb62e4d29d2605d30bceef137", + "7332e92a3f9d2792c4d444fac5ed888c39a073043a65eefb626318fd649328f8", + "", + "657a4a19711ce593c3844cb391b224f60124aba7e04266233bc50cafb971e26c7716b76e98376448f7d214dd11e629ef9a974d60e3770a695810a61c4ba66d78b936ee7892b98f0b48ddae9fcd8b599dca1c9b43e9b95e0226cf8d4459b8a7c2c4e6db80f1d58c7b20dd7208fa5c1057fb78734223ee801dbd851db601fee61e"); + TestBIP324PacketVector( + 1024, + "f90e080c64b05824c5a24b2501d5aeaf08af3872ee860aa80bdcd430f7b63494", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff115173765dc202cf029ad3f15479735d57697af12b0131dd21430d5772e4ef11474d58b9", + "12a50f3fafea7c1eeada4cf8d33777704b77361453afc83bda91eef349ae044d20126c6200547ea5a6911776c05dee2a7f1a9ba7dfbabbbd273c3ef29ef46e46", + true, + "5f67d15d22ca9b2804eeab0a66f7f8e3a10fa5de5809a046084348cbc5304e843ef96f59a59c7d7fdfe5946489f3ea297d941bac326225df316a25fc90f0e65b0d31a9c497e960fdbf8c482516bc8a9c1c77b7f6d0e1143810c737f76f9224e6f2c9af5186b4f7259c7e8d165b6e4fe3d38a60bdbdd4d06ecdcaaf62086070dbb68686b802d53dfd7db14b18743832605f5461ad81e2af4b7e8ff0eff0867a25b93cec7becf15c43131895fed09a83bf1ee4a87d44dd0f02a837bf5a1232e201cb882734eb9643dc2dc4d4e8b5690840766212c7ac8f38ad8a9ec47c7a9b3e022ae3eb6a32522128b518bd0d0085dd81c5", + 69615, + "", + true, + "b709dea25e0be287c50e3603482c2e98", + "1f677e9d7392ebe3633fd82c9efb0f16", + "889f339285564fd868401fac8380bb9887925122ec8f31c8ae51ce067def103b", + "", + "7c4b9e1e6c1ce69da7b01513cdc4588fd93b04dafefaf87f31561763d906c672bac3dfceb751ebd126728ac017d4d580e931b8e5c7d5dfe0123be4dc9b2d2238b655c8a7fadaf8082c31e310909b5b731efc12f0a56e849eae6bfeedcc86dd27ef9b91d159256aa8e8d2b71a311f73350863d70f18d0d7302cf551e4303c7733"); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 8332f54591..6fbe74a680 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -4,7 +4,7 @@ #include <crypto/aes.h> #include <crypto/chacha20.h> -#include <crypto/chacha_poly_aead.h> +#include <crypto/chacha20poly1305.h> #include <crypto/hkdf_sha256_32.h> #include <crypto/hmac_sha256.h> #include <crypto/hmac_sha512.h> @@ -182,6 +182,46 @@ static void TestChaCha20(const std::string &hex_message, const std::string &hexk } } +static void TestFSChaCha20(const std::string& hex_plaintext, const std::string& hexkey, uint32_t rekey_interval, const std::string& ciphertext_after_rotation) +{ + auto key = ParseHex<std::byte>(hexkey); + BOOST_CHECK_EQUAL(FSChaCha20::KEYLEN, key.size()); + + auto plaintext = ParseHex<std::byte>(hex_plaintext); + + auto fsc20 = FSChaCha20{key, rekey_interval}; + auto c20 = ChaCha20{UCharCast(key.data())}; + + std::vector<std::byte> fsc20_output; + fsc20_output.resize(plaintext.size()); + + std::vector<std::byte> c20_output; + c20_output.resize(plaintext.size()); + + for (size_t i = 0; i < rekey_interval; i++) { + fsc20.Crypt(plaintext, fsc20_output); + c20.Crypt(UCharCast(plaintext.data()), UCharCast(c20_output.data()), plaintext.size()); + BOOST_CHECK(c20_output == fsc20_output); + } + + // At the rotation interval, the outputs will no longer match + fsc20.Crypt(plaintext, fsc20_output); + auto c20_copy = c20; + c20.Crypt(UCharCast(plaintext.data()), UCharCast(c20_output.data()), plaintext.size()); + BOOST_CHECK(c20_output != fsc20_output); + + std::byte new_key[FSChaCha20::KEYLEN]; + c20_copy.Keystream(UCharCast(new_key), sizeof(new_key)); + c20.SetKey32(UCharCast(new_key)); + c20.Seek64({0, 1}, 0); + + // Outputs should match again after simulating key rotation + c20.Crypt(UCharCast(plaintext.data()), UCharCast(c20_output.data()), plaintext.size()); + BOOST_CHECK(c20_output == fsc20_output); + + BOOST_CHECK_EQUAL(HexStr(fsc20_output), ciphertext_after_rotation); +} + static void TestPoly1305(const std::string &hexmessage, const std::string &hexkey, const std::string& hextag) { auto key = ParseHex<std::byte>(hexkey); @@ -208,6 +248,93 @@ static void TestPoly1305(const std::string &hexmessage, const std::string &hexke } } +static void TestChaCha20Poly1305(const std::string& plain_hex, const std::string& aad_hex, const std::string& key_hex, ChaCha20::Nonce96 nonce, const std::string& cipher_hex) +{ + auto plain = ParseHex<std::byte>(plain_hex); + auto aad = ParseHex<std::byte>(aad_hex); + auto key = ParseHex<std::byte>(key_hex); + auto expected_cipher = ParseHex<std::byte>(cipher_hex); + + for (int i = 0; i < 10; ++i) { + // During i=0, use single-plain Encrypt/Decrypt; others use a split at prefix. + size_t prefix = i ? InsecureRandRange(plain.size() + 1) : plain.size(); + // Encrypt. + std::vector<std::byte> cipher(plain.size() + AEADChaCha20Poly1305::EXPANSION); + AEADChaCha20Poly1305 aead{key}; + if (i == 0) { + aead.Encrypt(plain, aad, nonce, cipher); + } else { + aead.Encrypt(Span{plain}.first(prefix), Span{plain}.subspan(prefix), aad, nonce, cipher); + } + BOOST_CHECK(cipher == expected_cipher); + + // Decrypt. + std::vector<std::byte> decipher(cipher.size() - AEADChaCha20Poly1305::EXPANSION); + bool ret{false}; + if (i == 0) { + ret = aead.Decrypt(cipher, aad, nonce, decipher); + } else { + ret = aead.Decrypt(cipher, aad, nonce, Span{decipher}.first(prefix), Span{decipher}.subspan(prefix)); + } + BOOST_CHECK(ret); + BOOST_CHECK(decipher == plain); + } + + // Test Keystream output. + std::vector<std::byte> keystream(plain.size()); + AEADChaCha20Poly1305 aead{key}; + aead.Keystream(nonce, keystream); + for (size_t i = 0; i < plain.size(); ++i) { + BOOST_CHECK_EQUAL(plain[i] ^ keystream[i], expected_cipher[i]); + } +} + +static void TestFSChaCha20Poly1305(const std::string& plain_hex, const std::string& aad_hex, const std::string& key_hex, uint64_t msg_idx, const std::string& cipher_hex) +{ + auto plain = ParseHex<std::byte>(plain_hex); + auto aad = ParseHex<std::byte>(aad_hex); + auto key = ParseHex<std::byte>(key_hex); + auto expected_cipher = ParseHex<std::byte>(cipher_hex); + std::vector<std::byte> cipher(plain.size() + FSChaCha20Poly1305::EXPANSION); + + for (int it = 0; it < 10; ++it) { + // During it==0 we use the single-plain Encrypt/Decrypt; others use a split at prefix. + size_t prefix = it ? InsecureRandRange(plain.size() + 1) : plain.size(); + std::byte dummy_tag[FSChaCha20Poly1305::EXPANSION] = {{}}; + + // Do msg_idx dummy encryptions to seek to the correct packet. + FSChaCha20Poly1305 enc_aead{key, 224}; + for (uint64_t i = 0; i < msg_idx; ++i) { + enc_aead.Encrypt(Span{dummy_tag}.first(0), Span{dummy_tag}.first(0), dummy_tag); + } + + // Invoke single-plain or plain1/plain2 Encrypt. + if (it == 0) { + enc_aead.Encrypt(plain, aad, cipher); + } else { + enc_aead.Encrypt(Span{plain}.first(prefix), Span{plain}.subspan(prefix), aad, cipher); + } + BOOST_CHECK(cipher == expected_cipher); + + // Do msg_idx dummy decryptions to seek to the correct packet. + FSChaCha20Poly1305 dec_aead{key, 224}; + for (uint64_t i = 0; i < msg_idx; ++i) { + dec_aead.Decrypt(dummy_tag, Span{dummy_tag}.first(0), Span{dummy_tag}.first(0)); + } + + // Invoke single-plain or plain1/plain2 Decrypt. + std::vector<std::byte> decipher(cipher.size() - AEADChaCha20Poly1305::EXPANSION); + bool ret{false}; + if (it == 0) { + ret = dec_aead.Decrypt(cipher, aad, decipher); + } else { + ret = dec_aead.Decrypt(cipher, aad, Span{decipher}.first(prefix), Span{decipher}.subspan(prefix)); + } + BOOST_CHECK(ret); + BOOST_CHECK(decipher == plain); + } +} + static void TestHKDF_SHA256_32(const std::string &ikm_hex, const std::string &salt_hex, const std::string &info_hex, const std::string &okm_check_hex) { std::vector<unsigned char> initial_key_material = ParseHex(ikm_hex); std::vector<unsigned char> salt = ParseHex(salt_hex); @@ -678,6 +805,20 @@ BOOST_AUTO_TEST_CASE(chacha20_testvector) "fd565dea5addbdb914208fde7950f23e0385f9a727143f6a6ac51d84b1c0fb3e" "2e3b00b63d6841a1cc6d1538b1d3a74bef1eb2f54c7b7281e36e484dba89b351" "c8f572617e61e342879f211b0e4c515df50ea9d0771518fad96cd0baee62deb6"); + + // Forward secure ChaCha20 + TestFSChaCha20("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "0000000000000000000000000000000000000000000000000000000000000000", + 256, + "a93df4ef03011f3db95f60d996e1785df5de38fc39bfcb663a47bb5561928349"); + TestFSChaCha20("01", + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + 5, + "ea"); + TestFSChaCha20("e93fdb5c762804b9a706816aca31e35b11d2aa3080108ef46a5b1f1508819c0a", + "8ec4c3ccdaea336bdeb245636970be01266509b33f3d2642504eaf412206207a", + 4096, + "8bfaa4eacff308fdb4a94a5ff25bd9d0c1f84b77f81239f67ff39d6e1ac280c9"); } BOOST_AUTO_TEST_CASE(chacha20_midblock) @@ -686,7 +827,7 @@ BOOST_AUTO_TEST_CASE(chacha20_midblock) ChaCha20 c20{key.data()}; // get one block of keystream unsigned char block[64]; - c20.Keystream(block, CHACHA20_ROUND_OUTPUT); + c20.Keystream(block, sizeof(block)); unsigned char b1[5], b2[7], b3[52]; c20 = ChaCha20{key.data()}; c20.Keystream(b1, 5); @@ -819,6 +960,87 @@ BOOST_AUTO_TEST_CASE(poly1305_testvector) "0e410fa9d7a40ac582e77546be9a72bb"); } +BOOST_AUTO_TEST_CASE(chacha20poly1305_testvectors) +{ + // Note that in our implementation, the authentication is suffixed to the ciphertext. + // The RFC test vectors specify them separately. + + // RFC 8439 Example from section 2.8.2 + TestChaCha20Poly1305("4c616469657320616e642047656e746c656d656e206f662074686520636c6173" + "73206f66202739393a204966204920636f756c64206f6666657220796f75206f" + "6e6c79206f6e652074697020666f7220746865206675747572652c2073756e73" + "637265656e20776f756c642062652069742e", + "50515253c0c1c2c3c4c5c6c7", + "808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9f", + {7, 0x4746454443424140}, + "d31a8d34648e60db7b86afbc53ef7ec2a4aded51296e08fea9e2b5a736ee62d6" + "3dbea45e8ca9671282fafb69da92728b1a71de0a9e060b2905d6a5b67ecd3b36" + "92ddbd7f2d778b8c9803aee328091b58fab324e4fad675945585808b4831d7bc" + "3ff4def08e4b7a9de576d26586cec64b61161ae10b594f09e26a7e902ecbd060" + "0691"); + + // RFC 8439 Test vector A.5 + TestChaCha20Poly1305("496e7465726e65742d4472616674732061726520647261667420646f63756d65" + "6e74732076616c696420666f722061206d6178696d756d206f6620736978206d" + "6f6e74687320616e64206d617920626520757064617465642c207265706c6163" + "65642c206f72206f62736f6c65746564206279206f7468657220646f63756d65" + "6e747320617420616e792074696d652e20497420697320696e617070726f7072" + "6961746520746f2075736520496e7465726e65742d4472616674732061732072" + "65666572656e6365206d6174657269616c206f7220746f206369746520746865" + "6d206f74686572207468616e206173202fe2809c776f726b20696e2070726f67" + "726573732e2fe2809d", + "f33388860000000000004e91", + "1c9240a5eb55d38af333888604f6b5f0473917c1402b80099dca5cbc207075c0", + {0, 0x0807060504030201}, + "64a0861575861af460f062c79be643bd5e805cfd345cf389f108670ac76c8cb2" + "4c6cfc18755d43eea09ee94e382d26b0bdb7b73c321b0100d4f03b7f355894cf" + "332f830e710b97ce98c8a84abd0b948114ad176e008d33bd60f982b1ff37c855" + "9797a06ef4f0ef61c186324e2b3506383606907b6a7c02b0f9f6157b53c867e4" + "b9166c767b804d46a59b5216cde7a4e99040c5a40433225ee282a1b0a06c523e" + "af4534d7f83fa1155b0047718cbc546a0d072b04b3564eea1b422273f548271a" + "0bb2316053fa76991955ebd63159434ecebb4e466dae5a1073a6727627097a10" + "49e617d91d361094fa68f0ff77987130305beaba2eda04df997b714d6c6f2c29" + "a6ad5cb4022b02709beead9d67890cbb22392336fea1851f38"); + + // Test vectors exercising aad and plaintext which are multiples of 16 bytes. + TestChaCha20Poly1305("8d2d6a8befd9716fab35819eaac83b33269afb9f1a00fddf66095a6c0cd91951" + "a6b7ad3db580be0674c3f0b55f618e34", + "", + "72ddc73f07101282bbbcf853b9012a9f9695fc5d36b303a97fd0845d0314e0c3", + {0x3432b75f, 0xb3585537eb7f4024}, + "f760b8224fb2a317b1b07875092606131232a5b86ae142df5df1c846a7f6341a" + "f2564483dd77f836be45e6230808ffe402a6f0a3e8be074b3d1f4ea8a7b09451"); + TestChaCha20Poly1305("", + "36970d8a704c065de16250c18033de5a400520ac1b5842b24551e5823a3314f3" + "946285171e04a81ebfbe3566e312e74ab80e94c7dd2ff4e10de0098a58d0f503", + "77adda51d6730b9ad6c995658cbd49f581b2547e7c0c08fcc24ceec797461021", + {0x1f90da88, 0x75dafa3ef84471a4}, + "aaae5bb81e8407c94b2ae86ae0c7efbe"); + + // FSChaCha20Poly1305 tests. + TestFSChaCha20Poly1305("d6a4cb04ef0f7c09c1866ed29dc24d820e75b0491032a51b4c3366f9ca35c19e" + "a3047ec6be9d45f9637b63e1cf9eb4c2523a5aab7b851ebeba87199db0e839cf" + "0d5c25e50168306377aedbe9089fd2463ded88b83211cf51b73b150608cc7a60" + "0d0f11b9a742948482e1b109d8faf15b450aa7322e892fa2208c6691e3fecf4c" + "711191b14d75a72147", + "786cb9b6ebf44288974cf0", + "5c9e1c3951a74fba66708bf9d2c217571684556b6a6a3573bff2847d38612654", + 500, + "9dcebbd3281ea3dd8e9a1ef7d55a97abd6743e56ebc0c190cb2c4e14160b385e" + "0bf508dddf754bd02c7c208447c131ce23e47a4a14dfaf5dd8bc601323950f75" + "4e05d46e9232f83fc5120fbbef6f5347a826ec79a93820718d4ec7a2b7cfaaa4" + "4b21e16d726448b62f803811aff4f6d827ed78e738ce8a507b81a8ae13131192" + "8039213de18a5120dc9b7370baca878f50ff254418de3da50c"); + TestFSChaCha20Poly1305("8349b7a2690b63d01204800c288ff1138a1d473c832c90ea8b3fc102d0bb3adc" + "44261b247c7c3d6760bfbe979d061c305f46d94c0582ac3099f0bf249f8cb234", + "", + "3bd2093fcbcb0d034d8c569583c5425c1a53171ea299f8cc3bbf9ae3530adfce", + 60000, + "30a6757ff8439b975363f166a0fa0e36722ab35936abd704297948f45083f4d4" + "99433137ce931f7fca28a0acd3bc30f57b550acbc21cbd45bbef0739d9caf30c" + "14b94829deb27f0b1923a2af704ae5d6"); +} + BOOST_AUTO_TEST_CASE(hkdf_hmac_sha256_l32_tests) { // Use rfc5869 test vectors but truncated to 32 bytes (our implementation only support length 32) @@ -839,129 +1061,6 @@ BOOST_AUTO_TEST_CASE(hkdf_hmac_sha256_l32_tests) "8da4e775a563c18f715f802a063c5a31b8a11f5c5ee1879ec3454e5f3c738d2d"); } -static void TestChaCha20Poly1305AEAD(bool must_succeed, unsigned int expected_aad_length, const std::string& hex_m, const std::string& hex_k1, const std::string& hex_k2, const std::string& hex_aad_keystream, const std::string& hex_encrypted_message, const std::string& hex_encrypted_message_seq_999) -{ - // we need two sequence numbers, one for the payload cipher instance... - uint32_t seqnr_payload = 0; - // ... and one for the AAD (length) cipher instance - uint32_t seqnr_aad = 0; - // we need to keep track of the position in the AAD cipher instance - // keystream since we use the same 64byte output 21 times - // (21 times 3 bytes length < 64) - int aad_pos = 0; - - std::vector<unsigned char> aead_K_1 = ParseHex(hex_k1); - std::vector<unsigned char> aead_K_2 = ParseHex(hex_k2); - std::vector<unsigned char> plaintext_buf = ParseHex(hex_m); - std::vector<unsigned char> expected_aad_keystream = ParseHex(hex_aad_keystream); - std::vector<unsigned char> expected_ciphertext_and_mac = ParseHex(hex_encrypted_message); - std::vector<unsigned char> expected_ciphertext_and_mac_sequence999 = ParseHex(hex_encrypted_message_seq_999); - - std::vector<unsigned char> ciphertext_buf(plaintext_buf.size() + Poly1305::TAGLEN, 0); - std::vector<unsigned char> plaintext_buf_new(plaintext_buf.size(), 0); - std::vector<unsigned char> cmp_ctx_buffer(64); - uint32_t out_len = 0; - - // create the AEAD instance - ChaCha20Poly1305AEAD aead(aead_K_1.data(), aead_K_1.size(), aead_K_2.data(), aead_K_2.size()); - - // create a chacha20 instance to compare against - ChaCha20 cmp_ctx(aead_K_1.data()); - - // encipher - bool res = aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, ciphertext_buf.data(), ciphertext_buf.size(), plaintext_buf.data(), plaintext_buf.size(), true); - // make sure the operation succeeded if expected to succeed - BOOST_CHECK_EQUAL(res, must_succeed); - if (!res) return; - - // verify ciphertext & mac against the test vector - BOOST_CHECK_EQUAL(expected_ciphertext_and_mac.size(), ciphertext_buf.size()); - BOOST_CHECK(memcmp(ciphertext_buf.data(), expected_ciphertext_and_mac.data(), ciphertext_buf.size()) == 0); - - // manually construct the AAD keystream - cmp_ctx.Seek64({0, seqnr_aad}, 0); - cmp_ctx.Keystream(cmp_ctx_buffer.data(), 64); - BOOST_CHECK(memcmp(expected_aad_keystream.data(), cmp_ctx_buffer.data(), expected_aad_keystream.size()) == 0); - // crypt the 3 length bytes and compare the length - uint32_t len_cmp = 0; - len_cmp = (ciphertext_buf[0] ^ cmp_ctx_buffer[aad_pos + 0]) | - (ciphertext_buf[1] ^ cmp_ctx_buffer[aad_pos + 1]) << 8 | - (ciphertext_buf[2] ^ cmp_ctx_buffer[aad_pos + 2]) << 16; - BOOST_CHECK_EQUAL(len_cmp, expected_aad_length); - - // encrypt / decrypt 1000 packets - for (size_t i = 0; i < 1000; ++i) { - res = aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, ciphertext_buf.data(), ciphertext_buf.size(), plaintext_buf.data(), plaintext_buf.size(), true); - BOOST_CHECK(res); - BOOST_CHECK(aead.GetLength(&out_len, seqnr_aad, aad_pos, ciphertext_buf.data())); - BOOST_CHECK_EQUAL(out_len, expected_aad_length); - res = aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, plaintext_buf_new.data(), plaintext_buf_new.size(), ciphertext_buf.data(), ciphertext_buf.size(), false); - BOOST_CHECK(res); - - // make sure we repetitive get the same plaintext - BOOST_CHECK(memcmp(plaintext_buf.data(), plaintext_buf_new.data(), plaintext_buf.size()) == 0); - - // compare sequence number 999 against the test vector - if (seqnr_payload == 999) { - BOOST_CHECK(memcmp(ciphertext_buf.data(), expected_ciphertext_and_mac_sequence999.data(), expected_ciphertext_and_mac_sequence999.size()) == 0); - } - // set nonce and block counter, output the keystream - cmp_ctx.Seek64({0, seqnr_aad}, 0); - cmp_ctx.Keystream(cmp_ctx_buffer.data(), 64); - - // crypt the 3 length bytes and compare the length - len_cmp = 0; - len_cmp = (ciphertext_buf[0] ^ cmp_ctx_buffer[aad_pos + 0]) | - (ciphertext_buf[1] ^ cmp_ctx_buffer[aad_pos + 1]) << 8 | - (ciphertext_buf[2] ^ cmp_ctx_buffer[aad_pos + 2]) << 16; - BOOST_CHECK_EQUAL(len_cmp, expected_aad_length); - - // increment the sequence number(s) - // always increment the payload sequence number - // increment the AAD keystream position by its size (3) - // increment the AAD sequence number if we would hit the 64 byte limit - seqnr_payload++; - aad_pos += CHACHA20_POLY1305_AEAD_AAD_LEN; - if (aad_pos + CHACHA20_POLY1305_AEAD_AAD_LEN > CHACHA20_ROUND_OUTPUT) { - aad_pos = 0; - seqnr_aad++; - } - } -} - -BOOST_AUTO_TEST_CASE(chacha20_poly1305_aead_testvector) -{ - /* test chacha20poly1305@bitcoin AEAD */ - - // must fail with no message - TestChaCha20Poly1305AEAD(false, 0, - "", - "0000000000000000000000000000000000000000000000000000000000000000", - "0000000000000000000000000000000000000000000000000000000000000000", "", "", ""); - - TestChaCha20Poly1305AEAD(true, 0, - /* m */ "0000000000000000000000000000000000000000000000000000000000000000", - /* k1 (AAD) */ "0000000000000000000000000000000000000000000000000000000000000000", - /* k2 (payload) */ "0000000000000000000000000000000000000000000000000000000000000000", - /* AAD keystream */ "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586", - /* encrypted message & MAC */ "76b8e09f07e7be5551387a98ba977c732d080dcb0f29a048e3656912c6533e32d2fc11829c1b6c1df1f551cd6131ff08", - /* encrypted message & MAC at sequence 999 */ "b0a03d5bd2855d60699e7d3a3133fa47be740fe4e4c1f967555e2d9271f31c3aaa7aa16ec62c5e24f040c08bb20c3598"); - TestChaCha20Poly1305AEAD(true, 1, - "0100000000000000000000000000000000000000000000000000000000000000", - "0000000000000000000000000000000000000000000000000000000000000000", - "0000000000000000000000000000000000000000000000000000000000000000", - "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586", - "77b8e09f07e7be5551387a98ba977c732d080dcb0f29a048e3656912c6533e32baf0c85b6dff8602b06cf52a6aefc62e", - "b1a03d5bd2855d60699e7d3a3133fa47be740fe4e4c1f967555e2d9271f31c3a8bd94d54b5ecabbc41ffbb0c90924080"); - TestChaCha20Poly1305AEAD(true, 255, - "ff0000f195e66982105ffb640bb7757f579da31602fc93ec01ac56f85ac3c134a4547b733b46413042c9440049176905d3be59ea1c53f15916155c2be8241a38008b9a26bc35941e2444177c8ade6689de95264986d95889fb60e84629c9bd9a5acb1cc118be563eb9b3a4a472f82e09a7e778492b562ef7130e88dfe031c79db9d4f7c7a899151b9a475032b63fc385245fe054e3dd5a97a5f576fe064025d3ce042c566ab2c507b138db853e3d6959660996546cc9c4a6eafdc777c040d70eaf46f76dad3979e5c5360c3317166a1c894c94a371876a94df7628fe4eaaf2ccb27d5aaae0ad7ad0f9d4b6ad3b54098746d4524d38407a6deb3ab78fab78c9", - "ff0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", - "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", - "c640c1711e3ee904ac35c57ab9791c8a1c408603a90b77a83b54f6c844cb4b06d94e7fc6c800e165acd66147e80ec45a567f6ce66d05ec0cae679dceeb890017", - "3940c1e92da4582ff6f92a776aeb14d014d384eeb30f660dacf70a14a23fd31e91212701334e2ce1acf5199dc84f4d61ddbe6571bca5af874b4c9226c26e650995d157644e1848b96ed6c2102d5489a050e71d29a5a66ece11de5fb5c9558d54da28fe45b0bc4db4e5b88030bfc4a352b4b7068eccf656bae7ad6a35615315fc7c49d4200388d5eca67c2e822e069336c69b40db67e0f3c81209c50f3216a4b89fb3ae1b984b7851a2ec6f68ab12b101ab120e1ea7313bb93b5a0f71185c7fea017ddb92769861c29dba4fbc432280d5dff21b36d1c4c790128b22699950bb18bf74c448cdfe547d8ed4f657d8005fdc0cd7a050c2d46050a44c4376355858981fbe8b184288276e7a93eabc899c4a", - "f039c6689eaeef0456685200feaab9d54bbd9acde4410a3b6f4321296f4a8ca2604b49727d8892c57e005d799b2a38e85e809f20146e08eec75169691c8d4f54a0d51a1e1c7b381e0474eb02f994be9415ef3ffcbd2343f0601e1f3b172a1d494f838824e4df570f8e3b0c04e27966e36c82abd352d07054ef7bd36b84c63f9369afe7ed79b94f953873006b920c3fa251a771de1b63da927058ade119aa898b8c97e42a606b2f6df1e2d957c22f7593c1e2002f4252f4c9ae4bf773499e5cfcfe14dfc1ede26508953f88553bf4a76a802f6a0068d59295b01503fd9a600067624203e880fdf53933b96e1f4d9eb3f4e363dd8165a278ff667a41ee42b9892b077cefff92b93441f7be74cf10e6cd"); -} - BOOST_AUTO_TEST_CASE(countbits_tests) { FastRandomContext ctx; diff --git a/src/test/fuzz/bip324.cpp b/src/test/fuzz/bip324.cpp new file mode 100644 index 0000000000..98ac10e364 --- /dev/null +++ b/src/test/fuzz/bip324.cpp @@ -0,0 +1,136 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bip324.h> +#include <chainparams.h> +#include <span.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/xoroshiro128plusplus.h> + +#include <cstdint> +#include <vector> + +namespace { + +void Initialize() +{ + ECC_Start(); + SelectParams(ChainType::MAIN); +} + +} // namespace + +FUZZ_TARGET(bip324_cipher_roundtrip, .init=Initialize) +{ + // Test that BIP324Cipher's encryption and decryption agree. + + // Load keys from fuzzer. + FuzzedDataProvider provider(buffer.data(), buffer.size()); + // Initiator key + auto init_key_data = provider.ConsumeBytes<unsigned char>(32); + init_key_data.resize(32); + CKey init_key; + init_key.Set(init_key_data.begin(), init_key_data.end(), true); + if (!init_key.IsValid()) return; + // Initiator entropy + auto init_ent = provider.ConsumeBytes<std::byte>(32); + init_ent.resize(32); + // Responder key + auto resp_key_data = provider.ConsumeBytes<unsigned char>(32); + resp_key_data.resize(32); + CKey resp_key; + resp_key.Set(resp_key_data.begin(), resp_key_data.end(), true); + if (!resp_key.IsValid()) return; + // Responder entropy + auto resp_ent = provider.ConsumeBytes<std::byte>(32); + resp_ent.resize(32); + + // Initialize ciphers by exchanging public keys. + BIP324Cipher initiator(init_key, init_ent); + assert(!initiator); + BIP324Cipher responder(resp_key, resp_ent); + assert(!responder); + initiator.Initialize(responder.GetOurPubKey(), true); + assert(initiator); + responder.Initialize(initiator.GetOurPubKey(), false); + assert(responder); + + // Initialize RNG deterministically, to generate contents and AAD. We assume that there are no + // (potentially buggy) edge cases triggered by specific values of contents/AAD, so we can avoid + // reading the actual data for those from the fuzzer input (which would need large amounts of + // data). + XoRoShiRo128PlusPlus rng(provider.ConsumeIntegral<uint64_t>()); + + // Compare session IDs and garbage terminators. + assert(initiator.GetSessionID() == responder.GetSessionID()); + assert(initiator.GetSendGarbageTerminator() == responder.GetReceiveGarbageTerminator()); + assert(initiator.GetReceiveGarbageTerminator() == responder.GetSendGarbageTerminator()); + + LIMITED_WHILE(provider.remaining_bytes(), 1000) { + // Mode: + // - Bit 0: whether the ignore bit is set in message + // - Bit 1: whether the responder (0) or initiator (1) sends + // - Bit 2: whether this ciphertext will be corrupted (making it the last sent one) + // - Bit 3-4: controls the maximum aad length (max 4095 bytes) + // - Bit 5-7: controls the maximum content length (max 16383 bytes, for performance reasons) + unsigned mode = provider.ConsumeIntegral<uint8_t>(); + bool ignore = mode & 1; + bool from_init = mode & 2; + bool damage = mode & 4; + unsigned aad_length_bits = 4 * ((mode >> 3) & 3); + unsigned aad_length = provider.ConsumeIntegralInRange<unsigned>(0, (1 << aad_length_bits) - 1); + unsigned length_bits = 2 * ((mode >> 5) & 7); + unsigned length = provider.ConsumeIntegralInRange<unsigned>(0, (1 << length_bits) - 1); + // Generate aad and content. + std::vector<std::byte> aad(aad_length); + for (auto& val : aad) val = std::byte{(uint8_t)rng()}; + std::vector<std::byte> contents(length); + for (auto& val : contents) val = std::byte{(uint8_t)rng()}; + + // Pick sides. + auto& sender{from_init ? initiator : responder}; + auto& receiver{from_init ? responder : initiator}; + + // Encrypt + std::vector<std::byte> ciphertext(length + initiator.EXPANSION); + sender.Encrypt(contents, aad, ignore, ciphertext); + + // Optionally damage 1 bit in either the ciphertext (corresponding to a change in transit) + // or the aad (to make sure that decryption will fail if the AAD mismatches). + if (damage) { + unsigned damage_bit = provider.ConsumeIntegralInRange<unsigned>(0, + (ciphertext.size() + aad.size()) * 8U - 1U); + unsigned damage_pos = damage_bit >> 3; + std::byte damage_val{(uint8_t)(1U << (damage_bit & 3))}; + if (damage_pos >= ciphertext.size()) { + aad[damage_pos - ciphertext.size()] ^= damage_val; + } else { + ciphertext[damage_pos] ^= damage_val; + } + } + + // Decrypt length + uint32_t dec_length = receiver.DecryptLength(Span{ciphertext}.first(initiator.LENGTH_LEN)); + if (!damage) { + assert(dec_length == length); + } else { + // For performance reasons, don't try to decode if length got increased too much. + if (dec_length > 16384 + length) break; + // Otherwise, just append zeros if dec_length > length. + ciphertext.resize(dec_length + initiator.EXPANSION); + } + + // Decrypt + std::vector<std::byte> decrypt(dec_length); + bool dec_ignore{false}; + bool ok = receiver.Decrypt(Span{ciphertext}.subspan(initiator.LENGTH_LEN), aad, dec_ignore, decrypt); + // Decryption *must* fail if the packet was damaged, and succeed if it wasn't. + assert(!ok == damage); + if (!ok) break; + assert(ignore == dec_ignore); + assert(decrypt == contents); + } +} diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index 723dc6420f..b088aa0bd7 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -155,14 +155,16 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) } assert((exists_using_access_coin && exists_using_have_coin_in_cache && exists_using_have_coin && exists_using_get_coin) || (!exists_using_access_coin && !exists_using_have_coin_in_cache && !exists_using_have_coin && !exists_using_get_coin)); + // If HaveCoin on the backend is true, it must also be on the cache if the coin wasn't spent. const bool exists_using_have_coin_in_backend = backend_coins_view.HaveCoin(random_out_point); - if (exists_using_have_coin_in_backend) { + if (!coin_using_access_coin.IsSpent() && exists_using_have_coin_in_backend) { assert(exists_using_have_coin); } Coin coin_using_backend_get_coin; if (backend_coins_view.GetCoin(random_out_point, coin_using_backend_get_coin)) { assert(exists_using_have_coin_in_backend); - assert(coin_using_get_coin == coin_using_backend_get_coin); + // Note we can't assert that `coin_using_get_coin == coin_using_backend_get_coin` because the coin in + // the cache may have been modified but not yet flushed. } else { assert(!exists_using_have_coin_in_backend); } diff --git a/src/test/fuzz/crypto_chacha20.cpp b/src/test/fuzz/crypto_chacha20.cpp index 63c7bf3b45..76370b4e57 100644 --- a/src/test/fuzz/crypto_chacha20.cpp +++ b/src/test/fuzz/crypto_chacha20.cpp @@ -8,6 +8,8 @@ #include <test/fuzz/util.h> #include <test/util/xoroshiro128plusplus.h> +#include <array> +#include <cstddef> #include <cstdint> #include <vector> @@ -151,3 +153,21 @@ FUZZ_TARGET(chacha20_split_keystream) FuzzedDataProvider provider{buffer.data(), buffer.size()}; ChaCha20SplitFuzz<false>(provider); } + +FUZZ_TARGET(crypto_fschacha20) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + + auto key = fuzzed_data_provider.ConsumeBytes<std::byte>(FSChaCha20::KEYLEN); + key.resize(FSChaCha20::KEYLEN); + + auto fsc20 = FSChaCha20{key, fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(1, 1024)}; + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + { + auto input = fuzzed_data_provider.ConsumeBytes<std::byte>(fuzzed_data_provider.ConsumeIntegralInRange(0, 4096)); + std::vector<std::byte> output; + output.resize(input.size()); + fsc20.Crypt(input, output); + } +} diff --git a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp deleted file mode 100644 index 84ac65dd88..0000000000 --- a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2020-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <crypto/chacha_poly_aead.h> -#include <crypto/poly1305.h> -#include <test/fuzz/FuzzedDataProvider.h> -#include <test/fuzz/fuzz.h> -#include <test/fuzz/util.h> -#include <util/overflow.h> - -#include <cassert> -#include <cstdint> -#include <limits> -#include <vector> - -FUZZ_TARGET(crypto_chacha20_poly1305_aead) -{ - FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - - const std::vector<uint8_t> k1 = ConsumeFixedLengthByteVector(fuzzed_data_provider, CHACHA20_POLY1305_AEAD_KEY_LEN); - const std::vector<uint8_t> k2 = ConsumeFixedLengthByteVector(fuzzed_data_provider, CHACHA20_POLY1305_AEAD_KEY_LEN); - - ChaCha20Poly1305AEAD aead(k1.data(), k1.size(), k2.data(), k2.size()); - uint64_t seqnr_payload = 0; - uint64_t seqnr_aad = 0; - int aad_pos = 0; - size_t buffer_size = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096); - std::vector<uint8_t> in(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + Poly1305::TAGLEN, 0); - std::vector<uint8_t> out(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + Poly1305::TAGLEN, 0); - bool is_encrypt = fuzzed_data_provider.ConsumeBool(); - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { - CallOneOf( - fuzzed_data_provider, - [&] { - buffer_size = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(64, 4096); - in = std::vector<uint8_t>(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + Poly1305::TAGLEN, 0); - out = std::vector<uint8_t>(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + Poly1305::TAGLEN, 0); - }, - [&] { - (void)aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffer_size, is_encrypt); - }, - [&] { - uint32_t len = 0; - const bool ok = aead.GetLength(&len, seqnr_aad, aad_pos, in.data()); - assert(ok); - }, - [&] { - if (AdditionOverflow(seqnr_payload, static_cast<uint64_t>(1))) { - return; - } - seqnr_payload += 1; - aad_pos += CHACHA20_POLY1305_AEAD_AAD_LEN; - if (aad_pos + CHACHA20_POLY1305_AEAD_AAD_LEN > CHACHA20_ROUND_OUTPUT) { - aad_pos = 0; - if (AdditionOverflow(seqnr_aad, static_cast<uint64_t>(1))) { - return; - } - seqnr_aad += 1; - } - }, - [&] { - seqnr_payload = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); - }, - [&] { - seqnr_aad = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); - }, - [&] { - is_encrypt = fuzzed_data_provider.ConsumeBool(); - }); - } -} diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 2782888dc3..9e76c7be3e 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -78,6 +78,7 @@ const std::vector<std::string> RPC_COMMANDS_NOT_SAFE_FOR_FUZZING{ "generatetoaddress", // avoid prohibitively slow execution (when `num_blocks` is large) "generatetodescriptor", // avoid prohibitively slow execution (when `nblocks` is large) "gettxoutproof", // avoid prohibitively slow execution + "importmempool", // avoid reading from disk "importwallet", // avoid reading from disk "loadwallet", // avoid reading from disk "savemempool", // disabled as a precautionary measure: may take a file path argument in the future diff --git a/src/test/fuzz/util/mempool.cpp b/src/test/fuzz/util/mempool.cpp index 4baca5ec77..8e7499a860 100644 --- a/src/test/fuzz/util/mempool.cpp +++ b/src/test/fuzz/util/mempool.cpp @@ -23,8 +23,9 @@ CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CAmount fee{ConsumeMoney(fuzzed_data_provider, /*max=*/std::numeric_limits<CAmount>::max() / CAmount{100'000})}; assert(MoneyRange(fee)); const int64_t time = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + const uint64_t entry_sequence{fuzzed_data_provider.ConsumeIntegral<uint64_t>()}; const unsigned int entry_height = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); const bool spends_coinbase = fuzzed_data_provider.ConsumeBool(); const unsigned int sig_op_cost = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, MAX_BLOCK_SIGOPS_COST); - return CTxMemPoolEntry{MakeTransactionRef(tx), fee, time, entry_height, spends_coinbase, sig_op_cost, {}}; + return CTxMemPoolEntry{MakeTransactionRef(tx), fee, time, entry_height, entry_sequence, spends_coinbase, sig_op_cost, {}}; } diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index c203dd4e39..5d020b4d59 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -20,6 +20,7 @@ #include <vector> using kernel::DumpMempool; +using kernel::LoadMempool; using node::MempoolPath; @@ -47,6 +48,10 @@ FUZZ_TARGET(validation_load_mempool, .init = initialize_validation_load_mempool) auto fuzzed_fopen = [&](const fs::path&, const char*) { return fuzzed_file_provider.open(); }; - (void)chainstate.LoadMempool(MempoolPath(g_setup->m_args), fuzzed_fopen); + (void)LoadMempool(pool, MempoolPath(g_setup->m_args), chainstate, + { + .mockable_fopen_function = fuzzed_fopen, + }); + pool.SetLoadTried(true); (void)DumpMempool(pool, MempoolPath(g_setup->m_args), fuzzed_fopen, true); } diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 08ef890ec4..ecae743d14 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -424,7 +424,7 @@ std::vector<CTransactionRef> TestChain100Setup::PopulateMempool(FastRandomContex LOCK2(cs_main, m_node.mempool->cs); LockPoints lp; m_node.mempool->addUnchecked(CTxMemPoolEntry(ptx, /*fee=*/(total_in - num_outputs * amount_per_output), - /*time=*/0, /*entry_height=*/1, + /*time=*/0, /*entry_height=*/1, /*entry_sequence=*/0, /*spends_coinbase=*/false, /*sigops_cost=*/4, lp)); } --num_transactions; @@ -454,7 +454,7 @@ void TestChain100Setup::MockMempoolMinFee(const CFeeRate& target_feerate) const auto tx_fee = target_feerate.GetFee(GetVirtualTransactionSize(*tx)) - m_node.mempool->m_incremental_relay_feerate.GetFee(GetVirtualTransactionSize(*tx)); m_node.mempool->addUnchecked(CTxMemPoolEntry(tx, /*fee=*/tx_fee, - /*time=*/0, /*entry_height=*/1, + /*time=*/0, /*entry_height=*/1, /*entry_sequence=*/0, /*spends_coinbase=*/true, /*sigops_cost=*/1, lp)); m_node.mempool->TrimToSize(0); assert(m_node.mempool->GetMinFee() == target_feerate); diff --git a/src/test/util/txmempool.cpp b/src/test/util/txmempool.cpp index 4797d9c310..c945f35d79 100644 --- a/src/test/util/txmempool.cpp +++ b/src/test/util/txmempool.cpp @@ -34,5 +34,5 @@ CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) co CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransactionRef& tx) const { - return CTxMemPoolEntry{tx, nFee, TicksSinceEpoch<std::chrono::seconds>(time), nHeight, spendsCoinbase, sigOpCost, lp}; + return CTxMemPoolEntry{tx, nFee, TicksSinceEpoch<std::chrono::seconds>(time), nHeight, m_sequence, spendsCoinbase, sigOpCost, lp}; } diff --git a/src/test/util/txmempool.h b/src/test/util/txmempool.h index 2fe7d69693..4b0daf0d42 100644 --- a/src/test/util/txmempool.h +++ b/src/test/util/txmempool.h @@ -19,6 +19,7 @@ struct TestMemPoolEntryHelper { CAmount nFee{0}; NodeSeconds time{}; unsigned int nHeight{1}; + uint64_t m_sequence{0}; bool spendsCoinbase{false}; unsigned int sigOpCost{4}; LockPoints lp; @@ -30,6 +31,7 @@ struct TestMemPoolEntryHelper { TestMemPoolEntryHelper& Fee(CAmount _fee) { nFee = _fee; return *this; } TestMemPoolEntryHelper& Time(NodeSeconds tp) { time = tp; return *this; } TestMemPoolEntryHelper& Height(unsigned int _height) { nHeight = _height; return *this; } + TestMemPoolEntryHelper& Sequence(uint64_t _seq) { m_sequence = _seq; return *this; } TestMemPoolEntryHelper& SpendsCoinbase(bool _flag) { spendsCoinbase = _flag; return *this; } TestMemPoolEntryHelper& SigOpsCost(unsigned int _sigopsCost) { sigOpCost = _sigopsCost; return *this; } }; diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index a8d6fb4b3f..9cf976a700 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -14,14 +14,26 @@ #include <net.h> #include <netaddress.h> #include <netbase.h> +#include <random.h> +#include <tinyformat.h> +#include <util/check.h> +#include <util/fs.h> #include <util/readwritefile.h> #include <util/strencodings.h> +#include <util/string.h> #include <util/thread.h> #include <util/time.h> +#include <algorithm> +#include <cassert> +#include <cstdlib> #include <deque> #include <functional> +#include <map> +#include <optional> #include <set> +#include <thread> +#include <utility> #include <vector> #include <event2/buffer.h> @@ -79,15 +91,15 @@ void TorControlConnection::readcb(struct bufferevent *bev, void *ctx) if (s.size() < 4) // Short line continue; // <status>(-|+| )<data><CRLF> - self->message.code = LocaleIndependentAtoi<int>(s.substr(0,3)); + self->message.code = ToIntegral<int>(s.substr(0, 3)).value_or(0); self->message.lines.push_back(s.substr(4)); char ch = s[3]; // '-','+' or ' ' if (ch == ' ') { // Final line, dispatch reply and clean up if (self->message.code >= 600) { + // (currently unused) // Dispatch async notifications to async handler // Synchronous and asynchronous messages are never interleaved - self->async_handler(*self, self->message); } else { if (!self->reply_handlers.empty()) { // Invoke reply handler with message diff --git a/src/torcontrol.h b/src/torcontrol.h index afc5413db0..1a9065b01a 100644 --- a/src/torcontrol.h +++ b/src/torcontrol.h @@ -11,19 +11,14 @@ #include <netaddress.h> #include <util/fs.h> -#include <boost/signals2/signal.hpp> +#include <event2/util.h> -#include <event2/bufferevent.h> -#include <event2/event.h> - -#include <cstdlib> +#include <cstdint> #include <deque> #include <functional> #include <string> #include <vector> -class CService; - extern const std::string DEFAULT_TOR_CONTROL; static const bool DEFAULT_LISTEN_ONION = true; @@ -83,8 +78,6 @@ public: */ bool Command(const std::string &cmd, const ReplyHandlerCB& reply_handler); - /** Response handlers for async replies */ - boost::signals2::signal<void(TorControlConnection &,const TorControlReply &)> async_handler; private: /** Callback when ready for use */ std::function<void(TorControlConnection&)> connected; diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 79b2b4ec94..497a41d6a9 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -853,6 +853,17 @@ TxMempoolInfo CTxMemPool::info(const GenTxid& gtxid) const return GetInfo(i); } +TxMempoolInfo CTxMemPool::info_for_relay(const GenTxid& gtxid, uint64_t last_sequence) const +{ + LOCK(cs); + indexed_transaction_set::const_iterator i = (gtxid.IsWtxid() ? get_iter_from_wtxid(gtxid.GetHash()) : mapTx.find(gtxid.GetHash())); + if (i != mapTx.end() && i->GetSequence() < last_sequence) { + return GetInfo(i); + } else { + return TxMempoolInfo(); + } +} + void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta) { { diff --git a/src/txmempool.h b/src/txmempool.h index a1867eb895..4d7d53ff68 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -663,13 +663,13 @@ public: void GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants, size_t* ancestorsize = nullptr, CAmount* ancestorfees = nullptr) const; /** - * @returns true if we've made an attempt to load the mempool regardless of + * @returns true if an initial attempt to load the persisted mempool was made, regardless of * whether the attempt was successful or not */ bool GetLoadTried() const; /** - * Set whether or not we've made an attempt to load the mempool (regardless + * Set whether or not an initial attempt to load the persisted mempool was made (regardless * of whether the attempt was successful or not) */ void SetLoadTried(bool load_tried); @@ -708,6 +708,10 @@ public: return mapTx.project<0>(mapTx.get<index_by_wtxid>().find(wtxid)); } TxMempoolInfo info(const GenTxid& gtxid) const; + + /** Returns info for a transaction if its entry_sequence < last_sequence */ + TxMempoolInfo info_for_relay(const GenTxid& gtxid, uint64_t last_sequence) const; + std::vector<TxMempoolInfo> infoAll() const; size_t DynamicMemoryUsage() const; diff --git a/src/validation.cpp b/src/validation.cpp index c45c847471..d6c4d49b34 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -69,7 +69,6 @@ using kernel::CCoinsStats; using kernel::CoinStatsHashType; using kernel::ComputeUTXOStats; -using kernel::LoadMempool; using kernel::Notifications; using fsbridge::FopenFn; @@ -834,7 +833,10 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) } } - entry.reset(new CTxMemPoolEntry(ptx, ws.m_base_fees, nAcceptTime, m_active_chainstate.m_chain.Height(), + // Set entry_sequence to 0 when bypass_limits is used; this allows txs from a block + // reorg to be marked earlier than any child txs that were already in the mempool. + const uint64_t entry_sequence = bypass_limits ? 0 : m_pool.GetSequence(); + entry.reset(new CTxMemPoolEntry(ptx, ws.m_base_fees, nAcceptTime, m_active_chainstate.m_chain.Height(), entry_sequence, fSpendsCoinbase, nSigOpsCost, lock_points.value())); ws.m_vsize = entry->GetTxSize(); @@ -4126,13 +4128,6 @@ void PruneBlockFilesManual(Chainstate& active_chainstate, int nManualPruneHeight } } -void Chainstate::LoadMempool(const fs::path& load_path, FopenFn mockable_fopen_function) -{ - if (!m_mempool) return; - ::LoadMempool(*m_mempool, load_path, *this, mockable_fopen_function); - m_mempool->SetLoadTried(!m_chainman.m_interrupt); -} - bool Chainstate::LoadChainTip() { AssertLockHeld(cs_main); diff --git a/src/validation.h b/src/validation.h index d7ad86a5e8..e4980c51a7 100644 --- a/src/validation.h +++ b/src/validation.h @@ -241,7 +241,8 @@ struct PackageMempoolAcceptResult * @param[in] tx The transaction to submit for mempool acceptance. * @param[in] accept_time The timestamp for adding the transaction to the mempool. * It is also used to determine when the entry expires. - * @param[in] bypass_limits When true, don't enforce mempool fee and capacity limits. + * @param[in] bypass_limits When true, don't enforce mempool fee and capacity limits, + * and set entry_sequence to zero. * @param[in] test_accept When true, run validation checks but don't submit to mempool. * * @returns a MempoolAcceptResult indicating whether the transaction was accepted/rejected with reason. @@ -712,9 +713,6 @@ public: /** Find the last common block of this chain and a locator. */ const CBlockIndex* FindForkInGlobalIndex(const CBlockLocator& locator) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** Load the persisted mempool from disk */ - void LoadMempool(const fs::path& load_path, fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen); - /** Update the chain tip based on database information, i.e. CoinsTip()'s best block. */ bool LoadChainTip() EXCLUSIVE_LOCKS_REQUIRED(cs_main); diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index a1335ff069..32a927084a 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -46,7 +46,7 @@ from test_framework.util import ( assert_greater_than_or_equal, assert_raises_rpc_error, ) -from test_framework.wallet import MiniWallet +from test_framework.wallet import MiniWallet, COIN class MempoolPersistTest(BitcoinTestFramework): @@ -159,6 +159,16 @@ class MempoolPersistTest(BitcoinTestFramework): assert self.nodes[0].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[0].getrawmempool()), 0) + self.log.debug("Import mempool at runtime to node0.") + assert_equal({}, self.nodes[0].importmempool(mempooldat0)) + assert_equal(len(self.nodes[0].getrawmempool()), 7) + fees = self.nodes[0].getmempoolentry(txid=last_txid)["fees"] + assert_equal(fees["base"], fees["modified"]) + assert_equal({}, self.nodes[0].importmempool(mempooldat0, {"apply_fee_delta_priority": True, "apply_unbroadcast_set": True})) + assert_equal(2, self.nodes[0].getmempoolinfo()["unbroadcastcount"]) + fees = self.nodes[0].getmempoolentry(txid=last_txid)["fees"] + assert_equal(fees["base"] + Decimal("0.00001000"), fees["modified"]) + self.log.debug("Stop-start node0. Verify that it has the transactions in its mempool.") self.stop_nodes() self.start_node(0) @@ -186,6 +196,7 @@ class MempoolPersistTest(BitcoinTestFramework): assert_raises_rpc_error(-1, "Unable to dump mempool to disk", self.nodes[1].savemempool) os.rmdir(mempooldotnew1) + self.test_importmempool_union() self.test_persist_unbroadcast() def test_persist_unbroadcast(self): @@ -210,6 +221,46 @@ class MempoolPersistTest(BitcoinTestFramework): node0.mockscheduler(16 * 60) # 15 min + 1 for buffer self.wait_until(lambda: len(conn.get_invs()) == 1) + def test_importmempool_union(self): + self.log.debug("Submit different transactions to node0 and node1's mempools") + self.start_node(0) + self.start_node(2) + tx_node0 = self.mini_wallet.send_self_transfer(from_node=self.nodes[0]) + tx_node1 = self.mini_wallet.send_self_transfer(from_node=self.nodes[1]) + tx_node01 = self.mini_wallet.create_self_transfer() + tx_node01_secret = self.mini_wallet.create_self_transfer() + self.nodes[0].prioritisetransaction(tx_node01["txid"], 0, COIN) + self.nodes[0].prioritisetransaction(tx_node01_secret["txid"], 0, 2 * COIN) + self.nodes[1].prioritisetransaction(tx_node01_secret["txid"], 0, 3 * COIN) + self.nodes[0].sendrawtransaction(tx_node01["hex"]) + self.nodes[1].sendrawtransaction(tx_node01["hex"]) + assert tx_node0["txid"] in self.nodes[0].getrawmempool() + assert not tx_node0["txid"] in self.nodes[1].getrawmempool() + assert not tx_node1["txid"] in self.nodes[0].getrawmempool() + assert tx_node1["txid"] in self.nodes[1].getrawmempool() + assert tx_node01["txid"] in self.nodes[0].getrawmempool() + assert tx_node01["txid"] in self.nodes[1].getrawmempool() + assert not tx_node01_secret["txid"] in self.nodes[0].getrawmempool() + assert not tx_node01_secret["txid"] in self.nodes[1].getrawmempool() + + self.log.debug("Check that importmempool can add txns without replacing the entire mempool") + mempooldat0 = str(self.nodes[0].chain_path / "mempool.dat") + result0 = self.nodes[0].savemempool() + assert_equal(mempooldat0, result0["filename"]) + assert_equal({}, self.nodes[1].importmempool(mempooldat0, {"apply_fee_delta_priority": True})) + # All transactions should be in node1's mempool now. + assert tx_node0["txid"] in self.nodes[1].getrawmempool() + assert tx_node1["txid"] in self.nodes[1].getrawmempool() + assert not tx_node1["txid"] in self.nodes[0].getrawmempool() + # For transactions that already existed, priority should be changed + entry_node01 = self.nodes[1].getmempoolentry(tx_node01["txid"]) + assert_equal(entry_node01["fees"]["base"] + 1, entry_node01["fees"]["modified"]) + # Deltas for not-yet-submitted transactions should be applied as well (prioritisation is stackable). + self.nodes[1].sendrawtransaction(tx_node01_secret["hex"]) + entry_node01_secret = self.nodes[1].getmempoolentry(tx_node01_secret["txid"]) + assert_equal(entry_node01_secret["fees"]["base"] + 5, entry_node01_secret["fees"]["modified"]) + self.stop_nodes() + if __name__ == "__main__": MempoolPersistTest().main() diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index 3a5bc1ebcd..28ba666dc7 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -8,6 +8,17 @@ Test re-org scenarios with a mempool that contains transactions that spend (directly or indirectly) coinbase transactions. """ +import time + +from test_framework.messages import ( + CInv, + MSG_WTX, + msg_getdata, +) +from test_framework.p2p import ( + P2PTxInvStore, + p2p_lock, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error from test_framework.wallet import MiniWallet @@ -22,8 +33,84 @@ class MempoolCoinbaseTest(BitcoinTestFramework): [] ] + def test_reorg_relay(self): + self.log.info("Test that transactions from disconnected blocks are available for relay immediately") + # Prevent time from moving forward + self.nodes[1].setmocktime(int(time.time())) + self.connect_nodes(0, 1) + self.generate(self.wallet, 3) + + # Disconnect node0 and node1 to create different chains. + self.disconnect_nodes(0, 1) + # Connect a peer to node1, which doesn't have immediate tx relay + peer1 = self.nodes[1].add_p2p_connection(P2PTxInvStore()) + + # Create a transaction that is included in a block. + tx_disconnected = self.wallet.send_self_transfer(from_node=self.nodes[1]) + self.generate(self.nodes[1], 1, sync_fun=self.no_op) + + # Create a transaction and submit it to node1's mempool. + tx_before_reorg = self.wallet.send_self_transfer(from_node=self.nodes[1]) + + # Create a child of that transaction and submit it to node1's mempool. + tx_child = self.wallet.send_self_transfer(utxo_to_spend=tx_disconnected["new_utxo"], from_node=self.nodes[1]) + assert_equal(self.nodes[1].getmempoolentry(tx_child["txid"])["ancestorcount"], 1) + assert_equal(len(peer1.get_invs()), 0) + + # node0 has a longer chain in which tx_disconnected was not confirmed. + self.generate(self.nodes[0], 3, sync_fun=self.no_op) + + # Reconnect the nodes and sync chains. node0's chain should win. + self.connect_nodes(0, 1) + self.sync_blocks() + + # Child now has an ancestor from the disconnected block + assert_equal(self.nodes[1].getmempoolentry(tx_child["txid"])["ancestorcount"], 2) + assert_equal(self.nodes[1].getmempoolentry(tx_before_reorg["txid"])["ancestorcount"], 1) + + # peer1 should not have received an inv for any of the transactions during this time, as not + # enough time has elapsed for those transactions to be announced. Likewise, it cannot + # request very recent, unanounced transactions. + assert_equal(len(peer1.get_invs()), 0) + # It's too early to request these two transactions + requests_too_recent = msg_getdata([CInv(t=MSG_WTX, h=int(tx["tx"].getwtxid(), 16)) for tx in [tx_before_reorg, tx_child]]) + peer1.send_and_ping(requests_too_recent) + for _ in range(len(requests_too_recent.inv)): + peer1.sync_with_ping() + with p2p_lock: + assert "tx" not in peer1.last_message + assert "notfound" in peer1.last_message + + # Request the tx from the disconnected block + request_disconnected_tx = msg_getdata([CInv(t=MSG_WTX, h=int(tx_disconnected["tx"].getwtxid(), 16))]) + peer1.send_and_ping(request_disconnected_tx) + + # The tx from the disconnected block was never announced, and it entered the mempool later + # than the transactions that are too recent. + assert_equal(len(peer1.get_invs()), 0) + with p2p_lock: + # However, the node will answer requests for the tx from the recently-disconnected block. + assert_equal(peer1.last_message["tx"].tx.getwtxid(),tx_disconnected["tx"].getwtxid()) + + self.nodes[1].setmocktime(int(time.time()) + 30) + peer1.sync_with_ping() + # the transactions are now announced + assert_equal(len(peer1.get_invs()), 3) + for _ in range(3): + # make sure all tx requests have been responded to + peer1.sync_with_ping() + last_tx_received = peer1.last_message["tx"] + + tx_after_reorg = self.wallet.send_self_transfer(from_node=self.nodes[1]) + request_after_reorg = msg_getdata([CInv(t=MSG_WTX, h=int(tx_after_reorg["tx"].getwtxid(), 16))]) + assert tx_after_reorg["txid"] in self.nodes[1].getrawmempool() + peer1.send_and_ping(request_after_reorg) + with p2p_lock: + assert_equal(peer1.last_message["tx"], last_tx_received) + def run_test(self): - wallet = MiniWallet(self.nodes[0]) + self.wallet = MiniWallet(self.nodes[0]) + wallet = self.wallet # Start with a 200 block chain assert_equal(self.nodes[0].getblockcount(), 200) @@ -103,6 +190,8 @@ class MempoolCoinbaseTest(BitcoinTestFramework): assert_equal(set(self.nodes[0].getrawmempool()), set()) self.sync_all() + self.test_reorg_relay() + if __name__ == '__main__': MempoolCoinbaseTest().main() diff --git a/test/functional/wallet_fundrawtransaction.py b/test/functional/wallet_fundrawtransaction.py index e367daae2c..fa4f009f34 100755 --- a/test/functional/wallet_fundrawtransaction.py +++ b/test/functional/wallet_fundrawtransaction.py @@ -23,6 +23,7 @@ from test_framework.util import ( assert_raises_rpc_error, count_bytes, find_vout_for_address, + get_fee, ) from test_framework.wallet_util import generate_keypair @@ -570,6 +571,8 @@ class RawTransactionsTest(BitcoinTestFramework): df_wallet = self.nodes[1].get_wallet_rpc(self.default_wallet_name) self.nodes[1].createwallet(wallet_name="locked_wallet", descriptors=self.options.descriptors) wallet = self.nodes[1].get_wallet_rpc("locked_wallet") + # This test is not meant to exercise fee estimation. Making sure all txs are sent at a consistent fee rate. + wallet.settxfee(self.min_relay_tx_fee) # Add some balance to the wallet (this will be reverted at the end of the test) df_wallet.sendall(recipients=[wallet.getnewaddress()]) @@ -599,8 +602,11 @@ class RawTransactionsTest(BitcoinTestFramework): # Choose input inputs = wallet.listunspent() - # Deduce fee to produce a changeless transaction - value = inputs[0]["amount"] - Decimal("0.00002200") + + # Deduce exact fee to produce a changeless transaction + tx_size = 110 # Total tx size: 110 vbytes, p2wpkh -> p2wpkh. Input 68 vbytes + rest of tx is 42 vbytes. + value = inputs[0]["amount"] - get_fee(tx_size, self.min_relay_tx_fee) + outputs = {self.nodes[0].getnewaddress():value} rawtx = wallet.createrawtransaction(inputs, outputs) # fund a transaction that does not require a new key for the change output diff --git a/test/functional/wallet_migration.py b/test/functional/wallet_migration.py index 925376e8cd..c565c879fb 100755 --- a/test/functional/wallet_migration.py +++ b/test/functional/wallet_migration.py @@ -134,13 +134,22 @@ class WalletMigrationTest(BitcoinTestFramework): self.generate(self.nodes[0], 1) bal = basic1.getbalance() txs = basic1.listtransactions() + addr_gps = basic1.listaddressgroupings() - basic1.migratewallet() + basic1_migrate = basic1.migratewallet() assert_equal(basic1.getwalletinfo()["descriptors"], True) self.assert_is_sqlite("basic1") assert_equal(basic1.getbalance(), bal) self.assert_list_txs_equal(basic1.listtransactions(), txs) + self.log.info("Test backup file can be successfully restored") + self.nodes[0].restorewallet("basic1_restored", basic1_migrate['backup_path']) + basic1_restored = self.nodes[0].get_wallet_rpc("basic1_restored") + basic1_restored_wi = basic1_restored.getwalletinfo() + assert_equal(basic1_restored_wi['balance'], bal) + assert_equal(basic1_restored.listaddressgroupings(), addr_gps) + self.assert_list_txs_equal(basic1_restored.listtransactions(), txs) + # restart node and verify that everything is still there self.restart_node(0) default = self.nodes[0].get_wallet_rpc(self.default_wallet_name) |