Merge bitcoin/bitcoin#29693: [27.x] Further Backports

a7116c8febb0747591b19a2f428d999984990873 ci: Bump msan to llvm-18 (MarcoFalke)
05f69b36d1ecf0b0de8387572e2590536c0de74f ci, macos: Use `--break-system-packages` with Homebrew's python (Hennadii Stepanov)
603f0368a5ad7fed69888af5446e565967f96d6a ci: Add workaround for Homebrew's python link error (Hennadii Stepanov)
5d381cfb6f870788048a8900daa7fb6d7b70a6c1 serfloat: improve/simplify tests (Pieter Wuille)
f4be4d7447c3b96f28f2bd2622979529fcc4b846 serfloat: do not test encode(bits)=bits anymore (Pieter Wuille)

Pull request description:

  Currently:
  * https://github.com/bitcoin/bitcoin/pull/29192
  * https://github.com/bitcoin/bitcoin/pull/29610
  * https://github.com/bitcoin/bitcoin/pull/29676

ACKs for top commit:
  stickies-v:
    ACK a7116c8febb0747591b19a2f428d999984990873 - all clean test backports

Tree-SHA512: f3508a2c20d336c8647ba16886859d6a070584c4739fc8b5cfce2041a0662794775fb0ce89c9bf848a29e70089bae05ad1c921bbe45afe3fd5cac2a5c6b76baf

6 files changed, 82 insertions, 54 deletions

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a86b00fdc6..77e4597b6d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -96,7 +96,10 @@ jobs:
       - name: Install Homebrew packages
         env:
           HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
-        run: brew install automake libtool pkg-config gnu-getopt ccache boost libevent miniupnpc libnatpmp zeromq qt@5 qrencode
+        run: |
+          # A workaround for "The `brew link` step did not complete successfully" error.
+          brew install python@3 || brew link --overwrite python@3
+          brew install automake libtool pkg-config gnu-getopt ccache boost libevent miniupnpc libnatpmp zeromq qt@5 qrencode
 
       - name: Set Ccache directory
         run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV"
diff --git a/ci/test/00_setup_env_mac_native.sh b/ci/test/00_setup_env_mac_native.sh
index 439fba16ef..c47f13f96e 100755
--- a/ci/test/00_setup_env_mac_native.sh
+++ b/ci/test/00_setup_env_mac_native.sh
@@ -7,7 +7,9 @@
 export LC_ALL=C.UTF-8
 
 export HOST=x86_64-apple-darwin
-export PIP_PACKAGES="zmq"
+# Homebrew's python@3.12 is marked as externally managed (PEP 668).
+# Therefore, `--break-system-packages` is needed.
+export PIP_PACKAGES="--break-system-packages zmq"
 export GOAL="install"
 export BITCOIN_CONFIG="--with-gui --with-miniupnpc --with-natpmp --enable-reduce-exports"
 export CI_OS_NAME="macos"
diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh
index 0b32049013..0a9dee2ed8 100755
--- a/ci/test/00_setup_env_native_fuzz_with_msan.sh
+++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh
@@ -6,7 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04"
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
 LIBCXX_DIR="/msan/cxx_build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
 LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"
diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh
index 60987f5011..cbcd51e218 100755
--- a/ci/test/00_setup_env_native_msan.sh
+++ b/ci/test/00_setup_env_native_msan.sh
@@ -6,7 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04"
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
 LIBCXX_DIR="/msan/cxx_build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
 LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"
diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh
index 99813da106..6f1498963a 100755
--- a/ci/test/01_base_install.sh
+++ b/ci/test/01_base_install.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018-2022 The Bitcoin Core developers
+# Copyright (c) 2018-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -36,7 +36,7 @@ if [ -n "$PIP_PACKAGES" ]; then
 fi
 
 if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
-  ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-17.0.6 /msan/llvm-project
+  ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-18.1.1" /msan/llvm-project
 
   cmake -G Ninja -B /msan/clang_build/ \
     -DLLVM_ENABLE_PROJECTS="clang" \
@@ -53,13 +53,14 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
   update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /msan/clang_build/bin/llvm-symbolizer 100
 
   cmake -G Ninja -B /msan/cxx_build/ \
-    -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \
+    -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
     -DCMAKE_BUILD_TYPE=Release \
     -DLLVM_USE_SANITIZER=MemoryWithOrigins \
     -DCMAKE_C_COMPILER=clang \
     -DCMAKE_CXX_COMPILER=clang++ \
     -DLLVM_TARGETS_TO_BUILD=Native \
     -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \
+    -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \
     -DLIBCXX_HARDENING_MODE=debug \
     -S /msan/llvm-project/runtimes
 
diff --git a/src/test/serfloat_tests.cpp b/src/test/serfloat_tests.cpp
index b36bdc02ca..304541074f 100644
--- a/src/test/serfloat_tests.cpp
+++ b/src/test/serfloat_tests.cpp
@@ -37,6 +37,7 @@ uint64_t TestDouble(double f) {
 } // namespace
 
 BOOST_AUTO_TEST_CASE(double_serfloat_tests) {
+    // Test specific values against their expected encoding.
     BOOST_CHECK_EQUAL(TestDouble(0.0), 0U);
     BOOST_CHECK_EQUAL(TestDouble(-0.0), 0x8000000000000000);
     BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::infinity()), 0x7ff0000000000000U);
@@ -46,55 +47,76 @@ BOOST_AUTO_TEST_CASE(double_serfloat_tests) {
     BOOST_CHECK_EQUAL(TestDouble(2.0), 0x4000000000000000ULL);
     BOOST_CHECK_EQUAL(TestDouble(4.0), 0x4010000000000000ULL);
     BOOST_CHECK_EQUAL(TestDouble(785.066650390625), 0x4088888880000000ULL);
+    BOOST_CHECK_EQUAL(TestDouble(3.7243058682384174), 0x400dcb60e0031440);
+    BOOST_CHECK_EQUAL(TestDouble(91.64070592566159), 0x4056e901536d447a);
+    BOOST_CHECK_EQUAL(TestDouble(-98.63087668642575), 0xc058a860489c007a);
+    BOOST_CHECK_EQUAL(TestDouble(4.908737756962054), 0x4013a28c268b2b70);
+    BOOST_CHECK_EQUAL(TestDouble(77.9247330021754), 0x40537b2ed3547804);
+    BOOST_CHECK_EQUAL(TestDouble(40.24732825357566), 0x40441fa873c43dfc);
+    BOOST_CHECK_EQUAL(TestDouble(71.39395607929222), 0x4051d936938f27b6);
+    BOOST_CHECK_EQUAL(TestDouble(58.80100710817612), 0x404d668766a2bd70);
+    BOOST_CHECK_EQUAL(TestDouble(-30.10665786964975), 0xc03e1b4dee1e01b8);
+    BOOST_CHECK_EQUAL(TestDouble(60.15231509068704), 0x404e137f0f969814);
+    BOOST_CHECK_EQUAL(TestDouble(-48.15848711335961), 0xc04814494e445bc6);
+    BOOST_CHECK_EQUAL(TestDouble(26.68450101125353), 0x403aaf3b755169b0);
+    BOOST_CHECK_EQUAL(TestDouble(-65.72071986604303), 0xc0506e2046378ede);
+    BOOST_CHECK_EQUAL(TestDouble(17.95575825512381), 0x4031f4ac92b0a388);
+    BOOST_CHECK_EQUAL(TestDouble(-35.27171863226279), 0xc041a2c7ad17a42a);
+    BOOST_CHECK_EQUAL(TestDouble(-8.58810329425124), 0xc0212d1bdffef538);
+    BOOST_CHECK_EQUAL(TestDouble(88.51393044338977), 0x405620e43c83b1c8);
+    BOOST_CHECK_EQUAL(TestDouble(48.07224932612732), 0x4048093f77466ffc);
+    BOOST_CHECK_EQUAL(TestDouble(9.867348871395659e+117), 0x586f4daeb2459b9f);
+    BOOST_CHECK_EQUAL(TestDouble(-1.5166424385129721e+206), 0xeabe3bbc484bd458);
+    BOOST_CHECK_EQUAL(TestDouble(-8.585156555624594e-275), 0x8707c76eee012429);
+    BOOST_CHECK_EQUAL(TestDouble(2.2794371091628822e+113), 0x5777b2184458f4ee);
+    BOOST_CHECK_EQUAL(TestDouble(-1.1290476594131867e+163), 0xe1c91893d3488bb0);
+    BOOST_CHECK_EQUAL(TestDouble(9.143848423979275e-246), 0x0d0ff76e5f2620a3);
+    BOOST_CHECK_EQUAL(TestDouble(-2.8366718125941117e+81), 0xd0d7ec7e754b394a);
+    BOOST_CHECK_EQUAL(TestDouble(-1.2754409481684012e+229), 0xef80d32f8ec55342);
+    BOOST_CHECK_EQUAL(TestDouble(6.000577060053642e-186), 0x197a1be7c8209b6a);
+    BOOST_CHECK_EQUAL(TestDouble(2.0839423284378986e-302), 0x014c94f8689cb0a5);
+    BOOST_CHECK_EQUAL(TestDouble(-1.422140051483753e+259), 0xf5bd99271d04bb35);
+    BOOST_CHECK_EQUAL(TestDouble(-1.0593973991188853e+46), 0xc97db0cdb72d1046);
+    BOOST_CHECK_EQUAL(TestDouble(2.62945125875249e+190), 0x67779b36366c993b);
+    BOOST_CHECK_EQUAL(TestDouble(-2.920377657275094e+115), 0xd7e7b7b45908e23b);
+    BOOST_CHECK_EQUAL(TestDouble(9.790289014855851e-118), 0x27a3c031cc428bcc);
+    BOOST_CHECK_EQUAL(TestDouble(-4.629317182034961e-114), 0xa866ccf0b753705a);
+    BOOST_CHECK_EQUAL(TestDouble(-1.7674605603846528e+279), 0xf9e8ed383ffc3e25);
+    BOOST_CHECK_EQUAL(TestDouble(2.5308171727712605e+120), 0x58ef5cd55f0ec997);
+    BOOST_CHECK_EQUAL(TestDouble(-1.05034156412799e+54), 0xcb25eea1b9350fa0);
 
-    // Roundtrip test on IEC559-compatible systems
-    if (std::numeric_limits<double>::is_iec559) {
-        BOOST_CHECK_EQUAL(sizeof(double), 8U);
-        BOOST_CHECK_EQUAL(sizeof(uint64_t), 8U);
-        // Test extreme values
-        TestDouble(std::numeric_limits<double>::min());
-        TestDouble(-std::numeric_limits<double>::min());
-        TestDouble(std::numeric_limits<double>::max());
-        TestDouble(-std::numeric_limits<double>::max());
-        TestDouble(std::numeric_limits<double>::lowest());
-        TestDouble(-std::numeric_limits<double>::lowest());
-        TestDouble(std::numeric_limits<double>::quiet_NaN());
-        TestDouble(-std::numeric_limits<double>::quiet_NaN());
-        TestDouble(std::numeric_limits<double>::signaling_NaN());
-        TestDouble(-std::numeric_limits<double>::signaling_NaN());
-        TestDouble(std::numeric_limits<double>::denorm_min());
-        TestDouble(-std::numeric_limits<double>::denorm_min());
-        // Test exact encoding: on currently supported platforms, EncodeDouble
-        // should produce exactly the same as the in-memory representation for non-NaN.
-        for (int j = 0; j < 1000; ++j) {
-            // Iterate over 9 specific bits exhaustively; the others are chosen randomly.
-            // These specific bits are the sign bit, and the 2 top and bottom bits of
-            // exponent and mantissa in the IEEE754 binary64 format.
-            for (int x = 0; x < 512; ++x) {
-                uint64_t v = InsecureRandBits(64);
-                v &= ~(uint64_t{1} << 0);
-                if (x & 1) v |= (uint64_t{1} << 0);
-                v &= ~(uint64_t{1} << 1);
-                if (x & 2) v |= (uint64_t{1} << 1);
-                v &= ~(uint64_t{1} << 50);
-                if (x & 4) v |= (uint64_t{1} << 50);
-                v &= ~(uint64_t{1} << 51);
-                if (x & 8) v |= (uint64_t{1} << 51);
-                v &= ~(uint64_t{1} << 52);
-                if (x & 16) v |= (uint64_t{1} << 52);
-                v &= ~(uint64_t{1} << 53);
-                if (x & 32) v |= (uint64_t{1} << 53);
-                v &= ~(uint64_t{1} << 61);
-                if (x & 64) v |= (uint64_t{1} << 61);
-                v &= ~(uint64_t{1} << 62);
-                if (x & 128) v |= (uint64_t{1} << 62);
-                v &= ~(uint64_t{1} << 63);
-                if (x & 256) v |= (uint64_t{1} << 63);
-                double f;
-                memcpy(&f, &v, 8);
-                uint64_t v2 = TestDouble(f);
-                if (!std::isnan(f)) BOOST_CHECK_EQUAL(v, v2);
+    // Test extreme values
+    BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::min()), 0x10000000000000);
+    BOOST_CHECK_EQUAL(TestDouble(-std::numeric_limits<double>::min()), 0x8010000000000000);
+    BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::max()), 0x7fefffffffffffff);
+    BOOST_CHECK_EQUAL(TestDouble(-std::numeric_limits<double>::max()), 0xffefffffffffffff);
+    BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::lowest()), 0xffefffffffffffff);
+    BOOST_CHECK_EQUAL(TestDouble(-std::numeric_limits<double>::lowest()), 0x7fefffffffffffff);
+    BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::denorm_min()), 0x1);
+    BOOST_CHECK_EQUAL(TestDouble(-std::numeric_limits<double>::denorm_min()), 0x8000000000000001);
+    // Note that all NaNs are encoded the same way.
+    BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::quiet_NaN()), 0x7ff8000000000000);
+    BOOST_CHECK_EQUAL(TestDouble(-std::numeric_limits<double>::quiet_NaN()), 0x7ff8000000000000);
+    BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::signaling_NaN()), 0x7ff8000000000000);
+    BOOST_CHECK_EQUAL(TestDouble(-std::numeric_limits<double>::signaling_NaN()), 0x7ff8000000000000);
+
+    // Construct doubles to test from the encoding.
+    static_assert(sizeof(double) == 8);
+    static_assert(sizeof(uint64_t) == 8);
+    for (int j = 0; j < 1000; ++j) {
+        // Iterate over 9 specific bits exhaustively; the others are chosen randomly.
+        // These specific bits are the sign bit, and the 2 top and bottom bits of
+        // exponent and mantissa in the IEEE754 binary64 format.
+        for (int x = 0; x < 512; ++x) {
+            uint64_t v = InsecureRandBits(64);
+            int x_pos = 0;
+            for (int v_pos : {0, 1, 50, 51, 52, 53, 61, 62, 63}) {
+                v &= ~(uint64_t{1} << v_pos);
+                if ((x >> (x_pos++)) & 1) v |= (uint64_t{1} << v_pos);
             }
+            double f;
+            memcpy(&f, &v, 8);
+            TestDouble(f);
         }
     }
 }