diff options
author | fanquake <fanquake@gmail.com> | 2023-06-02 10:34:40 +0100 |
---|---|---|
committer | fanquake <fanquake@gmail.com> | 2023-06-02 10:42:05 +0100 |
commit | 8a972813ba68307fa3a296e884b34928be96e246 (patch) | |
tree | b580c08ee613e133a5f316450e5f953f7206a793 /ci | |
parent | 83c7269965c330ae1f4d85b1ba652717c9080914 (diff) | |
parent | 5763b232e6e6a0f72d046f8aa322b39328be135b (diff) | |
download | bitcoin-8a972813ba68307fa3a296e884b34928be96e246.tar.xz |
Merge bitcoin/bitcoin#27737: ci: compile Clang and compiler-rt in msan jobs
5763b232e6e6a0f72d046f8aa322b39328be135b ci: return to using Ubuntu 22.04 in MSAN jobs (fanquake)
d3cbcbf62693ad7394b3f8693b1c08d4271903fa ci: compile clang and compiler-rt in MSAN jobs (fanquake)
796bd1d0d147ac90f921ce3831961f97748d4e1a ci: use LLVM 16.0.4 in MSAN jobs (fanquake)
883bc9f5611648c44956a09795afd924842c1d1d ci: remove extra CC & CXX from MSAN jobs (fanquake)
2d4f4b8f29c015c26cb02b26a517450bb6056ed4 ci: standardize custom libc++ usage in MSAN jobs (fanquake)
Pull request description:
This reworks the MSAN CIs, to first compile Clang and compiler-rt (using GCC 12), and then, compile an MSAN instrumented libc++ using the just-built Clang 16. This fixes the `native_fuzz_with_msan` job, working around https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1005341, by not using the Debian provided Clang/LLVM.
Also included are changes to streamline how we use our "custom libc++", according to upstream: https://releases.llvm.org/16.0.0/projects/libcxx/docs/UsingLibcxx.html#using-a-custom-built-libc, as well as other minor cleanups in the CI configs.
An example job is currently running in the qa-assets repo: https://github.com/bitcoin-core/qa-assets/pull/129 (https://cirrus-ci.com/task/4632561431871488).
ACKs for top commit:
dergoegge:
utACK 5763b232e6e6a0f72d046f8aa322b39328be135b
Tree-SHA512: 4f2a6e0b796bb1830b8346dd1e55eaa86a79037b8b4f16a336c1e29f4fc460acca2ecba076635459370bcbb4009333cb79d27ef1521c1fb5db7599cd5bdf558c
Diffstat (limited to 'ci')
-rwxr-xr-x | ci/test/00_setup_env_native_fuzz_with_msan.sh | 12 | ||||
-rwxr-xr-x | ci/test/00_setup_env_native_msan.sh | 12 | ||||
-rwxr-xr-x | ci/test/01_base_install.sh | 31 |
3 files changed, 38 insertions, 17 deletions
diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index dd694f818c..1f9adc0682 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -6,18 +6,18 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). -LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" +export CI_IMAGE_NAME_TAG="ubuntu:22.04" +LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" -LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" +LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_fuzz_msan" -export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake" +export PACKAGES="cmake ninja-build" # BDB generates false-positives and will be removed in future -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_UNIT_TESTS="false" export RUN_FUNCTIONAL_TESTS="false" diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index bdb9bd7b5d..6ebea46ad0 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -6,18 +6,18 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). -LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" +export CI_IMAGE_NAME_TAG="ubuntu:22.04" +LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" -LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" +LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" -export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake" +export PACKAGES="cmake ninja-build" # BDB generates false-positives and will be removed in future -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_FUNCTIONAL_TESTS="false" export CCACHE_SIZE=250M diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 98f96f0ece..43a76c3d56 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -42,11 +42,32 @@ if [ -n "$PIP_PACKAGES" ]; then fi if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - update-alternatives --install /usr/bin/clang++ clang++ "$(which clang++-16)" 100 - update-alternatives --install /usr/bin/clang clang "$(which clang-16)" 100 - git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.1 "${BASE_SCRATCH_DIR}"/msan/llvm-project - cmake -B "${BASE_SCRATCH_DIR}"/msan/build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF -DLIBCXX_ENABLE_DEBUG_MODE=ON -DLIBCXX_ENABLE_ASSERTIONS=ON -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes - make -C "${BASE_SCRATCH_DIR}"/msan/build/ "$MAKEJOBS" + git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.4 "${BASE_SCRATCH_DIR}"/msan/llvm-project + + cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/clang_build/ -DLLVM_ENABLE_PROJECTS="clang" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_TARGETS_TO_BUILD=X86 \ + -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \ + -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/llvm + + ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ "$MAKEJOBS" + ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ install-runtimes + + update-alternatives --install /usr/bin/clang++ clang++ "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang++ 100 + update-alternatives --install /usr/bin/clang clang "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang 100 + + cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/cxx_build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_USE_SANITIZER=MemoryWithOrigins \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DLLVM_TARGETS_TO_BUILD=X86 \ + -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ + -DLIBCXX_ENABLE_DEBUG_MODE=ON \ + -DLIBCXX_ENABLE_ASSERTIONS=ON \ + -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes + + ninja -C "${BASE_SCRATCH_DIR}"/msan/cxx_build/ "$MAKEJOBS" fi if [[ "${RUN_TIDY}" == "true" ]]; then |