From fad0b67c212dcb8a16fcbda5a74acc959ed4e284 Mon Sep 17 00:00:00 2001 From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz> Date: Mon, 17 Jul 2023 12:30:12 +0200 Subject: ci: Use qemu-user through container engine --- ci/test/00_setup_env.sh | 8 ++------ ci/test/00_setup_env_arm.sh | 10 +--------- ci/test/00_setup_env_i686_centos.sh | 2 +- ci/test/00_setup_env_i686_multiprocess.sh | 2 +- ci/test/00_setup_env_s390x.sh | 11 +---------- ci/test/04_install.sh | 4 ---- ci/test/06_script_b.sh | 12 +++++------- ci/test/wrap-qemu.sh | 18 ------------------ 8 files changed, 11 insertions(+), 56 deletions(-) delete mode 100755 ci/test/wrap-qemu.sh (limited to 'ci') diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index ba8fc861e8..75b47af0ed 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -32,10 +32,6 @@ fi echo "Fallback to default values in env (if not yet set)" # The number of parallel jobs to pass down to make and test_runner.py export MAKEJOBS=${MAKEJOBS:--j4} -# What host to compile for. See also ./depends/README.md -# Tests that need cross-compilation export the appropriate HOST. -# Tests that run natively guess the host -export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} # Whether to prefer BusyBox over GNU utilities export USE_BUSY_BOX=${USE_BUSY_BOX:-false} @@ -61,12 +57,12 @@ export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} # This folder exists only on the ci guest, and on the ci host as a volume. export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache} # Folder where the build result is put (bin and lib). -export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} +export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out} # Folder where the build is done (dist and out-of-tree build). export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build} # The folder for previous release binaries. # This folder exists only on the ci guest, and on the ci host as a volume. -export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST} +export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/prev_releases} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison} export GOAL=${GOAL:-install} diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index ac0c0be96a..65d37f01d9 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -7,18 +7,10 @@ export LC_ALL=C.UTF-8 export HOST=arm-linux-gnueabihf -# The host arch is unknown, so we run the tests through qemu. -# If the host is arm and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string. -if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-arm -L /usr/arm-linux-gnueabihf/"}"; fi export DPKG_ADD_ARCH="armhf" export PACKAGES="python3-zmq g++-arm-linux-gnueabihf busybox libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf" -if [ -n "$QEMU_USER_CMD" ]; then - # Likely cross-compiling, so install the needed gcc and qemu-user - export PACKAGES="$PACKAGES qemu-user" -fi export CONTAINER_NAME=ci_arm_linux -# Use debian to avoid 404 apt errors when cross compiling -export CI_IMAGE_NAME_TAG="debian:bullseye" +export CI_IMAGE_NAME_TAG="docker.io/arm64v8/debian:bookworm" export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index 606c28e252..a8bc0d0ca0 100755 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_centos -export CI_IMAGE_NAME_TAG="quay.io/centos/centos:stream9" +export CI_IMAGE_NAME_TAG="quay.io/centos/amd64:stream9" export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison util-linux" export PIP_PACKAGES="pyzmq" export GOAL="install" diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index 7911c1912f..b11a387660 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_multiprocess -export CI_IMAGE_NAME_TAG=ubuntu:20.04 +export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:20.04" export PACKAGES="cmake llvm clang g++-multilib" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 523e81c94a..a7516d6b4e 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -7,18 +7,9 @@ export LC_ALL=C.UTF-8 export HOST=s390x-linux-gnu -# The host arch is unknown, so we run the tests through qemu. -# If the host is s390x and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string. -if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-s390x"}"; fi export PACKAGES="python3-zmq" -if [ -n "$QEMU_USER_CMD" ]; then - # Likely cross-compiling, so install the needed gcc and qemu-user - export DPKG_ADD_ARCH="s390x" - export PACKAGES="$PACKAGES g++-s390x-linux-gnu qemu-user libc6:s390x libstdc++6:s390x" -fi -# Use debian to avoid 404 apt errors export CONTAINER_NAME=ci_s390x -export CI_IMAGE_NAME_TAG="debian:bookworm" +export CI_IMAGE_NAME_TAG="docker.io/s390x/debian:bookworm" export TEST_RUNNER_ENV="LC_ALL=C" export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export RUN_FUNCTIONAL_TESTS=true diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 205c79328a..7ecf00097c 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -6,10 +6,6 @@ export LC_ALL=C.UTF-8 -if [[ $QEMU_USER_CMD == qemu-s390* ]]; then - export LC_ALL=C -fi - # Create folders that are mounted into the docker mkdir -p "${CCACHE_DIR}" mkdir -p "${PREVIOUS_RELEASES_DIR}" diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 02358db789..eee4b8d2b3 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -19,6 +19,11 @@ fi echo "Free disk space:" df -h +# What host to compile for. See also ./depends/README.md +# Tests that need cross-compilation export the appropriate HOST. +# Tests that run natively guess the host +export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} + if [ "$RUN_FUZZ_TESTS" = "true" ]; then export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ if [ ! -d "$DIR_FUZZ_IN" ]; then @@ -124,13 +129,6 @@ if [[ $HOST = *-mingw32 ]]; then "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" fi -if [ -n "$QEMU_USER_CMD" ]; then - # Generate all binaries, so that they can be wrapped - make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 - make "$MAKEJOBS" -C src minisketch/test VERBOSE=1 - "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" -fi - if [ -n "$USE_VALGRIND" ]; then "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh" fi diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh deleted file mode 100755 index e028ede378..0000000000 --- a/ci/test/wrap-qemu.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{test_json,unitester,object}}; do - # shellcheck disable=SC2044 - for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do - echo "Wrap $b ..." - mv "$b" "${b}_orig" - echo '#!/usr/bin/env bash' > "$b" - echo "$QEMU_USER_CMD \"${b}_orig\" \"\$@\"" >> "$b" - chmod +x "$b" - done -done -- cgit v1.2.3