From fa00393bce0c6128c6188afc7a1d50cc01b0277f Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sat, 9 Nov 2019 09:16:16 -0500 Subject: ci: Make all filesystem operations inside docker --- ci/test/04_install.sh | 3 ++- ci/test/05_before_script.sh | 4 ++-- ci/test/06_script_a.sh | 8 ++++++-- ci/test/06_script_b.sh | 4 ---- 4 files changed, 10 insertions(+), 9 deletions(-) (limited to 'ci') diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 271ae82e5c..b394bd897d 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -40,7 +40,6 @@ if [ ! -d ${DIR_QA_ASSETS} ]; then fi export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ -mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/" export ASAN_OPTIONS="detect_stack_use_after_return=1" export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan" export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_BUILD_DIR}/sanitizer-output/tsan" @@ -85,6 +84,8 @@ if [ "$TRAVIS_OS_NAME" != "osx" ]; then ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES fi +DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/" + if [ "$USE_BUSY_BOX" = "true" ]; then echo "Setup to use BusyBox utils" DOCKER_EXEC mkdir -p $BASE_SCRATCH_DIR/bins/ diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index a0488f0807..3f0bba19ac 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -13,13 +13,13 @@ else DOCKER_EXEC echo \> \$HOME/.bitcoin fi -mkdir -p depends/SDKs depends/sdk-sources +DOCKER_EXEC mkdir -p depends/SDKs depends/sdk-sources if [ -n "$OSX_SDK" ] && [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz fi if [ -n "$OSX_SDK" ] && [ -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz + DOCKER_EXEC tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz fi if [[ $HOST = *-mingw32 ]]; then DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\) diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index c4dc22bdd8..34b8477197 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -19,7 +19,9 @@ else fi END_FOLD +# Create folder on host and docker, so that `cd` works mkdir -p build +DOCKER_EXEC mkdir -p build # Temporarily disable errexit, because Travis macOS fails without error message set +o errexit @@ -27,10 +29,12 @@ cd build || (echo "could not enter build directory"; exit 1) set -o errexit BEGIN_FOLD configure -DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) +DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) END_FOLD BEGIN_FOLD distdir +# Create folder on host and docker, so that `cd` works +mkdir -p "bitcoin-$HOST" DOCKER_EXEC make distdir VERSION=$HOST END_FOLD @@ -39,7 +43,7 @@ cd "bitcoin-$HOST" || (echo "could not enter distdir bitcoin-$HOST"; exit 1) set -o errexit BEGIN_FOLD configure -DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) +DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) END_FOLD set -o errtrace diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 048fe06c84..fa2a0f2f0e 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -48,7 +48,3 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then DOCKER_EXEC test/fuzz/test_runner.py -l DEBUG ${DIR_FUZZ_IN} END_FOLD fi - -set +o errexit -cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1) -set -o errexit -- cgit v1.2.3 From fab133329281cdaa3804585a2cdadd0478fefa4f Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sat, 9 Nov 2019 09:13:30 -0500 Subject: ci: Remove git from required packages on host --- ci/README.md | 2 +- ci/test/00_setup_env.sh | 2 +- ci/test/04_install.sh | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) (limited to 'ci') diff --git a/ci/README.md b/ci/README.md index fb1cd7460b..c1c7a0c851 100644 --- a/ci/README.md +++ b/ci/README.md @@ -12,7 +12,7 @@ To allow for a wide range of tested environments, but also ensure reproducibilit requires `docker` to be installed. To install all requirements on Ubuntu, run ``` -sudo apt install docker.io bash git +sudo apt install docker.io bash ``` To run the default test stage, diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 8bc50da2c3..2cd2085d01 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -41,7 +41,7 @@ export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_ROOT_DIR} export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export WINEDEBUG=${WINEDEBUG:-fixme-all} -export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3} +export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 git} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets} export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index b394bd897d..109f9c0b27 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -35,11 +35,6 @@ fi mkdir -p "${BASE_SCRATCH_DIR}" mkdir -p "${CCACHE_DIR}" -if [ ! -d ${DIR_QA_ASSETS} ]; then - git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} -fi -export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ - export ASAN_OPTIONS="detect_stack_use_after_return=1" export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan" export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_BUILD_DIR}/sanitizer-output/tsan" @@ -84,6 +79,11 @@ if [ "$TRAVIS_OS_NAME" != "osx" ]; then ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES fi +if [ ! -d ${DIR_QA_ASSETS} ]; then + DOCKER_EXEC git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} +fi +export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ + DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/" if [ "$USE_BUSY_BOX" = "true" ]; then -- cgit v1.2.3 From fa493ef08830efe493150d07411af85518959804 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sat, 9 Nov 2019 09:20:41 -0500 Subject: ci: Make ci system read-only on the git work tree --- ci/test/00_setup_env.sh | 2 +- ci/test/04_install.sh | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) (limited to 'ci') diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 2cd2085d01..58206dab59 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -41,7 +41,7 @@ export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_ROOT_DIR} export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export WINEDEBUG=${WINEDEBUG:-fixme-all} -export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 git} +export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets} export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 109f9c0b27..8f8317f93d 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -50,7 +50,13 @@ if [ -z "$RUN_CI_ON_HOST" ]; then echo "Creating $DOCKER_NAME_TAG container to run in" ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" - DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$BASE_BUILD_DIR,dst=$BASE_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $BASE_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG) + DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \ + --mount type=bind,src=$BASE_BUILD_DIR,dst=/ro_base,readonly \ + --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \ + --mount type=bind,src=$BASE_BUILD_DIR/depends,dst=$BASE_BUILD_DIR/depends \ + -w $BASE_BUILD_DIR \ + --env-file /tmp/env \ + $DOCKER_NAME_TAG) DOCKER_EXEC () { docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*" @@ -86,6 +92,11 @@ export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/" +if [ -z "$RUN_CI_ON_HOST" ]; then + echo "Create $BASE_BUILD_DIR" + DOCKER_EXEC rsync -a /ro_base/ $BASE_BUILD_DIR +fi + if [ "$USE_BUSY_BOX" = "true" ]; then echo "Setup to use BusyBox utils" DOCKER_EXEC mkdir -p $BASE_SCRATCH_DIR/bins/ -- cgit v1.2.3 From fa7523d3aa75b0266015af59901c2397b52265b5 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sun, 10 Nov 2019 22:38:02 -0500 Subject: ci: Extend docs --- ci/README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'ci') diff --git a/ci/README.md b/ci/README.md index c1c7a0c851..880e49b459 100644 --- a/ci/README.md +++ b/ci/README.md @@ -8,6 +8,16 @@ and numbered according to which stage and lifecycle step it belongs to. ### Running a stage locally +Be aware that the tests will be built and run in-place, so please run at your own risk. +If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first. + +The ci needs to perform various sysadmin tasks such as installing packages or writing to the user's home directory. +While most of the actions are done inside a docker container, this is not possible for all. Thus, cache directories, +such as the depends cache or ccache, are mounted as read-write into the docker container. While it should be fine to run +the ci system locally on you development box, the ci scripts can generally be assumed to have received less review and +testing compared to other parts of the codebase. If you want to keep the work tree clean, you might want to run the ci +system in a virtual machine with a Linux operating system of your choice. + To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage requires `docker` to be installed. To install all requirements on Ubuntu, run @@ -26,6 +36,3 @@ To run the test stage with a specific configuration, ``` FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh ``` - -Be aware that the tests will be build and run in-place, so please run at your own risk. -If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first. -- cgit v1.2.3