From fa31bc35eb8dd7e727e3e1eaf2c45017cd63bdcb Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sat, 10 Aug 2019 12:56:46 -0400 Subject: ci: Remove dependence on travis, use it as fallback env --- .travis.yml | 17 ++--------------- ci/README.md | 18 +++++++++++++++++- ci/retry/README.md | 2 +- ci/retry/retry | 4 ++-- ci/test/00_setup_env.sh | 33 +++++++++++++++++++++++++++++++++ ci/test/04_install.sh | 25 ++++++++++++++----------- ci/test/06_script_a.sh | 12 ++++-------- ci/test/06_script_b.sh | 4 ++-- ci/test_run_all.sh | 16 ++++++++++++++++ 9 files changed, 91 insertions(+), 40 deletions(-) create mode 100755 ci/test/00_setup_env.sh create mode 100755 ci/test_run_all.sh diff --git a/.travis.yml b/.travis.yml index b54ed67b02..7753dcc51f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,22 +40,10 @@ stages: - extended-lint env: global: - - MAKEJOBS=-j3 - - RUN_UNIT_TESTS=true - - RUN_FUNCTIONAL_TESTS=true - - RUN_FUZZ_TESTS=false - - DOCKER_NAME_TAG=ubuntu:18.04 - - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID - - CCACHE_SIZE=100M - - CCACHE_TEMPDIR=/tmp/.ccache-temp - - CCACHE_COMPRESS=1 - - CCACHE_DIR=$HOME/.ccache - - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out - - SDK_URL=https://bitcoincore.org/depends-sources/sdks - - WINEDEBUG=fixme-all - - DOCKER_PACKAGES="build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache" + - CI_RETRY_EXE="travis_retry" - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. Please manually re-run this job by using the travis restart button or asking a bitcoin maintainer to restart. The next run should not time out because the build cache has been saved." before_install: + - set -o errexit; source ./ci/test/00_setup_env.sh - set -o errexit; source ./ci/test/03_before_install.sh install: - set -o errexit; source ./ci/test/04_install.sh @@ -71,7 +59,6 @@ script: - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi after_script: - echo $TRAVIS_COMMIT_RANGE - - echo $TRAVIS_COMMIT_LOG jobs: include: diff --git a/ci/README.md b/ci/README.md index 0aed238c77..a50c7868a6 100644 --- a/ci/README.md +++ b/ci/README.md @@ -3,6 +3,22 @@ This directory contains scripts for each build step in each build stage. Currently three stages `lint`, `extended_lint` and `test` are defined. Each stage has its own lifecycle, similar to the -[Travis CI lifecycle](https://docs.travis-ci.com/user/job-lifecycle#the-job-lifecycle). Every script in here is named +[Travis CI lifecycle](https://docs.travis-ci.com/user/job-lifecycle#the-job-lifecycle). Every script in here is named and numbered according to which stage and lifecycle step it belongs to. +### Running a stage locally + +To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage +requires `docker` to be installed. To install all requirements on Ubuntu, run + +``` +sudo apt install docker.io ccache bash git +``` + +To run the test stage, + +``` +./ci/test_run_all.sh +``` + +Be aware that the tests will be build and run in-place, so please run at your own risk. diff --git a/ci/retry/README.md b/ci/retry/README.md index ead2271f64..983a498070 100644 --- a/ci/retry/README.md +++ b/ci/retry/README.md @@ -35,7 +35,7 @@ Help: -m, --min=secs Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3 -x, --max=secs Exponenetial Backoff: maximum sleep amount (seconds): Default 60 -f, --fail="script +cmds" Fail Script: run in case of final failure - + ### Examples No problem: diff --git a/ci/retry/retry b/ci/retry/retry index 03dc180ef2..0e5f6e9701 100755 --- a/ci/retry/retry +++ b/ci/retry/retry @@ -4,7 +4,7 @@ GETOPT_BIN=$IN_GETOPT_BIN GETOPT_BIN=${GETOPT_BIN:-getopt} __sleep_amount() { - if [ -n "$constant_sleep" ]; then + if [ -n "$constant_sleep" ]; then sleep_time=$constant_sleep else #TODO: check for awk @@ -159,5 +159,5 @@ EOF done retry "$max_tries" "$min_sleep" "$max_sleep" "$constant_sleep" "$fail_script" "$@" - + fi diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh new file mode 100755 index 0000000000..fae48ba176 --- /dev/null +++ b/ci/test/00_setup_env.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +echo "Setting default values in env" + +BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) +export BASE_ROOT_DIR + +export MAKEJOBS=${MAKEJOBS:--j3} +export HOST=${HOST:-x86_64-unknown-linux-gnu} +export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} +export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} +export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} +export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:18.04} +export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1$TRAVIS_BUILD_ID} +export CCACHE_SIZE=${CCACHE_SIZE:-100M} +export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} +export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} +export CCACHE_DIR=${CCACHE_DIR:-$HOME/.ccache} +export BASE_BUILD_DIR=${BASE_BUILD_DIR:-${TRAVIS_BUILD_DIR:-$BASE_ROOT_DIR}} +export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST} +export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} +export WINEDEBUG=${WINEDEBUG:-fixme-all} +export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3} +export GOAL=${GOAL:-install} +export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets} +export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH +export CI_RETRY_EXE=${CI_RETRY_EXE:retry} diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 319f2c5b21..2230f5c8eb 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -9,29 +9,32 @@ export LC_ALL=C.UTF-8 free -m -h echo "Number of CPUs (nproc): $(nproc)" -travis_retry docker pull "$DOCKER_NAME_TAG" +ccache echo "Creating ccache dir if it didn't already exist" -export DIR_FUZZ_IN=${TRAVIS_BUILD_DIR}/qa-assets -git clone https://github.com/bitcoin-core/qa-assets ${DIR_FUZZ_IN} -export DIR_FUZZ_IN=${DIR_FUZZ_IN}/fuzz_seed_corpus/ +${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" -mkdir -p "${TRAVIS_BUILD_DIR}/sanitizer-output/" +if [ ! -d ${DIR_QA_ASSETS} ]; then + git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} +fi +export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ + +mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/" export ASAN_OPTIONS="" -export LSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/lsan" -export TSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${TRAVIS_BUILD_DIR}/sanitizer-output/tsan" -export UBSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1" +export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan" +export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_BUILD_DIR}/sanitizer-output/tsan" +export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1" env | grep -E '^(BITCOIN_CONFIG|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env if [[ $HOST = *-mingw32 ]]; then DOCKER_ADMIN="--cap-add SYS_ADMIN" elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) DOCKER_ADMIN="--cap-add SYS_PTRACE" fi -DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$TRAVIS_BUILD_DIR,dst=$TRAVIS_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $TRAVIS_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG) +DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$BASE_BUILD_DIR,dst=$BASE_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $BASE_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG) DOCKER_EXEC () { docker exec $DOCKER_ID bash -c "cd $PWD && $*" } -travis_retry DOCKER_EXEC apt-get update -travis_retry DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES +${CI_RETRY_EXE} DOCKER_EXEC apt-get update +${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index 8cc593f936..eb6ade7919 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -6,11 +6,7 @@ export LC_ALL=C.UTF-8 -TRAVIS_COMMIT_LOG=$(git log --format=fuller -1) -export TRAVIS_COMMIT_LOG - -OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST -BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib" +BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$BASE_BUILD_DIR/depends/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" if [ -z "$NO_DEPENDS" ]; then DOCKER_EXEC ccache --max-size=$CCACHE_SIZE fi @@ -23,7 +19,7 @@ else fi END_FOLD -mkdir build +mkdir -p build cd build || (echo "could not enter build directory"; exit 1) BEGIN_FOLD configure @@ -41,10 +37,10 @@ DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOI END_FOLD set -o errtrace -trap 'DOCKER_EXEC "cat ${TRAVIS_BUILD_DIR}/sanitizer-output/* 2> /dev/null"' ERR +trap 'DOCKER_EXEC "cat ${BASE_BUILD_DIR}/sanitizer-output/* 2> /dev/null"' ERR BEGIN_FOLD build DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ) END_FOLD -cd ${TRAVIS_BUILD_DIR} || (echo "could not enter travis build dir $TRAVIS_BUILD_DIR"; exit 1) +cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1) diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index e40055a6ee..fafb6e0493 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -10,7 +10,7 @@ cd "build/bitcoin-$HOST" || (echo "could not enter distdir build/bitcoin-$HOST"; if [ "$RUN_UNIT_TESTS" = "true" ]; then BEGIN_FOLD unit-tests - DOCKER_EXEC LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1 + DOCKER_EXEC LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1 END_FOLD fi @@ -26,4 +26,4 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then END_FOLD fi -cd ${TRAVIS_BUILD_DIR} || (echo "could not enter travis build dir $TRAVIS_BUILD_DIR"; exit 1) +cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1) diff --git a/ci/test_run_all.sh b/ci/test_run_all.sh new file mode 100755 index 0000000000..a39f1f9f09 --- /dev/null +++ b/ci/test_run_all.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +echo "Setting default values in env" + +set -o errexit; source ./ci/test/00_setup_env.sh +set -o errexit; source ./ci/test/03_before_install.sh +set -o errexit; source ./ci/test/04_install.sh +set -o errexit; source ./ci/test/05_before_script.sh +set -o errexit; source ./ci/test/06_script_a.sh +set -o errexit; source ./ci/test/06_script_b.sh -- cgit v1.2.3