diff options
author | MarcoFalke <falke.marco@gmail.com> | 2021-04-11 09:51:13 +0200 |
---|---|---|
committer | MarcoFalke <falke.marco@gmail.com> | 2021-04-11 09:51:22 +0200 |
commit | 1e3db6807d1809182c799b5f7a78843f03fbe413 (patch) | |
tree | 25121f5a12f4609d0b21ada6f95069f840196aab | |
parent | dd01dc6a1d23e1f316940cd1092ad716216331aa (diff) | |
parent | fa41a917356a7f5e0f3286b227ced7f2e6797e67 (diff) |
Merge #21619: ci: Run self-hosted ci
fa41a917356a7f5e0f3286b227ced7f2e6797e67 ci: Run self-hosted ci (MarcoFalke)
fa52a40f0ebb7aa33af210a18e525eec7bb6887f ci: Make cirrus cache folders relative to cirrus base dir (MarcoFalke)
fa278412a023ac150da764a465567f2970348449 ci: Restart docker before run (MarcoFalke)
fad4f48e0763d111369656dea9575a789d2016e8 ci: [refactor] Create setting for ephemeral config in .cirrus.yml (MarcoFalke)
Pull request description:
Due to our heavy use of the Cirrus CI community cluster, some tasks may take a long time to get scheduled. While it is possible to use "Compute Credits" to get immediately scheduled on the cluster, I couldn't find a sponsor that'd be willing to cover the total cost, if all tasks were paid for with credits.
However, it is also possible to bring our own runners to Cirrus CI.
For testing purposes, a single task will be transformed to run on the DrahtBot infrastructure. If all goes well, the other tasks can be moved, too.
ACKs for top commit:
hebasto:
ACK fa41a917356a7f5e0f3286b227ced7f2e6797e67, I have reviewed the code and it looks OK.
Tree-SHA512: 513738daec36da8cd48a8f11e687ff0b7dfaba1ae4ed2fa77e7b043f88fd52bf5c0dbad2768e13df88518323917f08348cb62be6376a423142921f8d2c59a938
-rw-r--r-- | .cirrus.yml | 40 | ||||
-rwxr-xr-x | ci/test/04_install.sh | 7 |
2 files changed, 36 insertions, 11 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 99957f08c5..6148f56e0d 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -3,10 +3,19 @@ env: PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y" MAKEJOBS: "-j4" - DANGER_RUN_CI_ON_HOST: "1" # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache CCACHE_SIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" + CCACHE_NOHASHDIR: "1" # Debug info might contain a stale path if the build dir changes, but this is fine + +cirrus_ephemeral_worker_template_env: &CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + DANGER_RUN_CI_ON_HOST: "1" # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system + +persistent_worker_template_env: &PERSISTENT_WORKER_TEMPLATE_ENV + RESTART_CI_DOCKER_BEFORE_RUN: "1" + +persistent_worker_template: &PERSISTENT_WORKER_TEMPLATE + persistent_worker: {} # https://cirrus-ci.org/guide/persistent-workers/ # https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks base_template: &BASE_TEMPLATE @@ -31,9 +40,9 @@ global_task_template: &GLOBAL_TASK_TEMPLATE ccache_cache: folder: "/tmp/ccache_dir" depends_built_cache: - folder: "/tmp/cirrus-ci-build/depends/built" + folder: "depends/built" depends_sdk_cache: - folder: "/tmp/cirrus-ci-build/depends/sdk-sources" + folder: "depends/sdk-sources" ci_script: - ./ci/test_run_all.sh @@ -69,6 +78,8 @@ task: << : *CREDITS_TEMPLATE lint_script: - ./ci/lint_run_all.sh + env: + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV task: name: 'ARM [unit tests, no functional tests] [buster]' @@ -76,6 +87,7 @@ task: container: image: debian:buster env: + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_arm.sh" task: @@ -84,6 +96,7 @@ task: container: image: ubuntu:focal env: + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_win64.sh" task: @@ -92,19 +105,18 @@ task: container: image: centos:8 env: + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV PACKAGE_MANAGER_INSTALL: "yum install -y" FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh" task: name: '[previous releases, uses qt5 dev package and some depends packages] [unsigned char] [bionic]' - # For faster CI feedback, immediately schedule a task that compiles most modules - << : *CREDITS_TEMPLATE << : *GLOBAL_TASK_TEMPLATE depends_releases_cache: - folder: "/tmp/cirrus-ci-build/releases" - container: - image: ubuntu:bionic + folder: "releases" + << : *PERSISTENT_WORKER_TEMPLATE env: + << : *PERSISTENT_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh" task: @@ -115,6 +127,7 @@ task: cpu: 6 # Increase CPU and Memory to avoid timeout memory: 24G env: + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV MAKEJOBS: "-j8" FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" @@ -124,6 +137,7 @@ task: container: image: ubuntu:focal env: + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" task: @@ -132,6 +146,7 @@ task: container: image: ubuntu:focal env: + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" task: @@ -140,6 +155,7 @@ task: container: image: ubuntu:focal env: + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" task: @@ -148,6 +164,7 @@ task: container: image: ubuntu:focal env: + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_multiprocess.sh" task: @@ -156,6 +173,7 @@ task: container: image: ubuntu:bionic env: + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh" task: @@ -164,6 +182,7 @@ task: container: image: ubuntu:focal env: + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_mac.sh" task: @@ -175,7 +194,7 @@ task: # Use latest image, but hardcode version to avoid silent upgrades (and breaks) image: big-sur-xcode-12.4 # https://cirrus-ci.org/guide/macOS env: - DANGER_RUN_CI_ON_HOST: "true" + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV CI_USE_APT_INSTALL: "no" PACKAGE_MANAGER_INSTALL: "echo" # Nothing to do FILE_ENV: "./ci/test/00_setup_env_mac_host.sh" @@ -183,9 +202,10 @@ task: task: name: 'ARM64 Android APK [focal]' depends_sources_cache: - folder: "/tmp/cirrus-ci-build/depends/sources" + folder: "depends/sources" << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal env: + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_android.sh" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index f0ed314d19..608acfc2cf 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -33,7 +33,12 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then echo "Creating $DOCKER_NAME_TAG container to run in" ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" - DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \ + if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then + echo "Restart docker before run to stop and clear all containers started with --rm" + systemctl restart docker + fi + + DOCKER_ID=$(docker run $DOCKER_ADMIN --rm --interactive --detach --tty \ --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \ --mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \ |