aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMarcoFalke <falke.marco@gmail.com>2021-04-11 09:51:13 +0200
committerMarcoFalke <falke.marco@gmail.com>2021-04-11 09:51:22 +0200
commit1e3db6807d1809182c799b5f7a78843f03fbe413 (patch)
tree25121f5a12f4609d0b21ada6f95069f840196aab
parentdd01dc6a1d23e1f316940cd1092ad716216331aa (diff)
parentfa41a917356a7f5e0f3286b227ced7f2e6797e67 (diff)
Merge #21619: ci: Run self-hosted ci
fa41a917356a7f5e0f3286b227ced7f2e6797e67 ci: Run self-hosted ci (MarcoFalke) fa52a40f0ebb7aa33af210a18e525eec7bb6887f ci: Make cirrus cache folders relative to cirrus base dir (MarcoFalke) fa278412a023ac150da764a465567f2970348449 ci: Restart docker before run (MarcoFalke) fad4f48e0763d111369656dea9575a789d2016e8 ci: [refactor] Create setting for ephemeral config in .cirrus.yml (MarcoFalke) Pull request description: Due to our heavy use of the Cirrus CI community cluster, some tasks may take a long time to get scheduled. While it is possible to use "Compute Credits" to get immediately scheduled on the cluster, I couldn't find a sponsor that'd be willing to cover the total cost, if all tasks were paid for with credits. However, it is also possible to bring our own runners to Cirrus CI. For testing purposes, a single task will be transformed to run on the DrahtBot infrastructure. If all goes well, the other tasks can be moved, too. ACKs for top commit: hebasto: ACK fa41a917356a7f5e0f3286b227ced7f2e6797e67, I have reviewed the code and it looks OK. Tree-SHA512: 513738daec36da8cd48a8f11e687ff0b7dfaba1ae4ed2fa77e7b043f88fd52bf5c0dbad2768e13df88518323917f08348cb62be6376a423142921f8d2c59a938
-rw-r--r--.cirrus.yml40
-rwxr-xr-xci/test/04_install.sh7
2 files changed, 36 insertions, 11 deletions
diff --git a/.cirrus.yml b/.cirrus.yml
index 99957f08c5..6148f56e0d 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -3,10 +3,19 @@
env:
PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y"
MAKEJOBS: "-j4"
- DANGER_RUN_CI_ON_HOST: "1" # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system
TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache
CCACHE_SIZE: "200M"
CCACHE_DIR: "/tmp/ccache_dir"
+ CCACHE_NOHASHDIR: "1" # Debug info might contain a stale path if the build dir changes, but this is fine
+
+cirrus_ephemeral_worker_template_env: &CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+ DANGER_RUN_CI_ON_HOST: "1" # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system
+
+persistent_worker_template_env: &PERSISTENT_WORKER_TEMPLATE_ENV
+ RESTART_CI_DOCKER_BEFORE_RUN: "1"
+
+persistent_worker_template: &PERSISTENT_WORKER_TEMPLATE
+ persistent_worker: {} # https://cirrus-ci.org/guide/persistent-workers/
# https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks
base_template: &BASE_TEMPLATE
@@ -31,9 +40,9 @@ global_task_template: &GLOBAL_TASK_TEMPLATE
ccache_cache:
folder: "/tmp/ccache_dir"
depends_built_cache:
- folder: "/tmp/cirrus-ci-build/depends/built"
+ folder: "depends/built"
depends_sdk_cache:
- folder: "/tmp/cirrus-ci-build/depends/sdk-sources"
+ folder: "depends/sdk-sources"
ci_script:
- ./ci/test_run_all.sh
@@ -69,6 +78,8 @@ task:
<< : *CREDITS_TEMPLATE
lint_script:
- ./ci/lint_run_all.sh
+ env:
+ << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
task:
name: 'ARM [unit tests, no functional tests] [buster]'
@@ -76,6 +87,7 @@ task:
container:
image: debian:buster
env:
+ << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
FILE_ENV: "./ci/test/00_setup_env_arm.sh"
task:
@@ -84,6 +96,7 @@ task:
container:
image: ubuntu:focal
env:
+ << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
FILE_ENV: "./ci/test/00_setup_env_win64.sh"
task:
@@ -92,19 +105,18 @@ task:
container:
image: centos:8
env:
+ << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
PACKAGE_MANAGER_INSTALL: "yum install -y"
FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
task:
name: '[previous releases, uses qt5 dev package and some depends packages] [unsigned char] [bionic]'
- # For faster CI feedback, immediately schedule a task that compiles most modules
- << : *CREDITS_TEMPLATE
<< : *GLOBAL_TASK_TEMPLATE
depends_releases_cache:
- folder: "/tmp/cirrus-ci-build/releases"
- container:
- image: ubuntu:bionic
+ folder: "releases"
+ << : *PERSISTENT_WORKER_TEMPLATE
env:
+ << : *PERSISTENT_WORKER_TEMPLATE_ENV
FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh"
task:
@@ -115,6 +127,7 @@ task:
cpu: 6 # Increase CPU and Memory to avoid timeout
memory: 24G
env:
+ << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
MAKEJOBS: "-j8"
FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh"
@@ -124,6 +137,7 @@ task:
container:
image: ubuntu:focal
env:
+ << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
task:
@@ -132,6 +146,7 @@ task:
container:
image: ubuntu:focal
env:
+ << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
task:
@@ -140,6 +155,7 @@ task:
container:
image: ubuntu:focal
env:
+ << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
task:
@@ -148,6 +164,7 @@ task:
container:
image: ubuntu:focal
env:
+ << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
FILE_ENV: "./ci/test/00_setup_env_native_multiprocess.sh"
task:
@@ -156,6 +173,7 @@ task:
container:
image: ubuntu:bionic
env:
+ << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh"
task:
@@ -164,6 +182,7 @@ task:
container:
image: ubuntu:focal
env:
+ << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
FILE_ENV: "./ci/test/00_setup_env_mac.sh"
task:
@@ -175,7 +194,7 @@ task:
# Use latest image, but hardcode version to avoid silent upgrades (and breaks)
image: big-sur-xcode-12.4 # https://cirrus-ci.org/guide/macOS
env:
- DANGER_RUN_CI_ON_HOST: "true"
+ << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
CI_USE_APT_INSTALL: "no"
PACKAGE_MANAGER_INSTALL: "echo" # Nothing to do
FILE_ENV: "./ci/test/00_setup_env_mac_host.sh"
@@ -183,9 +202,10 @@ task:
task:
name: 'ARM64 Android APK [focal]'
depends_sources_cache:
- folder: "/tmp/cirrus-ci-build/depends/sources"
+ folder: "depends/sources"
<< : *GLOBAL_TASK_TEMPLATE
container:
image: ubuntu:focal
env:
+ << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
FILE_ENV: "./ci/test/00_setup_env_android.sh"
diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh
index f0ed314d19..608acfc2cf 100755
--- a/ci/test/04_install.sh
+++ b/ci/test/04_install.sh
@@ -33,7 +33,12 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
echo "Creating $DOCKER_NAME_TAG container to run in"
${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG"
- DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \
+ if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then
+ echo "Restart docker before run to stop and clear all containers started with --rm"
+ systemctl restart docker
+ fi
+
+ DOCKER_ID=$(docker run $DOCKER_ADMIN --rm --interactive --detach --tty \
--mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \
--mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \
--mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \