aboutsummaryrefslogtreecommitdiff
path: root/.cirrus.yml
blob: c0b85b24a7a1f94484847415ad3ff247ea3a1605 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
env:  # Global defaults
  CIRRUS_CLONE_DEPTH: 1
  PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y"
  MAKEJOBS: "-j10"
  TEST_RUNNER_PORT_MIN: "14000"  # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache
  CI_FAILFAST_TEST_LEAVE_DANGLING: "1"  # Cirrus CI does not care about dangling process and setting this variable avoids killing the CI script itself on error
  CCACHE_MAXSIZE: "200M"
  CCACHE_DIR: "/tmp/ccache_dir"
  CCACHE_NOHASHDIR: "1"  # Debug info might contain a stale path if the build dir changes, but this is fine

cirrus_ephemeral_worker_template_env: &CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
  DANGER_RUN_CI_ON_HOST: "1"  # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system

# https://cirrus-ci.org/guide/persistent-workers/
#
# It is possible to select a specific persistent worker by label. Refer to the
# Cirrus CI docs for more details.
#
# Generally, a persistent worker must run Ubuntu 23.04+ or Debian 12+.
# Specifically,
# - apt-get is required due to PACKAGE_MANAGER_INSTALL
# - podman-docker-4.1+ is required due to the use of `podman` when
#   RESTART_CI_DOCKER_BEFORE_RUN is set and 4.1+ due to the bugfix in 4.1
#   (https://github.com/bitcoin/bitcoin/pull/21652#issuecomment-1657098200)
# - The ./ci/ depedencies (with cirrus-cli) should be installed:
#
#   ```
#   apt update && apt install screen python3 bash podman-docker curl -y && curl -L -o cirrus "https://github.com/cirruslabs/cirrus-cli/releases/latest/download/cirrus-linux-$(dpkg --print-architecture)" && mv cirrus /usr/local/bin/cirrus && chmod +x /usr/local/bin/cirrus
#   ```
#
# - There are no strict requirements on the hardware, because having less CPUs
#   runs the same CI script (maybe slower). To avoid rare and intermittent OOM
#   due to short memory usage spikes, it is recommended to add (and persist)
#   swap:
#
#   ```
#   fallocate -l 16G /swapfile_ci && chmod 600 /swapfile_ci && mkswap /swapfile_ci && swapon /swapfile_ci && ( echo '/swapfile_ci none swap sw 0 0' | sudo tee -a /etc/fstab )
#   ```
#
# - To register the persistent worker, open a `screen` session and run:
#
#   ```
#   RESTART_CI_DOCKER_BEFORE_RUN=1 screen cirrus worker run --labels type=todo_fill_in_type --token todo_fill_in_token
#   ```
#
# The following specific types should exist, with the following requirements:
# - small: For an x86_64 machine, recommended to have 2 CPUs and 8 GB of memory.
# - medium: For an x86_64 machine, recommended to have 4 CPUs and 16 GB of memory.
# - lunar: For a machine running the Linux kernel shipped with Ubuntu Lunar 23.04. The machine is recommended to have 4 CPUs and 16 GB of memory.
# - arm64: For an aarch64 machine, recommended to have 2 CPUs and 8 GB of memory.

# https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks
filter_template: &FILTER_TEMPLATE
  skip: $CIRRUS_REPO_FULL_NAME == "bitcoin-core/gui" && $CIRRUS_PR == ""  # No need to run on the read-only mirror, unless it is a PR. https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution
  stateful: false  # https://cirrus-ci.org/guide/writing-tasks/#stateful-tasks

base_template: &BASE_TEMPLATE
  << : *FILTER_TEMPLATE
  merge_base_script:
    # Unconditionally install git (used in fingerprint_script).
    - bash -c "$PACKAGE_MANAGER_INSTALL git"
    - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi
    - git fetch --depth=1 $CIRRUS_REPO_CLONE_URL "pull/${CIRRUS_PR}/merge"
    - git checkout FETCH_HEAD  # Use merged changes to detect silent merge conflicts
                               # Also, the merge commit is used to lint COMMIT_RANGE="HEAD~..HEAD"

main_template: &MAIN_TEMPLATE
  timeout_in: 120m  # https://cirrus-ci.org/faq/#instance-timed-out
  ccache_cache:
    folder: "/tmp/ccache_dir"
  ci_script:
    - ./ci/test_run_all.sh

container_depends_template: &CONTAINER_DEPENDS_TEMPLATE
  << : *BASE_TEMPLATE
  container:
    # https://cirrus-ci.org/faq/#are-there-any-limits
    # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel
    cpu: 2
    memory: 8G  # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers
    dockerfile: ci/test_imagefile  # https://cirrus-ci.org/guide/docker-builder-vm/#dockerfile-as-a-ci-environment
  depends_built_cache:
    folder: "depends/built"
    fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-parse HEAD:depends)

global_task_template: &GLOBAL_TASK_TEMPLATE
  << : *CONTAINER_DEPENDS_TEMPLATE
  << : *MAIN_TEMPLATE

compute_credits_template: &CREDITS_TEMPLATE
  # https://cirrus-ci.org/pricing/#compute-credits
  # Only use credits for pull requests to the main repo
  use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'bitcoin/bitcoin' && $CIRRUS_PR != ""

task:
  name: 'lint [bookworm]'
  << : *BASE_TEMPLATE
  container:
    image: debian:bookworm
    cpu: 1
    memory: 1G
  # For faster CI feedback, immediately schedule the linters
  << : *CREDITS_TEMPLATE
  python_cache:
    folder: "/python_build"
    fingerprint_script: cat .python-version /etc/os-release
  unshallow_script:
    - git fetch --unshallow --no-tags
  lint_script:
    - ./ci/lint_run_all.sh
  env:
    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV

task:
  name: 'tidy [lunar]'
  << : *GLOBAL_TASK_TEMPLATE
  persistent_worker:
    labels:
      type: medium
  env:
    FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh"

task:
  name: "Win64 native [vs2022]"
  << : *FILTER_TEMPLATE
  windows_container:
    cpu: 6
    memory: 12G
    image: cirrusci/windowsservercore:visualstudio2022
  timeout_in: 120m
  env:
    PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\MSBuild\Current\Bin;%PATH%'
    PYTHONUTF8: 1
    CI_VCPKG_TAG: '2023.01.09'
    VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads'
    VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives'
    CCACHE_DIR: 'C:\Users\ContainerAdministrator\AppData\Local\ccache'
    WRAPPED_CL: 'C:\Users\ContainerAdministrator\AppData\Local\Temp\cirrus-ci-build\ci\test\wrapped-cl.bat'
    QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip'
    QT_LOCAL_PATH: 'C:\qt-everywhere-opensource-src-5.15.5.zip'
    QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.5'
    QTBASEDIR: 'C:\Qt_static'
    x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"'
    QT_CONFIGURE_COMMAND: '..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml'
    IgnoreWarnIntDirInTempDetected: 'true'
  merge_script:
    - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL pull/$env:CIRRUS_PR/merge; git reset --hard FETCH_HEAD; }
  msvc_qt_built_cache:
    folder: "%QTBASEDIR%"
    reupload_on_changes: false
    fingerprint_script:
      - echo %QT_DOWNLOAD_URL% %QT_CONFIGURE_COMMAND%
      - msbuild -version
    populate_script:
      - curl -L -o C:\jom.zip http://download.qt.io/official_releases/jom/jom.zip
      - mkdir C:\jom
      - tar -xf C:\jom.zip -C C:\jom
      - curl -L -o %QT_LOCAL_PATH% %QT_DOWNLOAD_URL%
      - tar -xf %QT_LOCAL_PATH% -C C:\
      - '%x64_NATIVE_TOOLS%'
      - cd %QT_SOURCE_DIR%
      - mkdir build
      - cd build
      - '%QT_CONFIGURE_COMMAND% -prefix %QTBASEDIR%'
      - jom
      - jom install
  vcpkg_tools_cache:
    folder: '%VCPKG_DOWNLOADS%\tools'
    reupload_on_changes: false
    fingerprint_script:
      - echo %CI_VCPKG_TAG%
      - msbuild -version
  vcpkg_binary_cache:
    folder: '%VCPKG_DEFAULT_BINARY_CACHE%'
    reupload_on_changes: true
    fingerprint_script:
      - echo %CI_VCPKG_TAG%
      - type build_msvc\vcpkg.json
      - msbuild -version
    populate_script:
      - mkdir %VCPKG_DEFAULT_BINARY_CACHE%
  ccache_cache:
    folder: '%CCACHE_DIR%'
  install_tools_script:
    - choco install --yes --no-progress ccache --version=4.7.4
    - choco install --yes --no-progress python3 --version=3.9.6
    - pip install zmq
    - ccache --version
    - python -VV
  install_vcpkg_script:
    - cd ..
    - git clone --quiet https://github.com/microsoft/vcpkg.git
    - cd vcpkg
    - git -c advice.detachedHead=false checkout %CI_VCPKG_TAG%
    - .\bootstrap-vcpkg -disableMetrics
    - echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake
    - .\vcpkg integrate install
    - .\vcpkg version
  build_script:
    - '%x64_NATIVE_TOOLS%'
    - cd %CIRRUS_WORKING_DIR%
    - ccache --zero-stats
    - python build_msvc\msvc-autogen.py
    - msbuild build_msvc\bitcoin.sln -property:CLToolExe=%WRAPPED_CL%;UseMultiToolTask=true;Configuration=Release -maxCpuCount -verbosity:minimal -noLogo
    - ccache --show-stats
  check_script:
    - src\test_bitcoin.exe -l test_suite
    - src\bench_bitcoin.exe --sanity-check
    - python test\util\test_runner.py
    - python test\util\rpcauth-test.py
  functional_tests_script:
    # Increase the dynamic port range to the maximum allowed value to mitigate "OSError: [WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted".
    # See: https://learn.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance
    - netsh int ipv4 set dynamicport tcp start=1025 num=64511
    - netsh int ipv6 set dynamicport tcp start=1025 num=64511
    # Exclude feature_dbcrash for now due to timeout
    - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=99999999 --jobs=6 --timeout-factor=8 --extended --exclude feature_dbcrash

task:
  name: 'ARM [unit tests, no functional tests] [bullseye]'
  << : *GLOBAL_TASK_TEMPLATE
  persistent_worker:
    labels:
      type: arm64  # Use arm64 worker to sidestep qemu and avoid a slow CI: https://github.com/bitcoin/bitcoin/pull/28087#issuecomment-1649399453
  env:
    FILE_ENV: "./ci/test/00_setup_env_arm.sh"

task:
  name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [jammy]'
  << : *GLOBAL_TASK_TEMPLATE
  container:
    docker_arguments:
      CI_IMAGE_NAME_TAG: ubuntu:jammy
      FILE_ENV: "./ci/test/00_setup_env_win64.sh"
  << : *CREDITS_TEMPLATE
  env:
    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV

task:
  name: '32-bit + dash [gui] [CentOS 9]'
  << : *GLOBAL_TASK_TEMPLATE
  container:
    docker_arguments:
      CI_IMAGE_NAME_TAG: "quay.io/centos/amd64:stream9"
      FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
  # For faster CI feedback, immediately schedule one task that runs all tests
  << : *CREDITS_TEMPLATE
  env:
    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
    PACKAGE_MANAGER_INSTALL: "yum install -y"

task:
  name: '[previous releases, qt5 dev package and depends packages, DEBUG] [focal]'
  previous_releases_cache:
    folder: "releases"
  << : *GLOBAL_TASK_TEMPLATE
  persistent_worker:
    labels:
      type: small
  env:
    FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh"

task:
  name: '[TSan, depends, gui] [lunar]'
  << : *GLOBAL_TASK_TEMPLATE
  container:
    cpu: 4
    memory: 16G  # The default memory is too small, so double everything
    docker_arguments:
      CI_IMAGE_NAME_TAG: ubuntu:lunar
      FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh"
  env:
    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV

task:
  name: '[MSan, depends] [jammy]'
  << : *GLOBAL_TASK_TEMPLATE
  container:
    docker_arguments:
      CI_IMAGE_NAME_TAG: ubuntu:jammy
      FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
  env:
    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
    MAKEJOBS: "-j4"  # Avoid excessive memory use due to MSan

task:
  name: '[ASan + LSan + UBSan + integer, no depends, USDT] [lunar]'
  enable_bpfcc_script:
    # In the image build step, no external environment variables are available,
    # so any settings will need to be written to the settings env file:
    - sed -i "s|\${CIRRUS_CI}|true|g" ./ci/test/00_setup_env_native_asan.sh
  << : *GLOBAL_TASK_TEMPLATE
  persistent_worker:
    labels:
      type: lunar  # Must use the lunar-specific worker (needed for USDT functional tests)
  env:
    FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"

task:
  name: '[fuzzer,address,undefined,integer, no depends] [lunar]'
  << : *GLOBAL_TASK_TEMPLATE
  container:
    cpu: 4  # Increase CPU and memory to avoid timeout
    memory: 16G
    docker_arguments:
      CI_IMAGE_NAME_TAG: ubuntu:lunar
      FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
  env:
    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV

task:
  name: '[multiprocess, i686, DEBUG] [focal]'
  << : *GLOBAL_TASK_TEMPLATE
  container:
    cpu: 4
    memory: 16G  # The default memory is too small, so double everything
    docker_arguments:
      CI_IMAGE_NAME_TAG: "docker.io/amd64/ubuntu:focal"
      FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh"
  env:
    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV

task:
  name: '[no wallet, libbitcoinkernel] [focal]'
  << : *GLOBAL_TASK_TEMPLATE
  container:
    docker_arguments:
      CI_IMAGE_NAME_TAG: ubuntu:focal
      FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh"
  << : *CREDITS_TEMPLATE
  env:
    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV

task:
  name: 'macOS 11.0 [gui, no tests] [jammy]'
  << : *CONTAINER_DEPENDS_TEMPLATE
  container:
    docker_arguments:
      CI_IMAGE_NAME_TAG: ubuntu:jammy
      FILE_ENV: "./ci/test/00_setup_env_mac.sh"
  macos_sdk_cache:
    folder: "depends/SDKs/$MACOS_SDK"
    fingerprint_key: "$MACOS_SDK"
  << : *MAIN_TEMPLATE
  env:
    MACOS_SDK: "Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers"
    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV

task:
  name: 'macOS 13 native arm64 [gui, sqlite only] [no depends]'
  macos_instance:
    # Use latest image, but hardcode version to avoid silent upgrades (and breaks)
    image: ghcr.io/cirruslabs/macos-ventura-xcode:14.3.1  # https://cirrus-ci.org/guide/macOS
  << : *BASE_TEMPLATE
  check_clang_script:
    - clang --version
  brew_install_script:
    - brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt
  << : *MAIN_TEMPLATE
  env:
    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
    CI_USE_APT_INSTALL: "no"
    PACKAGE_MANAGER_INSTALL: "echo"  # Nothing to do
    FILE_ENV: "./ci/test/00_setup_env_mac_native_arm64.sh"