aboutsummaryrefslogtreecommitdiff
path: root/contrib/guix/guix-build
blob: bfffbda742c6f8140dfe09ddd481f63ec7bec7fd (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
#!/usr/bin/env bash
export LC_ALL=C
set -e -o pipefail

# Source the common prelude, which:
#   1. Checks if we're at the top directory of the Bitcoin Core repository
#   2. Defines a few common functions and variables
#
# shellcheck source=libexec/prelude.bash
source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash"


###################
## SANITY CHECKS ##
###################

################
# Required non-builtin commands should be invocable
################

check_tools cat mkdir make getent curl git guix

################
# GUIX_BUILD_OPTIONS should be empty
################
#
# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that
# can perform builds. This seems like what we want instead of
# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually
# _appended_ to normal command-line options. Meaning that they will take
# precedence over the command-specific ADDITIONAL_GUIX_<CMD>_FLAGS.
#
# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's
# existence here and direct users of this script to use our (more flexible)
# custom environment variables.
if [ -n "$GUIX_BUILD_OPTIONS" ]; then
cat << EOF
Error: Environment variable GUIX_BUILD_OPTIONS is not empty:
  '$GUIX_BUILD_OPTIONS'

Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset
GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options
across guix commands or ADDITIONAL_GUIX_<CMD>_FLAGS to set build options for a
specific guix command.

See contrib/guix/README.md for more details.
EOF
exit 1
fi

################
# The git worktree should not be dirty
################

if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then
cat << EOF
ERR: The current git worktree is dirty, which may lead to broken builds.

     Aborting...

Hint: To make your git worktree clean, You may want to:
      1. Commit your changes,
      2. Stash your changes, or
      3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on
         using a dirty worktree
EOF
exit 1
fi

mkdir -p "$VERSION_BASE"

################
# Build directories should not exist
################

# Default to building for all supported HOSTs (overridable by environment)
export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu
                       x86_64-w64-mingw32
                       x86_64-apple-darwin arm64-apple-darwin}"

# Usage: distsrc_for_host HOST
#
#   HOST: The current platform triple we're building for
#
distsrc_for_host() {
    echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}"
}

# Accumulate a list of build directories that already exist...
hosts_distsrc_exists=""
for host in $HOSTS; do
    if [ -e "$(distsrc_for_host "$host")" ]; then
        hosts_distsrc_exists+=" ${host}"
    fi
done

if [ -n "$hosts_distsrc_exists" ]; then
# ...so that we can print them out nicely in an error message
cat << EOF
ERR: Build directories for this commit already exist for the following platform
     triples you're attempting to build, probably because of previous builds.
     Please remove, or otherwise deal with them prior to starting another build.

     Aborting...

Hint: To blow everything away, you may want to use:

  $ ./contrib/guix/guix-clean

Specifically, this will remove all files without an entry in the index,
excluding the SDK directory, the depends download cache, the depends built
packages cache, the garbage collector roots for Guix environments, and the
output directory.
EOF
for host in $hosts_distsrc_exists; do
    echo "     ${host} '$(distsrc_for_host "$host")'"
done
exit 1
else
    mkdir -p "$DISTSRC_BASE"
fi

################
# When building for darwin, the macOS SDK should exists
################

for host in $HOSTS; do
    case "$host" in
        *darwin*)
            OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')"
            if [ -e "$OSX_SDK" ]; then
                echo "Found macOS SDK at '${OSX_SDK}', using..."
            else
                echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, exiting..."
                exit 1
            fi
            ;;
    esac
done

################
# VERSION_BASE should have enough space
################

avail_KiB="$(df -Pk "$VERSION_BASE" | sed 1d | tr -s ' ' | cut -d' ' -f4)"
total_required_KiB=0
for host in $HOSTS; do
    case "$host" in
        *darwin*) required_KiB=440000 ;;
        *mingw*)  required_KiB=7600000 ;;
        *)        required_KiB=6400000 ;;
    esac
    total_required_KiB=$((total_required_KiB+required_KiB))
done

if (( total_required_KiB > avail_KiB )); then
    total_required_GiB=$((total_required_KiB / 1048576))
    avail_GiB=$((avail_KiB / 1048576))
    echo "Performing a Bitcoin Core Guix build for the selected HOSTS requires ${total_required_GiB} GiB, however, only ${avail_GiB} GiB is available. Please free up some disk space before performing the build."
    exit 1
fi

################
# Check that we can connect to the guix-daemon
################

cat << EOF
Checking that we can connect to the guix-daemon...

Hint: If this hangs, you may want to try turning your guix-daemon off and on
      again.

EOF
if ! guix gc --list-failures > /dev/null; then
cat << EOF

ERR: Failed to connect to the guix-daemon, please ensure that one is running and
     reachable.
EOF
exit 1
fi

# Developer note: we could use `guix repl` for this check and run:
#
#     (import (guix store)) (close-connection (open-connection))
#
# However, the internal API is likely to change more than the CLI invocation

################
# Services database must have basic entries
################

if ! getent services http https ftp > /dev/null 2>&1; then
cat << EOF
ERR: Your system's C library can not find service database entries for at least
     one of the following services: http, https, ftp.

Hint: Most likely, /etc/services does not exist yet (common for docker images
      and minimal distros), or you don't have permissions to access it.

      If /etc/services does not exist yet, you may want to install the
      appropriate package for your distro which provides it.

          On Debian/Ubuntu: netbase
          On Arch Linux: iana-etc

      For more information, see: getent(1), services(5)

EOF

fi

#########
# SETUP #
#########

# Determine the maximum number of jobs to run simultaneously (overridable by
# environment)
JOBS="${JOBS:-$(nproc)}"

# Usage: host_to_commonname HOST
#
#   HOST: The current platform triple we're building for
#
host_to_commonname() {
    case "$1" in
        *darwin*) echo osx ;;
        *mingw*)  echo win ;;
        *linux*)  echo linux ;;
        *)        exit 1 ;;
    esac
}

# Determine the reference time used for determinism (overridable by environment)
SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}"

# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility
# across time.
time-machine() {
    # shellcheck disable=SC2086
    guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \
                      --commit=1ef7a03a148cf5f83ab1820444f6bd50d8e732d1 \
                      --cores="$JOBS" \
                      --keep-failed \
                      --fallback \
                      ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \
                      ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \
                      -- "$@"
}


# Precious directories are those which should not be cleaned between successive
# guix builds
depends_precious_dir_names='SOURCES_PATH BASE_CACHE SDK_PATH'
precious_dir_names="${depends_precious_dir_names} OUTDIR_BASE PROFILES_BASE"

# Usage: contains IFS-SEPARATED-LIST ITEM
contains() {
    for i in ${1}; do
        if [ "$i" = "${2}" ]; then
            return 0  # Found!
        fi
    done
    return 1
}

# If the user explicitly specified a precious directory, create it so we
# can map it into the container
for precious_dir_name in $precious_dir_names; do
    precious_dir_path="${!precious_dir_name}"
    if [ -n "$precious_dir_path" ]; then
        if [ ! -e "$precious_dir_path" ]; then
            mkdir -p "$precious_dir_path"
        elif [ -L "$precious_dir_path" ]; then
            echo "ERR: ${precious_dir_name} cannot be a symbolic link"
            exit 1
        elif [ ! -d "$precious_dir_path" ]; then
            echo "ERR: ${precious_dir_name} must be a directory"
            exit 1
        fi
    fi
done

mkdir -p "$VAR_BASE"

# Record the _effective_ values of precious directories such that guix-clean can
# avoid clobbering them if appropriate.
#
# shellcheck disable=SC2046,SC2086
{
    # Get depends precious dir definitions from depends
    make -C "${PWD}/depends" \
         --no-print-directory \
         -- $(printf "print-%s\n" $depends_precious_dir_names)

    # Get remaining precious dir definitions from the environment
    for precious_dir_name in $precious_dir_names; do
        precious_dir_path="${!precious_dir_name}"
        if ! contains "$depends_precious_dir_names" "$precious_dir_name"; then
            echo "${precious_dir_name}=${precious_dir_path}"
        fi
    done
} > "${VAR_BASE}/precious_dirs"

# Make sure an output directory exists for our builds
OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}"
mkdir -p "$OUTDIR_BASE"

# Download the depends sources now as we won't have internet access in the build
# container
for host in $HOSTS; do
    make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"}
done

# Usage: outdir_for_host HOST SUFFIX
#
#   HOST: The current platform triple we're building for
#
outdir_for_host() {
    echo "${OUTDIR_BASE}/${1}${2:+-${2}}"
}

# Usage: profiledir_for_host HOST SUFFIX
#
#   HOST: The current platform triple we're building for
#
profiledir_for_host() {
    echo "${PROFILES_BASE}/${1}${2:+-${2}}"
}


#########
# BUILD #
#########

# Function to be called when building for host ${1} and the user interrupts the
# build
int_trap() {
cat << EOF
** INT received while building ${1}, you may want to clean up the relevant
   work directories (e.g. distsrc-*) before rebuilding

Hint: To blow everything away, you may want to use:

  $ ./contrib/guix/guix-clean

Specifically, this will remove all files without an entry in the index,
excluding the SDK directory, the depends download cache, the depends built
packages cache, the garbage collector roots for Guix environments, and the
output directory.
EOF
}

# Deterministically build Bitcoin Core
# shellcheck disable=SC2153
for host in $HOSTS; do

    # Display proper warning when the user interrupts the build
    trap 'int_trap ${host}' INT

    (
        # Required for 'contrib/guix/manifest.scm' to output the right manifest
        # for the particular $HOST we're building for
        export HOST="$host"

        # shellcheck disable=SC2030
cat << EOF
INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}:
      ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set}
      ...running at most ${JOBS:?not set} jobs
      ...from worktree directory: '${PWD}'
          ...bind-mounted in container to: '/bitcoin'
      ...in build directory: '$(distsrc_for_host "$HOST")'
          ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")'
      ...outputting in: '$(outdir_for_host "$HOST")'
          ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")'
EOF

        # Run the build script 'contrib/guix/libexec/build.sh' in the build
        # container specified by 'contrib/guix/manifest.scm'.
        #
        # Explanation of `guix environment` flags:
        #
        #   --container        run command within an isolated container
        #
        #     Running in an isolated container minimizes build-time differences
        #     between machines and improves reproducibility
        #
        #   --pure             unset existing environment variables
        #
        #     Same rationale as --container
        #
        #   --no-cwd           do not share current working directory with an
        #                      isolated container
        #
        #     When --container is specified, the default behavior is to share
        #     the current working directory with the isolated container at the
        #     same exact path (e.g. mapping '/home/satoshi/bitcoin/' to
        #     '/home/satoshi/bitcoin/'). This means that the $PWD inside the
        #     container becomes a source of irreproducibility. --no-cwd disables
        #     this behaviour.
        #
        #   --share=SPEC       for containers, share writable host file system
        #                      according to SPEC
        #
        #   --share="$PWD"=/bitcoin
        #
        #                     maps our current working directory to /bitcoin
        #                     inside the isolated container, which we later cd
        #                     into.
        #
        #     While we don't want to map our current working directory to the
        #     same exact path (as this introduces irreproducibility), we do want
        #     it to be at a _fixed_ path _somewhere_ inside the isolated
        #     container so that we have something to build. '/bitcoin' was
        #     chosen arbitrarily.
        #
        #   ${SOURCES_PATH:+--share="$SOURCES_PATH"}
        #
        #                     make the downloaded depends sources path available
        #                     inside the isolated container
        #
        #     The isolated container has no network access as it's in a
        #     different network namespace from the main machine, so we have to
        #     make the downloaded depends sources available to it. The sources
        #     should have been downloaded prior to this invocation.
        #
        #   --keep-failed     keep build tree of failed builds
        #
        #     When builds of the Guix environment itself (not Bitcoin Core)
        #     fail, it is useful for the build tree to be kept for debugging
        #     purposes.
        #
        #  ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"}
        #
        #                     fetch substitute from SUBSTITUTE_URLS if they are
        #                     authorized
        #
        #    Depending on the user's security model, it may be desirable to use
        #    substitutes (pre-built packages) from servers that the user trusts.
        #    Please read the README.md in the same directory as this file for
        #    more information.
        #
        # shellcheck disable=SC2086,SC2031
        time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \
                                 --container \
                                 --pure \
                                 --no-cwd \
                                 --share="$PWD"=/bitcoin \
                                 --share="$DISTSRC_BASE"=/distsrc-base \
                                 --share="$OUTDIR_BASE"=/outdir-base \
                                 --expose="$(git rev-parse --git-common-dir)" \
                                 ${SOURCES_PATH:+--share="$SOURCES_PATH"} \
                                 ${BASE_CACHE:+--share="$BASE_CACHE"} \
                                 ${SDK_PATH:+--share="$SDK_PATH"} \
                                 --cores="$JOBS" \
                                 --keep-failed \
                                 --fallback \
                                 --link-profile \
                                 --root="$(profiledir_for_host "${HOST}")" \
                                 ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \
                                 ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \
                                 -- env HOST="$host" \
                                        DISTNAME="$DISTNAME" \
                                        JOBS="$JOBS" \
                                        SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \
                                        ${V:+V=1} \
                                        ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \
                                        ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \
                                        ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \
                                        DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \
                                        OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \
                                        DIST_ARCHIVE_BASE=/outdir-base/dist-archive \
                                      bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh"
    )

done