From ca8bc873801c77f67378e542686d19ed388bba53 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 2 Nov 2022 14:04:08 +0000 Subject: Multi-stage Docker builds (#2850) This builds on @S7evinK's work to make multi-stage Docker builds. Now that we can build SQLite without Cgo this should be much simpler and should make Docker builds in CI significantly faster. Co-authored-by: Till Faelligen Co-authored-by: Till Faelligen Co-authored-by: Till Faelligen <2353100+S7evinK@users.noreply.github.com> --- .github/workflows/dendrite.yml | 6 +- .github/workflows/docker.yml | 151 +++++++++++++++++++++++++++++++---------- 2 files changed, 118 insertions(+), 39 deletions(-) (limited to '.github') diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index bfe80bc1..4725637e 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -297,7 +297,7 @@ jobs: SYTEST_BRANCH: ${{ github.head_ref }} CGO_ENABLED: ${{ matrix.cgo && 1 }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run Sytest run: /bootstrap.sh dendrite working-directory: /src @@ -364,8 +364,8 @@ jobs: sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest - - name: Run actions/checkout@v2 for dendrite - uses: actions/checkout@v2 + - name: Run actions/checkout@v3 for dendrite + uses: actions/checkout@v3 with: path: dendrite diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b4e24e52..95213f71 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,26 +21,32 @@ jobs: monolith: name: Monolith image runs-on: ubuntu-latest + needs: build-flags permissions: contents: read packages: write steps: - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag + uses: actions/checkout@v3 + - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -49,27 +55,41 @@ jobs: - name: Build main monolith image if: github.ref_name == 'main' id: docker_build_monolith - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.monolith + build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" + target: monolith platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} + format: "sarif" + output: "trivy-results.sarif" + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: "trivy-results.sarif" + - name: Build release monolith image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_monolith_release - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.monolith + build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" + target: monolith platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -81,26 +101,32 @@ jobs: polylith: name: Polylith image runs-on: ubuntu-latest + needs: build-flags permissions: contents: read packages: write steps: - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag + uses: actions/checkout@v3 + - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -109,27 +135,40 @@ jobs: - name: Build main polylith image if: github.ref_name == 'main' id: docker_build_polylith - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.polylith + build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" + target: polylith platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} + format: "sarif" + output: "trivy-results.sarif" + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: "trivy-results.sarif" + - name: Build release polylith image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_polylith_release - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.polylith + target: polylith platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -141,59 +180,99 @@ jobs: demo-pinecone: name: Pinecone demo image runs-on: ubuntu-latest + needs: build-flags permissions: contents: read packages: write steps: - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag + uses: actions/checkout@v3 + - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build main pinecone demo image + - name: Build main Pinecone demo image if: github.ref_name == 'main' id: docker_build_demo_pinecone - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.demo-pinecone + target: demo-pinecone platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }} - - name: Build release pinecone demo image + - name: Build release Pinecone demo image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_demo_pinecone_release - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 + with: + cache-from: type=gha + cache-to: type=gha,mode=max + context: . + build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" + target: demo-pinecone + platforms: ${{ env.PLATFORMS }} + push: true + tags: | + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} + + - name: Build main Yggdrasil demo image + if: github.ref_name == 'main' + id: docker_build_demo_yggdrasil + uses: docker/build-push-action@v3 + with: + cache-from: type=gha + cache-to: type=gha,mode=max + context: . + target: demo-yggdrasil + platforms: ${{ env.PLATFORMS }} + push: true + tags: | + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }} + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }} + + - name: Build release Yggdrasil demo image + if: github.event_name == 'release' # Only for GitHub releases + id: docker_build_demo_yggdrasil_release + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.demo-pinecone + build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" + target: demo-yggdrasil platforms: ${{ env.PLATFORMS }} push: true tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:latest - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ env.RELEASE_VERSION }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:latest - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ env.RELEASE_VERSION }} + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} -- cgit v1.2.3