Add Sytest/Complement coverage to scheduled runs (#2962)

This adds Sytest and Complement coverage reporting to the nightly
scheduled CI runs.

Fixes a few API mode related issues as well, since we seemingly never
really ran them with Complement.

Also fixes a bug related to device list changes: When we pass in an
empty `newlyLeftRooms` slice, we got a list of all currently joined
rooms with the corresponding members. When we then got the
`newlyJoinedRooms`, we wouldn't update the `changed` slice, because we
already got the user from the `newlyLeftRooms` query. This is fixed by
simply ignoring empty `newlyLeftRooms`.

2 files changed, 176 insertions, 3 deletions

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 1de39850..2dc5f74c 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -460,7 +460,8 @@ jobs:
         name: Run Complement Tests
         env:
           COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }}
-          API: ${{ matrix.api && 1 }}
+          COMPLEMENT_DENDRITE_API: ${{ matrix.api && 1 }}
+          COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_
         working-directory: complement
 
   integration-tests-done:
diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml
index 5636c4cf..fa2304c1 100644
--- a/.github/workflows/schedules.yaml
+++ b/.github/workflows/schedules.yaml
@@ -19,10 +19,17 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - label: SQLite
+          - label: SQLite native
 
-          - label: SQLite, full HTTP APIs
+          - label: SQLite Cgo
+            cgo: 1
+
+          - label: SQLite native, full HTTP APIs
+            api: full-http
+
+          - label: SQLite Cgo, full HTTP APIs
             api: full-http
+            cgo: 1
 
           - label: PostgreSQL
             postgres: postgres
@@ -41,6 +48,7 @@ jobs:
         API: ${{ matrix.api && 1 }}
         SYTEST_BRANCH: ${{ github.head_ref }}
         RACE_DETECTION: 1
+        COVER: 1
     steps:
       - uses: actions/checkout@v3
       - uses: actions/cache@v3
@@ -73,7 +81,171 @@ jobs:
           path: |
             /logs/results.tap
             /logs/**/*.log*
+
+  sytest-coverage:
+    timeout-minutes: 5
+    name: "Sytest Coverage"
+    runs-on: ubuntu-latest
+    needs: sytest # only run once Sytest is done
+    if: ${{ always() }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '>=1.19.0'
+          cache: true
+      - name: Download all artifacts
+        uses: actions/download-artifact@v3
+      - name: Install gocovmerge
+        run: go install github.com/wadey/gocovmerge@latest
+      - name: Run gocovmerge
+        run: |
+          find -name 'integrationcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > sytest.cov
+          go tool cover -func=sytest.cov
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          files: ./sytest.cov
+          flags: sytest
+          fail_ci_if_error: true
+
+  # run Complement
+  complement:
+    name: "Complement (${{ matrix.label }})"
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - label: SQLite native
+            cgo: 0
+
+          - label: SQLite Cgo
+            cgo: 1
+
+          - label: SQLite native, full HTTP APIs
+            api: full-http
+            cgo: 0
+
+          - label: SQLite Cgo, full HTTP APIs
+            api: full-http
+            cgo: 1
+
+          - label: PostgreSQL
+            postgres: Postgres
+            cgo: 0
+
+          - label: PostgreSQL, full HTTP APIs
+            postgres: Postgres
+            api: full-http
+            cgo: 0
+    steps:
+      # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
+      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
+      - name: "Set Go Version"
+        run: |
+          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
+          echo "~/go/bin" >> $GITHUB_PATH
+      - name: "Install Complement Dependencies"
+        # We don't need to install Go because it is included on the Ubuntu 20.04 image:
+        # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
+        run: |
+          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
+          go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
+      - name: Run actions/checkout@v3 for dendrite
+        uses: actions/checkout@v3
+        with:
+          path: dendrite
+
+      # Attempt to check out the same branch of Complement as the PR. If it
+      # doesn't exist, fallback to main.
+      - name: Checkout complement
+        shell: bash
+        run: |
+          mkdir -p complement
+          # Attempt to use the version of complement which best matches the current
+          # build. Depending on whether this is a PR or release, etc. we need to
+          # use different fallbacks.
+          #
+          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
+          #    for pull requests, otherwise GITHUB_REF).
+          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
+          #    (GITHUB_BASE_REF for pull requests).
+          # 3. Use the default complement branch ("master").
+          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
+            # Skip empty branch names and merge commits.
+            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
+              continue
+            fi
+            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
+          done
+      # Build initial Dendrite image
+      - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
+        working-directory: dendrite
+        env:
+          DOCKER_BUILDKIT: 1
+
+      - name: Create post test script
+        run: |
+          cat <<EOF > /tmp/posttest.sh
+          #!/bin/bash
+          mkdir -p /tmp/Complement/logs/\$2/\$1/
+          docker cp \$1:/dendrite/complementcover.log /tmp/Complement/logs/\$2/\$1/
+          EOF
           
+          chmod +x /tmp/posttest.sh
+      # Run Complement
+      - run: |
+          set -o pipefail &&
+          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
+        shell: bash
+        name: Run Complement Tests
+        env:
+          COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }}
+          COMPLEMENT_DENDRITE_API: ${{ matrix.api && 1 }}
+          COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_
+          COMPLEMENT_DENDRITE_COVER: 1
+          COMPLEMENT_POST_TEST_SCRIPT: /tmp/posttest.sh
+        working-directory: complement
+
+      - name: Upload Complement logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Complement Logs - (Dendrite, ${{ join(matrix.*, ', ') }})
+          path: |
+            /tmp/Complement/**/complementcover.log
+
+  complement-coverage:
+    timeout-minutes: 5
+    name: "Complement Coverage"
+    runs-on: ubuntu-latest
+    needs: complement # only run once Complement is done
+    if: ${{ always() }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '>=1.19.0'
+          cache: true
+      - name: Download all artifacts
+        uses: actions/download-artifact@v3
+      - name: Install gocovmerge
+        run: go install github.com/wadey/gocovmerge@latest
+      - name: Run gocovmerge
+        run: |
+          find -name 'complementcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > complement.cov
+          go tool cover -func=complement.cov
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          files: ./complement.cov
+          flags: complement
+          fail_ci_if_error: true
+
   element_web:
     timeout-minutes: 120
     runs-on: ubuntu-latest