diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml
index 781768d01..6917f6131 100644
--- a/.github/workflows/check.yaml
+++ b/.github/workflows/check.yaml
@@ -307,6 +307,8 @@ jobs:
     name: Build and push Docker image
     needs: [rustfmt, clippy]
     runs-on: ubuntu-latest
+    env:
+      IMAGE: ghcr.io/matrix-org/matrix-authentication-service
 
     steps:
       - name: Checkout the code
@@ -316,7 +318,7 @@ jobs:
         id: meta
         uses: docker/metadata-action@v3
         with:
-          images: ghcr.io/matrix-org/matrix-authentication-service
+          images: "${{ env.IMAGE }}"
           bake-target: docker-metadata-action
           tags: |
             type=ref,event=branch
@@ -330,7 +332,7 @@ jobs:
         id: meta-debug
         uses: docker/metadata-action@v3
         with:
-          images: ghcr.io/matrix-org/matrix-authentication-service
+          images: "${{ env.IMAGE }}"
           bake-target: docker-metadata-action-debug
           tags: |
             type=ref,event=branch,suffix=-debug
@@ -362,8 +364,20 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and push
+      - name: Build
         uses: docker/bake-action@v1
+        if: github.event_name == 'pull_request'
         with:
           targets: gha
-          set: gha-push.output=type=image,push=${{ github.event_name != 'pull_request' }}
+          set: |
+            gha-push.cache-from=type=registry,ref=${{ env.IMAGE }}:buildcache
+
+      - name: Build and push
+        uses: docker/bake-action@v1
+        if: github.event_name != 'pull_request'
+        with:
+          targets: gha
+          set: |
+            gha-push.output=type=image,push=true
+            gha-push.cache-from=type=registry,ref=${{ env.IMAGE }}:buildcache
+            gha-push.cache-to=type=registry,ref=${{ env.IMAGE }}:buildcache,mode=max
diff --git a/docker-bake.hcl b/docker-bake.hcl
index fb6c8940f..c134996f5 100644
--- a/docker-bake.hcl
+++ b/docker-bake.hcl
@@ -31,8 +31,6 @@ group "gha" { targets = ["gha-regular", "gha-debug", "gha-test"] }
 
 target "gha-base" {
   inherits = ["release"]
-  cache-from = ["type=gha"]
-  cache-to = ["type=gha,mode=max"]
 }
 
 // This is filled by GitHub Actions