CI: make sure Paparazzi test failures stop the test flow (#117)

* Run Paparazzi tests just once when koverMergedReport is used.
* Make sure a Paparazzi failure breaks the test flow and blocks merging.
* Change ./gradlew check call in quality.yml to custom runQualityChecks task.

.github/workflows/quality.yml

@@ -1,6 +1,7 @@
 name: Code Quality Checks
 
 on:
+  workflow_dispatch:
   pull_request: { }
   push:
     branches: [ main, develop ]
@@ -21,7 +22,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Run code quality check suite
-        run: ./gradlew check $CI_GRADLE_ARG_PROPERTIES
+        run: ./gradlew runQualityChecks $CI_GRADLE_ARG_PROPERTIES
       - name: Upload reports
         if: always()
         uses: actions/upload-artifact@v3
@@ -29,15 +30,6 @@ jobs:
           name: linting-report
           path: |
             */build/reports/**/*.*
-      - name: Check Kover rules
-        run: ./gradlew koverMergedVerify $CI_GRADLE_ARG_PROPERTIES
-      - name: Upload reports
-        if: failure()
-        uses: actions/upload-artifact@v3
-        with:
-          name: kover-report
-          path: |
-            **/kover/merged/verification/errors.txt
       - name: Prepare Danger
         if: always()
         run: |

.github/workflows/tests.yml

@@ -1,6 +1,7 @@
 name: Test
 
 on:
+  workflow_dispatch:
   pull_request: { }
   push:
     branches: [ main, develop ]
@@ -8,7 +9,7 @@ on:
 # Enrich gradle.properties for CI/CD
 env:
   GRADLE_OPTS: -Dorg.gradle.jvmargs="-Xmx3072m -Dfile.encoding=UTF-8 -XX:+HeapDumpOnOutOfMemoryError" -Dkotlin.daemon.jvm.options="-Xmx2560m" -Dkotlin.incremental=false
-  CI_GRADLE_ARG_PROPERTIES: --stacktrace -PpreDexEnable=false --max-workers 4 --no-daemon
+  CI_GRADLE_ARG_PROPERTIES: --stacktrace -PpreDexEnable=false --max-workers 4
 
 jobs:
   tests:
@@ -20,11 +21,42 @@ jobs:
       group: ${{ github.ref == 'refs/heads/main' && format('unit-tests-main-{0}', github.sha) || github.ref == 'refs/heads/develop' && format('unit-tests-develop-{0}', github.sha) || format('unit-tests-{0}', github.ref) }}
       cancel-in-progress: true
     steps:
-      - uses: actions/checkout@v3
-      - name: Run tests and generate kover report
-        run: ./gradlew koverMergedReport $CI_GRADLE_ARG_PROPERTIES
+      - name: ⏬ Checkout with LFS
+        uses: actions/checkout@v3
+        with:
+          lfs: 'true'
 
-      - name: Archive kover report
+      - name: ⚙️ Run unit & screenshot tests, generate kover report
+        run: ./gradlew koverMergedReport $CI_GRADLE_ARG_PROPERTIES -Pci-build=true
+
+      - name: 📈 Verify coverage
+        run: ./gradlew koverMergedVerify $CI_GRADLE_ARG_PROPERTIES -Pci-build=true
+
+      - name: 🚫 Upload kover failed coverage reports
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: kover-error-report
+          path: |
+            **/kover/merged/verification/errors.txt
+
+      - name: 📸 Upload Screenshot test report
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: reports
+          path: tests/uitests/build/reports/tests/testDebugUnitTest/
+          retention-days: 5
+
+      - name: 🚫 Upload Screenshot failure differences on error
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: failures
+          path: tests/uitests/out/failures/
+          retention-days: 5
+
+      - name: ✅ Upload kover report
         if: always()
         uses: actions/upload-artifact@v3
         with:
@@ -32,7 +64,7 @@ jobs:
           path: |
             **/build/reports/kover/merged
 
-      - name: Archive test results on error
+      - name: 🚫 Upload test results on error
         if: failure()
         uses: actions/upload-artifact@v3
         with:
@@ -41,7 +73,7 @@ jobs:
             **/out/failures/
             **/build/reports/tests/*UnitTest/
 
-      - name: Publish results to Sonar
+      - name: 🔊 Publish results to Sonar
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
           ORG_GRADLE_PROJECT_SONAR_LOGIN: ${{ secrets.SONAR_TOKEN }}
@@ -49,7 +81,7 @@ jobs:
         run: ./gradlew sonar $CI_GRADLE_ARG_PROPERTIES
 
       # https://github.com/codecov/codecov-action
-      - name: Upload coverage reports to codecov
+      - name: ☂️ Upload coverage reports to codecov
         if: always()
         uses: codecov/codecov-action@v3
         # with:

build.gradle.kts

@@ -1,3 +1,6 @@
+import kotlinx.kover.api.KoverTaskExtension
+import org.jetbrains.kotlin.cli.common.toBooleanLenient
+
 /*
  * Copyright (c) 2022 New Vector Ltd
  *
@@ -205,11 +208,11 @@ koverMerged {
             name = "Global minimum code coverage."
             target = kotlinx.kover.api.VerificationTarget.ALL
             bound {
-                minValue = 55
+                minValue = 50
                 // Setting a max value, so that if coverage is bigger, it means that we have to change minValue.
                 // For instance if we have minValue = 25 and maxValue = 30, and current code coverage is now 37.32%, update
                 // minValue to 35 and maxValue to 40.
-                maxValue = 60
+                maxValue = 55
                 counter = kotlinx.kover.api.CounterType.INSTRUCTION
                 valueType = kotlinx.kover.api.VerificationValueType.COVERED_PERCENTAGE
             }
@@ -257,3 +260,40 @@ koverMerged {
         }
     }
 }
+
+// Make Kover depend on Paparazzi
+tasks.whenTaskAdded {
+    if (name.startsWith("koverMerged")) {
+        dependsOn(":tests:uitests:verifyPaparazziDebug")
+    }
+}
+
+// When running on the CI, run only debug test variants
+val ciBuildProperty = "ci-build"
+val isCiBuild = if (project.hasProperty(ciBuildProperty)) {
+    val raw = project.property(ciBuildProperty) as? String
+    raw?.toBooleanLenient() == true || raw?.toIntOrNull() == 1
+} else false
+if (isCiBuild) {
+    allprojects {
+        afterEvaluate {
+            tasks.withType<Test>().configureEach {
+                extensions.configure<KoverTaskExtension> {
+                    val enabled = name.contains("debug", ignoreCase = true)
+                    isDisabled.set(!enabled)
+                }
+            }
+        }
+    }
+}
+
+// Register quality check tasks.
+tasks.register("runQualityChecks") {
+    project.subprojects {
+        // For some reason `findByName("lint")` doesn't work
+        tasks.findByPath("$path:lint")?.let { dependsOn(it) }
+        tasks.findByName("detekt")?.let { dependsOn(it) }
+        tasks.findByName("ktlintCheck")?.let { dependsOn(it) }
+    }
+    dependsOn(":app:knitCheck")
+}

changelog.d/117.misc

@@ -0,0 +1,4 @@
+Changes to CI:
+- Run Paparazzi tests just once when `koverMergedReport` is used.
+- Make sure a Paparazzi failure breaks the test flow and blocks merging.
+- Change `./gradlew check` call in `quality.yml` to custom `runQualityChecks` task.

tests/uitests/src/test/kotlin/io/element/android/tests/uitests/ScreenshotTest.kt

@@ -67,7 +67,7 @@ class ScreenshotTest {
 
     @get:Rule
     val paparazzi = Paparazzi(
-        maxPercentDifference = 0.0,
+        maxPercentDifference = 0.01,
     )
 
     @Test