From 494e425de0ea4ccc8017d978bb8bb210ec628cc8 Mon Sep 17 00:00:00 2001
From: Jorge Martin Espinosa <jorgem@element.io>
Date: Mon, 2 Mar 2026 11:23:41 +0100
Subject: [PATCH] CI: Add failed tests to summary (#6271)

* Fix maestro CI flow not writing the successful summary

* Add the test failures to the summary of the `test` CI flow
---
 .github/workflows/maestro-local.yml           |  2 +-
 .../workflows/scripts/parse_test_failures.py  | 77 +++++++++++++++++++
 .github/workflows/tests.yml                   |  9 +++
 3 files changed, 87 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/scripts/parse_test_failures.py

diff --git a/.github/workflows/maestro-local.yml b/.github/workflows/maestro-local.yml
index 9ff6f578c9..0c7dcc0418 100644
--- a/.github/workflows/maestro-local.yml
+++ b/.github/workflows/maestro-local.yml
@@ -126,7 +126,7 @@ jobs:
       - name: Update summary (success)
         if: steps.maestro_test.outcome == 'success'
         run: |
-          echo "### Maestro tests worked :rocket:!"
+          echo "### Maestro tests worked :rocket:!" >> $GITHUB_STEP_SUMMARY
       - name: Update summary (failure)
         if: steps.maestro_test.outcome != 'success'
         run: |
diff --git a/.github/workflows/scripts/parse_test_failures.py b/.github/workflows/scripts/parse_test_failures.py
new file mode 100644
index 0000000000..eb0a0ecafa
--- /dev/null
+++ b/.github/workflows/scripts/parse_test_failures.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+import xml.etree.ElementTree as ET
+import sys
+import glob
+
+screenshot_test_failures = []
+output = []
+
+def parse_test_failures(xml_file):
+    """Parse XML test results and print failures."""
+    tree = ET.parse(xml_file)
+    root = tree.getroot()
+
+    # Find all testcase elements with failure children
+    if root.get("failures", "0") == "0":
+        return
+
+    name = root.get('name', 'Test Suite')
+    is_screenshot_test = name.startswith('ui.Preview')
+
+    if not is_screenshot_test:
+        output.append(f"## {name}")
+
+    for testcase in root.findall('.//testcase'):
+        failure = testcase.find('failure')
+        if failure is not None:
+            # Get testcase attributes
+            classname = testcase.get('classname', '')
+            name = testcase.get('name', '')
+
+            if is_screenshot_test:
+                # For screenshot tests, we want to display the classname as well
+                screenshot_test_failures.append(f"{classname}.{name}")
+            else:
+                # Get failure content (text inside the failure element)
+                failure_message = failure.get('message', '')
+                failure_content = failure.text if failure.text else ''
+
+                # Print in the requested format
+                output.append(f"### {name}")
+                output.append("```")
+                output.append(failure_message)
+                output.append("```")
+                output.append("<details><summary>Stacktrace</summary>")
+                output.append(f"<pre><code>{failure_content}</code></pre>")
+                output.append("</details>")
+                output.append("\n")
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        output.append("Usage: parse_test_failures.py <file>", file=sys.stderr)
+        sys.exit(1)
+
+    file = sys.argv[1]
+
+    if file.endswith('xml'):
+        parse_test_failures(file)
+    else:
+        files = glob.glob("**/build/test-results/*UnitTest/*.xml", root_dir = file, recursive = True)
+        for file in files:
+            parse_test_failures(file)
+
+    if screenshot_test_failures:
+        output.append("## Screenshot Test Failures")
+        output.append("```")
+        for failure in screenshot_test_failures:
+            output.append(failure)
+        output.append("```")
+
+    text_output = '\n'.join(output)
+    # Trim output larger than 1MB to avoid GitHub Action log limits
+    while len(text_output.encode('utf-8')) > 1_040_000:
+        output.pop(-2)
+        output.append("## !!! Truncated output due to size limits. !!!")
+        text_output = '\n'.join(output)
+
+    print(text_output)
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 68339e8443..522de31daf 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -95,6 +95,15 @@ jobs:
             **/build/roborazzi/failures/
             **/build/reports/tests/*UnitTest/
 
+      - name: 🚫 Modify summary on error
+        if: failure()
+        run: |
+          echo """## Tests failed!
+          
+          """ >> $GITHUB_STEP_SUMMARY
+          python3 .github/workflows/scripts/parse_test_failures.py . >> $GITHUB_STEP_SUMMARY
+          echo "---" >> $GITHUB_STEP_SUMMARY
+
       # https://github.com/codecov/codecov-action
       - name: ☂️ Upload coverage reports to codecov
         uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2