Fix the UI tests. (#4085)

* Compile errors 😬 * Update server selection snapshots. The default footer string has changed. * Remove the TemplateScreen UI tests. We use preview tests for snapshots now, UI tests are for flow coordinators. * Add the same looping on the OIDC UI test (but disable the test in the end). * Run UI tests separately for iPhone and iPad. GitHub runners don't seem to have enough memory to handle 2 simulators at once.
2025-05-02 12:47:31 +01:00
parent 08e265220d
commit 834f62503e
16 changed files with 75 additions and 80 deletions
--- a/.github/workflows/ui_tests.yml
+++ b/.github/workflows/ui_tests.yml
@@ -14,10 +14,13 @@ jobs:
  tests:
    name: Tests
    runs-on: macos-15
+    strategy:
+      matrix:
+        device: [iPhone, iPad]

    concurrency:
      # Only allow a single run of this workflow on each branch, automatically cancelling older runs.
-      group: ${{ format('ui-tests-{0}', github.ref) }}
+      group: ${{ format('ui-tests-{0}-{1}', github.ref, matrix.device) }}
      cancel-in-progress: true

    steps:
@@ -40,9 +43,9 @@ jobs:
      - name: Run tests
        run: |
          if [[ -z "${{ github.event.inputs.test_name }}" ]]; then
-            bundle exec fastlane ui_tests
+            bundle exec fastlane ui_tests device:${{ matrix.device }} 
          else
-            bundle exec fastlane ui_tests test_name:${{ github.event.inputs.test_name }}
+            bundle exec fastlane ui_tests device:${{ matrix.device }} test_name:${{ github.event.inputs.test_name }}
          fi

      - name: Zip results # for faster upload
@@ -54,7 +57,7 @@ jobs:
        uses: actions/upload-artifact@v4
        if: failure()
        with:
-          name: Results
+          name: ${{ matrix.device }}
          path: fastlane/test_output/UITests.xcresult.zip
          retention-days: 7
          if-no-files-found: ignore
--- a/ElementX.xcodeproj/project.pbxproj
+++ b/ElementX.xcodeproj/project.pbxproj
@@ -1049,7 +1049,6 @@
 		CD6A72B65D3B6076F4045C30 /* PHGPostHogConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = A6B891A6DA826E2461DBB40F /* PHGPostHogConfiguration.swift */; };
 		CDAE3A37D4DF136F9D07DB61 /* RoomChangeRolesScreenSection.swift in Sources */ = {isa = PBXBuildFile; fileRef = EAF710CB1C31F8938EAA3A7D /* RoomChangeRolesScreenSection.swift */; };
 		CDCA8A559E098503DDE29477 /* AttributedStringBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2A5C6FBF97B6EED3D4FA5EFF /* AttributedStringBuilder.swift */; };
-		CDE21091BB1272C088509F59 /* TemplateScreenTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B287771477D9A0D94AAC1CE /* TemplateScreenTests.swift */; };
 		CE1694C7BB93C3311524EF28 /* Untranslated.strings in Resources */ = {isa = PBXBuildFile; fileRef = D2F7194F440375338F8E2487 /* Untranslated.strings */; };
 		CE3B7FC34FB2C279AAA5EA01 /* AVMetadataMachineReadableCodeObject.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3339B1DDB1341E833D2555BC /* AVMetadataMachineReadableCodeObject.swift */; };
 		CE6F237360875D3D573FD0B2 /* RoomNotificationSettingsProxy.swift in Sources */ = {isa = PBXBuildFile; fileRef = AD6B522BD637845AB9570B10 /* RoomNotificationSettingsProxy.swift */; };
@@ -1956,7 +1955,6 @@
 		7B04BD3874D736127A8156B8 /* it */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = it; path = it.lproj/Localizable.strings; sourceTree = "<group>"; };
 		7B19B2BCC779ED934E0BBC2A /* AudioPlayer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioPlayer.swift; sourceTree = "<group>"; };
 		7B25F959A434BB9923A3223F /* ExpiringTaskRunner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExpiringTaskRunner.swift; sourceTree = "<group>"; };
-		7B287771477D9A0D94AAC1CE /* TemplateScreenTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TemplateScreenTests.swift; sourceTree = "<group>"; };
 		7B3D16709ADD4F4BCC710B1E /* SecureBackupScreenModels.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SecureBackupScreenModels.swift; sourceTree = "<group>"; };
 		7B849D2FF2CC12BA411A1651 /* CreateRoomModels.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CreateRoomModels.swift; sourceTree = "<group>"; };
 		7B9FCA1CFD07B8CF9BD21266 /* FlowCoordinatorProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FlowCoordinatorProtocol.swift; sourceTree = "<group>"; };
@@ -3796,7 +3794,6 @@
 		4B5DC42A1DB20ECEB0FF67CB /* Tests */ = {
 			isa = PBXGroup;
 			children = (
-				AD5FCF9340D670C526AD17E4 /* UI */,
 				73AB116809AE89292624CD8E /* Unit */,
 			);
 			path = Tests;
@@ -5206,14 +5203,6 @@
 			path = View;
 			sourceTree = "<group>";
 		};
-		AD5FCF9340D670C526AD17E4 /* UI */ = {
-			isa = PBXGroup;
-			children = (
-				7B287771477D9A0D94AAC1CE /* TemplateScreenTests.swift */,
-			);
-			path = UI;
-			sourceTree = "<group>";
-		};
 		B04B538A859CD012755DC19C /* NSE */ = {
 			isa = PBXGroup;
 			children = (
@@ -7858,7 +7847,6 @@
 				E9985DCD1B0D026D7E8BF809 /* ServerSelectionTests.swift in Sources */,
 				D5C2DA52162A978743A183F2 /* SessionVerificationTests.swift in Sources */,
 				F9788AE0B9EA19F1190ED14F /* StartChatScreenTests.swift in Sources */,
-				CDE21091BB1272C088509F59 /* TemplateScreenTests.swift in Sources */,
 				54AE8860D668AFD96E7E177B /* UITestsScreenIdentifier.swift in Sources */,
 				84EFCB95F9DA2979C8042B26 /* UITestsSignalling.swift in Sources */,
 				B22D857D1E8FCA6DD74A58E3 /* UserSessionScreenTests.swift in Sources */,
--- a/ElementX/Sources/UITests/UITestsAppCoordinator.swift
+++ b/ElementX/Sources/UITests/UITestsAppCoordinator.swift
@@ -132,11 +132,6 @@ class MockScreen: Identifiable {
            flowCoordinator.start()
            retainedState.append(flowCoordinator)
            return nil
-        case .templateScreen:
-            let navigationStackCoordinator = NavigationStackCoordinator()
-            let coordinator = TemplateScreenCoordinator(parameters: .init())
-            navigationStackCoordinator.setRootCoordinator(coordinator)
-            return navigationStackCoordinator
        case .appLockFlow, .appLockFlowDisabled:
            // The tested coordinator is setup below in the alternate window.
            // Here we just return a blank screen to snapshot as the unlocked app.
--- a/ElementX/Sources/UITests/UITestsScreenIdentifier.swift
+++ b/ElementX/Sources/UITests/UITestsScreenIdentifier.swift
@@ -42,7 +42,6 @@ enum UITestsScreenIdentifier: String {
    case sessionVerification
    case startChat
    case startChatWithSearchResults
-    case templateScreen
    case userSessionScreen
    case userSessionScreenReply
    case autoUpdatingTimeline
--- a/Tools/Scripts/Templates/SimpleScreenExample/Tests/UI/TemplateScreenTests.swift
+++ b/Tools/Scripts/Templates/SimpleScreenExample/Tests/UI/TemplateScreenTests.swift
@@ -1,20 +0,0 @@
-//
-// Copyright 2022-2024 New Vector Ltd.
-//
-// SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-// Please see LICENSE files in the repository root for full details.
-//
-
-import XCTest
-
-@MainActor
-class TemplateScreenUITests: XCTestCase {
-    func testScreen() async throws {
-        let app = Application.launch(.templateScreen)
-        
-        let title = app.staticTexts["Template title"]
-        XCTAssert(title.exists)
-
-        try await app.assertScreenshot()
-    }
-}
--- a/UITests/Sources/AuthenticationFlowCoordinatorTests.swift
+++ b/UITests/Sources/AuthenticationFlowCoordinatorTests.swift
@@ -98,7 +98,13 @@ class AuthenticationFlowCoordinatorUITests: XCTestCase {
        try await app.assertScreenshot()
    }
    
-    func testSelectingOIDCServer() {
+    // Disabled for now as the looping isn't 100% fool-proof and we have OIDC on the integration tests
+    // so this mock version doesn't really add anything to the tests as a whole.
+    func disabled_testSelectingOIDCServer() {
+        // Allow this test to run for longer to help with the loop whilst waiting to resolve the
+        // webcredentials for the Web Authentication Session (see below).
+        executionTimeAllowance = 300
+        
        // Given the authentication flow.
        let app = Application.launch(.authenticationFlow)
        
@@ -111,11 +117,31 @@ class AuthenticationFlowCoordinatorUITests: XCTestCase {
        // Server Selection: Clear the default, enter OIDC server and continue.
        app.textFields[A11yIdentifiers.changeServerScreen.server].clearAndTypeText("company.com\n", app: app)
        
-        // Server Confirmation: Tap continue button
-        app.buttons[A11yIdentifiers.serverConfirmationScreen.continue].tap()
-        
        let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard")
-        XCTAssertTrue(springboard.staticTexts["“ElementX” Wants to Use “company.com” to Sign In"].waitForExistence(timeout: 4),
-                      "The web authentication prompt should be shown after selecting a homeserver with OIDC.")
+        let wasAlertText = springboard.staticTexts["“ElementX” Wants to Use “company.com” to Sign In"]
+        
+        // On a fresh simulator the webcredentials association is sometimes slow to be resolved.
+        // This results in an error alert being shown instead of the Web Authentication Session alert.
+        // Keep looping on the Continue button for ~5 minutes until the Authentication Session is happy.
+        var remainingAttempts = 30
+        while !wasAlertText.exists {
+            // Server Confirmation: Tap continue button
+            app.buttons[A11yIdentifiers.serverConfirmationScreen.continue].tap()
+            
+            if wasAlertText.waitForExistence(timeout: 10) {
+                break
+            }
+            
+            remainingAttempts -= 1
+            if remainingAttempts <= 0 {
+                XCTFail("Failed to present the web authentication session.")
+            }
+            
+            if app.alerts.count > 0 {
+                app.alerts.firstMatch.buttons["OK"].tap()
+            }
+        }
+        
+        XCTAssertTrue(wasAlertText.exists, "The web authentication prompt should be shown after selecting a homeserver with OIDC.")
    }
 }
--- a/UITests/Sources/PollFormScreenTests.swift
+++ b/UITests/Sources/PollFormScreenTests.swift
@@ -17,15 +17,15 @@ class PollFormScreenUITests: XCTestCase {
    func testFilledPoll() async throws {
        let app = Application.launch(.createPoll)
        let questionTextField = app.textFields[A11yIdentifiers.pollFormScreen.question]
-        questionTextField.tapCenter()
+        questionTextField.tap(.center)
        questionTextField.typeText("Do you like polls?")

        let option1TextField = app.textFields[A11yIdentifiers.pollFormScreen.optionID(0)]
-        option1TextField.tapCenter()
+        option1TextField.tap(.center)
        option1TextField.typeText("Yes")

        let option2TextField = app.textFields[A11yIdentifiers.pollFormScreen.optionID(1)]
-        option2TextField.tapCenter()
+        option2TextField.tap(.center)
        option2TextField.typeText("No")
        
        // Dismiss the keyboard
--- a/UITests/Sources/UserSessionScreenTests.swift
+++ b/UITests/Sources/UserSessionScreenTests.swift
@@ -23,7 +23,7 @@ class UserSessionScreenTests: XCTestCase {
        try await Task.sleep(for: .seconds(1))
        try await app.assertScreenshot(step: 2)

-        app.buttons[A11yIdentifiers.roomScreen.composerToolbar.openComposeOptions].tapCenter()
+        app.buttons[A11yIdentifiers.roomScreen.composerToolbar.openComposeOptions].tap(.center)
        try await app.assertScreenshot(step: 3)
    }

--- a/UITests/Sources/Snapshots/Application/serverSelection.testEmptyAddress-iPad-18-4-en-GB.png
+++ b/UITests/Sources/Snapshots/Application/serverSelection.testEmptyAddress-iPad-18-4-en-GB.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f41fca4268f52cffb50d6611762b3afe15074d56edd4dcc169dc48d4f4703aa
-size 178019
+oid sha256:09790b253032a3cba0bb2e3d0185239f314a9ca5261f222a8f206fec6c5b9803
+size 166204
--- a/UITests/Sources/Snapshots/Application/serverSelection.testEmptyAddress-iPhone-18-4-en-GB.png
+++ b/UITests/Sources/Snapshots/Application/serverSelection.testEmptyAddress-iPhone-18-4-en-GB.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2622cc5aa6802a0b33ac982b327a1cfeb51f082509188316801526a5bd3b144f
-size 185421
+oid sha256:292478daac4382c9fb63766daa92e8277656ccabb46ed9780abe88ffc5c32a94
+size 163034
--- a/UITests/Sources/Snapshots/Application/serverSelection.testNormalState-iPad-18-4-en-GB.png
+++ b/UITests/Sources/Snapshots/Application/serverSelection.testNormalState-iPad-18-4-en-GB.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03ec96c3117b93b21537a3e6a64f0b609722820d0fb967056adf2d4fabd6d0bb
-size 104825
+oid sha256:f1792419ba4926084b1862ed660385c9a288c30d5e819c041a7d203a133f0b7d
+size 92973
--- a/UITests/Sources/Snapshots/Application/serverSelection.testNormalState-iPhone-18-4-en-GB.png
+++ b/UITests/Sources/Snapshots/Application/serverSelection.testNormalState-iPhone-18-4-en-GB.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17c71fda8d1494f7fc67b51b304f77f53eba92ca19153aff3a4468f3fa6add80
-size 116342
+oid sha256:48bafe417ca3f7997f59592d8aa297faa7f0ddf6e0e9281513f835f77bba2670
+size 97386
--- a/UITests/Sources/Snapshots/Application/templateScreen.testScreen-iPad-18-4-en-GB.png
+++ b/UITests/Sources/Snapshots/Application/templateScreen.testScreen-iPad-18-4-en-GB.png
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3a08af0ee7b494b7d1d3932141c743d1e0700e4cbb734fb66abd72696472bf12
-size 72787
--- a/UITests/Sources/Snapshots/Application/templateScreen.testScreen-iPhone-18-4-en-GB.png
+++ b/UITests/Sources/Snapshots/Application/templateScreen.testScreen-iPhone-18-4-en-GB.png
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:569e5ddc8cc188045103a62124f65f1ad550ee8b52b9d0f56b3692d00497dca3
-size 70422
--- a/UITests/SupportingFiles/target.yml
+++ b/UITests/SupportingFiles/target.yml
@@ -64,4 +64,3 @@ targets:
    - path: ../../ElementX/Sources/Other/Extensions/XCUIElement.swift
    - path: ../../ElementX/Sources/UITests/UITestsScreenIdentifier.swift
    - path: ../../ElementX/Sources/UITests/UITestsSignalling.swift
-    - path: ../../Tools/Scripts/Templates/SimpleScreenExample/Tests/UI
--- a/fastlane/Fastfile
+++ b/fastlane/Fastfile
@@ -105,17 +105,28 @@ lane :unit_tests do |options|
 end

 lane :ui_tests do |options|
-  create_simulator_if_necessary(
-    name: "iPhone-18.4",
-    type: "com.apple.CoreSimulator.SimDeviceType.iPhone-16",
-    runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-4"
-  )
+  # We used to run these simultaneously on iPhone and iPad but it is *really* slow on GitHub runners.
+  # Presumably because launching 2 simulators uses more memory than the runner has available.
  
-  create_simulator_if_necessary(
-    name: "iPad-18.4",
-    type: "com.apple.CoreSimulator.SimDeviceType.iPad-10th-generation",
-    runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-4"
-  )
+  if options[:device] == "iPhone"
+    device = "iPhone-18.4"
+    
+    create_simulator_if_necessary(
+      name: "iPhone-18.4",
+      type: "com.apple.CoreSimulator.SimDeviceType.iPhone-16",
+      runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-4"
+    )
+  elsif options[:device] == "iPad"
+    device = "iPad-18.4"
+    
+    create_simulator_if_necessary(
+      name: "iPad-18.4",
+      type: "com.apple.CoreSimulator.SimDeviceType.iPad-10th-generation",
+      runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-4"
+    )
+  else
+    UI.user_error!("Please supply a device argument as device:iPhone or device:iPad")
+  end

  if options[:test_name]
    test_to_run = ["UITests/#{options[:test_name]}"]
@@ -124,12 +135,12 @@ lane :ui_tests do |options|
  end

  reset_simulator = ENV.key?('CI')
-
+  
  run_tests(
    scheme: "UITests",
-    devices: ["iPhone-18.4", "iPad-18.4"],
+    device: device,
    ensure_devices_found: true,
-    prelaunch_simulator: true,
+    prelaunch_simulator: false,
    result_bundle: true,
    only_testing: test_to_run,
    number_of_retries: 3,