Split up the tests into performance tests that are measured, and user flow tests that access more screens and aren't

IntegrationTests/Sources/Common.swift

@@ -0,0 +1,135 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import XCTest
+
+extension XCUIApplication {
+    func login(currentTestCase: XCTestCase) {
+        let getStartedButton = buttons[A11yIdentifiers.onboardingScreen.signIn]
+        
+        XCTAssertTrue(getStartedButton.waitForExistence(timeout: 10.0))
+        getStartedButton.tap()
+        
+        let changeHomeserverButton = buttons[A11yIdentifiers.serverConfirmationScreen.changeServer]
+        XCTAssertTrue(changeHomeserverButton.waitForExistence(timeout: 10.0))
+        changeHomeserverButton.tap()
+        
+        let homeserverTextField = textFields[A11yIdentifiers.changeServerScreen.server]
+        XCTAssertTrue(homeserverTextField.waitForExistence(timeout: 10.0))
+        
+        homeserverTextField.clearAndTypeText(homeserver)
+        
+        let confirmButton = buttons[A11yIdentifiers.changeServerScreen.continue]
+        XCTAssertTrue(confirmButton.waitForExistence(timeout: 10.0))
+        confirmButton.tap()
+        
+        let continueButton = buttons[A11yIdentifiers.serverConfirmationScreen.continue]
+        XCTAssertTrue(continueButton.waitForExistence(timeout: 10.0))
+        continueButton.tap()
+        
+        let usernameTextField = textFields[A11yIdentifiers.loginScreen.emailUsername]
+        XCTAssertTrue(usernameTextField.waitForExistence(timeout: 10.0))
+        
+        usernameTextField.clearAndTypeText(username)
+        
+        let passwordTextField = secureTextFields[A11yIdentifiers.loginScreen.password]
+        XCTAssertTrue(passwordTextField.waitForExistence(timeout: 10.0))
+        
+        passwordTextField.clearAndTypeText(password)
+        
+        let nextButton = buttons[A11yIdentifiers.loginScreen.continue]
+        XCTAssertTrue(nextButton.waitForExistence(timeout: 10.0))
+        XCTAssertTrue(nextButton.isEnabled)
+        
+        nextButton.tap()
+        
+        sleep(10)
+        
+        // Handle analytics prompt screen
+        if staticTexts[A11yIdentifiers.analyticsPromptScreen.title].waitForExistence(timeout: 1.0) {
+            // Wait for login and then handle save password sheet
+            let savePasswordButton = buttons["Save Password"]
+            if savePasswordButton.waitForExistence(timeout: 10.0) {
+                savePasswordButton.tap()
+            }
+            
+            let enableButton = buttons[A11yIdentifiers.analyticsPromptScreen.enable]
+            XCTAssertTrue(enableButton.waitForExistence(timeout: 10.0))
+            enableButton.tap()
+        }
+        
+        // This might come in a different order, wait for both.
+        let savePasswordButton = buttons["Save Password"]
+        if savePasswordButton.waitForExistence(timeout: 10.0) {
+            savePasswordButton.tap()
+        }
+        
+        // Handle the notifications permission alert https://stackoverflow.com/a/58171074/730924
+        let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard")
+        let alertAllowButton = springboard.buttons.element(boundBy: 1)
+        if alertAllowButton.waitForExistence(timeout: 10.0) {
+            alertAllowButton.tap()
+        }
+        
+        // Migration screen may be shown as an overlay.
+        // if that pops up soon enough, we just let that happen and wait
+        let message = staticTexts[A11yIdentifiers.migrationScreen.message]
+        
+        if message.waitForExistence(timeout: 10.0) {
+            let doesNotExistPredicate = NSPredicate(format: "exists == 0")
+            currentTestCase.expectation(for: doesNotExistPredicate, evaluatedWith: message)
+            currentTestCase.waitForExpectations(timeout: 300.0)
+        }
+        
+        // Welcome screen may be shown as an overlay.
+        if buttons[A11yIdentifiers.welcomeScreen.letsGo].waitForExistence(timeout: 1.0) {
+            let goButton = buttons[A11yIdentifiers.welcomeScreen.letsGo]
+            XCTAssertTrue(goButton.waitForExistence(timeout: 1.0))
+            goButton.tap()
+        }
+        
+        // Wait for the home screen to become visible.
+        let profileButton = buttons[A11yIdentifiers.homeScreen.userAvatar]
+        // Timeouts are huge because we're waiting for the server.
+        XCTAssertTrue(profileButton.waitForExistence(timeout: 300.0))
+    }
+    
+    func logout() {
+        let profileButton = buttons[A11yIdentifiers.homeScreen.userAvatar]
+        
+        // `Failed to scroll to visible (by AX action) Button` https://stackoverflow.com/a/33534187/730924
+        profileButton.forceTap()
+        
+        // Open the settings
+        let settingsButton = buttons["Settings"]
+        XCTAssertTrue(settingsButton.waitForExistence(timeout: 10.0))
+        settingsButton.tap()
+        
+        // Logout
+        let logoutButton = buttons[A11yIdentifiers.settingsScreen.logout]
+        XCTAssertTrue(logoutButton.waitForExistence(timeout: 10.0))
+        logoutButton.tap()
+        
+        // Confirm logout
+        let alertLogoutButton = alerts.firstMatch.buttons["Sign out"]
+        XCTAssertTrue(alertLogoutButton.waitForExistence(timeout: 10.0))
+        alertLogoutButton.tap()
+        
+        // Check that we're back on the login screen
+        let getStartedButton = buttons[A11yIdentifiers.onboardingScreen.signIn]
+        XCTAssertTrue(getStartedButton.waitForExistence(timeout: 10.0))
+    }
+}

IntegrationTests/Sources/LoginTests.swift

@@ -1,200 +0,0 @@
-//
-// Copyright 2022 New Vector Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-import XCTest
-
-class LoginTests: XCTestCase {
-    func testLoginFlow() throws {
-        let parser = TestMeasurementParser()
-        parser.capture(testCase: self) {
-            let metrics: [XCTMetric] = [
-                XCTApplicationLaunchMetric(),
-                XCTClockMetric(),
-                XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.login)"),
-                XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.firstSync)"),
-                XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.firstRooms)"),
-                XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.roomFlow)")
-            ]
-            
-            self.measure(metrics: metrics) {
-                self.runLoginLogoutFlow()
-            }
-        }
-    }
-    
-    private func runLoginLogoutFlow() {
-        let app = Application.launch()
-        
-        let getStartedButton = app.buttons[A11yIdentifiers.onboardingScreen.signIn]
-        
-        XCTAssertTrue(getStartedButton.waitForExistence(timeout: 10.0))
-        getStartedButton.tap()
-        
-        let changeHomeserverButton = app.buttons[A11yIdentifiers.serverConfirmationScreen.changeServer]
-        XCTAssertTrue(changeHomeserverButton.waitForExistence(timeout: 10.0))
-        changeHomeserverButton.tap()
-        
-        let homeserverTextField = app.textFields[A11yIdentifiers.changeServerScreen.server]
-        XCTAssertTrue(homeserverTextField.waitForExistence(timeout: 10.0))
-        
-        homeserverTextField.clearAndTypeText(app.homeserver)
-        
-        let confirmButton = app.buttons[A11yIdentifiers.changeServerScreen.continue]
-        XCTAssertTrue(confirmButton.waitForExistence(timeout: 10.0))
-        confirmButton.tap()
-        
-        let continueButton = app.buttons[A11yIdentifiers.serverConfirmationScreen.continue]
-        XCTAssertTrue(continueButton.waitForExistence(timeout: 10.0))
-        continueButton.tap()
-        
-        let usernameTextField = app.textFields[A11yIdentifiers.loginScreen.emailUsername]
-        XCTAssertTrue(usernameTextField.waitForExistence(timeout: 10.0))
-        
-        usernameTextField.clearAndTypeText(app.username)
-        
-        let passwordTextField = app.secureTextFields[A11yIdentifiers.loginScreen.password]
-        XCTAssertTrue(passwordTextField.waitForExistence(timeout: 10.0))
-        
-        passwordTextField.clearAndTypeText(app.password)
-        
-        let nextButton = app.buttons[A11yIdentifiers.loginScreen.continue]
-        XCTAssertTrue(nextButton.waitForExistence(timeout: 10.0))
-        XCTAssertTrue(nextButton.isEnabled)
-        
-        nextButton.tap()
-        
-        sleep(10)
-        
-        // Handle analytics prompt screen
-        if app.staticTexts[A11yIdentifiers.analyticsPromptScreen.title].waitForExistence(timeout: 1.0) {
-            // Wait for login and then handle save password sheet
-            let savePasswordButton = app.buttons["Save Password"]
-            if savePasswordButton.waitForExistence(timeout: 10.0) {
-                savePasswordButton.tap()
-            }
-            
-            let enableButton = app.buttons[A11yIdentifiers.analyticsPromptScreen.enable]
-            XCTAssertTrue(enableButton.waitForExistence(timeout: 10.0))
-            enableButton.tap()
-        }
-        
-        // This might come in a different order, wait for both.
-        let savePasswordButton = app.buttons["Save Password"]
-        if savePasswordButton.waitForExistence(timeout: 10.0) {
-            savePasswordButton.tap()
-        }
-        
-        // Handle the notifications permission alert https://stackoverflow.com/a/58171074/730924
-        let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard")
-        let alertAllowButton = springboard.buttons.element(boundBy: 1)
-        if alertAllowButton.waitForExistence(timeout: 10.0) {
-            alertAllowButton.tap()
-        }
-        
-        // Migration screen may be shown as an overlay.
-        // if that pops up soon enough, we just let that happen and wait
-        let message = app.staticTexts[A11yIdentifiers.migrationScreen.message]
-        
-        if message.waitForExistence(timeout: 10.0) {
-            let doesNotExistPredicate = NSPredicate(format: "exists == 0")
-            expectation(for: doesNotExistPredicate, evaluatedWith: message)
-            waitForExpectations(timeout: 300.0)
-        }
-        
-        // Welcome screen may be shown as an overlay.
-        if app.buttons[A11yIdentifiers.welcomeScreen.letsGo].waitForExistence(timeout: 1.0) {
-            let goButton = app.buttons[A11yIdentifiers.welcomeScreen.letsGo]
-            XCTAssertTrue(goButton.waitForExistence(timeout: 1.0))
-            goButton.tap()
-        }
-        
-        // Wait for the home screen to become visible.
-        let profileButton = app.buttons[A11yIdentifiers.homeScreen.userAvatar]
-        // Timeouts are huge because we're waiting for the server.
-        XCTAssertTrue(profileButton.waitForExistence(timeout: 300.0))
-        
-        // Open the first room in the list.
-        let firstRoom = app.buttons.matching(NSPredicate(format: "identifier BEGINSWITH %@", A11yIdentifiers.homeScreen.roomNamePrefix)).firstMatch
-        XCTAssertTrue(firstRoom.waitForExistence(timeout: 10.0))
-        firstRoom.tap()
-        
-        // Long press on the last message
-        let lastMessage = app.cells.firstMatch
-        XCTAssertTrue(lastMessage.waitForExistence(timeout: 10.0))
-        lastMessage.press(forDuration: 2.0)
-        
-        // Hide the bottom sheet
-        let timelineItemActionMenu = app.otherElements[A11yIdentifiers.roomScreen.timelineItemActionMenu].firstMatch
-        XCTAssertTrue(timelineItemActionMenu.waitForExistence(timeout: 10.0))
-        timelineItemActionMenu.swipeDown(velocity: .fast)
-        
-        // Open the room details
-        let roomHeader = app.staticTexts["room-name"]
-        XCTAssertTrue(roomHeader.waitForExistence(timeout: 10.0))
-        roomHeader.tap()
-        
-        // Open the room member details
-        let roomMembers = app.buttons["room_details-people"]
-        XCTAssertTrue(roomMembers.waitForExistence(timeout: 10.0))
-        roomMembers.tap()
-        
-        // Open the first member's details
-        let firstRoomMember = app.scrollViews.buttons.firstMatch
-        XCTAssertTrue(firstRoomMember.waitForExistence(timeout: 10.0))
-        firstRoomMember.tap()
-        
-        // Go back to the room member details
-        var backButton = app.buttons["People"]
-        XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
-        backButton.tap()
-        
-        // Go back to the room details
-        backButton = app.buttons["Back"]
-        XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
-        backButton.tap()
-        
-        // Go back to the room
-        backButton = app.buttons["Back"]
-        XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
-        backButton.tap()
-        
-        // Go back to the room list
-        backButton = app.navigationBars.firstMatch.buttons["All Chats"]
-        XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
-        backButton.tap()
-        
-        // `Failed to scroll to visible (by AX action) Button` https://stackoverflow.com/a/33534187/730924
-        profileButton.forceTap()
-        
-        // Open the settings
-        let settingsButton = app.buttons["Settings"]
-        XCTAssertTrue(settingsButton.waitForExistence(timeout: 10.0))
-        settingsButton.tap()
-        
-        // Logout
-        let logoutButton = app.buttons[A11yIdentifiers.settingsScreen.logout]
-        XCTAssertTrue(logoutButton.waitForExistence(timeout: 10.0))
-        logoutButton.tap()
-        
-        // Confirm logout
-        let alertLogoutButton = app.alerts.firstMatch.buttons["Sign out"]
-        XCTAssertTrue(alertLogoutButton.waitForExistence(timeout: 10.0))
-        alertLogoutButton.tap()
-        
-        // Check that we're back on the login screen
-        XCTAssertTrue(getStartedButton.waitForExistence(timeout: 10.0))
-    }
-}

IntegrationTests/Sources/PerformanceTests.swift

@@ -0,0 +1,55 @@
+//
+// Copyright 2022 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import XCTest
+
+class PerformanceTests: XCTestCase {
+    func testLoginFlow() throws {
+        let parser = TestMeasurementParser()
+        parser.capture(testCase: self) {
+            let metrics: [XCTMetric] = [
+                XCTApplicationLaunchMetric(),
+                XCTClockMetric(),
+                XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.login)"),
+                XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.firstSync)"),
+                XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.firstRooms)"),
+                XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.roomFlow)")
+            ]
+            
+            self.measure(metrics: metrics) {
+                self.runLoginLogoutFlow()
+            }
+        }
+    }
+    
+    private func runLoginLogoutFlow() {
+        let app = Application.launch()
+        
+        app.login(currentTestCase: self)
+        
+        // Open the first room in the list.
+        let firstRoom = app.buttons.matching(NSPredicate(format: "identifier BEGINSWITH %@", A11yIdentifiers.homeScreen.roomNamePrefix)).firstMatch
+        XCTAssertTrue(firstRoom.waitForExistence(timeout: 10.0))
+        firstRoom.tap()
+                
+        // Go back to the room list
+        let backButton = app.navigationBars.firstMatch.buttons["All Chats"]
+        XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
+        backButton.tap()
+        
+        app.logout()
+    }
+}

IntegrationTests/Sources/UserFlowTests.swift

@@ -0,0 +1,77 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import XCTest
+
+class UserFlowTests: XCTestCase {
+    func testUserFlow() {
+        let app = Application.launch()
+        
+        app.login(currentTestCase: self)
+        
+        // Open the first room in the list.
+        let firstRoom = app.buttons.matching(NSPredicate(format: "identifier BEGINSWITH %@", A11yIdentifiers.homeScreen.roomNamePrefix)).firstMatch
+        XCTAssertTrue(firstRoom.waitForExistence(timeout: 10.0))
+        firstRoom.tap()
+        
+        // Long press on the last message
+        let lastMessage = app.cells.firstMatch
+        XCTAssertTrue(lastMessage.waitForExistence(timeout: 10.0))
+        lastMessage.press(forDuration: 2.0)
+        
+        // Hide the bottom sheet
+        let timelineItemActionMenu = app.otherElements[A11yIdentifiers.roomScreen.timelineItemActionMenu].firstMatch
+        XCTAssertTrue(timelineItemActionMenu.waitForExistence(timeout: 10.0))
+        timelineItemActionMenu.swipeDown(velocity: .fast)
+        
+        // Open the room details
+        let roomHeader = app.staticTexts["room-name"]
+        XCTAssertTrue(roomHeader.waitForExistence(timeout: 10.0))
+        roomHeader.tap()
+        
+        // Open the room member details
+        let roomMembers = app.buttons["room_details-people"]
+        XCTAssertTrue(roomMembers.waitForExistence(timeout: 10.0))
+        roomMembers.tap()
+        
+        // Open the first member's details
+        let firstRoomMember = app.scrollViews.buttons.firstMatch
+        XCTAssertTrue(firstRoomMember.waitForExistence(timeout: 10.0))
+        firstRoomMember.tap()
+        
+        // Go back to the room member details
+        var backButton = app.buttons["People"]
+        XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
+        backButton.tap()
+        
+        // Go back to the room details
+        backButton = app.buttons["Back"]
+        XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
+        backButton.tap()
+        
+        // Go back to the room
+        backButton = app.buttons["Back"]
+        XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
+        backButton.tap()
+        
+        // Go back to the room list
+        backButton = app.navigationBars.firstMatch.buttons["All Chats"]
+        XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
+        backButton.tap()
+        
+        app.logout()
+    }
+}