Performance tests (#1301)

* Add signposts to performance tests.

- Update flow to include support for the migration screen.

* If the welcome screen shows, click on the button.

* Ensure a clean simulator each run.

* Add accessibility identifier for migration screen if required.

* Handle walking into the room and back out again.

* use iphone 14 pro to match what's used in xcode.

* Remove ApplicationTests as duplicated in LoginTests.

We measure app startup time in LoginTests as part of the flow - we may as well avoid spending 60s doing only that measurement in ApplicationTests

* Sleep 10s, the ui is otherwise showing up in random order.

* Revert "Remove ApplicationTests as duplicated in LoginTests."

This reverts commit 8670710315bcd0d6c3c3046f534b32b4c728b837.

* Update script to parse out correct values from results file.

* Allow cancellation of password prompt in any order.

* Remove test timeout, performance tests will always take a while.

* Adjust parsing further

* Remove ApplicationTests.

* Move to a more elegant way to wait for something to disappear.

* Linting.

* Fix unit tests.

---------

Co-authored-by: Doug <douglase@element.io>
Co-authored-by: Doug <6060466+pixlwave@users.noreply.github.com>

IntegrationTests/Sources/ApplicationTests.swift

@@ -1,34 +0,0 @@
-//
-// Copyright 2022 New Vector Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-import XCTest
-
-class ApplicationTests: XCTestCase {
-    func testLaunchPerformance() throws {
-        let parser = TestMeasurementParser()
-        
-        parser.capture(testCase: self) {
-            self.measure(metrics: [XCTApplicationLaunchMetric()]) {
-                Application.launch()
-            }
-        }
-        
-        guard let actualDuration = parser.valueForMetric(.appLaunch) else {
-            XCTFail("Couldn't retrieve app launch duration")
-            return
-        }
-    }
-}

IntegrationTests/Sources/LoginTests.swift

@@ -20,7 +20,15 @@ class LoginTests: XCTestCase {
     func testLoginFlow() throws {
         let parser = TestMeasurementParser()
         parser.capture(testCase: self) {
-            self.measure(metrics: [XCTClockMetric()]) {
+            let metrics: [XCTMetric] = [
+                XCTApplicationLaunchMetric(),
+                XCTClockMetric(),
+                XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.login)"),
+                XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.sync)"),
+                XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.roomFlow)")
+            ]
+            
+            self.measure(metrics: metrics) {
                 self.runLoginLogoutFlow()
             }
         }
@@ -47,7 +55,7 @@ class LoginTests: XCTestCase {
         XCTAssertTrue(homeserverTextField.waitForExistence(timeout: 10.0))
         
         homeserverTextField.clearAndTypeText(app.homeserver)
-                
+        
         let confirmButton = app.buttons[A11yIdentifiers.changeServerScreen.continue]
         XCTAssertTrue(confirmButton.waitForExistence(timeout: 10.0))
         confirmButton.tap()
@@ -71,26 +79,27 @@ class LoginTests: XCTestCase {
         XCTAssertTrue(nextButton.isEnabled)
         
         nextButton.tap()
-        
-        // Wait for login to finish
-        let doesNotExistPredicate = NSPredicate(format: "exists == 0")
-        expectation(for: doesNotExistPredicate, evaluatedWith: nextButton)
 
-        // timeout is huge because we're waiting for server actions as well.
-        waitForExpectations(timeout: 300.0)
+        sleep(10)
 
-        // Handle save password sheet
-        let savePasswordButton = app.buttons["Save Password"]
-        if savePasswordButton.waitForExistence(timeout: 10.0) {
-            savePasswordButton.tap()
-        }
-        
         // Handle analytics prompt screen
         if app.staticTexts[A11yIdentifiers.analyticsPromptScreen.title].waitForExistence(timeout: 1.0) {
+            // Wait for login and then handle save password sheet
+            let savePasswordButton = app.buttons["Save Password"]
+            if savePasswordButton.waitForExistence(timeout: 10.0) {
+                savePasswordButton.tap()
+            }
+        
             let enableButton = app.buttons[A11yIdentifiers.analyticsPromptScreen.enable]
             XCTAssertTrue(enableButton.waitForExistence(timeout: 10.0))
             enableButton.tap()
         }
+
+        // This might come in a different order, wait for both.
+        let savePasswordButton = app.buttons["Save Password"]
+        if savePasswordButton.waitForExistence(timeout: 10.0) {
+            savePasswordButton.tap()
+        }
         
         // Handle the notifications permission alert https://stackoverflow.com/a/58171074/730924
         let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard")
@@ -99,9 +108,36 @@ class LoginTests: XCTestCase {
             alertAllowButton.tap()
         }
         
-        let profileButton = app.buttons[A11yIdentifiers.homeScreen.userAvatar]
-        XCTAssertTrue(profileButton.waitForExistence(timeout: 10.0))
+        // Migration screen may be shown as an overlay.
+        // if that pops up soon enough, we just let that happen and wait
+        let message = app.staticTexts[A11yIdentifiers.migrationScreen.message]
 
+        if message.waitForExistence(timeout: 10.0) {
+            let doesNotExistPredicate = NSPredicate(format: "exists == 0")
+            expectation(for: doesNotExistPredicate, evaluatedWith: message)
+            waitForExpectations(timeout: 300.0)
+        }
+
+        // Welcome screen may be shown as an overlay.
+        if app.buttons[A11yIdentifiers.welcomeScreen.letsGo].waitForExistence(timeout: 1.0) {
+            let goButton = app.buttons[A11yIdentifiers.welcomeScreen.letsGo]
+            XCTAssertTrue(goButton.waitForExistence(timeout: 1.0))
+            goButton.tap()
+        }
+
+        // Wait for the home screen to become visible.
+        let profileButton = app.buttons[A11yIdentifiers.homeScreen.userAvatar]
+        // Timeouts are huge because we're waiting for the server.
+        XCTAssertTrue(profileButton.waitForExistence(timeout: 300.0))
+        
+        // Open the first room in the list.
+        let rooms = app.buttons.matching(NSPredicate(format: "identifier BEGINSWITH %@", A11yIdentifiers.homeScreen.roomNamePrefix))
+        rooms.firstMatch.tap()
+        // Temporary sleep to get it working.
+        sleep(20)
+        // Go back to the home screen.
+        app.navigationBars.firstMatch.buttons["All Chats"].tap()
+        
         // `Failed to scroll to visible (by AX action) Button` https://stackoverflow.com/a/33534187/730924
         profileButton.forceTap()