Split up the tests into performance tests that are measured, and user flow tests that access more screens and aren't
This commit is contained in:
Stefan Ceriu
committed by
Stefan Ceriu
Stefan Ceriu
parent
0512fcf329
commit
b037368a4f
135
IntegrationTests/Sources/Common.swift
Normal file
135
IntegrationTests/Sources/Common.swift
Normal file
@@ -0,0 +1,135 @@
|
||||
//
|
||||
// Copyright 2023 New Vector Ltd
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import XCTest
|
||||
|
||||
extension XCUIApplication {
|
||||
func login(currentTestCase: XCTestCase) {
|
||||
let getStartedButton = buttons[A11yIdentifiers.onboardingScreen.signIn]
|
||||
|
||||
XCTAssertTrue(getStartedButton.waitForExistence(timeout: 10.0))
|
||||
getStartedButton.tap()
|
||||
|
||||
let changeHomeserverButton = buttons[A11yIdentifiers.serverConfirmationScreen.changeServer]
|
||||
XCTAssertTrue(changeHomeserverButton.waitForExistence(timeout: 10.0))
|
||||
changeHomeserverButton.tap()
|
||||
|
||||
let homeserverTextField = textFields[A11yIdentifiers.changeServerScreen.server]
|
||||
XCTAssertTrue(homeserverTextField.waitForExistence(timeout: 10.0))
|
||||
|
||||
homeserverTextField.clearAndTypeText(homeserver)
|
||||
|
||||
let confirmButton = buttons[A11yIdentifiers.changeServerScreen.continue]
|
||||
XCTAssertTrue(confirmButton.waitForExistence(timeout: 10.0))
|
||||
confirmButton.tap()
|
||||
|
||||
let continueButton = buttons[A11yIdentifiers.serverConfirmationScreen.continue]
|
||||
XCTAssertTrue(continueButton.waitForExistence(timeout: 10.0))
|
||||
continueButton.tap()
|
||||
|
||||
let usernameTextField = textFields[A11yIdentifiers.loginScreen.emailUsername]
|
||||
XCTAssertTrue(usernameTextField.waitForExistence(timeout: 10.0))
|
||||
|
||||
usernameTextField.clearAndTypeText(username)
|
||||
|
||||
let passwordTextField = secureTextFields[A11yIdentifiers.loginScreen.password]
|
||||
XCTAssertTrue(passwordTextField.waitForExistence(timeout: 10.0))
|
||||
|
||||
passwordTextField.clearAndTypeText(password)
|
||||
|
||||
let nextButton = buttons[A11yIdentifiers.loginScreen.continue]
|
||||
XCTAssertTrue(nextButton.waitForExistence(timeout: 10.0))
|
||||
XCTAssertTrue(nextButton.isEnabled)
|
||||
|
||||
nextButton.tap()
|
||||
|
||||
sleep(10)
|
||||
|
||||
// Handle analytics prompt screen
|
||||
if staticTexts[A11yIdentifiers.analyticsPromptScreen.title].waitForExistence(timeout: 1.0) {
|
||||
// Wait for login and then handle save password sheet
|
||||
let savePasswordButton = buttons["Save Password"]
|
||||
if savePasswordButton.waitForExistence(timeout: 10.0) {
|
||||
savePasswordButton.tap()
|
||||
}
|
||||
|
||||
let enableButton = buttons[A11yIdentifiers.analyticsPromptScreen.enable]
|
||||
XCTAssertTrue(enableButton.waitForExistence(timeout: 10.0))
|
||||
enableButton.tap()
|
||||
}
|
||||
|
||||
// This might come in a different order, wait for both.
|
||||
let savePasswordButton = buttons["Save Password"]
|
||||
if savePasswordButton.waitForExistence(timeout: 10.0) {
|
||||
savePasswordButton.tap()
|
||||
}
|
||||
|
||||
// Handle the notifications permission alert https://stackoverflow.com/a/58171074/730924
|
||||
let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard")
|
||||
let alertAllowButton = springboard.buttons.element(boundBy: 1)
|
||||
if alertAllowButton.waitForExistence(timeout: 10.0) {
|
||||
alertAllowButton.tap()
|
||||
}
|
||||
|
||||
// Migration screen may be shown as an overlay.
|
||||
// if that pops up soon enough, we just let that happen and wait
|
||||
let message = staticTexts[A11yIdentifiers.migrationScreen.message]
|
||||
|
||||
if message.waitForExistence(timeout: 10.0) {
|
||||
let doesNotExistPredicate = NSPredicate(format: "exists == 0")
|
||||
currentTestCase.expectation(for: doesNotExistPredicate, evaluatedWith: message)
|
||||
currentTestCase.waitForExpectations(timeout: 300.0)
|
||||
}
|
||||
|
||||
// Welcome screen may be shown as an overlay.
|
||||
if buttons[A11yIdentifiers.welcomeScreen.letsGo].waitForExistence(timeout: 1.0) {
|
||||
let goButton = buttons[A11yIdentifiers.welcomeScreen.letsGo]
|
||||
XCTAssertTrue(goButton.waitForExistence(timeout: 1.0))
|
||||
goButton.tap()
|
||||
}
|
||||
|
||||
// Wait for the home screen to become visible.
|
||||
let profileButton = buttons[A11yIdentifiers.homeScreen.userAvatar]
|
||||
// Timeouts are huge because we're waiting for the server.
|
||||
XCTAssertTrue(profileButton.waitForExistence(timeout: 300.0))
|
||||
}
|
||||
|
||||
func logout() {
|
||||
let profileButton = buttons[A11yIdentifiers.homeScreen.userAvatar]
|
||||
|
||||
// `Failed to scroll to visible (by AX action) Button` https://stackoverflow.com/a/33534187/730924
|
||||
profileButton.forceTap()
|
||||
|
||||
// Open the settings
|
||||
let settingsButton = buttons["Settings"]
|
||||
XCTAssertTrue(settingsButton.waitForExistence(timeout: 10.0))
|
||||
settingsButton.tap()
|
||||
|
||||
// Logout
|
||||
let logoutButton = buttons[A11yIdentifiers.settingsScreen.logout]
|
||||
XCTAssertTrue(logoutButton.waitForExistence(timeout: 10.0))
|
||||
logoutButton.tap()
|
||||
|
||||
// Confirm logout
|
||||
let alertLogoutButton = alerts.firstMatch.buttons["Sign out"]
|
||||
XCTAssertTrue(alertLogoutButton.waitForExistence(timeout: 10.0))
|
||||
alertLogoutButton.tap()
|
||||
|
||||
// Check that we're back on the login screen
|
||||
let getStartedButton = buttons[A11yIdentifiers.onboardingScreen.signIn]
|
||||
XCTAssertTrue(getStartedButton.waitForExistence(timeout: 10.0))
|
||||
}
|
||||
}
|
||||
@@ -1,200 +0,0 @@
|
||||
//
|
||||
// Copyright 2022 New Vector Ltd
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import XCTest
|
||||
|
||||
class LoginTests: XCTestCase {
|
||||
func testLoginFlow() throws {
|
||||
let parser = TestMeasurementParser()
|
||||
parser.capture(testCase: self) {
|
||||
let metrics: [XCTMetric] = [
|
||||
XCTApplicationLaunchMetric(),
|
||||
XCTClockMetric(),
|
||||
XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.login)"),
|
||||
XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.firstSync)"),
|
||||
XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.firstRooms)"),
|
||||
XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.roomFlow)")
|
||||
]
|
||||
|
||||
self.measure(metrics: metrics) {
|
||||
self.runLoginLogoutFlow()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func runLoginLogoutFlow() {
|
||||
let app = Application.launch()
|
||||
|
||||
let getStartedButton = app.buttons[A11yIdentifiers.onboardingScreen.signIn]
|
||||
|
||||
XCTAssertTrue(getStartedButton.waitForExistence(timeout: 10.0))
|
||||
getStartedButton.tap()
|
||||
|
||||
let changeHomeserverButton = app.buttons[A11yIdentifiers.serverConfirmationScreen.changeServer]
|
||||
XCTAssertTrue(changeHomeserverButton.waitForExistence(timeout: 10.0))
|
||||
changeHomeserverButton.tap()
|
||||
|
||||
let homeserverTextField = app.textFields[A11yIdentifiers.changeServerScreen.server]
|
||||
XCTAssertTrue(homeserverTextField.waitForExistence(timeout: 10.0))
|
||||
|
||||
homeserverTextField.clearAndTypeText(app.homeserver)
|
||||
|
||||
let confirmButton = app.buttons[A11yIdentifiers.changeServerScreen.continue]
|
||||
XCTAssertTrue(confirmButton.waitForExistence(timeout: 10.0))
|
||||
confirmButton.tap()
|
||||
|
||||
let continueButton = app.buttons[A11yIdentifiers.serverConfirmationScreen.continue]
|
||||
XCTAssertTrue(continueButton.waitForExistence(timeout: 10.0))
|
||||
continueButton.tap()
|
||||
|
||||
let usernameTextField = app.textFields[A11yIdentifiers.loginScreen.emailUsername]
|
||||
XCTAssertTrue(usernameTextField.waitForExistence(timeout: 10.0))
|
||||
|
||||
usernameTextField.clearAndTypeText(app.username)
|
||||
|
||||
let passwordTextField = app.secureTextFields[A11yIdentifiers.loginScreen.password]
|
||||
XCTAssertTrue(passwordTextField.waitForExistence(timeout: 10.0))
|
||||
|
||||
passwordTextField.clearAndTypeText(app.password)
|
||||
|
||||
let nextButton = app.buttons[A11yIdentifiers.loginScreen.continue]
|
||||
XCTAssertTrue(nextButton.waitForExistence(timeout: 10.0))
|
||||
XCTAssertTrue(nextButton.isEnabled)
|
||||
|
||||
nextButton.tap()
|
||||
|
||||
sleep(10)
|
||||
|
||||
// Handle analytics prompt screen
|
||||
if app.staticTexts[A11yIdentifiers.analyticsPromptScreen.title].waitForExistence(timeout: 1.0) {
|
||||
// Wait for login and then handle save password sheet
|
||||
let savePasswordButton = app.buttons["Save Password"]
|
||||
if savePasswordButton.waitForExistence(timeout: 10.0) {
|
||||
savePasswordButton.tap()
|
||||
}
|
||||
|
||||
let enableButton = app.buttons[A11yIdentifiers.analyticsPromptScreen.enable]
|
||||
XCTAssertTrue(enableButton.waitForExistence(timeout: 10.0))
|
||||
enableButton.tap()
|
||||
}
|
||||
|
||||
// This might come in a different order, wait for both.
|
||||
let savePasswordButton = app.buttons["Save Password"]
|
||||
if savePasswordButton.waitForExistence(timeout: 10.0) {
|
||||
savePasswordButton.tap()
|
||||
}
|
||||
|
||||
// Handle the notifications permission alert https://stackoverflow.com/a/58171074/730924
|
||||
let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard")
|
||||
let alertAllowButton = springboard.buttons.element(boundBy: 1)
|
||||
if alertAllowButton.waitForExistence(timeout: 10.0) {
|
||||
alertAllowButton.tap()
|
||||
}
|
||||
|
||||
// Migration screen may be shown as an overlay.
|
||||
// if that pops up soon enough, we just let that happen and wait
|
||||
let message = app.staticTexts[A11yIdentifiers.migrationScreen.message]
|
||||
|
||||
if message.waitForExistence(timeout: 10.0) {
|
||||
let doesNotExistPredicate = NSPredicate(format: "exists == 0")
|
||||
expectation(for: doesNotExistPredicate, evaluatedWith: message)
|
||||
waitForExpectations(timeout: 300.0)
|
||||
}
|
||||
|
||||
// Welcome screen may be shown as an overlay.
|
||||
if app.buttons[A11yIdentifiers.welcomeScreen.letsGo].waitForExistence(timeout: 1.0) {
|
||||
let goButton = app.buttons[A11yIdentifiers.welcomeScreen.letsGo]
|
||||
XCTAssertTrue(goButton.waitForExistence(timeout: 1.0))
|
||||
goButton.tap()
|
||||
}
|
||||
|
||||
// Wait for the home screen to become visible.
|
||||
let profileButton = app.buttons[A11yIdentifiers.homeScreen.userAvatar]
|
||||
// Timeouts are huge because we're waiting for the server.
|
||||
XCTAssertTrue(profileButton.waitForExistence(timeout: 300.0))
|
||||
|
||||
// Open the first room in the list.
|
||||
let firstRoom = app.buttons.matching(NSPredicate(format: "identifier BEGINSWITH %@", A11yIdentifiers.homeScreen.roomNamePrefix)).firstMatch
|
||||
XCTAssertTrue(firstRoom.waitForExistence(timeout: 10.0))
|
||||
firstRoom.tap()
|
||||
|
||||
// Long press on the last message
|
||||
let lastMessage = app.cells.firstMatch
|
||||
XCTAssertTrue(lastMessage.waitForExistence(timeout: 10.0))
|
||||
lastMessage.press(forDuration: 2.0)
|
||||
|
||||
// Hide the bottom sheet
|
||||
let timelineItemActionMenu = app.otherElements[A11yIdentifiers.roomScreen.timelineItemActionMenu].firstMatch
|
||||
XCTAssertTrue(timelineItemActionMenu.waitForExistence(timeout: 10.0))
|
||||
timelineItemActionMenu.swipeDown(velocity: .fast)
|
||||
|
||||
// Open the room details
|
||||
let roomHeader = app.staticTexts["room-name"]
|
||||
XCTAssertTrue(roomHeader.waitForExistence(timeout: 10.0))
|
||||
roomHeader.tap()
|
||||
|
||||
// Open the room member details
|
||||
let roomMembers = app.buttons["room_details-people"]
|
||||
XCTAssertTrue(roomMembers.waitForExistence(timeout: 10.0))
|
||||
roomMembers.tap()
|
||||
|
||||
// Open the first member's details
|
||||
let firstRoomMember = app.scrollViews.buttons.firstMatch
|
||||
XCTAssertTrue(firstRoomMember.waitForExistence(timeout: 10.0))
|
||||
firstRoomMember.tap()
|
||||
|
||||
// Go back to the room member details
|
||||
var backButton = app.buttons["People"]
|
||||
XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
|
||||
backButton.tap()
|
||||
|
||||
// Go back to the room details
|
||||
backButton = app.buttons["Back"]
|
||||
XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
|
||||
backButton.tap()
|
||||
|
||||
// Go back to the room
|
||||
backButton = app.buttons["Back"]
|
||||
XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
|
||||
backButton.tap()
|
||||
|
||||
// Go back to the room list
|
||||
backButton = app.navigationBars.firstMatch.buttons["All Chats"]
|
||||
XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
|
||||
backButton.tap()
|
||||
|
||||
// `Failed to scroll to visible (by AX action) Button` https://stackoverflow.com/a/33534187/730924
|
||||
profileButton.forceTap()
|
||||
|
||||
// Open the settings
|
||||
let settingsButton = app.buttons["Settings"]
|
||||
XCTAssertTrue(settingsButton.waitForExistence(timeout: 10.0))
|
||||
settingsButton.tap()
|
||||
|
||||
// Logout
|
||||
let logoutButton = app.buttons[A11yIdentifiers.settingsScreen.logout]
|
||||
XCTAssertTrue(logoutButton.waitForExistence(timeout: 10.0))
|
||||
logoutButton.tap()
|
||||
|
||||
// Confirm logout
|
||||
let alertLogoutButton = app.alerts.firstMatch.buttons["Sign out"]
|
||||
XCTAssertTrue(alertLogoutButton.waitForExistence(timeout: 10.0))
|
||||
alertLogoutButton.tap()
|
||||
|
||||
// Check that we're back on the login screen
|
||||
XCTAssertTrue(getStartedButton.waitForExistence(timeout: 10.0))
|
||||
}
|
||||
}
|
||||
55
IntegrationTests/Sources/PerformanceTests.swift
Normal file
55
IntegrationTests/Sources/PerformanceTests.swift
Normal file
@@ -0,0 +1,55 @@
|
||||
//
|
||||
// Copyright 2022 New Vector Ltd
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import XCTest
|
||||
|
||||
class PerformanceTests: XCTestCase {
|
||||
func testLoginFlow() throws {
|
||||
let parser = TestMeasurementParser()
|
||||
parser.capture(testCase: self) {
|
||||
let metrics: [XCTMetric] = [
|
||||
XCTApplicationLaunchMetric(),
|
||||
XCTClockMetric(),
|
||||
XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.login)"),
|
||||
XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.firstSync)"),
|
||||
XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.firstRooms)"),
|
||||
XCTOSSignpostMetric(subsystem: Signposter.subsystem, category: Signposter.category, name: "\(Signposter.Name.roomFlow)")
|
||||
]
|
||||
|
||||
self.measure(metrics: metrics) {
|
||||
self.runLoginLogoutFlow()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func runLoginLogoutFlow() {
|
||||
let app = Application.launch()
|
||||
|
||||
app.login(currentTestCase: self)
|
||||
|
||||
// Open the first room in the list.
|
||||
let firstRoom = app.buttons.matching(NSPredicate(format: "identifier BEGINSWITH %@", A11yIdentifiers.homeScreen.roomNamePrefix)).firstMatch
|
||||
XCTAssertTrue(firstRoom.waitForExistence(timeout: 10.0))
|
||||
firstRoom.tap()
|
||||
|
||||
// Go back to the room list
|
||||
let backButton = app.navigationBars.firstMatch.buttons["All Chats"]
|
||||
XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
|
||||
backButton.tap()
|
||||
|
||||
app.logout()
|
||||
}
|
||||
}
|
||||
77
IntegrationTests/Sources/UserFlowTests.swift
Normal file
77
IntegrationTests/Sources/UserFlowTests.swift
Normal file
@@ -0,0 +1,77 @@
|
||||
//
|
||||
// Copyright 2023 New Vector Ltd
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import XCTest
|
||||
|
||||
class UserFlowTests: XCTestCase {
|
||||
func testUserFlow() {
|
||||
let app = Application.launch()
|
||||
|
||||
app.login(currentTestCase: self)
|
||||
|
||||
// Open the first room in the list.
|
||||
let firstRoom = app.buttons.matching(NSPredicate(format: "identifier BEGINSWITH %@", A11yIdentifiers.homeScreen.roomNamePrefix)).firstMatch
|
||||
XCTAssertTrue(firstRoom.waitForExistence(timeout: 10.0))
|
||||
firstRoom.tap()
|
||||
|
||||
// Long press on the last message
|
||||
let lastMessage = app.cells.firstMatch
|
||||
XCTAssertTrue(lastMessage.waitForExistence(timeout: 10.0))
|
||||
lastMessage.press(forDuration: 2.0)
|
||||
|
||||
// Hide the bottom sheet
|
||||
let timelineItemActionMenu = app.otherElements[A11yIdentifiers.roomScreen.timelineItemActionMenu].firstMatch
|
||||
XCTAssertTrue(timelineItemActionMenu.waitForExistence(timeout: 10.0))
|
||||
timelineItemActionMenu.swipeDown(velocity: .fast)
|
||||
|
||||
// Open the room details
|
||||
let roomHeader = app.staticTexts["room-name"]
|
||||
XCTAssertTrue(roomHeader.waitForExistence(timeout: 10.0))
|
||||
roomHeader.tap()
|
||||
|
||||
// Open the room member details
|
||||
let roomMembers = app.buttons["room_details-people"]
|
||||
XCTAssertTrue(roomMembers.waitForExistence(timeout: 10.0))
|
||||
roomMembers.tap()
|
||||
|
||||
// Open the first member's details
|
||||
let firstRoomMember = app.scrollViews.buttons.firstMatch
|
||||
XCTAssertTrue(firstRoomMember.waitForExistence(timeout: 10.0))
|
||||
firstRoomMember.tap()
|
||||
|
||||
// Go back to the room member details
|
||||
var backButton = app.buttons["People"]
|
||||
XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
|
||||
backButton.tap()
|
||||
|
||||
// Go back to the room details
|
||||
backButton = app.buttons["Back"]
|
||||
XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
|
||||
backButton.tap()
|
||||
|
||||
// Go back to the room
|
||||
backButton = app.buttons["Back"]
|
||||
XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
|
||||
backButton.tap()
|
||||
|
||||
// Go back to the room list
|
||||
backButton = app.navigationBars.firstMatch.buttons["All Chats"]
|
||||
XCTAssertTrue(backButton.waitForExistence(timeout: 10.0))
|
||||
backButton.tap()
|
||||
|
||||
app.logout()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user