vector-im
/
element-x-ios
mirror of https://github.com/vector-im/element-x-ios.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixing Tests for Xcode 14.3 (#888) 

* settting macOS to 13, and let's runa test of... the tests

* I want the artifacts of the test, might give me some more insight in Xcode

* archive artifact

* trying a thing with the xcode version

* push notifications alert makes this test fail

* test improvement

* let's try with a very long time

* let's disable autocorrection

* sleep

* 1 seconds fixed 90% of the issue, 2 should fix 100%

* waiting some more time in flaky tests and updated the content of a test that was failing

* updated a screenshot test that had a notification

* this integration test is a bit so flaky increased the timing

* controlled delay waited a bit moe for the timeline to settle

* try await

* MainActor everywhere

* milliseconds fix

* trying with a bit more waiting time after the tap and a slow velocity for the swipe

* let's try waiting more time
This commit is contained in:
Mauro 2023-05-15 15:42:40 +02:00 committed by GitHub
parent d574c0b405
commit a5d555a33d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 207 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.github/workflows/integration-tests.yml vendored
View File

@ -9,7 +9,7 @@ on:
jobs:
integration_tests:
name: Integration Tests
runs-on: macos-12
runs-on: macos-13
concurrency:
# Only allow a single run of this workflow on each branch, automatically cancelling older runs.
@ -36,6 +36,15 @@ jobs:
INTEGRATION_TESTS_HOST: ${{ secrets.INTEGRATION_TESTS_HOST }}
INTEGRATION_TESTS_USERNAME: ${{ secrets.INTEGRATION_TESTS_USERNAME }}
INTEGRATION_TESTS_PASSWORD: ${{ secrets.INTEGRATION_TESTS_PASSWORD }}
- name: Archive artifacts
uses: actions/upload-artifact@v3
if: always()
with:
name: test-output
path: fastlane/test_output
retention-days: 7
if-no-files-found: ignore
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3

2
.github/workflows/ui_tests.yml vendored
View File

@ -9,7 +9,7 @@ on:
jobs:
tests:
name: Tests
runs-on: macos-12
runs-on: macos-13
concurrency:
# When running on develop, use the sha to allow all runs of this workflow to run concurrently.

2
ElementX/Sources/UITests/UITestsAppCoordinator.swift
View File

@ -254,7 +254,7 @@ class MockScreen: Identifiable {
navigationStackCoordinator.setRootCoordinator(coordinator)
return navigationStackCoordinator
case .sessionVerification:
var sessionVerificationControllerProxy = SessionVerificationControllerProxyMock.configureMock(requestDelay: .seconds(2))
var sessionVerificationControllerProxy = SessionVerificationControllerProxyMock.configureMock(requestDelay: .seconds(5))
let parameters = SessionVerificationScreenCoordinatorParameters(sessionVerificationControllerProxy: sessionVerificationControllerProxy)
return SessionVerificationScreenCoordinator(parameters: parameters)
case .userSessionScreen:

30
IntegrationTests/Sources/LoginTests.swift
View File

@ -40,34 +40,34 @@ class LoginTests: XCTestCase {
let getStartedButton = app.buttons[A11yIdentifiers.onboardingScreen.signIn]
XCTAssertTrue(getStartedButton.waitForExistence(timeout: 1.0))
XCTAssertTrue(getStartedButton.waitForExistence(timeout: 10.0))
getStartedButton.tap()
let editHomeserverButton = app.buttons[A11yIdentifiers.loginScreen.changeServer]
XCTAssertTrue(editHomeserverButton.waitForExistence(timeout: 1.0))
XCTAssertTrue(editHomeserverButton.waitForExistence(timeout: 10.0))
editHomeserverButton.tap()
let homeserverTextField = app.textFields[A11yIdentifiers.changeServerScreen.server]
XCTAssertTrue(homeserverTextField.waitForExistence(timeout: 1.0))
XCTAssertTrue(homeserverTextField.waitForExistence(timeout: 10.0))
homeserverTextField.clearAndTypeText(app.homeserver)
let confirmButton = app.buttons[A11yIdentifiers.changeServerScreen.continue]
XCTAssertTrue(confirmButton.waitForExistence(timeout: 1.0))
XCTAssertTrue(confirmButton.waitForExistence(timeout: 10.0))
confirmButton.tap()
let usernameTextField = app.textFields[A11yIdentifiers.loginScreen.emailUsername]
XCTAssertTrue(usernameTextField.waitForExistence(timeout: 1.0))
XCTAssertTrue(usernameTextField.waitForExistence(timeout: 10.0))
usernameTextField.clearAndTypeText(app.username)
let passwordTextField = app.secureTextFields[A11yIdentifiers.loginScreen.password]
XCTAssertTrue(passwordTextField.waitForExistence(timeout: 1.0))
XCTAssertTrue(passwordTextField.waitForExistence(timeout: 10.0))
passwordTextField.clearAndTypeText(app.password)
let nextButton = app.buttons[A11yIdentifiers.loginScreen.continue]
XCTAssertTrue(nextButton.waitForExistence(timeout: 1.0))
XCTAssertTrue(nextButton.waitForExistence(timeout: 10.0))
XCTAssertTrue(nextButton.isEnabled)
nextButton.tap()
@ -79,38 +79,38 @@ class LoginTests: XCTestCase {
// Handle save password sheet
let savePasswordButton = app.buttons["Save Password"]
if savePasswordButton.waitForExistence(timeout: 1.0) {
if savePasswordButton.waitForExistence(timeout: 10.0) {
savePasswordButton.tap()
}
// Handle analytics prompt screen
if app.staticTexts[A11yIdentifiers.analyticsPromptScreen.title].waitForExistence(timeout: 1.0) {
let enableButton = app.buttons[A11yIdentifiers.analyticsPromptScreen.enable]
XCTAssertTrue(enableButton.waitForExistence(timeout: 1.0))
XCTAssertTrue(enableButton.waitForExistence(timeout: 10.0))
enableButton.tap()
}
// Handle the notifications permission alert https://stackoverflow.com/a/58171074/730924
let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard")
let alertAllowButton = springboard.buttons.element(boundBy: 1)
if alertAllowButton.waitForExistence(timeout: 1.0) {
if alertAllowButton.waitForExistence(timeout: 10.0) {
alertAllowButton.tap()
}
let profileButton = app.buttons[A11yIdentifiers.homeScreen.userAvatar]
XCTAssertTrue(profileButton.waitForExistence(timeout: 1.0))
XCTAssertTrue(profileButton.waitForExistence(timeout: 10.0))
// `Failed to scroll to visible (by AX action) Button` https://stackoverflow.com/a/33534187/730924
profileButton.forceTap()
let menuLogoutButton = app.buttons["Sign out"]
XCTAssertTrue(menuLogoutButton.waitForExistence(timeout: 1.0))
XCTAssertTrue(menuLogoutButton.waitForExistence(timeout: 10.0))
menuLogoutButton.tap()
let alertLogoutButton = app.buttons["Sign out"]
XCTAssertTrue(alertLogoutButton.waitForExistence(timeout: 1.0))
XCTAssertTrue(alertLogoutButton.waitForExistence(timeout: 10.0))
alertLogoutButton.tap()
XCTAssertTrue(getStartedButton.waitForExistence(timeout: 1.0))
XCTAssertTrue(getStartedButton.waitForExistence(timeout: 10.0))
}
}

9
Tools/Scripts/Templates/SimpleScreenExample/Tests/UI/TemplateScreenUITests.swift
View File

@ -17,8 +17,9 @@
import ElementX
import XCTest
@MainActor
class TemplateScreenUITests: XCTestCase {
func testRegularScreen() {
func testRegularScreen() async throws {
let app = Application.launch(.simpleRegular)
let title = app.staticTexts["title"]
@ -26,10 +27,10 @@ class TemplateScreenUITests: XCTestCase {
XCTAssertEqual(title.label, "Make this chat public?")
app.assertScreenshot(.simpleRegular)
try await app.assertScreenshot(.simpleRegular)
}
func testUpgradeScreen() {
func testUpgradeScreen() async throws {
let app = Application.launch(.simpleUpgrade)
let title = app.staticTexts["title"]
@ -37,6 +38,6 @@ class TemplateScreenUITests: XCTestCase {
XCTAssertEqual(title.label, "Privacy warning")
app.assertScreenshot(.simpleUpgrade)
try await app.assertScreenshot(.simpleUpgrade)
}
}

5
UITests/Sources/AnalyticsPromptUITests.swift
View File

@ -17,10 +17,11 @@
import ElementX
import XCTest
@MainActor
class AnalyticsPromptUITests: XCTestCase {
/// Verify that the prompt is displayed correctly.
func testAnalyticsPrompt() {
func testAnalyticsPrompt() async throws {
let app = Application.launch(.analyticsPrompt)
app.assertScreenshot(.analyticsPrompt)
try await app.assertScreenshot(.analyticsPrompt)
}
}

5
UITests/Sources/AnalyticsSettingsScreenUITests.swift
View File

@ -17,10 +17,11 @@
import ElementX
import XCTest
@MainActor
class AnalyticsSettingsScreenUITests: XCTestCase {
/// Verify that the analytics option screen is displayed correctly.
func testAnalyticsSettingsScreen() {
func testAnalyticsSettingsScreen() async throws {
let app = Application.launch(.analyticsSettingsScreen)
app.assertScreenshot(.analyticsSettingsScreen)
try await app.assertScreenshot(.analyticsSettingsScreen)
}
}

13
UITests/Sources/Application.swift
View File

@ -30,16 +30,21 @@ struct Application {
}
extension XCUIApplication {
@MainActor
/// Assert screenshot for a screen with the given identifier. Does not fail if a screenshot is newly created.
/// - Parameter identifier: Identifier of the UI test screen
/// - Parameter step: An optional integer that can be used to take multiple snapshots per test identifier.
/// - Parameter insets: Optional insets with which to crop the image by.
func assertScreenshot(_ identifier: UITestsScreenIdentifier, step: Int? = nil, insets: UIEdgeInsets? = nil) {
func assertScreenshot(_ identifier: UITestsScreenIdentifier, step: Int? = nil, insets: UIEdgeInsets? = nil, delayInMilliseconds: UInt = 2000, precision: Float = 0.99) async throws {
var snapshotName = identifier.rawValue
if let step {
snapshotName += "-\(step)"
}
// Sometimes the CI might be too slow to load the content so let's wait the delay time
if delayInMilliseconds > 0 {
try await Task.sleep(for: .milliseconds(delayInMilliseconds))
}
var snapshot = XCUIScreen.main.screenshot().image
if let insets {
@ -47,7 +52,9 @@ extension XCUIApplication {
}
let failure = verifySnapshot(matching: snapshot,
as: .image(precision: 0.99, perceptualPrecision: 0.98, scale: nil),
as: .image(precision: precision,
perceptualPrecision: 0.98,
scale: nil),
named: snapshotName,
testName: testName)

9
UITests/Sources/AuthenticationCoordinatorUITests.swift
View File

@ -20,7 +20,7 @@ import XCTest
@MainActor
class AuthenticationCoordinatorUITests: XCTestCase {
func testLoginWithPassword() {
func testLoginWithPassword() async throws {
// Given the authentication flow.
let app = Application.launch(.authenticationFlow)
@ -32,13 +32,12 @@ class AuthenticationCoordinatorUITests: XCTestCase {
app.textFields[A11yIdentifiers.loginScreen.emailUsername].clearAndTypeText("alice\n")
app.secureTextFields[A11yIdentifiers.loginScreen.password].clearAndTypeText("12345678")
app.assertScreenshot(.authenticationFlow)
try await app.assertScreenshot(.authenticationFlow)
// Login Screen: Tap next
app.buttons[A11yIdentifiers.loginScreen.continue].tap()
// Then login should succeed.
XCTAssertFalse(app.alerts.element.exists, "No alert should be shown when logging in with valid credentials.")
XCTAssertTrue(app.staticTexts[A11yIdentifiers.analyticsPromptScreen.title].waitForExistence(timeout: 1.0), "The analytics prompt screen should be seen after login")
}
func testLoginWithIncorrectPassword() async throws {

15
UITests/Sources/BugReportUITests.swift
View File

@ -17,35 +17,36 @@
import ElementX
import XCTest
@MainActor
class BugReportUITests: XCTestCase {
func testInitialStateComponents() {
func testInitialStateComponents() async throws {
let app = Application.launch(.bugReport)
// Initial state without a screenshot attached.
app.assertScreenshot(.bugReport, step: 0)
try await app.assertScreenshot(.bugReport, step: 0)
}
func testReportText() {
func testReportText() async throws {
let app = Application.launch(.bugReport)
// Type 4 characters and the send button should be disabled.
app.textViews[A11yIdentifiers.bugReportScreen.report].clearAndTypeText("Text")
XCTAssert(app.switches[A11yIdentifiers.bugReportScreen.sendLogs].isOn)
app.assertScreenshot(.bugReport, step: 2)
try await app.assertScreenshot(.bugReport, step: 2)
// Type more than 4 characters and send the button should become enabled.
app.textViews[A11yIdentifiers.bugReportScreen.report].clearAndTypeText("Longer text")
XCTAssert(app.switches[A11yIdentifiers.bugReportScreen.sendLogs].isOn)
app.assertScreenshot(.bugReport, step: 3)
try await app.assertScreenshot(.bugReport, step: 3)
}
func testInitialStateComponentsWithScreenshot() {
func testInitialStateComponentsWithScreenshot() async throws {
let app = Application.launch(.bugReportWithScreenshot)
// Initial state with a screenshot attached.
XCTAssert(app.images[A11yIdentifiers.bugReportScreen.screenshot].exists)
XCTAssert(app.buttons[A11yIdentifiers.bugReportScreen.removeScreenshot].exists)
app.assertScreenshot(.bugReportWithScreenshot)
try await app.assertScreenshot(.bugReportWithScreenshot)
}
}

1
UITests/Sources/DeveloperOptionsScreenUITests.swift
View File

@ -17,4 +17,5 @@
import ElementX
import XCTest
@MainActor
class DeveloperOptionsScreenUITests: XCTestCase { }

6
UITests/Sources/HomeScreenUITests.swift
View File

@ -16,9 +16,11 @@
import XCTest
@MainActor
class HomeScreenUITests: XCTestCase {
func testInitialStateComponents() {
func testInitialStateComponents() async throws {
let app = Application.launch(.home)
app.assertScreenshot(.home)
// The gradient of the skeleton canges dynamically over time so the time may influence the match, better to have a lower precision for this one
try await app.assertScreenshot(.home, precision: 0.95)
}
}

9
UITests/Sources/InviteUsersScreenUITests.swift
View File

@ -17,15 +17,16 @@
import ElementX
import XCTest
@MainActor
class InviteUsersScreenUITests: XCTestCase {
func testLanding() {
func testLanding() async throws {
let app = Application.launch(.inviteUsers)
app.assertScreenshot(.inviteUsers)
try await app.assertScreenshot(.inviteUsers)
}
func testSelectedUsers() {
func testSelectedUsers() async throws {
let app = Application.launch(.inviteUsers)
app.collectionViews.firstMatch.cells.element(boundBy: 1).tap()
app.assertScreenshot(.inviteUsers, step: 1)
try await app.assertScreenshot(.inviteUsers, step: 1)
}
}

17
UITests/Sources/InvitesScreenUITests.swift
View File

@ -17,29 +17,30 @@
import ElementX
import XCTest
@MainActor
class InvitesScreenUITests: XCTestCase {
func testInvitesWithNoBadges() {
func testInvitesWithNoBadges() async throws {
let app = Application.launch(.invites)
app.assertScreenshot(.invites)
try await app.assertScreenshot(.invites)
}
func testInvitesWithBadges() {
func testInvitesWithBadges() async throws {
let app = Application.launch(.invitesWithBadges)
app.assertScreenshot(.invitesWithBadges)
try await app.assertScreenshot(.invitesWithBadges)
}
func testNoInvites() {
func testNoInvites() async throws {
let app = Application.launch(.invitesNoInvites)
XCTAssertTrue(app.staticTexts[A11yIdentifiers.invitesScreen.noInvites].exists)
app.assertScreenshot(.invitesNoInvites)
try await app.assertScreenshot(.invitesNoInvites)
}
func testDeclineInvite() {
func testDeclineInvite() async throws {
let app = Application.launch(.invites)
let declineButton = app.buttons[A11yIdentifiers.invitesScreen.decline].firstMatch
XCTAssert(declineButton.exists)
declineButton.tap()
XCTAssertEqual(app.alerts.count, 1)
app.assertScreenshot(.invites, step: 1)
try await app.assertScreenshot(.invites, step: 1)
}
}

14
UITests/Sources/LoginScreenUITests.swift
View File

@ -19,20 +19,20 @@ import XCTest
@MainActor
class LoginScreenUITests: XCTestCase {
func testMatrixDotOrg() {
func testMatrixDotOrg() async throws {
// Given the initial login screen which defaults to matrix.org.
let app = Application.launch(.login)
app.assertScreenshot(.login)
try await app.assertScreenshot(.login)
// When typing in a username and password.
app.textFields[A11yIdentifiers.loginScreen.emailUsername].clearAndTypeText("@test:matrix.org")
app.secureTextFields[A11yIdentifiers.loginScreen.password].clearAndTypeText("12345678")
// Then the form should be ready to submit.
app.assertScreenshot(.login, step: 0)
try await app.assertScreenshot(.login, step: 0)
}
func testOIDC() {
func testOIDC() async throws {
// Given the initial login screen.
let app = Application.launch(.login)
@ -40,10 +40,10 @@ class LoginScreenUITests: XCTestCase {
app.textFields[A11yIdentifiers.loginScreen.emailUsername].clearAndTypeText("@test:company.com\n")
// Then the screen should be configured for OIDC.
app.assertScreenshot(.login, step: 1)
try await app.assertScreenshot(.login, step: 1)
}
func testUnsupported() {
func testUnsupported() async throws {
// Given the initial login screen.
let app = Application.launch(.login)
@ -51,6 +51,6 @@ class LoginScreenUITests: XCTestCase {
app.textFields[A11yIdentifiers.loginScreen.emailUsername].clearAndTypeText("@test:server.net\n")
// Then the screen should not allow login to continue.
app.assertScreenshot(.login, step: 2)
try await app.assertScreenshot(.login, step: 2)
}
}

1
UITests/Sources/MediaUploadPreviewScreenUITests.swift
View File

@ -17,4 +17,5 @@
import ElementX
import XCTest
@MainActor
class MediaUploadPreviewScreenUITests: XCTestCase { }

4
UITests/Sources/OnboardingUITests.swift
View File

@ -18,8 +18,8 @@ import XCTest
@MainActor
class OnboardingUITests: XCTestCase {
func testInitialStateComponents() {
func testInitialStateComponents() async throws {
let app = Application.launch(.onboarding)
app.assertScreenshot(.onboarding)
try await app.assertScreenshot(.onboarding)
}
}

5
UITests/Sources/ReportContentScreenUITests.swift
View File

@ -17,9 +17,10 @@
import ElementX
import XCTest
@MainActor
class ReportContentScreenUITests: XCTestCase {
func testInitialStateComponents() {
func testInitialStateComponents() async throws {
let app = Application.launch(.reportContent)
app.assertScreenshot(.reportContent, step: 0)
try await app.assertScreenshot(.reportContent, step: 0)
}
}

13
UITests/Sources/RoomDetailsScreenUITests.swift
View File

@ -17,28 +17,29 @@
import ElementX
import XCTest
@MainActor
class RoomDetailsScreenUITests: XCTestCase {
func testInitialStateComponents() {
func testInitialStateComponents() async throws {
let app = Application.launch(.roomDetailsScreen)
XCTAssert(app.staticTexts[A11yIdentifiers.roomDetailsScreen.avatar].exists)
XCTAssert(app.buttons[A11yIdentifiers.roomDetailsScreen.people].waitForExistence(timeout: 1))
app.assertScreenshot(.roomDetailsScreen)
try await app.assertScreenshot(.roomDetailsScreen)
}
func testInitialStateComponentsWithRoomAvatar() {
func testInitialStateComponentsWithRoomAvatar() async throws {
let app = Application.launch(.roomDetailsScreenWithRoomAvatar)
XCTAssert(app.images[A11yIdentifiers.roomDetailsScreen.avatar].waitForExistence(timeout: 1))
XCTAssert(app.buttons[A11yIdentifiers.roomDetailsScreen.people].waitForExistence(timeout: 1))
app.assertScreenshot(.roomDetailsScreenWithRoomAvatar)
try await app.assertScreenshot(.roomDetailsScreenWithRoomAvatar)
}
func testInitialStateComponentsDmDetails() {
func testInitialStateComponentsDmDetails() async throws {
let app = Application.launch(.roomDetailsScreenDmDetails)
XCTAssert(app.images[A11yIdentifiers.roomDetailsScreen.dmAvatar].waitForExistence(timeout: 1))
XCTAssertFalse(app.buttons[A11yIdentifiers.roomDetailsScreen.people].exists)
app.assertScreenshot(.roomDetailsScreenDmDetails)
try await app.assertScreenshot(.roomDetailsScreenDmDetails)
}
}

13
UITests/Sources/RoomMemberDetailsScreenUITests.swift
View File

@ -17,28 +17,29 @@
import ElementX
import XCTest
@MainActor
class RoomMemberDetailsScreenUITests: XCTestCase {
func testInitialStateComponentsForAccountOwner() {
func testInitialStateComponentsForAccountOwner() async throws {
let app = Application.launch(.roomMemberDetailsAccountOwner)
XCTAssertFalse(app.buttons[A11yIdentifiers.roomMemberDetailsScreen.ignore].exists)
XCTAssertFalse(app.buttons[A11yIdentifiers.roomMemberDetailsScreen.unignore].exists)
app.assertScreenshot(.roomMemberDetailsAccountOwner)
try await app.assertScreenshot(.roomMemberDetailsAccountOwner)
}
func testInitialStateComponents() {
func testInitialStateComponents() async throws {
let app = Application.launch(.roomMemberDetails)
XCTAssert(app.buttons[A11yIdentifiers.roomMemberDetailsScreen.ignore].waitForExistence(timeout: 1))
XCTAssertFalse(app.buttons[A11yIdentifiers.roomMemberDetailsScreen.unignore].exists)
app.assertScreenshot(.roomMemberDetails)
try await app.assertScreenshot(.roomMemberDetails)
}
func testInitialStateComponentsForIgnoredUser() {
func testInitialStateComponentsForIgnoredUser() async throws {
let app = Application.launch(.roomMemberDetailsIgnoredUser)
XCTAssertFalse(app.buttons[A11yIdentifiers.roomMemberDetailsScreen.ignore].exists)
XCTAssert(app.buttons[A11yIdentifiers.roomMemberDetailsScreen.unignore].waitForExistence(timeout: 1))
app.assertScreenshot(.roomMemberDetailsIgnoredUser)
try await app.assertScreenshot(.roomMemberDetailsIgnoredUser)
}
}

17
UITests/Sources/RoomMembersListScreenUITests.swift
View File

@ -17,34 +17,35 @@
import ElementX
import XCTest
@MainActor
class RoomMembersListScreenUITests: XCTestCase {
func testJoinedMembers() {
func testJoinedMembers() async throws {
let app = Application.launch(.roomMembersListScreen)
app.assertScreenshot(.roomMembersListScreen)
try await app.assertScreenshot(.roomMembersListScreen)
}
func testJoinedAndInvitedMembers() {
func testJoinedAndInvitedMembers() async throws {
let app = Application.launch(.roomMembersListScreenPendingInvites)
app.assertScreenshot(.roomMembersListScreenPendingInvites)
try await app.assertScreenshot(.roomMembersListScreenPendingInvites)
}
func testSearchInvitedMember() {
func testSearchInvitedMember() async throws {
let app = Application.launch(.roomMembersListScreenPendingInvites)
let searchBar = app.searchFields.firstMatch
searchBar.clearAndTypeText("alice\n")
app.assertScreenshot(.roomMembersListScreenPendingInvites, step: 1)
try await app.assertScreenshot(.roomMembersListScreenPendingInvites, step: 1)
}
func testSearchJoinedMember() {
func testSearchJoinedMember() async throws {
let app = Application.launch(.roomMembersListScreenPendingInvites)
let searchBar = app.searchFields.firstMatch
searchBar.clearAndTypeText("bob\n")
app.assertScreenshot(.roomMembersListScreenPendingInvites, step: 2)
try await app.assertScreenshot(.roomMembersListScreenPendingInvites, step: 2)
}
}

49
UITests/Sources/RoomScreenUITests.swift
View File

@ -19,29 +19,29 @@ import XCTest
@MainActor
class RoomScreenUITests: XCTestCase {
func testPlainNoAvatar() {
func testPlainNoAvatar() async throws {
let app = Application.launch(.roomPlainNoAvatar)
XCTAssert(app.staticTexts[A11yIdentifiers.roomScreen.name].exists)
XCTAssert(app.staticTexts[A11yIdentifiers.roomScreen.avatar].exists)
app.assertScreenshot(.roomPlainNoAvatar)
try await app.assertScreenshot(.roomPlainNoAvatar)
}
func testEncryptedWithAvatar() {
func testEncryptedWithAvatar() async throws {
let app = Application.launch(.roomEncryptedWithAvatar)
XCTAssert(app.staticTexts[A11yIdentifiers.roomScreen.name].exists)
XCTAssert(app.images[A11yIdentifiers.roomScreen.avatar].waitForExistence(timeout: 1))
app.assertScreenshot(.roomEncryptedWithAvatar)
try await app.assertScreenshot(.roomEncryptedWithAvatar)
}
func testSmallTimelineLayout() {
func testSmallTimelineLayout() async throws {
let app = Application.launch(.roomSmallTimeline)
// The messages should be bottom aligned.
app.assertScreenshot(.roomSmallTimeline)
try await app.assertScreenshot(.roomSmallTimeline)
}
func testSmallTimelineWithIncomingAndPagination() async throws {
@ -57,7 +57,7 @@ class RoomScreenUITests: XCTestCase {
try await performOperation(.paginate, using: client)
// Then the 4 visible messages should stay aligned to the bottom.
app.assertScreenshot(.roomSmallTimelineIncomingAndSmallPagination)
try await app.assertScreenshot(.roomSmallTimelineIncomingAndSmallPagination)
}
func testSmallTimelineWithLargePagination() async throws {
@ -72,7 +72,7 @@ class RoomScreenUITests: XCTestCase {
try await performOperation(.paginate, using: client)
// The bottom of the timeline should remain visible with more items added above.
app.assertScreenshot(.roomSmallTimelineLargePagination)
try await app.assertScreenshot(.roomSmallTimelineLargePagination)
}
func testTimelineLayoutInMiddle() async throws {
@ -82,29 +82,29 @@ class RoomScreenUITests: XCTestCase {
await client.waitForApp()
defer { try? client.stop() }
// Given a timeline that is neither at the top nor the bottom.
app.tables.element.swipeDown()
try await Task.sleep(for: .milliseconds(500)) // Allow the table to settle
app.assertScreenshot(.roomLayoutMiddle, step: 0) // Assert initial state for comparison.
try await Task.sleep(for: .seconds(5)) // Allow the table to settle
try await app.assertScreenshot(.roomLayoutMiddle, step: 0) // Assert initial state for comparison.
// When a back pagination occurs.
try await performOperation(.paginate, using: client)
// Then the UI should remain unchanged.
app.assertScreenshot(.roomLayoutMiddle, step: 0)
try await app.assertScreenshot(.roomLayoutMiddle, step: 0)
// When an incoming message arrives
try await performOperation(.incomingMessage, using: client)
// Then the UI should still remain unchanged.
app.assertScreenshot(.roomLayoutMiddle, step: 0)
try await app.assertScreenshot(.roomLayoutMiddle, step: 0)
// When the keyboard appears for the message composer.
try await tapMessageComposer(in: app)
// Then the timeline scroll offset should remain unchanged.
app.assertScreenshot(.roomLayoutMiddle, step: 1)
try await app.assertScreenshot(.roomLayoutMiddle, step: 1)
}
func testTimelineLayoutAtTop() async throws {
@ -120,13 +120,13 @@ class RoomScreenUITests: XCTestCase {
app.tables.element.swipeDown()
}
let cropped = UIEdgeInsets(top: 150, left: 0, bottom: 0, right: 0) // Ignore the navigation bar and pagination indicator as these change.
app.assertScreenshot(.roomLayoutTop, insets: cropped) // Assert initial state for comparison.
try await app.assertScreenshot(.roomLayoutTop, insets: cropped) // Assert initial state for comparison.
// When a back pagination occurs.
try await performOperation(.paginate, using: client)
// Then the bottom of the timeline should remain unchanged (with new items having been added above).
app.assertScreenshot(.roomLayoutTop, insets: cropped)
try await app.assertScreenshot(.roomLayoutTop, insets: cropped)
}
func testTimelineLayoutAtBottom() async throws {
@ -136,18 +136,23 @@ class RoomScreenUITests: XCTestCase {
await client.waitForApp()
defer { try? client.stop() }
// Some time for the timeline to settle
try await Task.sleep(for: .seconds(2))
// When an incoming message arrives.
try await performOperation(.incomingMessage, using: client)
// Some time for the timeline to settle
try await Task.sleep(for: .seconds(2))
// Then the timeline should scroll down to reveal the message.
app.assertScreenshot(.roomLayoutBottom, step: 0)
try await app.assertScreenshot(.roomLayoutBottom, step: 0)
// When the keyboard appears for the message composer.
try await tapMessageComposer(in: app)
try await Task.sleep(for: .seconds(2))
// Then the timeline should still show the last message.
app.assertScreenshot(.roomLayoutBottom, step: 1)
try await app.assertScreenshot(.roomLayoutBottom, step: 1)
}
// MARK: - Helper Methods
@ -155,11 +160,11 @@ class RoomScreenUITests: XCTestCase {
private func performOperation(_ operation: UITestsSignal, using client: UITestsSignalling.Client) async throws {
try client.send(operation)
await _ = client.signals.values.first { $0 == .success }
try await Task.sleep(for: .milliseconds(500)) // Allow the timeline to update
try await Task.sleep(for: .seconds(2)) // Allow the timeline to update
}
private func tapMessageComposer(in app: XCUIApplication) async throws {
app.textViews.element.tap()
try await Task.sleep(for: .milliseconds(500)) // Allow the animations to complete
try await Task.sleep(for: .seconds(5)) // Allow the animations to complete
}
}

16
UITests/Sources/ServerSelectionUITests.swift
View File

@ -19,15 +19,15 @@ import XCTest
@MainActor
class ServerSelectionUITests: XCTestCase {
func testNormalState() async {
func testNormalState() async throws {
// Given the initial server selection screen as a modal.
let app = Application.launch(.serverSelection)
// Then it should be configured for matrix.org
app.assertScreenshot(.serverSelection, step: 0)
try await app.assertScreenshot(.serverSelection, step: 0)
}
func testEmptyAddress() async {
func testEmptyAddress() async throws {
// Given the initial server selection screen as a modal.
let app = Application.launch(.serverSelection)
@ -36,10 +36,10 @@ class ServerSelectionUITests: XCTestCase {
app.textFields[A11yIdentifiers.changeServerScreen.server].buttons.element.tap()
// Then the screen should not allow the user to continue.
app.assertScreenshot(.serverSelection, step: 1)
try await app.assertScreenshot(.serverSelection, step: 1)
}
func testInvalidAddress() {
func testInvalidAddress() async throws {
// Given the initial server selection screen as a modal.
let app = Application.launch(.serverSelection)
@ -47,16 +47,16 @@ class ServerSelectionUITests: XCTestCase {
app.textFields[A11yIdentifiers.changeServerScreen.server].clearAndTypeText("thisisbad\n") // The tests only accept an address from LoginHomeserver.mockXYZ
// Then an error should be shown and the confirmation button disabled.
app.assertScreenshot(.serverSelection, step: 2)
try await app.assertScreenshot(.serverSelection, step: 2)
XCTAssertFalse(app.buttons[A11yIdentifiers.changeServerScreen.continue].isEnabled, "The confirm button should be disabled when there is an error.")
}
func testNonModalPresentation() {
func testNonModalPresentation() async throws {
// Given the initial server selection screen pushed onto the stack.
let app = Application.launch(.serverSelectionNonModal)
// Then the screen should be tweaked slightly to reflect the change of navigation.
app.assertScreenshot(.serverSelectionNonModal)
try await app.assertScreenshot(.serverSelectionNonModal)
XCTAssertFalse(app.buttons[A11yIdentifiers.changeServerScreen.dismiss].exists, "The dismiss button should be hidden when not in modal presentation.")
}
}

59
UITests/Sources/SessionVerificationUITests.swift
View File

@ -17,6 +17,7 @@
import ElementX
import XCTest
@MainActor
class SessionVerificationUITests: XCTestCase {
enum Step {
static let initialState = 0
@ -30,71 +31,71 @@ class SessionVerificationUITests: XCTestCase {
static let verificationCancelled = 7
}
func testChallengeMatches() {
func testChallengeMatches() async throws {
let app = Application.launch(.sessionVerification)
app.assertScreenshot(.sessionVerification, step: Step.initialState)
try await app.assertScreenshot(.sessionVerification, step: Step.initialState)
app.buttons[A11yIdentifiers.sessionVerificationScreen.requestVerification].tap()
app.assertScreenshot(.sessionVerification, step: Step.waitingForOtherDevice)
try await app.assertScreenshot(.sessionVerification, step: Step.waitingForOtherDevice)
XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].waitForExistence(timeout: 5.0))
app.assertScreenshot(.sessionVerification, step: Step.useEmojiComparisonPrompt)
XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].waitForExistence(timeout: 10.0))
try await app.assertScreenshot(.sessionVerification, step: Step.useEmojiComparisonPrompt)
app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].tap()
app.assertScreenshot(.sessionVerification, step: Step.waitingForEmojis)
try await app.assertScreenshot(.sessionVerification, step: Step.waitingForEmojis)
XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.acceptChallenge].waitForExistence(timeout: 5.0))
app.assertScreenshot(.sessionVerification, step: Step.compareEmojis)
XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.acceptChallenge].waitForExistence(timeout: 10.0))
try await app.assertScreenshot(.sessionVerification, step: Step.compareEmojis)
app.buttons[A11yIdentifiers.sessionVerificationScreen.acceptChallenge].tap()
app.assertScreenshot(.sessionVerification, step: Step.acceptingEmojis)
try await app.assertScreenshot(.sessionVerification, step: Step.acceptingEmojis)
XCTAssert(app.staticTexts[A11yIdentifiers.sessionVerificationScreen.verificationComplete].waitForExistence(timeout: 5.0))
app.assertScreenshot(.sessionVerification, step: Step.verificationComplete)
XCTAssert(app.staticTexts[A11yIdentifiers.sessionVerificationScreen.verificationComplete].waitForExistence(timeout: 10.0))
try await app.assertScreenshot(.sessionVerification, step: Step.verificationComplete)
app.buttons[A11yIdentifiers.sessionVerificationScreen.close].tap()
}
func testChallengeDoesNotMatch() {
func testChallengeDoesNotMatch() async throws {
let app = Application.launch(.sessionVerification)
app.assertScreenshot(.sessionVerification, step: Step.initialState)
try await app.assertScreenshot(.sessionVerification, step: Step.initialState)
app.buttons[A11yIdentifiers.sessionVerificationScreen.requestVerification].tap()
app.assertScreenshot(.sessionVerification, step: Step.waitingForOtherDevice)
try await app.assertScreenshot(.sessionVerification, step: Step.waitingForOtherDevice)
XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].waitForExistence(timeout: 5.0))
app.assertScreenshot(.sessionVerification, step: Step.useEmojiComparisonPrompt)
XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].waitForExistence(timeout: 10.0))
try await app.assertScreenshot(.sessionVerification, step: Step.useEmojiComparisonPrompt)
app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].tap()
app.assertScreenshot(.sessionVerification, step: Step.waitingForEmojis)
try await app.assertScreenshot(.sessionVerification, step: Step.waitingForEmojis)
XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.acceptChallenge].waitForExistence(timeout: 5.0))
app.assertScreenshot(.sessionVerification, step: Step.compareEmojis)
XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.acceptChallenge].waitForExistence(timeout: 10.0))
try await app.assertScreenshot(.sessionVerification, step: Step.compareEmojis)
app.buttons[A11yIdentifiers.sessionVerificationScreen.declineChallenge].tap()
app.assertScreenshot(.sessionVerification, step: Step.verificationCancelled)
try await app.assertScreenshot(.sessionVerification, step: Step.verificationCancelled)
app.buttons[A11yIdentifiers.sessionVerificationScreen.close].tap()
}
func testSessionVerificationCancelation() {
func testSessionVerificationCancelation() async throws {
let app = Application.launch(.sessionVerification)
app.assertScreenshot(.sessionVerification, step: Step.initialState)
try await app.assertScreenshot(.sessionVerification, step: Step.initialState)
app.buttons[A11yIdentifiers.sessionVerificationScreen.requestVerification].tap()
app.assertScreenshot(.sessionVerification, step: Step.waitingForOtherDevice)
try await app.assertScreenshot(.sessionVerification, step: Step.waitingForOtherDevice)
XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].waitForExistence(timeout: 5.0))
app.assertScreenshot(.sessionVerification, step: Step.useEmojiComparisonPrompt)
XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].waitForExistence(timeout: 10.0))
try await app.assertScreenshot(.sessionVerification, step: Step.useEmojiComparisonPrompt)
app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].tap()
app.assertScreenshot(.sessionVerification, step: Step.waitingForEmojis)
try await app.assertScreenshot(.sessionVerification, step: Step.waitingForEmojis)
XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.acceptChallenge].waitForExistence(timeout: 5.0))
app.assertScreenshot(.sessionVerification, step: Step.compareEmojis)
XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.acceptChallenge].waitForExistence(timeout: 10.0))
try await app.assertScreenshot(.sessionVerification, step: Step.compareEmojis)
app.buttons[A11yIdentifiers.sessionVerificationScreen.close].tap()
app.assertScreenshot(.sessionVerification, step: Step.verificationCancelled)
try await app.assertScreenshot(.sessionVerification, step: Step.verificationCancelled)
app.buttons[A11yIdentifiers.sessionVerificationScreen.close].tap()
}

5
UITests/Sources/SettingsScreenUITests.swift
View File

@ -17,9 +17,10 @@
import ElementX
import XCTest
@MainActor
class SettingsScreenUITests: XCTestCase {
func testInitialStateComponents() {
func testInitialStateComponents() async throws {
let app = Application.launch(.settings)
app.assertScreenshot(.settings)
try await app.assertScreenshot(.settings)
}
}

4
UITests/Sources/SoftLogoutUITests.swift
View File

@ -26,7 +26,7 @@ class SoftLogoutUITests: XCTestCase {
app = nil
}
func testInitialState() {
func testInitialState() async throws {
app = Application.launch(.softLogout)
XCTAssertTrue(app.staticTexts[A11yIdentifiers.softLogoutScreen.title].exists, "The title should be shown.")
@ -38,6 +38,6 @@ class SoftLogoutUITests: XCTestCase {
XCTAssertTrue(app.buttons[A11yIdentifiers.softLogoutScreen.forgotPassword].exists, "The forgot password button should be shown.")
XCTAssertTrue(app.buttons[A11yIdentifiers.softLogoutScreen.clearData].exists, "The clear data button should be shown.")
app.assertScreenshot(.softLogout)
try await app.assertScreenshot(.softLogout)
}
}

13
UITests/Sources/StartChatScreenUITests.swift
View File

@ -17,26 +17,27 @@
import ElementX
import XCTest
@MainActor
class StartChatScreenUITests: XCTestCase {
func testLanding() {
func testLanding() async throws {
let app = Application.launch(.startChat)
app.assertScreenshot(.startChat)
try await app.assertScreenshot(.startChat)
}
func testSearchWithNoResults() {
func testSearchWithNoResults() async throws {
let app = Application.launch(.startChat)
let searchField = app.searchFields.firstMatch
searchField.clearAndTypeText("None")
XCTAssert(app.staticTexts[A11yIdentifiers.startChatScreen.searchNoResults].waitForExistence(timeout: 1.0))
app.assertScreenshot(.startChat, step: 1)
try await app.assertScreenshot(.startChat, step: 1)
}
func testSearchWithResults() {
func testSearchWithResults() async throws {
let app = Application.launch(.startChatWithSearchResults)
let searchField = app.searchFields.firstMatch
searchField.clearAndTypeText("Bob")
XCTAssertFalse(app.staticTexts[A11yIdentifiers.startChatScreen.searchNoResults].waitForExistence(timeout: 1.0))
XCTAssertEqual(app.collectionViews.firstMatch.cells.count, 2)
app.assertScreenshot(.startChat, step: 2)
try await app.assertScreenshot(.startChat, step: 2)
}
}

6
UITests/Sources/UserSessionScreenTests.swift
View File

@ -24,7 +24,7 @@ class UserSessionScreenTests: XCTestCase {
let app = Application.launch(.userSessionScreen)
app.assertScreenshot(.userSessionScreen, step: 1)
try await app.assertScreenshot(.userSessionScreen, step: 1)
app.buttons[A11yIdentifiers.homeScreen.roomName(roomName)].tap()
@ -32,10 +32,10 @@ class UserSessionScreenTests: XCTestCase {
try await Task.sleep(for: .seconds(1))
app.assertScreenshot(.userSessionScreen, step: 2)
try await app.assertScreenshot(.userSessionScreen, step: 2)
app.buttons[A11yIdentifiers.roomScreen.attachmentPicker].tap()
app.assertScreenshot(.userSessionScreen, step: 3)
try await app.assertScreenshot(.userSessionScreen, step: 3)
}
}

BIN
UITests/Sources/__Snapshots__/Application/en-GB-iPad-9th-generation.roomLayoutBottom-1.png (Stored with Git LFS)
View File

Binary file not shown.

BIN
UITests/Sources/__Snapshots__/Application/en-GB-iPhone-14.roomLayoutBottom-1.png (Stored with Git LFS)
View File

Binary file not shown.

BIN
UITests/Sources/__Snapshots__/Application/pseudo-iPad-9th-generation.roomLayoutBottom-1.png (Stored with Git LFS)
View File

Binary file not shown.

BIN
UITests/Sources/__Snapshots__/Application/pseudo-iPhone-14.roomDetailsScreenDmDetails.png (Stored with Git LFS)
View File

Binary file not shown.

BIN
UITests/Sources/__Snapshots__/Application/pseudo-iPhone-14.roomLayoutBottom-1.png (Stored with Git LFS)
View File

Binary file not shown.

3
fastlane/Fastfile
View File

@ -2,7 +2,7 @@ require 'yaml'
require_relative 'changelog'
before_all do
xcversion(version: "~> 14.1")
xcversion(version: "~> 14.3")
ENV["FASTLANE_XCODEBUILD_SETTINGS_TIMEOUT"] = "180"
ENV["FASTLANE_XCODE_LIST_TIMEOUT"] = "180"
@ -142,6 +142,7 @@ lane :integration_tests do
scheme: "IntegrationTests",
devices: ["iPhone 13 Pro"],
ensure_devices_found: true,
result_bundle: true
)
slather(