From 1333ab6643d12768426ee4d071c7e851400675f3 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Thu, 12 Jan 2023 17:36:42 +0100 Subject: [PATCH] Setup Maestro and add first tests. Also configure CI --- .github/workflows/maestro.yml | 35 ++++++++++++++ .maestro/README.md | 48 +++++++++++++++++++ .maestro/allTests.yaml | 7 +++ .maestro/tests/account/changeServer.yaml | 5 ++ .maestro/tests/account/login.yaml | 13 +++++ .maestro/tests/account/logout.yaml | 9 ++++ .../tests/assertions/assertHomeDisplayed.yaml | 5 ++ .../tests/assertions/assertInitDisplayed.yaml | 5 ++ .../assertions/assertLoginDisplayed.yaml | 5 ++ .maestro/tests/init.yaml | 7 +++ .maestro/tests/roomList/roomList.yaml | 6 +++ .maestro/tests/roomList/searchRoomList.yaml | 15 ++++++ .../tests/roomList/timeline/timeline.yaml | 14 ++++++ .maestro/tests/settings/settings.yaml | 12 +++++ .../android/x/component/ShowkaseButton.kt | 2 +- 15 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/maestro.yml create mode 100644 .maestro/README.md create mode 100644 .maestro/allTests.yaml create mode 100644 .maestro/tests/account/changeServer.yaml create mode 100644 .maestro/tests/account/login.yaml create mode 100644 .maestro/tests/account/logout.yaml create mode 100644 .maestro/tests/assertions/assertHomeDisplayed.yaml create mode 100644 .maestro/tests/assertions/assertInitDisplayed.yaml create mode 100644 .maestro/tests/assertions/assertLoginDisplayed.yaml create mode 100644 .maestro/tests/init.yaml create mode 100644 .maestro/tests/roomList/roomList.yaml create mode 100644 .maestro/tests/roomList/searchRoomList.yaml create mode 100644 .maestro/tests/roomList/timeline/timeline.yaml create mode 100644 .maestro/tests/settings/settings.yaml diff --git a/.github/workflows/maestro.yml b/.github/workflows/maestro.yml new file mode 100644 index 0000000000..da7d35a080 --- /dev/null +++ b/.github/workflows/maestro.yml @@ -0,0 +1,35 @@ +name: Meastro + +on: + pull_request: { } + push: + branches: [ main, develop ] + +# Enrich gradle.properties for CI/CD +env: + GRADLE_OPTS: -Dorg.gradle.jvmargs="-Xmx3072m -Dfile.encoding=UTF-8 -XX:+HeapDumpOnOutOfMemoryError" -Dkotlin.daemon.jvm.options="-Xmx2560m" -Dkotlin.incremental=false + CI_GRADLE_ARG_PROPERTIES: --stacktrace -PpreDexEnable=false --max-workers 2 --no-daemon + +jobs: + maestro-cloud: + name: Build debug APKs + runs-on: ubuntu-latest + if: github.ref != 'refs/heads/main' + strategy: + fail-fast: false + # Allow all jobs on develop. Just one per PR. + concurrency: + group: ${{ github.ref == 'refs/heads/develop' && format('build-develop-{0}', github.sha) || format('build-debug-{0}', github.ref) }} + cancel-in-progress: true + steps: + - uses: actions/checkout@v3 + - name: Assemble debug APK + run: ./gradlew assembleDebug $CI_GRADLE_ARG_PROPERTIES + - uses: mobile-dev-inc/action-maestro-cloud@v1.1.1 + with: + api-key: ${{ secrets.MAESTRO_CLOUD_API_KEY }} + app-file: app/build/outputs/apk/debug/app-debug.apk + env: | + USERNAME=maestroelement + PASSWORD=${{ secrets.MATRIX_MAESTRO_ACCOUNT_PASSWORD }} + APP_ID=io.element.android.x.debug diff --git a/.maestro/README.md b/.maestro/README.md new file mode 100644 index 0000000000..119aad7ba3 --- /dev/null +++ b/.maestro/README.md @@ -0,0 +1,48 @@ +# Maestro + +Maestro is a framework that we are using to test navigation across the application. +To setup, please refer at [https://maestro.mobile.dev](https://maestro.mobile.dev) + + + +* [Run test](#run-test) + * [Output](#output) +* [Write test](#write-test) + + + +## Run test + +From root dir of the project + +*Note: Since ElementX does not allow account creation nor room creation, we have to use an existing account with an existing room to run maestro test suite. So to run locally, please replace `user` and `123` with your test matrix.org account credentials, and `my room` with one of a room this account has join. Note that the test will send messages to this room.* + +```shell +maestro test \ + -e APP_ID=io.element.android.x.debug \ + -e USERNAME=user \ + -e PASSWORD=123 \ + -e ROOM_NAME="my room" \ + .maestro/allTest.yaml +``` + +### Output + +Test result will be printed on the console, and screenshots will be generated at `./build/maestro` + +## Write test + +Tests are yaml file. Generally each yaml file should leave the app in the same screen than at the beginning. + +Start the app and run this command to help writing test. + +```shell +maestro studio +``` + +Note that sometimes, this prevent running the test. So kill the `meastro studio` process to be able to run the test again. + +## CI + +The CI is running maestro using the workflow `.github/worflow/maestro.yaml` and [maestro cloud](https://cloud.mobile.dev/). For now we are limited to 100 runs a month. +Some GitHub secret are used to be able to do that: `MAESTRO_CLOUD_API_KEY`, for now api key from `benoitm@element.io` maestro cloud account, and `MATRIX_MAESTRO_ACCOUNT_PASSWORD` which is the password of the account @maestroelement:matrix.org. This account contains a room `MyRoom` to ba able to run the maestro test suite. diff --git a/.maestro/allTests.yaml b/.maestro/allTests.yaml new file mode 100644 index 0000000000..8283e9fed5 --- /dev/null +++ b/.maestro/allTests.yaml @@ -0,0 +1,7 @@ +appId: ${APP_ID} +--- +- runFlow: tests/init.yaml +- runFlow: tests/account/login.yaml +- runFlow: tests/settings/settings.yaml +- runFlow: tests/roomList/roomList.yaml +- runFlow: tests/account/logout.yaml diff --git a/.maestro/tests/account/changeServer.yaml b/.maestro/tests/account/changeServer.yaml new file mode 100644 index 0000000000..3f360e70a1 --- /dev/null +++ b/.maestro/tests/account/changeServer.yaml @@ -0,0 +1,5 @@ +appId: ${APP_ID} +--- +- tapOn: "Change" +- takeScreenshot: build/maestro/200-ChangeServer +- tapOn: "Continue" diff --git a/.maestro/tests/account/login.yaml b/.maestro/tests/account/login.yaml new file mode 100644 index 0000000000..1541ee37d6 --- /dev/null +++ b/.maestro/tests/account/login.yaml @@ -0,0 +1,13 @@ +appId: ${APP_ID} +--- +- tapOn: "Sign in" +- runFlow: ../assertions/assertLoginDisplayed.yaml +- takeScreenshot: build/maestro/100-SignIn +- runFlow: changeServer.yaml +- runFlow: ../assertions/assertLoginDisplayed.yaml +- tapOn: "Email or username" +- inputText: ${USERNAME} +- tapOn: "Password" +- inputText: ${PASSWORD} +- tapOn: "Continue" +- runFlow: ../assertions/assertHomeDisplayed.yaml diff --git a/.maestro/tests/account/logout.yaml b/.maestro/tests/account/logout.yaml new file mode 100644 index 0000000000..894b5ec03c --- /dev/null +++ b/.maestro/tests/account/logout.yaml @@ -0,0 +1,9 @@ +appId: ${APP_ID} +--- +- tapOn: "Settings" +- tapOn: "Sign out" +- takeScreenshot: build/maestro/900-SignOutDialg +- tapOn: + text: "Sign out" + index: 1 +- runFlow: ../assertions/assertInitDisplayed.yaml diff --git a/.maestro/tests/assertions/assertHomeDisplayed.yaml b/.maestro/tests/assertions/assertHomeDisplayed.yaml new file mode 100644 index 0000000000..6e9eec50db --- /dev/null +++ b/.maestro/tests/assertions/assertHomeDisplayed.yaml @@ -0,0 +1,5 @@ +appId: ${APP_ID} +--- +- extendedWaitUntil: + visible: "All Chats" + timeout: 10_000 diff --git a/.maestro/tests/assertions/assertInitDisplayed.yaml b/.maestro/tests/assertions/assertInitDisplayed.yaml new file mode 100644 index 0000000000..0bcef846c6 --- /dev/null +++ b/.maestro/tests/assertions/assertInitDisplayed.yaml @@ -0,0 +1,5 @@ +appId: ${APP_ID} +--- +- extendedWaitUntil: + visible: "Own your conversations." + timeout: 10_000 diff --git a/.maestro/tests/assertions/assertLoginDisplayed.yaml b/.maestro/tests/assertions/assertLoginDisplayed.yaml new file mode 100644 index 0000000000..0d6461de86 --- /dev/null +++ b/.maestro/tests/assertions/assertLoginDisplayed.yaml @@ -0,0 +1,5 @@ +appId: ${APP_ID} +--- +- extendedWaitUntil: + visible: "Welcome back" + timeout: 10_000 diff --git a/.maestro/tests/init.yaml b/.maestro/tests/init.yaml new file mode 100644 index 0000000000..178cb85d34 --- /dev/null +++ b/.maestro/tests/init.yaml @@ -0,0 +1,7 @@ +appId: ${APP_ID} +--- +- clearState +- launchApp +- tapOn: "Close showkase button" +- runFlow: ./assertions/assertInitDisplayed.yaml +- takeScreenshot: build/maestro/000-FirstScreen diff --git a/.maestro/tests/roomList/roomList.yaml b/.maestro/tests/roomList/roomList.yaml new file mode 100644 index 0000000000..3a26048791 --- /dev/null +++ b/.maestro/tests/roomList/roomList.yaml @@ -0,0 +1,6 @@ +appId: ${APP_ID} +--- +- takeScreenshot: build/maestro/300-RoomList +- runFlow: searchRoomList.yaml +- runFlow: timeline/timeline.yaml + diff --git a/.maestro/tests/roomList/searchRoomList.yaml b/.maestro/tests/roomList/searchRoomList.yaml new file mode 100644 index 0000000000..fe859defbb --- /dev/null +++ b/.maestro/tests/roomList/searchRoomList.yaml @@ -0,0 +1,15 @@ +appId: ${APP_ID} +--- +- tapOn: "search" +- inputText: ${ROOM_NAME.substring(0, 3)} +- takeScreenshot: build/maestro/400-SearchRoom +- tapOn: ${ROOM_NAME} +# Close keyboard +- hideKeyboard +# Back from timeline +- back +# Close keyboard +- hideKeyboard +# Back from search +- back +- runFlow: ../assertions/assertHomeDisplayed.yaml diff --git a/.maestro/tests/roomList/timeline/timeline.yaml b/.maestro/tests/roomList/timeline/timeline.yaml new file mode 100644 index 0000000000..95d1c1ff57 --- /dev/null +++ b/.maestro/tests/roomList/timeline/timeline.yaml @@ -0,0 +1,14 @@ +appId: ${APP_ID} +--- +# This is the name of one room +# TODO Create a room on a new account +- tapOn: ${ROOM_NAME} +- takeScreenshot: build/maestro/500-Timeline +- tapOn: "Message…" +- inputText: "Hello world!" +- tapOn: "Toggle full screen mode" +- tapOn: "Toggle full screen mode" +- tapOn: "Send" +- hideKeyboard +- back +- runFlow: ../../assertions/assertHomeDisplayed.yaml diff --git a/.maestro/tests/settings/settings.yaml b/.maestro/tests/settings/settings.yaml new file mode 100644 index 0000000000..6d54953242 --- /dev/null +++ b/.maestro/tests/settings/settings.yaml @@ -0,0 +1,12 @@ +appId: ${APP_ID} +--- +- tapOn: "Settings" +- assertVisible: "Rage shake to report bug" +- takeScreenshot: build/maestro/600-Settings +- tapOn: + text: "Report bug" + index: 1 +- assertVisible: "Describe your problem here" +- back +- back +- runFlow: ../assertions/assertHomeDisplayed.yaml diff --git a/app/src/main/java/io/element/android/x/component/ShowkaseButton.kt b/app/src/main/java/io/element/android/x/component/ShowkaseButton.kt index bd8906c2a7..689b46dcdc 100644 --- a/app/src/main/java/io/element/android/x/component/ShowkaseButton.kt +++ b/app/src/main/java/io/element/android/x/component/ShowkaseButton.kt @@ -48,7 +48,7 @@ internal fun ShowkaseButton( .size(16.dp), onClick = onCloseClicked, ) { - Icon(imageVector = Icons.Filled.Close, contentDescription = "") + Icon(imageVector = Icons.Filled.Close, contentDescription = "Close showkase button") } } }