From 1a42dbda97927fe5efa1f9c544ba35eac13f6fcb Mon Sep 17 00:00:00 2001 From: Stefan Ceriu Date: Sun, 1 Mar 2026 18:05:19 +0200 Subject: [PATCH] Add swift command for running integration tests --- .github/workflows/integration-tests.yml | 48 ++----------- Tools/Sources/Commands/CI/CI.swift | 1 + .../Commands/CI/IntegrationTests.swift | 71 +++++++++++++++++++ fastlane/Fastfile | 15 ---- 4 files changed, 76 insertions(+), 59 deletions(-) create mode 100644 Tools/Sources/Commands/CI/IntegrationTests.swift diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 96f9e7499..7fdfe0420 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -20,79 +20,39 @@ jobs: steps: - uses: actions/checkout@v6 - - uses: actions/cache@v5 - with: - path: vendor/bundle - key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }} - restore-keys: | - ${{ runner.os }}-gems- - - name: Setup environment run: source ci_scripts/ci_common.sh && setup_github_actions_environment - - name: Delete old log files - run: find '/Users/Shared' -name 'console*' -delete - - name: Run tests - run: bundle exec fastlane integration_tests + run: swift run tools ci integration-tests env: INTEGRATION_TESTS_HOST: ${{ secrets.INTEGRATION_TESTS_HOST }} INTEGRATION_TESTS_USERNAME: ${{ secrets.INTEGRATION_TESTS_USERNAME }} INTEGRATION_TESTS_PASSWORD: ${{ secrets.INTEGRATION_TESTS_PASSWORD }} - - name: Check logs are set to the `trace` level - run: (grep ' TRACE ' /Users/Shared -qR) - - - name: Check logs don't contain private messages - run: "! grep 'Go down in flames' /Users/Shared -R" - - - name: Zip results # Faster upload - if: failure() - working-directory: fastlane/test_output - run: zip -r IntegrationTests.xcresult.zip IntegrationTests.xcresult - - name: Archive artifacts uses: actions/upload-artifact@v7 # We only care about artefacts if the tests fail if: failure() with: name: Results - path: fastlane/test_output/IntegrationTests.xcresult.zip + path: test_output/IntegrationTests.xcresult.zip retention-days: 7 if-no-files-found: ignore - - name: Archive raw log file - uses: actions/upload-artifact@v7 - if: always() - with: - name: raw.log - path: ~/Library/Logs/scan/IntegrationTests-IntegrationTests.log - retention-days: 2 - if-no-files-found: ignore - - - name: Collect coverage - # Skip if not successful and in forks - if: ${{ success() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} - run: xcresultparser -q -o cobertura -t ElementX -p $(pwd) fastlane/test_output/IntegrationTests.xcresult > fastlane/test_output/integration-cobertura.xml - - name: Upload coverage to Codecov # Skip if not successful and in forks if: ${{ success() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 with: report_type: coverage - files: fastlane/test_output/integration-cobertura.xml + files: test_output/integration-cobertura.xml disable_search: true fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} flags: integrationtests - - name: Collect test results - # Skip if cancelled and in forks - if: ${{ !cancelled() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} - run: xcresultparser -q -o junit -p $(pwd) fastlane/test_output/IntegrationTests.xcresult > fastlane/test_output/integration-junit.xml - - name: Upload test results to Codecov # Skip if cancelled and in forks if: ${{ !cancelled() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} @@ -100,7 +60,7 @@ jobs: continue-on-error: true with: report_type: test_results - files: fastlane/test_output/integration-junit.xml + files: test_output/integration-junit.xml disable_search: true fail_ci_if_error: false token: ${{ secrets.CODECOV_TOKEN }} diff --git a/Tools/Sources/Commands/CI/CI.swift b/Tools/Sources/Commands/CI/CI.swift index 2c388553e..9906fa7f4 100644 --- a/Tools/Sources/Commands/CI/CI.swift +++ b/Tools/Sources/Commands/CI/CI.swift @@ -9,6 +9,7 @@ struct CI: ParsableCommand { AccessibilityTests.self, UnitTests.self, UITests.self, + IntegrationTests.self, RunTests.self, ConfigureNightly.self ]) diff --git a/Tools/Sources/Commands/CI/IntegrationTests.swift b/Tools/Sources/Commands/CI/IntegrationTests.swift new file mode 100644 index 000000000..1798bc964 --- /dev/null +++ b/Tools/Sources/Commands/CI/IntegrationTests.swift @@ -0,0 +1,71 @@ +import ArgumentParser +import CommandLineTools +import Foundation + +struct IntegrationTests: AsyncParsableCommand { + static let configuration = CommandConfiguration(commandName: "integration-tests", + abstract: "Runs the integration test CI workflow.", + discussion: """ + Deletes old log files, runs integration tests, validates that logs are set + to the trace level and don't contain private messages, then collects results. + """) + + @Option(help: "Device name for tests.") + var device = "iPhone 17" + + @Option(help: "iOS version for the simulator.") + var osVersion = "26.1" + + func run() async throws { + // Delete old log files + logger.info("🗑️ Deleting old log files…") + try await CI.run(.path("/bin/zsh"), ["-cu", "find '/Users/Shared' -name 'console*' -delete"]) + + var testsFailed = false + do { + logger.info("\n🧪 Running integration tests…\n") + try await RunTests.parse([ + "--scheme", "IntegrationTests", + "--device", device, + "--os-version", osVersion, + "--retries", "0" + ]).run() + } catch { + testsFailed = true + logger.error("\n❌ Integration tests failed.\n") + } + + // Validate logs only when tests passed — log files won't be meaningful otherwise + if !testsFailed { + do { + logger.info("🔍 Checking logs are set to the trace level…") + try await CI.run(.path("/bin/zsh"), ["-cu", "grep ' TRACE ' /Users/Shared -qR"]) + logger.info("✅ Trace level logging verified.") + } catch { + testsFailed = true + logger.error("❌ Logs are not set to the trace level.") + } + + do { + logger.info("🔍 Checking logs don't contain private messages…") + try await CI.run(.path("/bin/zsh"), ["-cu", "! grep 'Go down in flames' /Users/Shared -R"]) + logger.info("✅ No private messages found in logs.") + } catch { + testsFailed = true + logger.error("❌ Private messages found in logs.") + } + } + + await CI.zipResults(bundles: ["IntegrationTests.xcresult"], + outputName: "IntegrationTests.xcresult.zip") + + await CI.collectCoverage(resultBundle: "IntegrationTests.xcresult", outputName: "integration-cobertura.xml") + await CI.collectTestResults(resultBundle: "IntegrationTests.xcresult", outputName: "integration-junit.xml") + + if testsFailed { + throw ExitCode.failure + } + + logger.info("\n✅ Accessibility tests passed.\n") + } +} diff --git a/fastlane/Fastfile b/fastlane/Fastfile index c4301b50b..797927a55 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -45,21 +45,6 @@ lane :unit_tests do |options| # We use xcresultparser in the workflow to collect coverage from both result bundles. end -lane :integration_tests do - clear_derived_data() - - reset_simulator = ENV.key?('CI') - - run_tests( - scheme: "IntegrationTests", - device: "iPhone 17 (#{simulator_version})", - ensure_devices_found: true, - result_bundle: true, - reset_simulator: reset_simulator, - xcodebuild_formatter: "xcbeautify --quiet --is-ci --renderer github-actions" - ) -end - lane :config_production do sh("(cd .. && swift run pipeline update-foss-secrets)") xcodegen(spec: "project.yml")