diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 184ce65f8..24de114b9 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -22,65 +22,34 @@ jobs: steps: - uses: nschloe/action-cached-lfs-checkout@f46300cd8952454b9f0a21a3d133d4bd5684cfc2 #v1.2.3 - - uses: actions/cache@v5 - with: - path: vendor/bundle - key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }} - restore-keys: | - ${{ runner.os }}-gems- - - name: Setup environment - run: - source ci_scripts/ci_common.sh && setup_github_actions_environment - - - name: SwiftFormat - run: - swiftformat --lint . - + run: source ci_scripts/ci_common.sh && setup_github_actions_environment + - name: Run tests - run: bundle exec fastlane unit_tests - - - name: Zip results # Faster uploads - if: failure() - working-directory: fastlane/test_output - run: zip -r UnitTests.zip UnitTests.xcresult PreviewTests.xcresult - + run: swift run -q tools ci unit-tests + - name: Archive artifacts uses: actions/upload-artifact@v6 # We only care about artefacts if the tests fail if: failure() with: - name: Results - path: fastlane/test_output/UnitTests.zip - retention-days: 1 - if-no-files-found: ignore - - - name: Collect coverage - # Skip if not successful and in forks - if: ${{ success() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} - run: | - xcresultparser -q -o cobertura -t ElementX -p $(pwd) fastlane/test_output/UnitTests.xcresult > fastlane/test_output/unit-cobertura.xml - xcresultparser -q -o cobertura -t ElementX -p $(pwd) fastlane/test_output/PreviewTests.xcresult > fastlane/test_output/preview-cobertura.xml - + name: Results + path: test_output/UnitTests.zip + retention-days: 1 + if-no-files-found: ignore + - name: Upload coverage to Codecov # Skip if not successful and in forks if: ${{ success() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 with: report_type: coverage - files: fastlane/test_output/unit-cobertura.xml,fastlane/test_output/preview-cobertura.xml + files: test_output/unit-cobertura.xml,test_output/preview-cobertura.xml disable_search: true fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} flags: unittests - - name: Collect test results - # Skip if cancelled and in forks - if: ${{ !cancelled() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} - run: | - xcresultparser -q -o junit -p $(pwd) fastlane/test_output/UnitTests.xcresult > fastlane/test_output/unit-junit.xml - xcresultparser -q -o junit -p $(pwd) fastlane/test_output/PreviewTests.xcresult > fastlane/test_output/preview-junit.xml - - name: Upload test results to Codecov # Skip if cancelled and in forks if: ${{ !cancelled() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} @@ -88,7 +57,7 @@ jobs: continue-on-error: true with: report_type: test_results - files: fastlane/test_output/unit-junit.xml,fastlane/test_output/preview-junit.xml + files: test_output/unit-junit.xml,test_output/preview-junit.xml disable_search: true fail_ci_if_error: false token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index e9b336020..bf70e4e9a 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,9 @@ build ## Editors .nova +## Project specific +test_output + # Only commit Pkl packages (there may be other caches in here). Secrets/vendor/* !Secrets/vendor/package-2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 02e229005..979609725 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -85,9 +85,11 @@ Please do **not** manually edit the `Localizable.strings`, `Localizable.stringsd ### Continuous Integration -Element X uses Fastlane for running actions on the CI and tries to keep the configuration confined to either [fastlane](fastlane/Fastfile) or [xcodegen](project.yml). +Element X uses a combination of Swift command line tools and Fastlane for running actions on the CI and tries to keep the configuration confined to either [Tools/Sources](Tools/Sources) or the [FastFile](fastlane/Fastfile) alongside the project's [xcodegen](project.yml) configuration. -Please run `bundle exec fastlane` to see available options. +Please run `swift run tools ci --help` and `bundle exec fastlane` to see available options. + +Note: We are in the process of converting our Fastlane lanes to Swift and so long-term are intending to remove Fastlane from the project all together. ### Network debugging proxy diff --git a/Package.resolved b/Package.resolved index db185fa75..0de178e96 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "9ad9b7de4b8539442c6bbc918a26130ff2c685c4a9d5596bdaf021e33cd45c6c", + "originHash" : "de3b5dfdd30df13c55deff097064bd887e3cc34e6259ed01bd9d29091b158262", "pins" : [ { "identity" : "swift-argument-parser", @@ -18,6 +18,33 @@ "revision" : "e5eaab1558ef664e6cd80493f64259381670fb3a" } }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log", + "state" : { + "revision" : "bbd81b6725ae874c69e9b8c8804d462356b55523", + "version" : "1.10.1" + } + }, + { + "identity" : "swift-subprocess", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-subprocess", + "state" : { + "revision" : "ba5888ad7758cbcbe7abebac37860b1652af2d9c", + "version" : "0.3.0" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system", + "state" : { + "revision" : "7c6ad0fc39d0763e0b699210e4124afd5041c5df", + "version" : "1.6.4" + } + }, { "identity" : "yams", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index a165ab42e..91bbdf9dd 100644 --- a/Package.swift +++ b/Package.swift @@ -15,14 +15,18 @@ let package = Package( .package(url: "https://github.com/apple/swift-argument-parser", .upToNextMinor(from: "1.7.0")), .package(url: "https://github.com/element-hq/swift-command-line-tools.git", revision: "e5eaab1558ef664e6cd80493f64259381670fb3a"), // .package(path: "../../../swift-command-line-tools"), - .package(url: "https://github.com/jpsim/Yams", .upToNextMinor(from: "6.2.1")) + .package(url: "https://github.com/swiftlang/swift-subprocess", .upToNextMinor(from: "0.3.0")), + .package(url: "https://github.com/jpsim/Yams", .upToNextMinor(from: "6.2.1")), + .package(url: "https://github.com/apple/swift-log", .upToNextMinor(from: "1.10.1")) ], targets: [ .executableTarget(name: "Tools", dependencies: [ .product(name: "ArgumentParser", package: "swift-argument-parser"), .product(name: "CommandLineTools", package: "swift-command-line-tools"), - .product(name: "Yams", package: "Yams") + .product(name: "Subprocess", package: "swift-subprocess"), + .product(name: "Yams", package: "Yams"), + .product(name: "Logging", package: "swift-log") ], path: "Tools/Sources") ] diff --git a/Tools/Sources/Commands/CI/CI.swift b/Tools/Sources/Commands/CI/CI.swift new file mode 100644 index 000000000..e435ec476 --- /dev/null +++ b/Tools/Sources/Commands/CI/CI.swift @@ -0,0 +1,103 @@ +import ArgumentParser +import Foundation +import Subprocess + +struct CI: ParsableCommand { + static let configuration = CommandConfiguration(abstract: "CI workflow commands that can be run both locally and in CI environments.", + subcommands: [ + UnitTests.self, + RunTests.self + ]) + + static let testOutputDirectory = "test_output" + + // MARK: - Linting + + /// Runs SwiftFormat in lint mode against the current directory. + static func lint() async throws { + logger.info("\n🔍 Running SwiftFormat lint…\n") + + do { + try await run(.name("swiftformat"), ["--lint", "."]) + } catch { + logger.error("\n❌ SwiftFormat failed.\n") + throw error + } + logger.info("\n✅ SwiftFormat passed.\n") + } + + // MARK: - Test Results + + /// Collects coverage from an xcresult bundle using xcresultparser (cobertura format). + /// Failures are non-fatal — the output file simply won't be created. + static func collectCoverage(resultBundle: String, target: String = "ElementX", outputName: String) async { + let projectPath = URL.projectDirectory.path + let resultBundlePath = "\(testOutputDirectory)/\(resultBundle)" + let outputPath = "\(testOutputDirectory)/\(outputName)" + + guard FileManager.default.fileExists(atPath: resultBundlePath) else { + logger.error("\n❌ Result bundle not found at \(resultBundlePath), skipping coverage collection.\n") + return + } + + do { + try await run(.path("/bin/zsh"), ["-cu", "xcresultparser -q -o cobertura -t \(target) -p \(projectPath) \(resultBundlePath) > \(outputPath)"]) + logger.info("\n📊 Coverage report: \(outputPath)\n") + } catch { + logger.error("\n❌ Failed to collect coverage for \(resultBundle): \(error.localizedDescription)\n") + } + } + + /// Collects test results from an xcresult bundle using xcresultparser (junit format). + /// Failures are non-fatal — the output file simply won't be created. + static func collectTestResults(resultBundle: String, outputName: String) async { + let projectPath = URL.projectDirectory.path + let resultBundlePath = "\(testOutputDirectory)/\(resultBundle)" + let outputPath = "\(testOutputDirectory)/\(outputName)" + + guard FileManager.default.fileExists(atPath: resultBundlePath) else { + logger.info(" Result bundle not found at \(resultBundlePath), skipping test result collection.") + return + } + + do { + try await run(.path("/bin/zsh"), ["-cu", "xcresultparser -q -o junit -p \(projectPath) \(resultBundlePath) > \(outputPath)"]) + logger.info("📋 Test results: \(outputPath)") + } catch { + logger.error("\n❌ Failed to collect test results for \(resultBundle): \(error.localizedDescription)\n") + } + } + + /// Zips xcresult bundles in the test output directory for faster artifact uploads. + static func zipResults(bundles: [String], outputName: String) async { + let bundleArgs = bundles.joined(separator: " ") + do { + logger.info("\n📦 Zipping test results…") + try await run(.path("/bin/zsh"), ["-cu", "cd \(testOutputDirectory) && zip -rq \(outputName) \(bundleArgs)"]) + logger.info("📦 Zipped: \(testOutputDirectory)/\(outputName)\n") + } catch { + logger.error("\n❌ Failed to zip results: \(error.localizedDescription)\n") + } + } + + // MARK: - Shell Interaction + + @discardableResult + static func run(_ executable: Executable, + _ arguments: Arguments = [], + environment: Environment = .inherit, + output: Output = .standardOutput, + error: Error = .standardError) async throws -> CollectedResult { + let result = try await Subprocess.run(executable, + arguments: arguments, + environment: environment, + output: output, + error: error) + + if case let .exited(code) = result.terminationStatus, code != 0 { + throw ExitCode.failure + } + + return result + } +} diff --git a/Tools/Sources/Commands/CI/RunTests.swift b/Tools/Sources/Commands/CI/RunTests.swift new file mode 100644 index 000000000..ab77ce809 --- /dev/null +++ b/Tools/Sources/Commands/CI/RunTests.swift @@ -0,0 +1,148 @@ +import ArgumentParser +import CommandLineTools +import Foundation + +struct RunTests: AsyncParsableCommand { + static let configuration = CommandConfiguration(abstract: "Runs xcodebuild tests with simulator management, retries, and formatting.", + discussion: """ + Uses xcodebuild's native -retry-tests-on-failure flag to retry only \ + failing tests instead of re-running the entire suite. + + Examples: + swift run tools run-tests --scheme UnitTests + swift run tools run-tests --scheme UITests --device iPhone --os-version 26.1 + swift run tools run-tests --scheme PreviewTests --create-simulator-name "iPhone SE (3rd generation)" \ + --create-simulator-type com.apple.CoreSimulator.SimDeviceType.iPhone-SE-3rd-generation + """) + + @Option(help: "The Xcode scheme to test.") + var scheme: String + + @Option(help: "The simulator device name to run tests on (e.g. 'iPhone 17').") + var device = "iPhone 17" + + @Option(help: "The iOS version to use for the simulator runtime (e.g. '26.1').") + var osVersion = "26.1" + + @Option(help: "Number of times to retry failed tests. Only the failing tests are re-run, not the entire suite.") + var retries = 3 + + @Option(help: "When set, create a simulator with this name if one doesn't already exist.") + var createSimulatorName: String? + + @Option(help: "The simulator device type identifier for creating a new simulator (e.g. 'com.apple.CoreSimulator.SimDeviceType.iPhone-SE-3rd-generation').") + var createSimulatorType: String? + + @Option(help: "Only run a specific test (format: 'ClassName/testName').") + var testName: String? + + private var isCI: Bool { + ProcessInfo.processInfo.environment["CI"] != nil + } + + private var resultBundlePath: String { + "test_output/\(scheme).xcresult" + } + + private var formatter: String { + "xcbeautify -q --disable-logging --is-ci --renderer github-actions" + } + + private var simulatorRuntime: String { + "com.apple.CoreSimulator.SimRuntime.iOS-\(osVersion.replacingOccurrences(of: ".", with: "-"))" + } + + func run() async throws { + if let createName = createSimulatorName { + guard let createType = createSimulatorType else { + throw ValidationError("--create-simulator-type must be provided when --create-simulator-name is set.") + } + try await createSimulatorIfNecessary(name: createName, type: createType) + } + + // Remove any previous result bundle at this path + let resultBundleURL = URL.projectDirectory.appendingPathComponent(resultBundlePath) + if FileManager.default.fileExists(atPath: resultBundleURL.path) { + try? FileManager.default.removeItem(at: resultBundleURL) + } + + // Ensure the output directory exists + let outputDirectory = resultBundleURL.deletingLastPathComponent() + try FileManager.default.createDirectory(at: outputDirectory, withIntermediateDirectories: true) + + try await executeXcodeBuild() + + try await shutdownSimulator() + + logger.info("\n✅ Tests passed.\n") + } + + // MARK: - Simulator Management + + private func createSimulatorIfNecessary(name: String, type: String) async throws { + logger.info("Checking for simulator '\(name)'…") + + guard let simulators = try await CI.run(.path("/bin/zsh"), ["-cu", "xcrun simctl list devices \"iOS \(osVersion)\" available"], + output: .string(limit: 4096)).standardOutput else { + logger.info("No simulators found for iOS \(osVersion). Creating '\(name)'…") + try await createSimulator(name: name, type: type) + return + } + + // Use a `(` to avoid matching e.g. "iPhone 14 Pro" on "iPhone 14 Pro Max" + let hasExisting = simulators.components(separatedBy: "\n").contains { line in + line.contains("\(name) (") + } + + if hasExisting { + logger.info("Simulator '\(name)' already exists.") + } else { + logger.info("Simulator '\(name)' not found. Creating…") + try await createSimulator(name: name, type: type) + } + } + + private func createSimulator(name: String, type: String) async throws { + let deviceID = try await CI.run(.path("/bin/zsh"), ["-cu", "xcrun simctl create '\(name)' \(type) \(simulatorRuntime)"], + output: .string(limit: 4096)).standardOutput + logger.info("Created simulator '\(name)' (\(deviceID?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "unknown")).") + } + + // MARK: - Simulator Shutdown + + private func shutdownSimulator() async throws { + print("Shutting down simulator '\(device)'…") + + let command = "xcrun simctl shutdown '\(device)' 2>/dev/null || true" + try await CI.run(.path("/bin/zsh"), ["-cu", command]) + + print("Simulator shut down.") + } + + // MARK: - Test Running + + private func executeXcodeBuild() async throws { + var command = "set -o pipefail && xcodebuild test" + command += " -scheme \(scheme)" + command += " -sdk iphonesimulator" + command += " -destination 'platform=iOS Simulator,name=\(device),OS=\(osVersion),arch=arm64'" + command += " -resultBundlePath \(resultBundlePath)" + command += " -skipPackagePluginValidation" + + // Use xcodebuild's native retry support to re-run only failing tests + // instead of re-running the entire suite. retries=0 means no retries (single run). + if retries > 0 { + // -test-iterations is the total number of attempts (initial + retries) + command += " -retry-tests-on-failure" + command += " -test-iterations \(retries + 1)" + } + + if let testName { + command += " -only-testing:\(scheme)/\(testName)" + } + + command += " | \(formatter)" + + try await CI.run(.path("/bin/zsh"), ["-cu", command]) + } +} diff --git a/Tools/Sources/Commands/CI/UnitTests.swift b/Tools/Sources/Commands/CI/UnitTests.swift new file mode 100644 index 000000000..71f132ffe --- /dev/null +++ b/Tools/Sources/Commands/CI/UnitTests.swift @@ -0,0 +1,67 @@ +import ArgumentParser +import CommandLineTools +import Foundation + +struct UnitTests: AsyncParsableCommand { + static let configuration = CommandConfiguration(commandName: "unit-tests", + abstract: "Runs the unit test CI workflow: lint, unit tests, preview tests, and result collection.") + + @Option(help: "Device name for unit tests.") + var device = "iPhone 17" + + @Option(help: "iOS version for the simulator.") + var osVersion = "26.1" + + func run() async throws { + try await CI.lint() + + var failures: [String] = [] + + // Run unit tests + do { + logger.info("\n🧪 Running unit tests…\n") + try await RunTests.parse([ + "--scheme", "UnitTests", + "--device", device, + "--os-version", osVersion + ]).run() + } catch { + failures.append("Unit tests failed: \(error)") + logger.error("\n❌ Unit tests failed. \(error)\n") + } + + // Run preview tests on a smaller device + do { + logger.info("\n🧪 Running preview tests…") + try await RunTests.parse([ + "--scheme", "PreviewTests", + "--device", "iPhone SE (3rd generation)", + "--os-version", osVersion, + "--create-simulator-name", "iPhone SE (3rd generation)", + "--create-simulator-type", "com.apple.CoreSimulator.SimDeviceType.iPhone-SE-3rd-generation" + ]).run() + } catch { + failures.append("Preview tests failed: \(error)") + logger.error("\n❌ Preview tests failed.\n") + } + + // Zip results (best-effort, useful for CI artifact uploads) + await CI.zipResults(bundles: ["UnitTests.xcresult", "PreviewTests.xcresult"], + outputName: "UnitTests.zip") + + // Collect coverage reports + await CI.collectCoverage(resultBundle: "UnitTests.xcresult", outputName: "unit-cobertura.xml") + await CI.collectCoverage(resultBundle: "PreviewTests.xcresult", outputName: "preview-cobertura.xml") + + // Collect JUnit test results + await CI.collectTestResults(resultBundle: "UnitTests.xcresult", outputName: "unit-junit.xml") + await CI.collectTestResults(resultBundle: "PreviewTests.xcresult", outputName: "preview-junit.xml") + + if !failures.isEmpty { + logger.error("\n❌ \(failures.count) test suite(s) failed.\n") + throw ExitCode.failure + } + + logger.info("\n✅ All unit test suites passed.\n") + } +} diff --git a/Tools/Sources/Tools.swift b/Tools/Sources/Tools.swift index 761ff0dae..772826f0f 100644 --- a/Tools/Sources/Tools.swift +++ b/Tools/Sources/Tools.swift @@ -1,5 +1,8 @@ import ArgumentParser import Foundation +import Logging + +let logger = Logger(label: "🚀") @main struct Tools: AsyncParsableCommand { @@ -13,5 +16,6 @@ struct Tools: AsyncParsableCommand { GenerateSAS.self, AppIconBanner.self, UnusedStrings.self, - BumpCalendarVersion.self]) + BumpCalendarVersion.self, + CI.self]) }