diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 15c9d94..ed3e67e 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -234,7 +234,10 @@ steps: queue: macos artifact_paths: - swift-xcuitest/artifacts/SwiftXCUITest.xcresult/**/* + - swift-xcuitest/artifacts/xcodebuild.log + soft_fail: true env: + BUILDKITE_ANALYTICS_DEBUG_ENABLED: "true" BUILDKITE_ANALYTICS_TAGS: '{"test.framework.name":"xcuitest","language.name":"swift","custom.tag.from":"upload"}' plugins: - tests#v1.0.0: diff --git a/swift-xcuitest/ExampleApp/ExampleApp.swift b/swift-xcuitest/ExampleApp/ExampleApp.swift index 9d0a04a..d8dc18f 100644 --- a/swift-xcuitest/ExampleApp/ExampleApp.swift +++ b/swift-xcuitest/ExampleApp/ExampleApp.swift @@ -1,12 +1,44 @@ +import Foundation import SwiftUI @main struct ExampleApp: App { var body: some Scene { WindowGroup { + ContentView() + } + } +} + +private struct ContentView: View { + private let runID: String + @State private var runnerCrashRecorded: Bool + + init() { + let runID = ProcessInfo.processInfo.environment["XCUITEST_RUN_ID"] ?? "local" + self.runID = runID + self._runnerCrashRecorded = State( + initialValue: UserDefaults.standard.string(forKey: "runner-crash-run-id") == runID + ) + } + + var body: some View { + VStack { Text("Swift XCUITest example") .accessibilityIdentifier("status") - .padding() + + if runnerCrashRecorded { + Text("Runner crash recorded") + .accessibilityIdentifier("runner-crash-recorded") + } else { + Button("Record runner crash") { + UserDefaults.standard.set(runID, forKey: "runner-crash-run-id") + UserDefaults.standard.synchronize() + runnerCrashRecorded = true + } + .accessibilityIdentifier("record-runner-crash") + } } + .padding() } } diff --git a/swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift b/swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift index 500ebcf..68285d6 100644 --- a/swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift +++ b/swift-xcuitest/ExampleAppUITests/ExampleAppUITests.swift @@ -1,4 +1,5 @@ import Core +import Darwin import XCTest final class ExampleAppUITests: XCTestCase { @@ -7,8 +8,7 @@ final class ExampleAppUITests: XCTestCase { } func test01AppLaunches() { - let app = XCUIApplication() - app.launch() + let app = launchApp() XCTAssertTrue(app.staticTexts["status"].waitForExistence(timeout: 5)) } @@ -16,9 +16,48 @@ final class ExampleAppUITests: XCTestCase { func test02ExecutionTag() { tagExecution("custom.tag.from", "execution") - let app = XCUIApplication() - app.launch() + let app = launchApp() XCTAssertEqual(app.staticTexts["status"].label, "Swift XCUITest example") } + + func test03RunnerCrashesOnce() { + let app = launchApp() + + if app.buttons["record-runner-crash"].waitForExistence(timeout: 2) { + app.buttons["record-runner-crash"].tap() + XCTAssertTrue(app.staticTexts["runner-crash-recorded"].waitForExistence(timeout: 2)) + + _ = kill(getpid(), SIGKILL) + XCTFail("The runner process should have terminated") + } + + XCTAssertTrue(app.staticTexts["runner-crash-recorded"].waitForExistence(timeout: 2)) + } + + func test04RunsInReplacementRunner() { + let app = launchApp() + + XCTAssertTrue(app.staticTexts["runner-crash-recorded"].waitForExistence(timeout: 5)) + } + + func test05FailureIncludesDiagnostics() { + tagExecution("runner.replacement", "true") + + let app = launchApp() + XCTAssertTrue(app.staticTexts["status"].waitForExistence(timeout: 5)) + XCTAssertEqual( + app.staticTexts["status"].label, + "Intentional failure", + "This failure demonstrates native XCTest diagnostics" + ) + } + + private func launchApp() -> XCUIApplication { + let app = XCUIApplication() + let runID = ProcessInfo.processInfo.environment["XCUITEST_RUN_ID"] ?? "local" + app.launchEnvironment["XCUITEST_RUN_ID"] = runID + app.launch() + return app + } } diff --git a/swift-xcuitest/README.md b/swift-xcuitest/README.md index be7ffb4..6e176e6 100644 --- a/swift-xcuitest/README.md +++ b/swift-xcuitest/README.md @@ -1,9 +1,16 @@ # Swift XCUITest example -This example runs passing UI tests against a minimal iOS app and uploads the -results with [`test-collector-swift`](https://github.com/buildkite/test-collector-swift). -It uses a shared scheme, a lexical test plan, and an explicit `.xcresult` -bundle so the same command works locally and on a macOS Buildkite agent. +This example reproduces the result loss that occurs when XCTest replaces an +XCUITest runner. Two tests finish before a third test terminates the runner. +Xcode continues the remaining tests in a replacement process and keeps all +five tests in the `.xcresult`, but `test-collector-swift` loses the completed +executions buffered by the terminated process. The final test also fails with +an execution tag, backtrace, and source location to exercise the richer native +collector metadata. + +The verification script compares the `.xcresult` test count with the unique +test names logged when the collector uploads. It intentionally fails while +the counts differ. The Buildkite step is therefore soft-failed. Run it with: diff --git a/swift-xcuitest/bin/test b/swift-xcuitest/bin/test index c33db23..180f2ee 100755 --- a/swift-xcuitest/bin/test +++ b/swift-xcuitest/bin/test @@ -5,6 +5,7 @@ set -euo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" artifacts_dir="${XCUITEST_ARTIFACTS_DIR:-$root/artifacts}" result_bundle="$artifacts_dir/SwiftXCUITest.xcresult" +test_log="$artifacts_dir/xcodebuild.log" for name in \ BUILDKITE_ANALYTICS_BASE_URL \ @@ -34,6 +35,9 @@ do fi done +run_id="${XCUITEST_RUN_ID:-${BUILDKITE_BUILD_ID:-local}-${BUILDKITE_JOB_ID:-$(date +%s)}}" +export "TEST_RUNNER_XCUITEST_RUN_ID=$run_id" + destination="${XCUITEST_DESTINATION:-}" if [[ -z "$destination" ]]; then device_id="$({ @@ -57,6 +61,7 @@ xcodebuild \ -scheme SwiftXCUITest \ -resolvePackageDependencies +set +e xcodebuild \ -project "$root/SwiftXCUITest.xcodeproj" \ -scheme SwiftXCUITest \ @@ -64,4 +69,17 @@ xcodebuild \ -destination "$destination" \ -derivedDataPath "$artifacts_dir/DerivedData" \ -resultBundlePath "$result_bundle" \ - test + -retry-tests-on-failure \ + -test-iterations 2 \ + test 2>&1 | tee "$test_log" +xcodebuild_status="${PIPESTATUS[0]}" +set -e + +verification_status=0 +"$root/bin/verify-results" "$result_bundle" "$test_log" || verification_status="$?" + +if [[ "$xcodebuild_status" -ne 0 ]]; then + exit "$xcodebuild_status" +fi + +exit "$verification_status" diff --git a/swift-xcuitest/bin/verify-results b/swift-xcuitest/bin/verify-results new file mode 100755 index 0000000..39c29ba --- /dev/null +++ b/swift-xcuitest/bin/verify-results @@ -0,0 +1,52 @@ +#!/bin/bash + +set -euo pipefail + +result_bundle="${1:?usage: verify-results RESULT_BUNDLE XCODEBUILD_LOG}" +test_log="${2:?usage: verify-results RESULT_BUNDLE XCODEBUILD_LOG}" +expected_count=5 + +result_count="$({ + xcrun xcresulttool get test-results summary --path "$result_bundle" +} | plutil -extract totalTestCount raw -o - -)" +uploaded_count="$(awk ' + { + line = $0 + while (match(line, /name: Optional\("test[^"]*"\)/)) { + names[substr(line, RSTART, RLENGTH)] = 1 + line = substr(line, RSTART + RLENGTH) + } + } + END { + for (name in names) count++ + print count + 0 + } +' "$test_log")" + +echo "Expected unique tests: $expected_count" +echo ".xcresult unique tests: $result_count" +echo "Collector unique tests recorded for upload: $uploaded_count" + +status=0 + +if [[ "$result_count" -ne "$expected_count" ]]; then + echo "Expected .xcresult to retain all $expected_count tests" >&2 + status=1 +fi + +if grep -Fq 'location: Optional("ExampleAppUITests.swift:' "$test_log" \ + && grep -Fq 'failureExpanded: [Core.Trace.FailureExpanded' "$test_log" \ + && grep -Fq 'tags: Optional(["runner.replacement": "true"])' "$test_log" +then + echo "Collector retained the failure location, expanded backtrace, and execution tag" +else + echo "Collector did not log all expected native failure metadata" >&2 + status=1 +fi + +if [[ "$uploaded_count" -ne "$result_count" ]]; then + echo "Collector lost executions when the XCTest runner was replaced" >&2 + status=1 +fi + +exit "$status"