From f6afffbdf532efa8451b59ca67ec7b63dc941c65 Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Thu, 27 Aug 2026 19:10:21 +0930 Subject: [PATCH 01/20] Isolate Swift default collector test Prevent the no-configuration test from reading ambient process or Info.plist values. Hosted macOS agents may provide telemetry variables, so relying on the runner environment makes the assertion depend on CI configuration. This keeps production lookup behavior unchanged and confines test setup to the empty environment it claims to exercise. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- test-collector-swift/Tests/CoreTests/CollectorTests.swift | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test-collector-swift/Tests/CoreTests/CollectorTests.swift b/test-collector-swift/Tests/CoreTests/CollectorTests.swift index eaf999ec..394adfab 100644 --- a/test-collector-swift/Tests/CoreTests/CollectorTests.swift +++ b/test-collector-swift/Tests/CoreTests/CollectorTests.swift @@ -3,7 +3,11 @@ import XCTest final class CollectorTests: XCTestCase { func testDefaultCollector() throws { - let environment = EnvironmentValues(values: [:]) + let environment = EnvironmentValues( + values: [:], + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ) let collector = TestCollector(environment: environment) let observer = try XCTUnwrap(collector.observer, "Observer should be initialised") XCTAssertNil(observer.uploader, "Uploader should not be initialised without an api key") From 049b3f05d84fbdab2926b6cedd3996b367a8a955 Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 17:20:08 +0930 Subject: [PATCH 02/20] feat(swift): submit test executions through OTLP Replace the collector's proprietary JSON execution uploads with OpenTelemetry test.execution root spans accepted by the Test Engine OTLP endpoint. Keep execution and run tagging APIs compatible by emitting buildkite.tag.* attributes, preserve XCTest failure details as span status and exception events, and forward sampled application spans beneath each test root. Export each completed root synchronously and retain failed requests in the exporter's in-memory queue so later tests or the bundle flush can retry. This also lets the Test Engine Client relay acknowledge completed tests before an XCTest runner can restart. Support standard OTLP HTTP environment configuration, retain the suite token authentication path, and document the remaining in-memory delivery limitations. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- test-collector-swift/.buildkite/pipeline.yml | 2 +- test-collector-swift/Package.swift | 28 +- test-collector-swift/README.md | 73 ++- .../TestCollector.swift | 9 +- .../Core/ApiClient/ApiClient+Live.swift | 35 -- .../Sources/Core/ApiClient/ApiClient.swift | 35 -- .../Sources/Core/ApiClient/ApiSession.swift | 48 -- .../Sources/Core/Models/Api/ApiRoute.swift | 9 - .../Core/Models/Api/RunEnvironment.swift | 102 ---- .../Sources/Core/Models/Api/TestResults.swift | 37 -- .../Models/Api/Trace+FailureExpanded.swift | 17 - .../Core/Models/Api/Trace+Result.swift | 18 - .../Sources/Core/Models/Api/Trace+Span.swift | 35 -- .../Sources/Core/Models/Api/Trace.swift | 73 --- .../Core/Models/Api/UploadResponse.swift | 21 - .../Environment/EnvironmentValues.swift | 18 +- .../Models/Environment/RunEnvironment.swift | 49 ++ .../ExecutionChildSpanProcessor.swift | 75 +++ .../SynchronousExecutionSpanProcessor.swift | 53 +++ .../Core/Telemetry/TelemetryClient+Live.swift | 446 ++++++++++++++++++ .../Core/Telemetry/TelemetryClient.swift | 42 ++ .../Sources/Core/TestCollector.swift | 26 +- .../Sources/Core/TestObserver.swift | 40 +- .../Sources/Core/Tracer/Tracer+Live.swift | 43 -- .../Sources/Core/Tracer/Tracer.swift | 41 -- .../Core/UploadClient/UploadClient+Live.swift | 121 ----- .../Core/UploadClient/UploadClient.swift | 32 -- .../Tests/CoreTests/ApiClientTests.swift | 36 -- .../Tests/CoreTests/CollectorTests.swift | 16 +- .../Tests/CoreTests/RunEnvironmentTests.swift | 61 --- .../CoreTests/TelemetryClientTests.swift | 235 +++++++++ .../CoreTests/TestHelpers/ApiClient.swift | 23 - .../CoreTests/TestHelpers/URLRequest.swift | 32 -- .../Tests/CoreTests/TestObserverTests.swift | 62 +-- .../Tests/CoreTests/TestResultsTests.swift | 177 ------- .../Tests/CoreTests/UploadClientTests.swift | 140 ------ 36 files changed, 1085 insertions(+), 1225 deletions(-) delete mode 100644 test-collector-swift/Sources/Core/ApiClient/ApiClient+Live.swift delete mode 100644 test-collector-swift/Sources/Core/ApiClient/ApiClient.swift delete mode 100644 test-collector-swift/Sources/Core/ApiClient/ApiSession.swift delete mode 100644 test-collector-swift/Sources/Core/Models/Api/ApiRoute.swift delete mode 100644 test-collector-swift/Sources/Core/Models/Api/RunEnvironment.swift delete mode 100644 test-collector-swift/Sources/Core/Models/Api/TestResults.swift delete mode 100644 test-collector-swift/Sources/Core/Models/Api/Trace+FailureExpanded.swift delete mode 100644 test-collector-swift/Sources/Core/Models/Api/Trace+Result.swift delete mode 100644 test-collector-swift/Sources/Core/Models/Api/Trace+Span.swift delete mode 100644 test-collector-swift/Sources/Core/Models/Api/Trace.swift delete mode 100644 test-collector-swift/Sources/Core/Models/Api/UploadResponse.swift create mode 100644 test-collector-swift/Sources/Core/Models/Environment/RunEnvironment.swift create mode 100644 test-collector-swift/Sources/Core/Telemetry/ExecutionChildSpanProcessor.swift create mode 100644 test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift create mode 100644 test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift create mode 100644 test-collector-swift/Sources/Core/Telemetry/TelemetryClient.swift delete mode 100644 test-collector-swift/Sources/Core/Tracer/Tracer+Live.swift delete mode 100644 test-collector-swift/Sources/Core/Tracer/Tracer.swift delete mode 100644 test-collector-swift/Sources/Core/UploadClient/UploadClient+Live.swift delete mode 100644 test-collector-swift/Sources/Core/UploadClient/UploadClient.swift delete mode 100644 test-collector-swift/Tests/CoreTests/ApiClientTests.swift create mode 100644 test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift delete mode 100644 test-collector-swift/Tests/CoreTests/TestHelpers/ApiClient.swift delete mode 100644 test-collector-swift/Tests/CoreTests/TestHelpers/URLRequest.swift delete mode 100644 test-collector-swift/Tests/CoreTests/TestResultsTests.swift delete mode 100644 test-collector-swift/Tests/CoreTests/UploadClientTests.swift diff --git a/test-collector-swift/.buildkite/pipeline.yml b/test-collector-swift/.buildkite/pipeline.yml index 0b6b710c..ad19354b 100644 --- a/test-collector-swift/.buildkite/pipeline.yml +++ b/test-collector-swift/.buildkite/pipeline.yml @@ -23,7 +23,7 @@ steps: - docker#v3.7.0: image: "swift:{{matrix}}" matrix: - - "5.9.2" + - "6.0.3" - label: ":swift: Tests (macOS, {{matrix}})" agents: diff --git a/test-collector-swift/Package.swift b/test-collector-swift/Package.swift index b2888da7..12f6c696 100644 --- a/test-collector-swift/Package.swift +++ b/test-collector-swift/Package.swift @@ -1,11 +1,11 @@ -// swift-tools-version: 5.7.1 +// swift-tools-version: 6.0 import PackageDescription let package = Package( name: "BuildkiteTestCollector", platforms: [ - .macOS("10.15"), + .macOS("12.0"), .iOS("13.0"), .tvOS("13.0"), .watchOS("6.0") @@ -13,11 +13,29 @@ let package = Package( products: [ .library(name: "BuildkiteTestCollector", targets: ["BuildkiteTestCollector"]) ], + dependencies: [ + .package(url: "https://github.com/open-telemetry/opentelemetry-swift", from: "2.5.1"), + .package(url: "https://github.com/open-telemetry/opentelemetry-swift-core", from: "2.5.1") + ], targets: [ .target(name: "BuildkiteTestCollector", dependencies: ["Core", "Loader"]), - .target(name: "Core"), + .target( + name: "Core", + dependencies: [ + .product(name: "OpenTelemetryProtocolExporterHTTP", package: "opentelemetry-swift"), + .product(name: "OpenTelemetrySdk", package: "opentelemetry-swift-core") + ] + ), .target(name: "Loader"), .testTarget(name: "BuildkiteTestCollectorTests", dependencies: ["BuildkiteTestCollector"]), - .testTarget(name: "CoreTests", dependencies: ["Core"]) - ] + .testTarget( + name: "CoreTests", + dependencies: [ + "Core", + .product(name: "InMemoryExporter", package: "opentelemetry-swift"), + .product(name: "OpenTelemetrySdk", package: "opentelemetry-swift-core") + ] + ) + ], + swiftLanguageModes: [.v5] ) diff --git a/test-collector-swift/README.md b/test-collector-swift/README.md index df0187c2..de63bcea 100644 --- a/test-collector-swift/README.md +++ b/test-collector-swift/README.md @@ -6,6 +6,14 @@ Official [Buildkite Test Engine](https://buildkite.com/platform/test-engine/) co 📦 **Supported CI systems:** Buildkite, GitHub Actions, CircleCI, Xcode Cloud, and others via the `BUILDKITE_ANALYTICS_*` environment variables. +Each test execution is exported as a parentless OpenTelemetry `test.execution` +root span. Sampled OpenTelemetry spans created by the code under test can appear +as its children. Test execution data is submitted only through OTLP; the +collector does not use the legacy proprietary execution upload API. + +The collector requires Swift 6 or newer. Its macOS deployment target is macOS +12 or newer. + ## 👉 Installing ### Step 1 @@ -40,6 +48,25 @@ let package = Package( Set the `BUILDKITE_ANALYTICS_TOKEN` secret on your CI to the API token from earlier. +By default, traces are sent using OTLP/HTTP protobuf to +`https://tests-otlp.buildkite.com/v1/traces`. The collector adds the suite token +and run key as request headers. + +To send traces through an OpenTelemetry endpoint such as the Buildkite Test +Engine Client relay, set the standard exporter variables instead: + +```bash +export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="http://127.0.0.1:4318/v1/traces" +export OTEL_EXPORTER_OTLP_TRACES_HEADERS="authorization=Bearer%20local-token" +export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL="http/protobuf" +``` + +The signal-specific variables take precedence over +`OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_HEADERS`, and +`OTEL_EXPORTER_OTLP_PROTOCOL`. A generic OTLP endpoint has `/v1/traces` +appended. `BUILDKITE_ANALYTICS_OTLP_ENDPOINT` remains available as a +collector-specific endpoint override. + ### Step 3 If you're testing an Xcode project there's an extra step, Xcode doesn't pass environment variables from the process to the test runner so we need to manually map them. Open your test scheme or test plan(whichever you are using) and under the environment variable section add the following entry: @@ -50,11 +77,17 @@ key: value: `$(BUILDKITE_ANALYTICS_TOKEN)` +If you configure an OTLP endpoint, headers, or protocol, map those +`OTEL_EXPORTER_OTLP_TRACES_*` variables into the test runner in the same way. + The same key value pair can be specified in your main bundle's `info.plist` file if you would rather specify it there. Note variables in the environment take precedent over those in the `info.plist` file. ### Step 3.5 (Optional) -The only required environment variable is the analytics token but if you're using one of the supported CI platforms they can pass extra information to the test-collector to enrich the reports. Things like commit messages, branch names, build numbers, etc. Open your test scheme or test plan again and add the following key value pairs depending on your CI platform. +When using the default endpoint, only the analytics token is required. Supported +CI platforms can pass extra information to enrich the reports, such as commit +messages, branch names, and build numbers. Open your test scheme or test plan +again and add the following key-value pairs for your CI platform. **Buildkite** @@ -65,7 +98,12 @@ Key: BUILDKITE_BRANCH, Value: $(BUILDKITE_BRANCH) Key: BUILDKITE_COMMIT, Value: $(BUILDKITE_COMMIT) Key: BUILDKITE_BUILD_NUMBER, Value: $(BUILDKITE_BUILD_NUMBER) Key: BUILDKITE_JOB_ID, Value: $(BUILDKITE_JOB_ID) +Key: BUILDKITE_STEP_ID, Value: $(BUILDKITE_STEP_ID) Key: BUILDKITE_MESSAGE, Value: $(BUILDKITE_MESSAGE) +Key: BUILDKITE_ORGANIZATION_SLUG, Value: $(BUILDKITE_ORGANIZATION_SLUG) +Key: BUILDKITE_TEST_ENGINE_SUITE_SLUG, Value: $(BUILDKITE_TEST_ENGINE_SUITE_SLUG) +Key: TRACEPARENT, Value: $(TRACEPARENT) +Key: TRACESTATE, Value: $(TRACESTATE) ``` **Circle CI** @@ -118,9 +156,9 @@ git push origin add-buildkite-test-engine You can tag test executions with key-value pairs to filter and group results in [Test Engine](https://buildkite.com/docs/test-engine/test-suites/tags). -### Upload-level tags +### Run-level tags -Upload-level tags apply to all test executions in a run. Set the `BUILDKITE_ANALYTICS_TAGS` environment variable to a JSON object: +Run-level tags apply to all test executions in a run. Set the `BUILDKITE_ANALYTICS_TAGS` environment variable to a JSON object: ```bash export BUILDKITE_ANALYTICS_TAGS='{"host.arch":"arm64","cloud.region":"us-east-1"}' @@ -128,12 +166,14 @@ export BUILDKITE_ANALYTICS_TAGS='{"host.arch":"arm64","cloud.region":"us-east-1" If you're using an Xcode project, add this to your test scheme or test plan environment variables like the other `BUILDKITE_ANALYTICS_*` variables. -Upload-level tags can also be set programmatically by passing them to `load`. Environment variable tags take precedence over programmatic tags when keys collide: +Run-level tags can also be set programmatically by passing them to `load`. Environment variable tags take precedence over programmatic tags when keys collide: ```swift TestCollector.load(uploadTags: ["host.arch": "arm64"]) ``` +Run tags are exported as `buildkite.tag.` resource attributes. + ### Execution-level tags Tag individual tests from within a test method using the `tagExecution` extension on `XCTestCase`: @@ -149,6 +189,31 @@ class PaymentTests: XCTestCase { } ``` +The existing `tagExecution` API is unchanged. Execution tags are exported as +`buildkite.tag.` attributes on that test's root span. + +## OpenTelemetry behavior and delivery + +The collector creates each `test.execution` span through a private AlwaysOn +provider, so it remains a trace root regardless of an application's sampling +configuration. When Buildkite Agent trace context is available, the execution +links to the job span instead of becoming its child. If the test suite has an +OpenTelemetry SDK provider, the collector adds a forwarding processor without +replacing its sampler or exporters. Otherwise, it installs a provider so spans +created under a test can be exported as execution children. + +Finishing a test waits for its root span to be accepted by the configured OTLP +endpoint. A failed root export stays in the process's in-memory exporter queue +and is retried by the next execution or the bundle-end flush. When the endpoint +is the Test Engine Client relay, acceptance transfers the span to the +longer-lived parent process, which protects completed executions from an XCTest +runner restart. + +This is not disk-backed delivery. A hard exit before a test finishes, a process +exit while the endpoint remains unavailable, or a machine restart can still +lose spans. An ambiguous network timeout may also result in a retry after the +server accepted the first request. + ## 🔍 Debugging To enable debugging output, set the `BUILDKITE_ANALYTICS_DEBUG_ENABLED` environment variable to `true`. This also needs diff --git a/test-collector-swift/Sources/BuildkiteTestCollector/TestCollector.swift b/test-collector-swift/Sources/BuildkiteTestCollector/TestCollector.swift index 621b80f8..2e8733cf 100644 --- a/test-collector-swift/Sources/BuildkiteTestCollector/TestCollector.swift +++ b/test-collector-swift/Sources/BuildkiteTestCollector/TestCollector.swift @@ -1,8 +1,13 @@ import Core public enum TestCollector { - /// The base URL for the Buildkite Test Engine API. + /// The default OpenTelemetry traces endpoint for Buildkite Test Engine. + public static var endpoint: String { + Core.TestCollector.endpoint + } + + @available(*, deprecated, renamed: "endpoint") public static var baseURL: String { - Core.TestCollector.baseURL + self.endpoint } } diff --git a/test-collector-swift/Sources/Core/ApiClient/ApiClient+Live.swift b/test-collector-swift/Sources/Core/ApiClient/ApiClient+Live.swift deleted file mode 100644 index c2ba2e5c..00000000 --- a/test-collector-swift/Sources/Core/ApiClient/ApiClient+Live.swift +++ /dev/null @@ -1,35 +0,0 @@ -import Foundation - -#if canImport(FoundationNetworking) -import FoundationNetworking -#endif - -extension ApiClient { - static func live( - apiToken: String, - baseURL: URL, - encoder: JSONEncoder = .init(), - decoder: JSONDecoder = .init(), - session: ApiSession = .urlSession(.shared) - ) -> ApiClient { - func makeRequest(from route: ApiRoute) throws -> URLRequest { - var request = URLRequest(url: baseURL) - request.setValue("Token token=\"\(apiToken)\"", forHTTPHeaderField: "Authorization") - - switch route { - case .upload(let testData): - let data = try encoder.encode(testData) - request.url?.appendPathComponent("uploads") - request.setValue("application/json", forHTTPHeaderField: "Content-Type") - request.httpMethod = "POST" - request.httpBody = data - return request - } - } - - return ApiClient(decoder: decoder) { route in - let request = try makeRequest(from: route) - return try await session.data(for: request) - } - } -} diff --git a/test-collector-swift/Sources/Core/ApiClient/ApiClient.swift b/test-collector-swift/Sources/Core/ApiClient/ApiClient.swift deleted file mode 100644 index 59317d4e..00000000 --- a/test-collector-swift/Sources/Core/ApiClient/ApiClient.swift +++ /dev/null @@ -1,35 +0,0 @@ -import Foundation - -#if canImport(FoundationNetworking) -import FoundationNetworking -#endif - -struct ApiClient { - let decoder: JSONDecoder - var request: (ApiRoute) async throws -> (Data, HTTPURLResponse) - - init( - decoder: JSONDecoder = JSONDecoder(), - request: @escaping (ApiRoute) async throws -> (Data, HTTPURLResponse) - ) { - self.request = request - self.decoder = decoder - } - - func data(for route: ApiRoute) async throws -> (value: Data, response: HTTPURLResponse) { - try await self.request(route) - } - - func data( - for route: ApiRoute, - as type: Value.Type - ) async throws -> (value: Value, response: HTTPURLResponse) { - let (data, response) = try await self.data(for: route) - let value = try decode(data, as: type) - return (value, response) - } - - func decode(_ data: Data, as type: Value.Type) throws -> Value { - try self.decoder.decode(type, from: data) - } -} diff --git a/test-collector-swift/Sources/Core/ApiClient/ApiSession.swift b/test-collector-swift/Sources/Core/ApiClient/ApiSession.swift deleted file mode 100644 index 664def17..00000000 --- a/test-collector-swift/Sources/Core/ApiClient/ApiSession.swift +++ /dev/null @@ -1,48 +0,0 @@ -import Foundation - -#if canImport(FoundationNetworking) -import FoundationNetworking -#endif - -struct ApiSession { - var data: (URLRequest) async throws -> (Data, HTTPURLResponse) - - func data(for request: URLRequest) async throws -> (Data, HTTPURLResponse) { - try await self.data(request) - } -} - -extension ApiSession { - static func urlSession(_ session: URLSession) -> ApiSession { - ApiSession { request in - #if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) - if #available(macOS 12, iOS 15, tvOS 15, watchOS 8, *) { - let (data, response) = try await session.data(for: request) - guard let httpResponse = response as? HTTPURLResponse else { - throw URLError(.unsupportedURL) - } - return (data, httpResponse) - } - #endif - - var dataTask: URLSessionDataTask? - let cancel: () -> Void = { dataTask?.cancel() } - - return try await withTaskCancellationHandler( - operation: { - try await withCheckedThrowingContinuation { continuation in - dataTask = session.dataTask(with: request) { data, response, error in - if let data, let response { - continuation.resume(returning: (data, response as! HTTPURLResponse)) - } else { - continuation.resume(throwing: error ?? URLError(.badServerResponse)) - } - } - dataTask?.resume() - } - }, - onCancel: { cancel() } - ) - } - } -} diff --git a/test-collector-swift/Sources/Core/Models/Api/ApiRoute.swift b/test-collector-swift/Sources/Core/Models/Api/ApiRoute.swift deleted file mode 100644 index 6b941bd7..00000000 --- a/test-collector-swift/Sources/Core/Models/Api/ApiRoute.swift +++ /dev/null @@ -1,9 +0,0 @@ -import Foundation - -/// Different endpoints for Buildkite Test Engine -/// -/// Currently there is only a single upload endpoint, in the future this could be expanded to include others though. The new route would be handled in -/// `ApiClient`'s `makeRequest` function -enum ApiRoute: Equatable { - case upload(TestResults) -} diff --git a/test-collector-swift/Sources/Core/Models/Api/RunEnvironment.swift b/test-collector-swift/Sources/Core/Models/Api/RunEnvironment.swift deleted file mode 100644 index eb49ca81..00000000 --- a/test-collector-swift/Sources/Core/Models/Api/RunEnvironment.swift +++ /dev/null @@ -1,102 +0,0 @@ -/// A type containing information about the environment performing the test run. -/// -/// The only required property is `key` which represents a unique identifier for a test run. -struct RunEnvironment: Equatable { - /// The continuous integration platform. - var ci: String? - - /// A unique identifier. - var key: String - - /// The URL associated with the test run. - var url: String? - - /// The branch name. - var branch: String? - - /// The commit hash. - var commitSha: String? - - /// The run number. - var number: String? - - /// The job identifier. - var jobId: String? - - /// A message associated with the test run. - var message: String? - - /// A value indicating if the collector ran in debug mode. - var debug: String? - - /// A tag added to the start of the execution name. - var executionNamePrefix: String? - - /// A tag added to the end of the execution name. - var executionNameSuffix: String? - - /// The version of the collector used. - var version: String? - - /// The name of the collector used. - var collector: String? - - /// A dictionary that contains custom values associated with the test run. - /// - /// - Note: Used internally for testing experimental features. If an existing key - /// is used, the custom environment value will take precedence. - var customEnvironment: [String: AnyCodable]? -} - -extension RunEnvironment: Encodable { - struct CodingKey: Swift.CodingKey { - var stringValue: String - var intValue: Int? { Int(self.stringValue) } - - init(_ stringValue: String) { - self.stringValue = stringValue - } - - init(stringValue: String) { - self.stringValue = stringValue - } - - init(intValue: Int) { - self.stringValue = String(intValue) - } - - static var ci = Self("CI") - static var key = Self("key") - static var url = Self("url") - static var branch = Self("branch") - static var commitSha = Self("commit_sha") - static var number = Self("number") - static var jobId = Self("job_id") - static var message = Self("message") - static var debug = Self("debug") - static var executionNamePrefix = Self("execution_name_prefix") - static var executionNameSuffix = Self("execution_name_suffix") - static var version = Self("version") - static var collector = Self("collector") - } - - func encode(to encoder: Encoder) throws { - var container = encoder.container(keyedBy: CodingKey.self) - try container.encodeIfPresent(self.ci, forKey: .ci) - try container.encode(self.key, forKey: .key) - try container.encodeIfPresent(self.url, forKey: .url) - try container.encodeIfPresent(self.branch, forKey: .branch) - try container.encodeIfPresent(self.commitSha, forKey: .commitSha) - try container.encodeIfPresent(self.number, forKey: .number) - try container.encodeIfPresent(self.jobId, forKey: .jobId) - try container.encodeIfPresent(self.message, forKey: .message) - try container.encodeIfPresent(self.debug, forKey: .debug) - try container.encodeIfPresent(self.executionNamePrefix, forKey: .executionNamePrefix) - try container.encodeIfPresent(self.executionNameSuffix, forKey: .executionNameSuffix) - try container.encodeIfPresent(self.version, forKey: .version) - try container.encodeIfPresent(self.collector, forKey: .collector) - for (key, value) in self.customEnvironment ?? [:] { - try container.encode(value, forKey: CodingKey(key)) - } - } -} diff --git a/test-collector-swift/Sources/Core/Models/Api/TestResults.swift b/test-collector-swift/Sources/Core/Models/Api/TestResults.swift deleted file mode 100644 index 9f19f1c2..00000000 --- a/test-collector-swift/Sources/Core/Models/Api/TestResults.swift +++ /dev/null @@ -1,37 +0,0 @@ -/// A type containing a test run's metadata and test results. -/// -/// This is the type expected by the Test Engine upload API. -struct TestResults: Equatable { - /// The format for the included data - var format: String - - /// Information used to identify the test run. - /// Test results with matching run_env[key] will be grouped into a single run by Test Engine - var runEnv: RunEnvironment - - /// Tags to apply to the upload - var tags: [String: String]? - - /// An array of test executions - var data: [Trace] - - /// Constructs test results compatible with the JSON API - /// - /// - Parameters: - /// - runEnv: The run environment for when the data was captured. - /// - tags: Optional tags to apply to the upload. - /// - data: An array of test executions. - /// - Returns: Test results compatible with the JSON API - static func json(runEnv: RunEnvironment, tags: [String: String]? = nil, data: [Trace]) -> TestResults { - TestResults(format: "json", runEnv: runEnv, tags: tags, data: data) - } -} - -extension TestResults: Encodable { - enum CodingKeys: String, CodingKey { - case format - case runEnv = "run_env" - case tags - case data - } -} diff --git a/test-collector-swift/Sources/Core/Models/Api/Trace+FailureExpanded.swift b/test-collector-swift/Sources/Core/Models/Api/Trace+FailureExpanded.swift deleted file mode 100644 index e46aa5da..00000000 --- a/test-collector-swift/Sources/Core/Models/Api/Trace+FailureExpanded.swift +++ /dev/null @@ -1,17 +0,0 @@ -extension Trace { - /// A type containing details related to a test failure that occurred during a trace. - struct FailureExpanded: Encodable, Equatable { - /// An array of strings containing a line separated failure message or additional details. - var expanded: [String] - - /// An array of strings representing frames on the call stack when the failure occurred. - var backtrace: [String] - } -} - -extension Trace.FailureExpanded { - init(issue: TestIssue) { - self.expanded = issue.description.components(separatedBy: "\n") - self.backtrace = issue.sourceCodeContext.callStack.enumerated().map { "\($0) \($1)" } - } -} diff --git a/test-collector-swift/Sources/Core/Models/Api/Trace+Result.swift b/test-collector-swift/Sources/Core/Models/Api/Trace+Result.swift deleted file mode 100644 index 375c9787..00000000 --- a/test-collector-swift/Sources/Core/Models/Api/Trace+Result.swift +++ /dev/null @@ -1,18 +0,0 @@ -extension Trace { - /// The result for the test associated with a trace. - enum Result: String, Encodable, Equatable { - case passed - case failed - case skipped - } -} - -extension Trace.Result { - init(_ result: TestResult) { - switch result { - case .passed: self = .passed - case .skipped: self = .skipped - case .failed: self = .failed - } - } -} diff --git a/test-collector-swift/Sources/Core/Models/Api/Trace+Span.swift b/test-collector-swift/Sources/Core/Models/Api/Trace+Span.swift deleted file mode 100644 index 9ea5c8ca..00000000 --- a/test-collector-swift/Sources/Core/Models/Api/Trace+Span.swift +++ /dev/null @@ -1,35 +0,0 @@ -import Foundation - -extension Trace { - /// A type containing information for a timed block of work. - struct Span: Equatable { - /// The type of span. Used for categorising spans. - var section: String - - /// A monotonic timestamp in seconds for when the span started. - var startAt: TimeInterval? - - /// A monotonic timestamp in seconds for when the span ended. - var endAt: TimeInterval? - - /// The number of seconds that elapsed during this span. - var duration: TimeInterval? - - /// Any information related to this span. - var detail: [String: String] = [:] - - /// Any spans that occurred during this span. - var children: [Span] = [] - } -} - -extension Trace.Span: Encodable { - enum CodingKeys: String, CodingKey { - case section - case startAt = "start_at" - case endAt = "end_at" - case duration - case detail - case children - } -} diff --git a/test-collector-swift/Sources/Core/Models/Api/Trace.swift b/test-collector-swift/Sources/Core/Models/Api/Trace.swift deleted file mode 100644 index b7414e14..00000000 --- a/test-collector-swift/Sources/Core/Models/Api/Trace.swift +++ /dev/null @@ -1,73 +0,0 @@ -/// A type containing the span information and result of a test -struct Trace: Equatable { - /// A unique identifier. - var id: String - - /// The scope of the test. - /// - /// In XCTest this is usually the type name of an XCTestCase. - var scope: String? - - /// The name of the test. - /// - /// In XCTest this is usually a method of an XCTestCase. - var name: String? - - /// The source location of the test. - var location: String? - - /// The name of the file containing the test. - var fileName: String? - - /// The result of the test. - var result: Result? - - /// A short description of the failure. - var failureReason: String? - - /// An array of additional details related failures. - var failureExpanded: [FailureExpanded] = [] - - /// The span for the duration of the test. - var history: Span - - /// Tags associated with this test execution. - var tags: [String: String]? -} - -extension Trace: Encodable { - enum CodingKeys: String, CodingKey { - case id - case scope - case name - case location - case fileName = "file_name" - case result - case failureReason = "failure_reason" - case failureExpanded = "failure_expanded" - case history - case tags - } -} - -extension Trace { - init(test: TestState, span: Span, tags: [String: String]? = nil) { - self.id = test.id.uuidString - self.scope = test.className - self.name = test.testName - self.result = test.result.map(Trace.Result.init) ?? .failed - if test.issues.count > 1 { - let failureReasons = test.issues.map(\.compactDescription).joined(separator: ", ") - self.failureReason = "\(test.issues.count) failures: \(failureReasons)" - } else { - self.failureReason = test.issues.first?.compactDescription - } - if let issueLocation = test.issues.first?.sourceCodeContext.location { - self.fileName = issueLocation.fileName - self.location = "\(issueLocation.fileName):\(issueLocation.line)" - } - self.failureExpanded = test.issues.map(Trace.FailureExpanded.init(issue:)) - self.history = span - self.tags = tags - } -} diff --git a/test-collector-swift/Sources/Core/Models/Api/UploadResponse.swift b/test-collector-swift/Sources/Core/Models/Api/UploadResponse.swift deleted file mode 100644 index 0ce21f7d..00000000 --- a/test-collector-swift/Sources/Core/Models/Api/UploadResponse.swift +++ /dev/null @@ -1,21 +0,0 @@ -import Foundation - -/// Response from Test Engine API after uploading test results. -struct UploadResponse: Equatable { - /// The UUID allocated to this upload - var uploadID: String? - - /// The URL that can be used to view upload details - var uploadURL: String? -} - -extension UploadResponse: Decodable { - enum CodingKeys: String, CodingKey { - case uploadID = "upload_id" - case uploadURL = "upload_url" - } -} - -struct UploadFailureResponse: Equatable, Codable { - var message: String -} diff --git a/test-collector-swift/Sources/Core/Models/Environment/EnvironmentValues.swift b/test-collector-swift/Sources/Core/Models/Environment/EnvironmentValues.swift index 4323a0d9..3437e35f 100644 --- a/test-collector-swift/Sources/Core/Models/Environment/EnvironmentValues.swift +++ b/test-collector-swift/Sources/Core/Models/Environment/EnvironmentValues.swift @@ -81,11 +81,20 @@ struct EnvironmentValues { extension EnvironmentValues { var isAnalyticsEnabled: Bool { self.bool(for: "BUILDKITE_ANALYTICS_ENABLED") ?? true } var analyticsToken: String? { self.string(for: "BUILDKITE_ANALYTICS_TOKEN", private: true) } - var analyticsBaseURL: URL? { self.url(for: "BUILDKITE_ANALYTICS_BASE_URL") } + var analyticsOTLPEndpoint: String? { self.string(for: "BUILDKITE_ANALYTICS_OTLP_ENDPOINT") } var isAnalyticsDebugEnabled: Bool { self.bool(for: "BUILDKITE_ANALYTICS_DEBUG_ENABLED") ?? false } var analyticsTags: [String: String]? { self.stringDictionary(for: "BUILDKITE_ANALYTICS_TAGS") } + var otelEndpoint: String? { self.string(for: "OTEL_EXPORTER_OTLP_ENDPOINT") } + var otelHeaders: String? { self.string(for: "OTEL_EXPORTER_OTLP_HEADERS", private: true) } + var otelProtocol: String? { self.string(for: "OTEL_EXPORTER_OTLP_PROTOCOL") } + var otelTracesEndpoint: String? { self.string(for: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") } + var otelTracesHeaders: String? { + self.string(for: "OTEL_EXPORTER_OTLP_TRACES_HEADERS", private: true) + } + var otelTracesProtocol: String? { self.string(for: "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL") } + var analyticsKey: String? { self.string(for: "BUILDKITE_ANALYTICS_KEY") } var analyticsUrl: String? { self.string(for: "BUILDKITE_ANALYTICS_URL") } var analyticsBranch: String? { self.string(for: "BUILDKITE_ANALYTICS_BRANCH") } @@ -101,6 +110,13 @@ extension EnvironmentValues { var buildkiteBuildNumber: String? { self.string(for: "BUILDKITE_BUILD_NUMBER") } var buildkiteJobId: String? { self.string(for: "BUILDKITE_JOB_ID") } var buildkiteMessage: String? { self.string(for: "BUILDKITE_MESSAGE") } + var buildkiteOrganizationSlug: String? { self.string(for: "BUILDKITE_ORGANIZATION_SLUG") } + var buildkiteStepId: String? { self.string(for: "BUILDKITE_STEP_ID") } + var buildkiteTag: String? { self.string(for: "BUILDKITE_TAG") } + var testEngineSuiteSlug: String? { self.string(for: "BUILDKITE_TEST_ENGINE_SUITE_SLUG") } + + var traceParent: String? { self.string(for: "TRACEPARENT") } + var traceState: String? { self.string(for: "TRACESTATE") } var ci: String? { self.string(for: "CI") } diff --git a/test-collector-swift/Sources/Core/Models/Environment/RunEnvironment.swift b/test-collector-swift/Sources/Core/Models/Environment/RunEnvironment.swift new file mode 100644 index 00000000..72885089 --- /dev/null +++ b/test-collector-swift/Sources/Core/Models/Environment/RunEnvironment.swift @@ -0,0 +1,49 @@ +/// A type containing information about the environment performing the test run. +/// +/// The only required property is `key` which represents a unique identifier for a test run. +struct RunEnvironment: Equatable { + /// The continuous integration platform. + var ci: String? + + /// A unique identifier. + var key: String + + /// The URL associated with the test run. + var url: String? + + /// The branch name. + var branch: String? + + /// The commit hash. + var commitSha: String? + + /// The run number. + var number: String? + + /// The job identifier. + var jobId: String? + + /// A message associated with the test run. + var message: String? + + /// A value indicating if the collector ran in debug mode. + var debug: String? + + /// A tag added to the start of the execution name. + var executionNamePrefix: String? + + /// A tag added to the end of the execution name. + var executionNameSuffix: String? + + /// The version of the collector used. + var version: String? + + /// The name of the collector used. + var collector: String? + + /// A dictionary that contains custom values associated with the test run. + /// + /// - Note: Used internally for testing experimental features. If an existing key + /// is used, the custom environment value will take precedence. + var customEnvironment: [String: AnyCodable]? +} diff --git a/test-collector-swift/Sources/Core/Telemetry/ExecutionChildSpanProcessor.swift b/test-collector-swift/Sources/Core/Telemetry/ExecutionChildSpanProcessor.swift new file mode 100644 index 00000000..e9597e38 --- /dev/null +++ b/test-collector-swift/Sources/Core/Telemetry/ExecutionChildSpanProcessor.swift @@ -0,0 +1,75 @@ +import Foundation +import OpenTelemetryApi +import OpenTelemetrySdk + +final class ExecutionTraceRegistry { + private let traceIDs = LockIsolated(Set()) + + func insert(_ traceID: TraceId) { + _ = self.traceIDs.withValue { $0.insert(traceID) } + } + + func remove(_ traceID: TraceId) { + _ = self.traceIDs.withValue { $0.remove(traceID) } + } + + func contains(_ traceID: TraceId) -> Bool { + self.traceIDs.withValue { $0.contains(traceID) } + } +} + +final class ExecutionChildSpanProcessor: SpanProcessor { + let isStartRequired = true + let isEndRequired = true + + private let registry: ExecutionTraceRegistry + private let state: LockIsolated + + private struct State { + var active = true + var acceptedSpanIDs = Set() + var processor: any SpanProcessor + } + + init(processor: any SpanProcessor, registry: ExecutionTraceRegistry) { + self.registry = registry + self.state = LockIsolated(State(processor: processor)) + } + + func onStart(parentContext: SpanContext?, span: any ReadableSpan) { + guard + let parentContext, + parentContext.traceId == span.context.traceId, + self.registry.contains(span.context.traceId) + else { return } + + self.state.withValue { state in + if state.active { + state.acceptedSpanIDs.insert(span.context.spanId) + } + } + } + + func onEnd(span: any ReadableSpan) { + self.state.withValue { state in + guard state.active, state.acceptedSpanIDs.remove(span.context.spanId) != nil else { return } + state.processor.onEnd(span: span) + } + } + + func forceFlush(timeout: TimeInterval?) { + self.state.withValue { state in + if state.active { + state.processor.forceFlush(timeout: timeout) + } + } + } + + func shutdown(explicitTimeout: TimeInterval?) { + self.state.withValue { state in + state.active = false + state.acceptedSpanIDs.removeAll() + state.processor.shutdown(explicitTimeout: explicitTimeout) + } + } +} diff --git a/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift b/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift new file mode 100644 index 00000000..e0530475 --- /dev/null +++ b/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift @@ -0,0 +1,53 @@ +import Foundation +import OpenTelemetryApi +import OpenTelemetrySdk + +final class SynchronousExecutionSpanProcessor: SpanProcessor { + let isStartRequired = false + let isEndRequired = true + + private let exporter: any SpanExporter + private let exporterLock = NSLock() + private let logger: Logger? + + init(exporter: any SpanExporter, logger: Logger?) { + self.exporter = exporter + self.logger = logger + } + + func onStart(parentContext: SpanContext?, span: any ReadableSpan) {} + + func onEnd(span: any ReadableSpan) { + self.exporterLock.lock() + defer { self.exporterLock.unlock() } + + let name = span.getAttributes()[BuildkiteTelemetryAttribute.testName]?.description ?? span.name + switch self.exporter.export(spans: [span.toSpanData()]) { + case .success: + self.logger?.debug("Exported OpenTelemetry test execution: \(name)") + case .failure: + self.logger?.error( + "OpenTelemetry export failed for \(name); the execution remains queued in this process" + ) + } + } + + func forceFlush(timeout: TimeInterval?) { + self.exporterLock.lock() + defer { self.exporterLock.unlock() } + self.flush(timeout: timeout) + } + + func shutdown(explicitTimeout: TimeInterval?) { + self.exporterLock.lock() + defer { self.exporterLock.unlock() } + self.flush(timeout: explicitTimeout) + self.exporter.shutdown(explicitTimeout: explicitTimeout) + } + + private func flush(timeout: TimeInterval?) { + if case .failure = self.exporter.flush(explicitTimeout: timeout) { + self.logger?.error("OpenTelemetry export failed while flushing pending test executions") + } + } +} diff --git a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift new file mode 100644 index 00000000..57957746 --- /dev/null +++ b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift @@ -0,0 +1,446 @@ +import Foundation +import OpenTelemetryApi +import OpenTelemetryProtocolExporterCommon +import OpenTelemetryProtocolExporterHttp +import OpenTelemetrySdk + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +enum BuildkiteTelemetryAttribute { + static let annotation = "buildkite.annotation" + static let buildID = "buildkite.build_id" + static let buildNumber = "buildkite.build_number" + static let collectorName = "buildkite.collector.name" + static let collectorVersion = "buildkite.collector.version" + static let executionExternalID = "buildkite.test.execution.external_id" + static let executionLocation = "buildkite.test.location" + static let executionScope = "buildkite.test.scope" + static let executionVia = "buildkite.execution.via" + static let frameworkName = "buildkite.test.framework.name" + static let jobID = "buildkite.job_id" + static let message = "buildkite.message" + static let runKey = "buildkite.run_key" + static let runURL = "buildkite.run_url" + static let stepID = "buildkite.step_id" + static let tagPrefix = "buildkite.tag." + static let testName = "buildkite.test.name" +} + +private enum TelemetryValue { + static let executionVia = "otlp" + static let frameworkName = "xctest" + static let rootSpanName = "test.execution" + static let skippedResult = "skipped" +} + +struct CollectorOTLPConfiguration { + let endpoint: URL + let headers: [(String, String)] + let runEnvironment: RunEnvironment + + init?( + environment: EnvironmentValues, + logger: Logger? + ) { + let runEnvironment = environment.runEnvironment() + let rawHeaders = environment.otelTracesHeaders ?? environment.otelHeaders + let hasExplicitConfiguration = environment.otelTracesEndpoint != nil + || environment.otelEndpoint != nil + || environment.analyticsOTLPEndpoint != nil + || rawHeaders != nil + + guard environment.analyticsToken != nil || hasExplicitConfiguration else { return nil } + + if let protocolName = environment.otelTracesProtocol ?? environment.otelProtocol, + protocolName.lowercased() != "http/protobuf" { + logger?.error( + "Unsupported OpenTelemetry traces protocol \(protocolName); expected http/protobuf" + ) + return nil + } + + let endpoint: URL + if let value = environment.otelTracesEndpoint ?? environment.analyticsOTLPEndpoint { + guard let url = Self.absoluteURL(value) else { + logger?.error("OpenTelemetry traces endpoint is not a valid absolute URL") + return nil + } + endpoint = url + } else if let value = environment.otelEndpoint { + guard let url = Self.absoluteURL(value) else { + logger?.error("OpenTelemetry endpoint is not a valid absolute URL") + return nil + } + endpoint = url.appendingPathComponent("v1/traces") + } else { + endpoint = URL(string: TestCollector.endpoint)! + } + + var headers = [("Buildkite-Tests-Run-Key", runEnvironment.key)] + if let token = environment.analyticsToken { + headers.append(("Authorization", "Token token=\"\(token)\"")) + } + + if let rawHeaders { + guard let parsedHeaders = Self.parseHeaders(rawHeaders) else { + logger?.error("OpenTelemetry exporter headers are invalid") + return nil + } + for header in parsedHeaders { + headers.removeAll { $0.0.caseInsensitiveCompare(header.0) == .orderedSame } + headers.append(header) + } + } + + self.endpoint = endpoint + self.headers = headers + self.runEnvironment = runEnvironment + } + + private static func absoluteURL(_ value: String) -> URL? { + guard let url = URL(string: value), url.scheme != nil, url.host != nil else { return nil } + return url + } + + private static func parseHeaders(_ value: String) -> [(String, String)]? { + let entries = value.split(separator: ",", omittingEmptySubsequences: false) + guard !entries.isEmpty else { return nil } + + var headers: [(String, String)] = [] + for entry in entries { + let parts = entry.split(separator: "=", maxSplits: 1, omittingEmptySubsequences: false) + guard parts.count == 2 else { return nil } + + let name = String(parts[0]).removingPercentEncoding? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let value = String(parts[1]).removingPercentEncoding? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard !name.isEmpty, !value.isEmpty else { return nil } + + headers.removeAll { $0.0.caseInsensitiveCompare(name) == .orderedSame } + headers.append((name, value)) + } + return headers + } +} + +extension TelemetryClient { + static func live( + environment: EnvironmentValues, + uploadTags: [String: String], + logger: Logger?, + rootExporter: (any SpanExporter)? = nil, + childExporter: (any SpanExporter)? = nil + ) -> TelemetryClient? { + guard let configuration = CollectorOTLPConfiguration( + environment: environment, + logger: logger + ) else { return nil } + + let exporterConfiguration = OtlpConfiguration( + timeout: 10, + compression: .gzip, + headers: configuration.headers, + exportAsJson: false + ) + let makeExporter = { + OtlpHttpTraceExporter( + endpoint: configuration.endpoint, + config: exporterConfiguration, + envVarHeaders: nil, + requeueOnFailure: true + ) as any SpanExporter + } + let live = LiveTelemetryClient( + configuration: configuration, + environment: environment, + uploadTags: uploadTags, + logger: logger, + rootExporter: rootExporter ?? makeExporter(), + childExporter: childExporter ?? makeExporter() + ) + + return TelemetryClient( + start: live.startExecution, + annotate: live.annotate, + finish: live.finishExecution, + flush: live.forceFlush + ) + } +} + +private final class LiveTelemetryClient { + private static let providerEnvironmentLock = NSLock() + + private let childForwarder: ExecutionChildSpanProcessor? + private let executionProvider: TracerProviderSdk + private let logger: Logger? + private let registry = ExecutionTraceRegistry() + private let spans = LockIsolated([UUID: any Span]()) + private let tracer: any OpenTelemetryApi.Tracer + private let jobSpanContext: SpanContext? + + init( + configuration: CollectorOTLPConfiguration, + environment: EnvironmentValues, + uploadTags: [String: String], + logger: Logger?, + rootExporter: any SpanExporter, + childExporter: any SpanExporter + ) { + self.logger = logger + self.jobSpanContext = Self.jobSpanContext(environment: environment) + + let resource = Self.resource( + configuration: configuration, + environment: environment, + uploadTags: uploadTags + ) + let rootProcessor = SynchronousExecutionSpanProcessor(exporter: rootExporter, logger: logger) + self.executionProvider = Self.makeProvider( + resource: resource, + processor: rootProcessor + ) + self.tracer = self.executionProvider.get( + instrumentationName: TestCollector.name, + instrumentationVersion: TestCollector.version + ) + + let childProcessor = BatchSpanProcessor( + spanExporter: childExporter, + scheduleDelay: 1, + exportTimeout: 30, + maxQueueSize: 8192, + maxExportBatchSize: 512 + ) + let forwarder = ExecutionChildSpanProcessor(processor: childProcessor, registry: self.registry) + let globalProvider = OpenTelemetry.instance.tracerProvider + if globalProvider is DefaultTracerProvider { + let provider = Self.makeProvider(resource: resource, processor: forwarder) + OpenTelemetry.registerTracerProvider(tracerProvider: provider) + self.childForwarder = forwarder + } else if let provider = globalProvider as? TracerProviderSdk { + provider.addSpanProcessor(forwarder) + self.childForwarder = forwarder + } else { + logger?.error( + "OpenTelemetry child span export is disabled because the existing tracer provider cannot accept a span processor" + ) + self.childForwarder = nil + } + } + + func startExecution(_ test: TestState) -> UUID { + let builder = self.tracer + .spanBuilder(spanName: TelemetryValue.rootSpanName) + .setNoParent() + .setAttribute(key: BuildkiteTelemetryAttribute.executionVia, value: TelemetryValue.executionVia) + .setAttribute(key: BuildkiteTelemetryAttribute.executionScope, value: test.className) + .setAttribute(key: BuildkiteTelemetryAttribute.testName, value: test.testName) + .setAttribute( + key: BuildkiteTelemetryAttribute.executionExternalID, + value: test.id.uuidString + ) + .setAttribute( + key: SemanticConventions.Test.caseName.rawValue, + value: "\(test.className).\(test.testName)" + ) + .setAttribute(key: SemanticConventions.Test.suiteName.rawValue, value: test.className) + + if let jobSpanContext = self.jobSpanContext { + builder.addLink(spanContext: jobSpanContext) + } + + let span = builder.startSpan() + self.registry.insert(span.context.traceId) + self.spans.withValue { $0[test.id] = span } + OpenTelemetry.instance.contextProvider.setActiveSpan(span) + return test.id + } + + func annotate(executionID: UUID, content: String) { + guard let span = self.spans.withValue({ $0[executionID] }) else { return } + span.addEvent( + name: "test.annotation", + attributes: [BuildkiteTelemetryAttribute.annotation: .string(content)] + ) + } + + func finishExecution(executionID: UUID, test: TestState, tags: [String: String]?) { + guard let span = self.spans.withValue({ $0.removeValue(forKey: executionID) }) else { return } + + for (key, value) in tags ?? [:] { + span.setAttribute(key: BuildkiteTelemetryAttribute.tagPrefix + key, value: value) + } + + let result = test.result ?? .failed + span.setAttribute( + key: SemanticConventions.Test.caseResultStatus.rawValue, + value: Self.resultStatus(result) + ) + + if let location = test.issues.first?.sourceCodeContext.location { + span.setAttribute(key: SemanticConventions.Code.filePath.rawValue, value: location.filePath) + span.setAttribute(key: SemanticConventions.Code.lineNumber.rawValue, value: Int(location.line)) + span.setAttribute( + key: BuildkiteTelemetryAttribute.executionLocation, + value: "\(location.fileName):\(location.line)" + ) + } + + if result == .failed { + let reason = Self.failureReason(test.issues) + span.status = .error(description: reason) + for issue in test.issues { + var attributes: [String: AttributeValue] = [ + SemanticConventions.Exception.message.rawValue: .string(issue.description), + ] + let backtrace = issue.sourceCodeContext.callStack.enumerated() + .map { "\($0.offset) \($0.element)" } + .joined(separator: "\n") + if !backtrace.isEmpty { + attributes[SemanticConventions.Exception.stacktrace.rawValue] = .string(backtrace) + } + if let error = issue.associatedError { + attributes[SemanticConventions.Exception.type.rawValue] = .string( + String(reflecting: type(of: error)) + ) + } + span.addEvent( + name: SemanticConventions.Exception.exception.rawValue, + attributes: attributes + ) + } + } + + OpenTelemetry.instance.contextProvider.removeContextForSpan(span) + self.registry.remove(span.context.traceId) + span.end() + } + + func forceFlush() { + self.executionProvider.forceFlush(timeout: 30) + self.childForwarder?.forceFlush(timeout: 30) + } + + private static func resultStatus(_ result: TestResult) -> String { + switch result { + case .passed: + return SemanticConventions.Test.CaseResultStatusValues.pass.description + case .failed: + return SemanticConventions.Test.CaseResultStatusValues.fail.description + case .skipped: + return TelemetryValue.skippedResult + } + } + + private static func failureReason(_ issues: [TestIssue]) -> String { + if issues.count > 1 { + return "\(issues.count) failures: \(issues.map(\.compactDescription).joined(separator: ", "))" + } + return issues.first?.compactDescription ?? "Test failed" + } + + private static func makeProvider( + resource: Resource, + processor: any SpanProcessor + ) -> TracerProviderSdk { + self.withoutInheritedTraceContext { + TracerProviderBuilder() + .with(resource: resource) + .with(sampler: Samplers.alwaysOn) + .add(spanProcessor: processor) + .build() + } + } + + private static func withoutInheritedTraceContext(_ operation: () -> T) -> T { + self.providerEnvironmentLock.lock() + defer { self.providerEnvironmentLock.unlock() } + + let names = ["TRACEPARENT", "TRACESTATE"] + let previous = Dictionary(uniqueKeysWithValues: names.map { name in + (name, getenv(name).map { String(cString: $0) }) + }) + names.forEach { unsetenv($0) } + defer { + for name in names { + if let value = previous[name] ?? nil { + setenv(name, value, 1) + } else { + unsetenv(name) + } + } + } + return operation() + } + + private static func resource( + configuration: CollectorOTLPConfiguration, + environment: EnvironmentValues, + uploadTags: [String: String] + ) -> Resource { + let run = configuration.runEnvironment + var attributes = [String: AttributeValue]() + + func set(_ key: String, _ value: String?) { + if let value, Self.validAttributeKey(key) { + attributes[key] = .string(value) + } + } + + set(SemanticConventions.Service.name.rawValue, environment.testEngineSuiteSlug ?? TestCollector.name) + set(SemanticConventions.Service.namespace.rawValue, environment.buildkiteOrganizationSlug) + set(SemanticConventions.Service.instanceId.rawValue, run.jobId) + set(BuildkiteTelemetryAttribute.runKey, run.key) + set(BuildkiteTelemetryAttribute.runURL, run.url) + set(SemanticConventions.Vcs.refHeadName.rawValue, run.branch) + set(SemanticConventions.Vcs.refHeadRevision.rawValue, run.commitSha) + if run.branch != nil { + set( + SemanticConventions.Vcs.refHeadType.rawValue, + environment.buildkiteTag == nil ? "branch" : "tag" + ) + } + set(BuildkiteTelemetryAttribute.buildNumber, run.number) + set(BuildkiteTelemetryAttribute.buildID, environment.buildkiteBuildId) + set(BuildkiteTelemetryAttribute.jobID, run.jobId) + set(BuildkiteTelemetryAttribute.stepID, environment.buildkiteStepId) + set(BuildkiteTelemetryAttribute.message, run.message) + set(BuildkiteTelemetryAttribute.collectorName, run.collector) + set(BuildkiteTelemetryAttribute.collectorVersion, run.version) + set(BuildkiteTelemetryAttribute.frameworkName, TelemetryValue.frameworkName) + + for (key, value) in uploadTags { + set(BuildkiteTelemetryAttribute.tagPrefix + key, value) + } + for (key, value) in run.customEnvironment ?? [:] where Self.validAttributeKey(key) { + attributes[key] = AttributeValue(value.base) ?? .string(value.description) + } + + return EnvVarResource.get().merging(other: Resource(attributes: attributes)) + } + + private static func validAttributeKey(_ key: String) -> Bool { + !key.isEmpty && key.count <= 255 && key.unicodeScalars.allSatisfy { (32...126).contains($0.value) } + } + + private static func jobSpanContext(environment: EnvironmentValues) -> SpanContext? { + guard let traceParent = environment.traceParent else { return nil } + let carrier = [ + "traceparent": traceParent, + "tracestate": environment.traceState ?? "", + ] + return W3CTraceContextPropagator().extract(carrier: carrier, getter: DictionaryGetter()) + } + + private struct DictionaryGetter: Getter { + func get(carrier: [String: String], key: String) -> [String]? { + carrier[key].map { [$0] } + } + } +} diff --git a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient.swift b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient.swift new file mode 100644 index 00000000..238935b7 --- /dev/null +++ b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient.swift @@ -0,0 +1,42 @@ +import Foundation + +struct TelemetryClient { + typealias ExecutionID = UUID + + private let start: (TestState) -> ExecutionID + private let annotateExecution: (ExecutionID, String) -> Void + private let finish: (ExecutionID, TestState, [String: String]?) -> Void + private let flushPending: () -> Void + + init( + start: @escaping (TestState) -> ExecutionID, + annotate: @escaping (ExecutionID, String) -> Void, + finish: @escaping (ExecutionID, TestState, [String: String]?) -> Void, + flush: @escaping () -> Void + ) { + self.start = start + self.annotateExecution = annotate + self.finish = finish + self.flushPending = flush + } + + func startExecution(_ test: TestState) -> ExecutionID { + self.start(test) + } + + func annotate(_ content: String, executionID: ExecutionID) { + self.annotateExecution(executionID, content) + } + + func finishExecution( + _ executionID: ExecutionID, + test: TestState, + tags: [String: String]? + ) { + self.finish(executionID, test, tags) + } + + func forceFlush() { + self.flushPending() + } +} diff --git a/test-collector-swift/Sources/Core/TestCollector.swift b/test-collector-swift/Sources/Core/TestCollector.swift index 4ee8af36..e367b92e 100644 --- a/test-collector-swift/Sources/Core/TestCollector.swift +++ b/test-collector-swift/Sources/Core/TestCollector.swift @@ -23,20 +23,18 @@ public struct TestCollector { let envTags = environment.analyticsTags ?? [:] let tags = uploadTags.merging(envTags) { _, env in env } - let tracer = Tracer.live() - - let uploader: UploadClient? - if let apiToken = environment.analyticsToken { - let baseURL = environment.analyticsBaseURL ?? URL(string: Self.baseURL)! - let api = ApiClient.live(apiToken: apiToken, baseURL: baseURL) - let runEnvironment = environment.runEnvironment() - uploader = .live(api: api, runEnvironment: runEnvironment, tags: tags.isEmpty ? nil : tags, logger: logger) - } else { - logger?.info("TestCollector unable to locate API key. Test results will not be uploaded.") - uploader = nil + let telemetry = TelemetryClient.live( + environment: environment, + uploadTags: tags, + logger: logger + ) + if telemetry == nil { + logger?.info( + "TestCollector requires an API token or OpenTelemetry exporter configuration. Test results will not be exported." + ) } - self.observer = TestObserver(logger: logger, tracer: tracer, uploader: uploader) + self.observer = TestObserver(logger: logger, telemetry: telemetry) } /// Annotates the current test. @@ -45,7 +43,7 @@ public struct TestCollector { /// /// - Parameter content: The content of this annotation public func annotate(_ content: @autoclosure () -> String) { - self.observer?.tracer.annotate(content()) + self.observer?.annotate(content()) } /// Tags the execution of the given test case with a key-value pair. @@ -80,7 +78,7 @@ public struct TestCollector { public private(set) static var shared: TestCollector? - public static let baseURL = "https://analytics-api.buildkite.com/v1/" + public static let endpoint = "https://tests-otlp.buildkite.com/v1/traces" static let name = "test-collector-swift" static let version = "0.6.0" } diff --git a/test-collector-swift/Sources/Core/TestObserver.swift b/test-collector-swift/Sources/Core/TestObserver.swift index d578a749..df5d8ec4 100644 --- a/test-collector-swift/Sources/Core/TestObserver.swift +++ b/test-collector-swift/Sources/Core/TestObserver.swift @@ -1,17 +1,16 @@ import XCTest -/// An object that captures test data and uploads it in real time. +/// An object that captures test data and exports it in real time. final class TestObserver: NSObject, XCTestObservation { let logger: Logger? - let tracer: Tracer - let uploader: UploadClient? + let telemetry: TelemetryClient? let uuid: () -> UUID /// The state of the current test. var test: TestState? /// The id associated with the root span of the current test. - var spanId: AnyHashable? + var executionID: TelemetryClient.ExecutionID? /// Per-test execution tags, keyed by test case identity. private let executionTags = LockIsolated([ObjectIdentifier: [String: String]]()) @@ -20,18 +19,15 @@ final class TestObserver: NSObject, XCTestObservation { /// /// - Parameters: /// - logger: A logger. - /// - tracer: The tracer for recording span data. - /// - uploader: The upload client for uploading test results + /// - telemetry: The client for exporting OpenTelemetry spans. /// - uuid: A closure that returns a unique id to associate with an executed test case. init( logger: Logger? = .init(), - tracer: Tracer = .live(), - uploader: UploadClient? = nil, + telemetry: TelemetryClient? = nil, uuid: @escaping () -> UUID = UUID.init ) { self.logger = logger - self.tracer = tracer - self.uploader = uploader + self.telemetry = telemetry self.uuid = uuid } @@ -54,12 +50,13 @@ final class TestObserver: NSObject, XCTestObservation { /// Called exactly once per test case. func testCaseWillStart(_ testCase: XCTestCase) { self.executionTags.withValue { $0[ObjectIdentifier(testCase)] = [:] } - self.spanId = self.tracer.startSpan(section: "top") - self.test = TestState( + let test = TestState( id: self.uuid(), className: XCTestCase.className(of: testCase), testName: XCTestCase.testName(of: testCase) ) + self.test = test + self.executionID = self.telemetry?.startExecution(test) } #if canImport(ObjectiveC) @@ -106,19 +103,13 @@ final class TestObserver: NSObject, XCTestObservation { return result?.isEmpty == true ? nil : result } defer { - spanId = nil + executionID = nil test = nil } - guard - let span = self.spanId.map(self.tracer.endSpan(id:)), - var test = self.test - else { return } + guard let executionID = self.executionID, var test = self.test else { return } test.result = testCase.result - - let trace = Trace(test: test, span: span, tags: tags) - - self.uploader?.record(trace: trace) + self.telemetry?.finishExecution(executionID, test: test, tags: tags) } /// Notifies the observer immediately after all tests in a test bundle finish executing. @@ -128,7 +119,12 @@ final class TestObserver: NSObject, XCTestObservation { /// - Note: The test process will generally exit after this method returns, so it must block until all asynchronous /// work is complete. func testBundleDidFinish(_ testBundle: Bundle) { - self.uploader?.waitForUploads() + self.telemetry?.forceFlush() self.logger?.waitForLogs() } + + func annotate(_ content: String) { + guard let executionID = self.executionID else { return } + self.telemetry?.annotate(content, executionID: executionID) + } } diff --git a/test-collector-swift/Sources/Core/Tracer/Tracer+Live.swift b/test-collector-swift/Sources/Core/Tracer/Tracer+Live.swift deleted file mode 100644 index a9ff71fc..00000000 --- a/test-collector-swift/Sources/Core/Tracer/Tracer+Live.swift +++ /dev/null @@ -1,43 +0,0 @@ -import Foundation - -extension Tracer { - /// Constructs a "live" tracer that uses the system uptime for the start and end time of spans. - /// - /// - Parameters: - /// - systemUptime: A closure that returns the system uptime. - /// - spanId: A closure that returns a unique identifier. - /// - Returns: A tracer that use the system uptime for span times. - static func live( - systemUptime: @escaping () -> TimeInterval = { ProcessInfo.processInfo.systemUptime }, - spanId: @escaping () -> AnyHashable = { UUID() } - ) -> Tracer { - var stack: [(id: AnyHashable, span: Trace.Span)] = [] - - return Tracer( - startSpan: { section, detail in - let id = spanId() - let span = Trace.Span(section: section, startAt: systemUptime(), detail: detail) - stack.append((id, span)) - return id - - }, - endSpan: { id in - assert(!stack.isEmpty, "No active spans") - assert(stack.last!.id == id, "Id does not match current span") - - var span = stack.last!.span - let endAt = systemUptime() - - span.endAt = endAt - span.duration = span.startAt.map { endAt - $0 } - stack.removeLast() - - if !stack.isEmpty { - stack[stack.endIndex - 1].span.children.append(span) - } - - return span - } - ) - } -} diff --git a/test-collector-swift/Sources/Core/Tracer/Tracer.swift b/test-collector-swift/Sources/Core/Tracer/Tracer.swift deleted file mode 100644 index dde4ecf2..00000000 --- a/test-collector-swift/Sources/Core/Tracer/Tracer.swift +++ /dev/null @@ -1,41 +0,0 @@ -import Foundation - -/// A type for capturing spans over time. -struct Tracer { - private var startSpan: (String, [String: String]) -> AnyHashable - private var endSpan: (AnyHashable) -> Trace.Span - - init( - startSpan: @escaping (String, [String: String]) -> AnyHashable, - endSpan: @escaping (AnyHashable) -> Trace.Span - ) { - self.startSpan = startSpan - self.endSpan = endSpan - } - - /// Starts recording a span. - /// - /// - Parameters: - /// - section: The section's name. - /// - detail: Information attached to the span. - /// - Returns: The identifier associated with the started span. - func startSpan(section: String, detail: [String: String] = [:]) -> AnyHashable { - self.startSpan(section, detail) - } - - /// Ends the span that corresponds to the specified id. - /// - /// - Parameter id: The span's identifier. - /// - Returns: The completed span. - func endSpan(id: AnyHashable) -> Trace.Span { - self.endSpan(id) - } - - /// Adds an annotation span as a child to the current span . - /// - /// - Parameter content: The annotation's content. - func annotate(_ content: String) { - let id = self.startSpan(section: "annotation", detail: ["content": content]) - _ = self.endSpan(id: id) - } -} diff --git a/test-collector-swift/Sources/Core/UploadClient/UploadClient+Live.swift b/test-collector-swift/Sources/Core/UploadClient/UploadClient+Live.swift deleted file mode 100644 index 6f2f68d6..00000000 --- a/test-collector-swift/Sources/Core/UploadClient/UploadClient+Live.swift +++ /dev/null @@ -1,121 +0,0 @@ -import Dispatch -import Foundation - -#if canImport(FoundationNetworking) -import FoundationNetworking -#endif - -extension UploadClient { - /// Constructs a "live" upload client that uploads traces using an API client. - /// - /// - Parameters: - /// - api: An API client. - /// - logger: A logger. - /// - runEnvironment: The run environment to accompany uploaded traces. - /// - batchSize: The maximum number of traces per upload. - /// - group: A dispatch group to associate with upload tasks. - /// - Returns: A upload client that uses an api client. - static func live( - api: ApiClient, - runEnvironment: RunEnvironment, - tags: [String: String]? = nil, - logger: Logger? = nil, - batchSize: Int = maximumBatchSize, - group: DispatchGroup = DispatchGroup() - ) -> UploadClient { - let client = LiveClient( - api: api, - batchSize: batchSize, - logger: logger, - runEnvironment: runEnvironment, - tags: tags, - taskGroup: group - ) - - return UploadClient( - record: { client.record(trace: $0) }, - waitForUploads: { client.waitForUploads(timeout: $0) } - ) - } - - private struct LiveClient { - let api: ApiClient - let batchSize: Int - let logger: Logger? - let runEnvironment: RunEnvironment - let tags: [String: String]? - let taskGroup: DispatchGroup - - private let traces = LockIsolated([Trace]()) - - func record(trace: Trace) { - self.traces.withValue { traces in - traces.append(trace) - guard traces.count >= self.batchSize else { return } - self.upload(traces: traces) - traces = [] - } - } - - private func upload(traces: [Trace]) { - // NB: Uploads must enter the task group synchronously to ensure they are waited for - self.taskGroup.enter() - Task { - defer { self.taskGroup.leave() } - let testData = TestResults.json(runEnv: runEnvironment, tags: tags, data: traces) - try await self.upload(testData: testData) - } - } - - private func upload(testData: TestResults) async throws { - self.logger?.debug("Uploading \(testData)") - - var data: Data - var response: HTTPURLResponse - do { - (data, response) = try await self.api.data(for: .upload(testData)) - } catch { - self.logger?.error("Error uploading: \(error.localizedDescription)") - throw error - } - - // Ideally “HTTP 200 OK” etc, but maybe actually “HTTP 200 no error” etc. - let statusString = "HTTP \(response.statusCode) \(HTTPURLResponse.localizedString(forStatusCode: response.statusCode))" - - // Currently this should get HTTP 202 Accepted, but let's be a bit permissive to future changes. - guard (201...204).contains(response.statusCode) else { - if let body = try? self.api.decode(data, as: UploadFailureResponse.self) { - self.logger?.error("Unexpected \(statusString), \(body.message)") - } else { - self.logger?.error("Unexpected \(statusString), (no message)") - } - throw URLError(.badServerResponse, userInfo: [ NSLocalizedDescriptionKey: "Unexpected \(statusString)" ]) - } - - do { - let result = try self.api.decode(data, as: UploadResponse.self) - let uploadID = result.uploadID ?? "(missing)" - let uploadURL = result.uploadURL ?? "(missing)" - self.logger?.debug("\(statusString), ID: \(uploadID), URL: \(uploadURL)") - } catch let decodingError as DecodingError { - self.logger?.error("Warning: error decoding body of \(statusString): \(decodingError)") - // proceed anyway, since we got an HTTP 2xx, and decoding the response isn't critical - } - } - - func waitForUploads(timeout: TimeInterval) { - self.traces.withValue { traces in - guard !traces.isEmpty else { return } - self.upload(traces: traces) - traces = [] - } - let result = self.taskGroup.wait(timeout: timeout) - if result == .timedOut { - self.logger?.error("Upload client timed out before completing all uploads") - } - } - } -} - -// The maximum number of traces that can be sent per upload -private let maximumBatchSize = 5000 diff --git a/test-collector-swift/Sources/Core/UploadClient/UploadClient.swift b/test-collector-swift/Sources/Core/UploadClient/UploadClient.swift deleted file mode 100644 index bc4af9ca..00000000 --- a/test-collector-swift/Sources/Core/UploadClient/UploadClient.swift +++ /dev/null @@ -1,32 +0,0 @@ -import Foundation - -/// A type used to upload traces asynchronously during a test run. -struct UploadClient { - private var record: (Trace) -> Void - private var waitForUploads: (TimeInterval) -> Void - - init( - record: @escaping (Trace) -> Void, - waitForUploads: @escaping (TimeInterval) -> Void - ) { - self.record = record - self.waitForUploads = waitForUploads - } - - /// Records a trace to be included in the next upload. - /// - /// - Parameter trace: The trace to record - func record(trace: Trace) { - self.record(trace) - } - - /// Waits synchronously for the previously submitted traces to be uploaded. - /// - /// - Parameter timeout: The maximum duration in seconds to wait for uploads to complete. - func waitForUploads(timeout: TimeInterval = twoMinutes) { - self.waitForUploads(timeout) - } -} - -// Default timeout used by waitForUploads -private let twoMinutes: TimeInterval = 120 diff --git a/test-collector-swift/Tests/CoreTests/ApiClientTests.swift b/test-collector-swift/Tests/CoreTests/ApiClientTests.swift deleted file mode 100644 index fc6bd08f..00000000 --- a/test-collector-swift/Tests/CoreTests/ApiClientTests.swift +++ /dev/null @@ -1,36 +0,0 @@ -@testable import Core -import XCTest - -#if canImport(FoundationNetworking) -import FoundationNetworking -#endif - -final class ApiClientTests: XCTestCase { - func testApiClientMakesUploadRequest() async throws { - struct Response: Decodable { var status: String } - var requests = [URLRequest]() - let session = ApiSession { request in - requests.append(request) - return ("{\"status\":\"OK\"}".data(using: .utf8)!, .stub()) - } - let api = ApiClient.live( - apiToken: "token", - baseURL: URL(string: "http://api.test.com/")!, - session: session - ) - let results = TestResults.json(runEnv: .init(key: "key"), data: []) - - let (value, _) = try await api.data(for: .upload(results), as: Response.self) - - XCTAssertEqual(value.status, "OK") - XCTAssertEqual(requests.count, 1) - let request = try XCTUnwrap(requests.first) - XCTAssertEqual(request.url?.absoluteString, "http://api.test.com/uploads") - XCTAssertEqual(request.authorizationHeader, "Token token=\"token\"") - XCTAssertEqual(request.httpMethod, "POST") - XCTAssertEqual( - request.httpBodyDictionary, - ["format": "json", "run_env": ["key": "key"], "data": NSArray()] - ) - } -} diff --git a/test-collector-swift/Tests/CoreTests/CollectorTests.swift b/test-collector-swift/Tests/CoreTests/CollectorTests.swift index 394adfab..72c39d89 100644 --- a/test-collector-swift/Tests/CoreTests/CollectorTests.swift +++ b/test-collector-swift/Tests/CoreTests/CollectorTests.swift @@ -10,14 +10,24 @@ final class CollectorTests: XCTestCase { ) let collector = TestCollector(environment: environment) let observer = try XCTUnwrap(collector.observer, "Observer should be initialised") - XCTAssertNil(observer.uploader, "Uploader should not be initialised without an api key") + XCTAssertNil(observer.telemetry, "Telemetry should not be initialised without export configuration") } - func testDefaultCollectorWithUploader() throws { + func testDefaultCollectorWithTelemetry() throws { let environment = EnvironmentValues(values: ["BUILDKITE_ANALYTICS_TOKEN": "SECRET"]) let collector = TestCollector(environment: environment) let observer = try XCTUnwrap(collector.observer, "Observer should be created by default") - XCTAssertNotNil(observer.uploader, "Uploader should be initialised when provided an api key") + XCTAssertNotNil(observer.telemetry, "Telemetry should be initialised when provided an api key") + } + + func testCollectorWithOTLPConfigurationDoesNotRequireApiToken() throws { + let environment = EnvironmentValues(values: [ + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://127.0.0.1:4318/v1/traces", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS": "Authorization=Bearer%20relay-token", + ]) + let collector = TestCollector(environment: environment) + let observer = try XCTUnwrap(collector.observer, "Observer should be created by default") + XCTAssertNotNil(observer.telemetry) } func testCollectorIsDisabled() { diff --git a/test-collector-swift/Tests/CoreTests/RunEnvironmentTests.swift b/test-collector-swift/Tests/CoreTests/RunEnvironmentTests.swift index c2cf9760..14328870 100644 --- a/test-collector-swift/Tests/CoreTests/RunEnvironmentTests.swift +++ b/test-collector-swift/Tests/CoreTests/RunEnvironmentTests.swift @@ -121,65 +121,4 @@ final class RunEnvironmentTests: XCTestCase { ) } - func testJSONEncoding() throws { - let runEnvironment = RunEnvironment( - ci: "ci", - key: "key", - url: "url", - branch: "branch", - commitSha: "commitSha", - number: "number", - jobId: "jobId", - message: "message", - debug: "debug", - executionNamePrefix: "executionNamePrefix", - executionNameSuffix: "executionNameSuffix", - version: "version", - collector: "collector", - customEnvironment: [ - "custom_key": "customKey", - "tags": ["A", "B", "C"], - ] - ) - - let data = try JSONEncoder().encode(runEnvironment) - - let json = try JSONSerialization.jsonObject(with: data) - - XCTAssertEqual( - json as? NSDictionary, - [ - "CI": "ci", - "key": "key", - "url": "url", - "branch": "branch", - "commit_sha": "commitSha", - "number": "number", - "job_id": "jobId", - "message": "message", - "debug": "debug", - "execution_name_prefix": "executionNamePrefix", - "execution_name_suffix": "executionNameSuffix", - "version": "version", - "collector": "collector", - "custom_key": "customKey", - "tags": ["A", "B", "C"], - ] - ) - } - - func testCustomEnvironmentTakesPrecedence() throws { - let runEnvironment = RunEnvironment( - key: "key", - customEnvironment: [ - "key": "customKey", - ] - ) - - let data = try JSONEncoder().encode(runEnvironment) - - let json = try JSONSerialization.jsonObject(with: data) - - XCTAssertEqual(json as? NSDictionary, ["key": "customKey"]) - } } diff --git a/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift new file mode 100644 index 00000000..cb0c5296 --- /dev/null +++ b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift @@ -0,0 +1,235 @@ +@testable import Core +import InMemoryExporter +import OpenTelemetryApi +import OpenTelemetrySdk +import XCTest + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +final class TelemetryClientTests: XCTestCase { + override func setUp() { + super.setUp() + OpenTelemetry.registerTracerProvider(tracerProvider: DefaultTracerProvider.instance) + } + + override func tearDown() { + OpenTelemetry.registerTracerProvider(tracerProvider: DefaultTracerProvider.instance) + super.tearDown() + } + + func testExportsExecutionRootSynchronouslyWithAttributesAndRunTags() throws { + let rootExporter = InMemoryExporter() + let childExporter = InMemoryExporter() + let environment = EnvironmentValues( + values: [ + "BUILDKITE_ANALYTICS_TOKEN": "SECRET", + "BUILDKITE_ANALYTICS_KEY": "run-key", + "BUILDKITE_BUILD_ID": "build-id", + "BUILDKITE_BUILD_NUMBER": "123", + "BUILDKITE_BUILD_URL": "https://buildkite.example/builds/123", + "BUILDKITE_BRANCH": "main", + "BUILDKITE_COMMIT": "abc123", + "BUILDKITE_JOB_ID": "job-id", + "BUILDKITE_STEP_ID": "step-id", + "BUILDKITE_TEST_ENGINE_SUITE_SLUG": "swift-suite", + ], + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ) + let client = try XCTUnwrap(TelemetryClient.live( + environment: environment, + uploadTags: ["language": "swift"], + logger: nil, + rootExporter: rootExporter, + childExporter: childExporter + )) + var test = TestState( + id: UUID(uuidString: "00000000-0000-0000-0000-000000000001")!, + className: "PaymentTests", + testName: "testChargeCard" + ) + + let executionID = client.startExecution(test) + client.annotate("checkpoint", executionID: executionID) + test.result = .passed + client.finishExecution(executionID, test: test, tags: ["feature": "payments"]) + + let span = try XCTUnwrap(rootExporter.getFinishedSpanItems().only) + XCTAssertEqual(span.name, "test.execution") + XCTAssertNil(span.parentSpanId) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.executionVia], .string("otlp")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.executionScope], .string("PaymentTests")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.testName], .string("testChargeCard")) + XCTAssertEqual(span.attributes["test.case.result.status"], .string("pass")) + XCTAssertEqual(span.attributes["buildkite.tag.feature"], .string("payments")) + XCTAssertEqual(span.resource.attributes[BuildkiteTelemetryAttribute.runKey], .string("run-key")) + XCTAssertEqual(span.resource.attributes[BuildkiteTelemetryAttribute.buildID], .string("build-id")) + XCTAssertEqual(span.resource.attributes["buildkite.tag.language"], .string("swift")) + XCTAssertEqual(span.events.only?.name, "test.annotation") + XCTAssertEqual( + span.events.only?.attributes[BuildkiteTelemetryAttribute.annotation], + .string("checkpoint") + ) + } + + func testExportsFailureLocationStatusAndException() throws { + let rootExporter = InMemoryExporter() + let client = try XCTUnwrap(TelemetryClient.live( + environment: self.configuredEnvironment, + uploadTags: [:], + logger: nil, + rootExporter: rootExporter, + childExporter: InMemoryExporter() + )) + var test = TestState(id: UUID(), className: "FailureTests", testName: "testFailure") + test.result = .failed + test.issues = [ + TestIssue( + compactDescription: "Expected true but was false", + description: "XCTAssertTrue failed", + sourceCodeContext: SourceCodeContext( + callStack: [SourceCodeFrame(address: 1, symbolInfo: SourceCodeSymbolInfo( + imageName: "Tests", + symbolName: "FailureTests.testFailure()" + ))], + location: SourceCodeLocation( + filePath: "/src/FailureTests.swift", + fileName: "FailureTests.swift", + line: 42 + ) + ) + ), + ] + + let executionID = client.startExecution(test) + client.finishExecution(executionID, test: test, tags: nil) + + let span = try XCTUnwrap(rootExporter.getFinishedSpanItems().only) + XCTAssertEqual(span.attributes["test.case.result.status"], .string("fail")) + XCTAssertEqual(span.attributes["code.file.path"], .string("/src/FailureTests.swift")) + XCTAssertEqual(span.attributes["code.line.number"], .int(42)) + XCTAssertEqual(span.status, .error(description: "Expected true but was false")) + XCTAssertEqual(span.events.only?.name, "exception") + XCTAssertEqual(span.events.only?.attributes["exception.message"], .string("XCTAssertTrue failed")) + XCTAssertEqual( + span.events.only?.attributes["exception.stacktrace"], + .string("0 FailureTests.testFailure()") + ) + } + + func testExecutionRemainsRootAndLinksToInheritedJobContext() throws { + let traceParent = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" + let previousTraceParent = getenv("TRACEPARENT").map { String(cString: $0) } + setenv("TRACEPARENT", traceParent, 1) + defer { + if let previousTraceParent { + setenv("TRACEPARENT", previousTraceParent, 1) + } else { + unsetenv("TRACEPARENT") + } + } + + let rootExporter = InMemoryExporter() + let client = try XCTUnwrap(TelemetryClient.live( + environment: EnvironmentValues( + values: [ + "BUILDKITE_ANALYTICS_KEY": "run-key", + "BUILDKITE_ANALYTICS_TOKEN": "SECRET", + ], + getFromInfoDictionary: { _ in nil } + ), + uploadTags: [:], + logger: nil, + rootExporter: rootExporter, + childExporter: InMemoryExporter() + )) + var test = TestState(id: UUID(), className: "LinkedTests", testName: "testLink") + + let executionID = client.startExecution(test) + test.result = .passed + client.finishExecution(executionID, test: test, tags: nil) + + let span = try XCTUnwrap(rootExporter.getFinishedSpanItems().only) + XCTAssertNil(span.parentSpanId) + XCTAssertEqual(span.links.only?.context.traceId.hexString, "4bf92f3577b34da6a3ce929d0e0e4736") + XCTAssertEqual(span.links.only?.context.spanId.hexString, "00f067aa0ba902b7") + } + + #if canImport(os.activity) + func testExportsSpansCreatedUnderTheActiveExecutionAsChildren() throws { + let rootExporter = InMemoryExporter() + let childExporter = InMemoryExporter() + let client = try XCTUnwrap(TelemetryClient.live( + environment: self.configuredEnvironment, + uploadTags: [:], + logger: nil, + rootExporter: rootExporter, + childExporter: childExporter + )) + var test = TestState(id: UUID(), className: "ChildTests", testName: "testChild") + + let executionID = client.startExecution(test) + let child = OpenTelemetry.instance.tracerProvider + .get(instrumentationName: "example-app") + .spanBuilder(spanName: "database.query") + .startSpan() + child.end() + test.result = .passed + client.finishExecution(executionID, test: test, tags: nil) + client.forceFlush() + + let root = try XCTUnwrap(rootExporter.getFinishedSpanItems().only) + let exportedChild = try XCTUnwrap(childExporter.getFinishedSpanItems().only) + XCTAssertEqual(exportedChild.traceId, root.traceId) + XCTAssertEqual(exportedChild.parentSpanId, root.spanId) + } + #endif + + func testUsesStandardTraceEndpointAndHeadersOverCollectorToken() throws { + let configuration = try XCTUnwrap(CollectorOTLPConfiguration( + environment: EnvironmentValues( + values: [ + "BUILDKITE_ANALYTICS_KEY": "run-key", + "BUILDKITE_ANALYTICS_TOKEN": "collector-token", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://127.0.0.1:1234/v1/traces", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS": "authorization=Bearer%20relay-token,x-test=value%2Fone", + ], + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ), + logger: nil + )) + + XCTAssertEqual(configuration.endpoint.absoluteString, "http://127.0.0.1:1234/v1/traces") + XCTAssertEqual(configuration.header(named: "Buildkite-Tests-Run-Key"), "run-key") + XCTAssertEqual(configuration.header(named: "Authorization"), "Bearer relay-token") + XCTAssertEqual(configuration.header(named: "x-test"), "value/one") + } + + private var configuredEnvironment: EnvironmentValues { + EnvironmentValues( + values: [ + "BUILDKITE_ANALYTICS_KEY": "run-key", + "BUILDKITE_ANALYTICS_TOKEN": "SECRET", + ], + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ) + } +} + +private extension Array { + var only: Element? { + self.count == 1 ? self[0] : nil + } +} + +private extension CollectorOTLPConfiguration { + func header(named name: String) -> String? { + self.headers.first { $0.0.caseInsensitiveCompare(name) == .orderedSame }?.1 + } +} diff --git a/test-collector-swift/Tests/CoreTests/TestHelpers/ApiClient.swift b/test-collector-swift/Tests/CoreTests/TestHelpers/ApiClient.swift deleted file mode 100644 index 7719b3ba..00000000 --- a/test-collector-swift/Tests/CoreTests/TestHelpers/ApiClient.swift +++ /dev/null @@ -1,23 +0,0 @@ -@testable import Core -import Foundation -import XCTest - -#if canImport(FoundationNetworking) -import FoundationNetworking -#endif - -extension ApiClient { - static func fulfill(_ expectation: XCTestExpectation, after seconds: TimeInterval) -> ApiClient { - ApiClient { _ in - try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) - expectation.fulfill() - return (Data(), .stub()) - } - } -} - -extension HTTPURLResponse { - static func stub(from url: URL = URL(string: "test")!, status: Int = 200) -> HTTPURLResponse { - HTTPURLResponse(url: url, statusCode: status, httpVersion: "1.1", headerFields: nil)! - } -} diff --git a/test-collector-swift/Tests/CoreTests/TestHelpers/URLRequest.swift b/test-collector-swift/Tests/CoreTests/TestHelpers/URLRequest.swift deleted file mode 100644 index a18e35ad..00000000 --- a/test-collector-swift/Tests/CoreTests/TestHelpers/URLRequest.swift +++ /dev/null @@ -1,32 +0,0 @@ -import Foundation -import XCTest - -#if canImport(FoundationNetworking) -import FoundationNetworking -#endif - -extension URLRequest { - var authorizationHeader: String? { - self.value(forHTTPHeaderField: "Authorization") - } - - var httpBodyDictionary: NSDictionary? { - guard let data = self.httpBody ?? self.httpBodyStream?.read() else { return nil } - return try? JSONSerialization.jsonObject(with: data, options: []) as? NSDictionary - } -} - -extension InputStream { - fileprivate func read(maxLength: Int = 1024) -> Data { - var data = Data() - self.open() - let buffer = UnsafeMutablePointer.allocate(capacity: maxLength) - while self.hasBytesAvailable { - let bytesRead = self.read(buffer, maxLength: maxLength) - data.append(buffer, count: bytesRead) - } - buffer.deallocate() - self.close() - return data - } -} diff --git a/test-collector-swift/Tests/CoreTests/TestObserverTests.swift b/test-collector-swift/Tests/CoreTests/TestObserverTests.swift index f07e0e8a..cbd09ec8 100644 --- a/test-collector-swift/Tests/CoreTests/TestObserverTests.swift +++ b/test-collector-swift/Tests/CoreTests/TestObserverTests.swift @@ -2,57 +2,36 @@ import XCTest final class TestObserverTests: XCTestCase { - func testExecutionTagsFlowToTrace() { - let recordedTraces = LockIsolated([Trace]()) - let uploader = UploadClient( - record: { trace in recordedTraces.withValue { $0.append(trace) } }, - waitForUploads: { _ in } - ) - let tracer = Tracer( - startSpan: { section, _ in section as AnyHashable }, - endSpan: { id in Trace.Span(section: id as! String) } - ) - let observer = TestObserver(logger: nil, tracer: tracer, uploader: uploader) + func testExecutionTagsFlowToTelemetry() { + let recordedTags = LockIsolated([[String: String]?]()) + let telemetry = self.telemetry { tags in recordedTags.withValue { $0.append(tags) } } + let observer = TestObserver(logger: nil, telemetry: telemetry) observer.testCaseWillStart(self) observer.setTag(for: self, key: "suite", value: "smoke") observer.setTag(for: self, key: "feature", value: "payments") observer.testCaseDidFinish(self) - XCTAssertEqual(recordedTraces.value.count, 1) - XCTAssertEqual(recordedTraces.value.first?.tags, ["suite": "smoke", "feature": "payments"]) + XCTAssertEqual(recordedTags.value.count, 1) + XCTAssertEqual(recordedTags.value.first!, ["suite": "smoke", "feature": "payments"]) } func testExecutionTagsNilWhenEmpty() { - let recordedTraces = LockIsolated([Trace]()) - let uploader = UploadClient( - record: { trace in recordedTraces.withValue { $0.append(trace) } }, - waitForUploads: { _ in } - ) - let tracer = Tracer( - startSpan: { section, _ in section as AnyHashable }, - endSpan: { id in Trace.Span(section: id as! String) } - ) - let observer = TestObserver(logger: nil, tracer: tracer, uploader: uploader) + let recordedTags = LockIsolated([[String: String]?]()) + let telemetry = self.telemetry { tags in recordedTags.withValue { $0.append(tags) } } + let observer = TestObserver(logger: nil, telemetry: telemetry) observer.testCaseWillStart(self) observer.testCaseDidFinish(self) - XCTAssertEqual(recordedTraces.value.count, 1) - XCTAssertNil(recordedTraces.value.first?.tags) + XCTAssertEqual(recordedTags.value.count, 1) + XCTAssertNil(recordedTags.value.first!) } func testLateTagIsIgnored() { - let recordedTraces = LockIsolated([Trace]()) - let uploader = UploadClient( - record: { trace in recordedTraces.withValue { $0.append(trace) } }, - waitForUploads: { _ in } - ) - let tracer = Tracer( - startSpan: { section, _ in section as AnyHashable }, - endSpan: { id in Trace.Span(section: id as! String) } - ) - let observer = TestObserver(logger: nil, tracer: tracer, uploader: uploader) + let recordedTags = LockIsolated([[String: String]?]()) + let telemetry = self.telemetry { tags in recordedTags.withValue { $0.append(tags) } } + let observer = TestObserver(logger: nil, telemetry: telemetry) observer.testCaseWillStart(self) observer.setTag(for: self, key: "before", value: "yes") @@ -61,7 +40,16 @@ final class TestObserverTests: XCTestCase { // Tag set after finish should be ignored observer.setTag(for: self, key: "after", value: "should-not-appear") - XCTAssertEqual(recordedTraces.value.count, 1) - XCTAssertEqual(recordedTraces.value.first?.tags, ["before": "yes"]) + XCTAssertEqual(recordedTags.value.count, 1) + XCTAssertEqual(recordedTags.value.first!, ["before": "yes"]) + } + + private func telemetry(onFinish: @escaping ([String: String]?) -> Void) -> TelemetryClient { + TelemetryClient( + start: { $0.id }, + annotate: { _, _ in }, + finish: { _, _, tags in onFinish(tags) }, + flush: {} + ) } } diff --git a/test-collector-swift/Tests/CoreTests/TestResultsTests.swift b/test-collector-swift/Tests/CoreTests/TestResultsTests.swift deleted file mode 100644 index 24b28f23..00000000 --- a/test-collector-swift/Tests/CoreTests/TestResultsTests.swift +++ /dev/null @@ -1,177 +0,0 @@ -@testable import Core -import XCTest - -final class TestResultsTests: XCTestCase { - func testJSONEncoding() throws { - let testSuccess = TestState( - id: UUID(uuidString: "00000000-0000-0000-0000-000000000000")!, - className: "TestResultsTests", - testName: "testSuccess", - result: .passed, - issues: [], - expectedFailures: [] - ) - - let testFailure = TestState( - id: UUID(uuidString: "00000000-0000-0000-0000-000000000001")!, - className: "TestResultsTests", - testName: "testFailure", - result: .failed, - issues: [ - .init( - compactDescription: "The test failed", - description: """ - The thing you expected to happen didn't happen. - I wish I could provide you with more information. - """, - sourceCodeContext: .init( - callStack: [ - .init( - address: 1, - symbolInfo: .init( - imageName: "Library", - symbolName: "Foo.bar()" - ) - ), - .init( - address: 0, - symbolInfo: .init( - imageName: "CoreTests", - symbolName: "TestResultsTests.testJSONEncoding()" - ) - ), - ], - location: .init( - filePath: "/foo/bar", - fileName: "bar.swift", - line: 10 - ) - ) - ), - ], - expectedFailures: [] - ) - - let testMultipleFailures = TestState( - id: UUID(uuidString: "00000000-0000-0000-0000-000000000002")!, - className: "TestResultsTests", - testName: "testMultipleFailures", - result: .failed, - issues: [ - .init(compactDescription: "First failure", description: "Failure 1", sourceCodeContext: .init( - location: .init( - filePath: "/foo/first", - fileName: "first.swift", - line: 50 - ) - )), - .init(compactDescription: "Second failure", description: "Failure 2", sourceCodeContext: .init( - location: .init( - filePath: "/foo/second", - fileName: "second.swift", - line: 60 - ) - )), - .init(compactDescription: "Third failure", description: "Failure 3", sourceCodeContext: .init( - location: .init( - filePath: "/foo/third", - fileName: "third.swift", - line: 70 - ) - )), - ], - expectedFailures: [] - ) - - let testResults = TestResults.json( - runEnv: RunEnvironment(key: "test"), - tags: ["host.arch": "arm64"], - data: [ - .init(test: testSuccess, span: .init(section: "span0"), tags: ["suite": "smoke"]), - .init(test: testFailure, span: .init(section: "span1")), - .init(test: testMultipleFailures, span: .init(section: "span2")), - ] - ) - - let data = try JSONEncoder().encode(testResults) - - let json = try JSONSerialization.jsonObject(with: data) - - XCTAssertEqual( - json as? NSDictionary, - [ - "format": "json", - "run_env": ["key": "test"], - "tags": ["host.arch": "arm64"], - "data": [ - NSDictionary( - dictionary: [ - "id": "00000000-0000-0000-0000-000000000000", - "scope": "TestResultsTests", - "name": "testSuccess", - "result": "passed", - "tags": ["suite": "smoke"], - "failure_expanded": NSArray(), - "history": NSDictionary( - dictionary: [ - "section": "span0", - "detail": NSDictionary(), - "children": NSArray(), - ] - ), - ] - ), - [ - "id": "00000000-0000-0000-0000-000000000001", - "scope": "TestResultsTests", - "location": "bar.swift:10", - "file_name": "bar.swift", - "name": "testFailure", - "result": "failed", - "failure_reason": "The test failed", - "failure_expanded": [ - [ - "backtrace": [ - "0 Foo.bar()", - "1 TestResultsTests.testJSONEncoding()", - ], - "expanded": [ - "The thing you expected to happen didn't happen.", - "I wish I could provide you with more information.", - ], - ], - ], - "history": NSDictionary( - dictionary: [ - "section": "span1", - "detail": NSDictionary(), - "children": NSArray(), - ] - ), - ], - [ - "id": "00000000-0000-0000-0000-000000000002", - "scope": "TestResultsTests", - "location": "first.swift:50", - "file_name": "first.swift", - "name": "testMultipleFailures", - "result": "failed", - "failure_reason": "3 failures: First failure, Second failure, Third failure", - "failure_expanded": [ - ["backtrace": NSArray(), "expanded": ["Failure 1"]], - ["backtrace": NSArray(), "expanded": ["Failure 2"]], - ["backtrace": NSArray(), "expanded": ["Failure 3"]], - ], - "history": NSDictionary( - dictionary: [ - "section": "span2", - "detail": NSDictionary(), - "children": NSArray(), - ] - ), - ], - ], - ] - ) - } -} diff --git a/test-collector-swift/Tests/CoreTests/UploadClientTests.swift b/test-collector-swift/Tests/CoreTests/UploadClientTests.swift deleted file mode 100644 index 1841a066..00000000 --- a/test-collector-swift/Tests/CoreTests/UploadClientTests.swift +++ /dev/null @@ -1,140 +0,0 @@ -@testable import Core -import XCTest - -#if canImport(FoundationNetworking) -import FoundationNetworking -#endif - -final class UploadClientTests: XCTestCase { - func testWaitSynchronouslyForUploads() throws { - let uploadCompleted = self.expectation(description: "upload completed") - let uploadClient = UploadClient.live( - api: .fulfill(uploadCompleted, after: 0.5), - runEnvironment: EnvironmentValues().runEnvironment() - ) - - uploadClient.record(trace: .mock()) - uploadClient.waitForUploads() - - self.wait(for: [uploadCompleted], timeout: 0) - } - - func testWaitShouldTimeout() throws { - let uploadCompleted = self.expectation(description: "upload completed") - uploadCompleted.isInverted = true - let uploadClient = UploadClient.live( - api: .fulfill(uploadCompleted, after: 0.5), - runEnvironment: EnvironmentValues().runEnvironment() - ) - - uploadClient.record(trace: .mock()) - uploadClient.waitForUploads(timeout: 0.1) - - self.wait(for: [uploadCompleted], timeout: 0) - } - - func testFailureResponseLogsError() throws { - let errorMessage = LockIsolated("") - let logger = Logger(logLevel: .error) { errorMessage.setValue($0) } - - let data = try JSONEncoder().encode(UploadFailureResponse(message: "Something went wrong")) - let api = ApiClient { _ in (data, .stub(status: 500)) } - - let uploadClient = UploadClient.live( - api: api, - runEnvironment: EnvironmentValues().runEnvironment(), - logger: logger - ) - - uploadClient.record(trace: .mock()) - - uploadClient.waitForUploads() - logger.waitForLogs() - - // this varies e.g. macOS “internal server error” vs linux “Internal Server Error” - let statusName = HTTPURLResponse.localizedString(forStatusCode: 500) - XCTAssertEqual(errorMessage.value, "[BuildkiteTestCollector] error: Unexpected HTTP 500 \(statusName), Something went wrong") - } - - func testUploadsInBatchesOf5000ByDefault() throws { - let testResults = LockIsolated([TestResults]()) - - let api = ApiClient { route in - if case let .upload(results) = route { - testResults.withValue { $0.append(results) } - } - return (Data(), .stub()) - } - - let uploadTasks = DispatchGroup() - - let uploadClient = UploadClient.live( - api: api, - runEnvironment: EnvironmentValues().runEnvironment(), - group: uploadTasks - ) - - // Record 4999 traces - for id in 1...4999 { - uploadClient.record(trace: .mock(id: "\(id)")) - } - - // Wait to make sure no uploads were started - XCTAssertEqual(uploadTasks.wait(timeout: 0.1), .success) - XCTAssertEqual(testResults.count, 0) - - // Record one more trace to trigger the first batch of 5000 - uploadClient.record(trace: .mock(id: "5000")) - - // Wait for upload to complete - XCTAssertEqual(uploadTasks.wait(timeout: 0.1), .success) - XCTAssertEqual(testResults.count, 1) - - // Send the remaining traces - for id in 5001...12345 { - let trace = Trace(id: "\(id)", history: .init(section: "section")) - uploadClient.record(trace: trace) - } - - // A second batch will be sent for ids 5001...10000 - XCTAssertEqual(uploadTasks.wait(timeout: 0.1), .success) - XCTAssertEqual(testResults.count, 2) - - // Uploads any remaining traces regardless of batch size - uploadClient.waitForUploads() - - XCTAssertEqual(testResults.count, 3) - XCTAssertEqual(testResults[0].data.map(\.id), (1...5000).map { "\($0)" }) - XCTAssertEqual(testResults[1].data.map(\.id), (5001...10000).map { "\($0)" }) - XCTAssertEqual(testResults[2].data.map(\.id), (10001...12345).map { "\($0)" }) - } - - func testUploadIncludesTags() throws { - let testResults = LockIsolated([TestResults]()) - - let api = ApiClient { route in - if case let .upload(results) = route { - testResults.withValue { $0.append(results) } - } - return (Data(), .stub()) - } - - let uploadClient = UploadClient.live( - api: api, - runEnvironment: EnvironmentValues().runEnvironment(), - tags: ["host.arch": "arm64", "cloud.region": "us-east-1"] - ) - - uploadClient.record(trace: .mock()) - uploadClient.waitForUploads() - - XCTAssertEqual(testResults.count, 1) - XCTAssertEqual(testResults[0].tags, ["host.arch": "arm64", "cloud.region": "us-east-1"]) - } -} - -extension Trace { - fileprivate static func mock(id: String = "id") -> Self { - Trace(id: id, history: .init(section: "stub")) - } -} From f83192800124daed302401a5a6526181ba9e7b42 Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 17:21:01 +0930 Subject: [PATCH 03/20] docs(swift): record OTLP-only execution export Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- test-collector-swift/CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test-collector-swift/CHANGELOG.md b/test-collector-swift/CHANGELOG.md index 2de16b88..98fd8331 100644 --- a/test-collector-swift/CHANGELOG.md +++ b/test-collector-swift/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## Unreleased + +- Submit XCTest executions as OpenTelemetry root spans instead of proprietary JSON uploads +- Export run and execution tags as `buildkite.tag.*` attributes +- Support standard OTLP/HTTP exporter configuration and child-span forwarding +- Require Swift 6 and macOS 12 or newer + ## 0.6.0 - Add tagging support at upload and execution levels From 8fdc6e4949de266e630057228b5006fc1f96b33a Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 17:40:31 +0930 Subject: [PATCH 04/20] fix(swift): support the Swift 6.0 CI toolchain Pin OpenTelemetry to its Swift 6.0-compatible releases and constrain swift-metrics below the Swift 6.1 tools-version bump. Own failed root-span requeueing in the synchronous execution processor instead of depending on the newer exporter API. Retry pending roots with the next execution or final flush, and cover the retry batch behavior. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- test-collector-swift/Package.swift | 6 ++- test-collector-swift/README.md | 2 +- .../SynchronousExecutionSpanProcessor.swift | 13 +++++- .../Core/Telemetry/TelemetryClient+Live.swift | 3 +- .../CoreTests/TelemetryClientTests.swift | 40 +++++++++++++++++++ 5 files changed, 58 insertions(+), 6 deletions(-) diff --git a/test-collector-swift/Package.swift b/test-collector-swift/Package.swift index 12f6c696..413c0bf4 100644 --- a/test-collector-swift/Package.swift +++ b/test-collector-swift/Package.swift @@ -14,8 +14,10 @@ let package = Package( .library(name: "BuildkiteTestCollector", targets: ["BuildkiteTestCollector"]) ], dependencies: [ - .package(url: "https://github.com/open-telemetry/opentelemetry-swift", from: "2.5.1"), - .package(url: "https://github.com/open-telemetry/opentelemetry-swift-core", from: "2.5.1") + .package(url: "https://github.com/open-telemetry/opentelemetry-swift", exact: "2.4.1"), + .package(url: "https://github.com/open-telemetry/opentelemetry-swift-core", exact: "2.4.2"), + // Keep Swift 6.0 compatibility; swift-metrics 2.11 requires Swift 6.1. + .package(url: "https://github.com/apple/swift-metrics", exact: "2.10.1") ], targets: [ .target(name: "BuildkiteTestCollector", dependencies: ["Core", "Loader"]), diff --git a/test-collector-swift/README.md b/test-collector-swift/README.md index de63bcea..de22255a 100644 --- a/test-collector-swift/README.md +++ b/test-collector-swift/README.md @@ -203,7 +203,7 @@ replacing its sampler or exporters. Otherwise, it installs a provider so spans created under a test can be exported as execution children. Finishing a test waits for its root span to be accepted by the configured OTLP -endpoint. A failed root export stays in the process's in-memory exporter queue +endpoint. A failed root export stays in the collector's in-memory queue and is retried by the next execution or the bundle-end flush. When the endpoint is the Test Engine Client relay, acceptance transfers the span to the longer-lived parent process, which protects completed executions from an XCTest diff --git a/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift b/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift index e0530475..669b82d3 100644 --- a/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift +++ b/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift @@ -9,6 +9,7 @@ final class SynchronousExecutionSpanProcessor: SpanProcessor { private let exporter: any SpanExporter private let exporterLock = NSLock() private let logger: Logger? + private var pending = [SpanData]() init(exporter: any SpanExporter, logger: Logger?) { self.exporter = exporter @@ -22,8 +23,10 @@ final class SynchronousExecutionSpanProcessor: SpanProcessor { defer { self.exporterLock.unlock() } let name = span.getAttributes()[BuildkiteTelemetryAttribute.testName]?.description ?? span.name - switch self.exporter.export(spans: [span.toSpanData()]) { + self.pending.append(span.toSpanData()) + switch self.exporter.export(spans: self.pending) { case .success: + self.pending.removeAll() self.logger?.debug("Exported OpenTelemetry test execution: \(name)") case .failure: self.logger?.error( @@ -46,6 +49,14 @@ final class SynchronousExecutionSpanProcessor: SpanProcessor { } private func flush(timeout: TimeInterval?) { + if !self.pending.isEmpty { + switch self.exporter.export(spans: self.pending, explicitTimeout: timeout) { + case .success: + self.pending.removeAll() + case .failure: + self.logger?.error("OpenTelemetry export failed while retrying pending test executions") + } + } if case .failure = self.exporter.flush(explicitTimeout: timeout) { self.logger?.error("OpenTelemetry export failed while flushing pending test executions") } diff --git a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift index 57957746..6be1fda8 100644 --- a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift +++ b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift @@ -151,8 +151,7 @@ extension TelemetryClient { OtlpHttpTraceExporter( endpoint: configuration.endpoint, config: exporterConfiguration, - envVarHeaders: nil, - requeueOnFailure: true + envVarHeaders: nil ) as any SpanExporter } let live = LiveTelemetryClient( diff --git a/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift index cb0c5296..ef31ca19 100644 --- a/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift +++ b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift @@ -210,6 +210,31 @@ final class TelemetryClientTests: XCTestCase { XCTAssertEqual(configuration.header(named: "x-test"), "value/one") } + func testRetriesFailedExecutionWithTheNextExecution() throws { + let rootExporter = FailOnceExporter() + let client = try XCTUnwrap(TelemetryClient.live( + environment: self.configuredEnvironment, + uploadTags: [:], + logger: nil, + rootExporter: rootExporter, + childExporter: InMemoryExporter() + )) + + for name in ["testOne", "testTwo"] { + var test = TestState(id: UUID(), className: "RetryTests", testName: name) + let executionID = client.startExecution(test) + test.result = .passed + client.finishExecution(executionID, test: test, tags: nil) + } + + XCTAssertEqual(rootExporter.exports.count, 2) + XCTAssertEqual(rootExporter.exports[0].map(\.name), ["test.execution"]) + XCTAssertEqual( + rootExporter.exports[1].map(\.attributes[BuildkiteTelemetryAttribute.testName]), + [.string("testOne"), .string("testTwo")] + ) + } + private var configuredEnvironment: EnvironmentValues { EnvironmentValues( values: [ @@ -233,3 +258,18 @@ private extension CollectorOTLPConfiguration { self.headers.first { $0.0.caseInsensitiveCompare(name) == .orderedSame }?.1 } } + +private final class FailOnceExporter: SpanExporter, @unchecked Sendable { + var exports = [[SpanData]]() + + func export(spans: [SpanData], explicitTimeout: TimeInterval?) -> SpanExporterResultCode { + self.exports.append(spans) + return self.exports.count == 1 ? .failure : .success + } + + func flush(explicitTimeout: TimeInterval?) -> SpanExporterResultCode { + .success + } + + func shutdown(explicitTimeout: TimeInterval?) {} +} From 9ce042b390e91900a50854d7706ac3aecc6c3bf2 Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 17:47:35 +0930 Subject: [PATCH 05/20] fix(swift): use Swift 6.0-compatible OTel releases OpenTelemetry 2.4 requires swift-metrics 2.10, whose package manifest needs Swift 6.1. Pin the last compatible OpenTelemetry pair and swift-metrics 2.8 so the collector remains buildable on its declared Swift 6.0 floor. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- test-collector-swift/Package.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test-collector-swift/Package.swift b/test-collector-swift/Package.swift index 413c0bf4..7cb5e304 100644 --- a/test-collector-swift/Package.swift +++ b/test-collector-swift/Package.swift @@ -14,10 +14,10 @@ let package = Package( .library(name: "BuildkiteTestCollector", targets: ["BuildkiteTestCollector"]) ], dependencies: [ - .package(url: "https://github.com/open-telemetry/opentelemetry-swift", exact: "2.4.1"), - .package(url: "https://github.com/open-telemetry/opentelemetry-swift-core", exact: "2.4.2"), - // Keep Swift 6.0 compatibility; swift-metrics 2.11 requires Swift 6.1. - .package(url: "https://github.com/apple/swift-metrics", exact: "2.10.1") + .package(url: "https://github.com/open-telemetry/opentelemetry-swift", exact: "2.3.0"), + .package(url: "https://github.com/open-telemetry/opentelemetry-swift-core", exact: "2.3.0"), + // Keep Swift 6.0 compatibility; swift-metrics 2.9 requires Swift 6.1. + .package(url: "https://github.com/apple/swift-metrics", exact: "2.8.0") ], targets: [ .target(name: "BuildkiteTestCollector", dependencies: ["Core", "Loader"]), From 7a0ebc0c5fb025817a623d8711770d61111b9d1e Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 17:49:22 +0930 Subject: [PATCH 06/20] fix(swift): propagate the Swift 6.0 metrics pin Attach CoreMetrics to the collector target so downstream SwiftPM consumers inherit the swift-metrics 2.8 constraint instead of resolving the Swift 6.1-only 2.9+ manifests. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- test-collector-swift/Package.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test-collector-swift/Package.swift b/test-collector-swift/Package.swift index 7cb5e304..b814cfe5 100644 --- a/test-collector-swift/Package.swift +++ b/test-collector-swift/Package.swift @@ -25,7 +25,8 @@ let package = Package( name: "Core", dependencies: [ .product(name: "OpenTelemetryProtocolExporterHTTP", package: "opentelemetry-swift"), - .product(name: "OpenTelemetrySdk", package: "opentelemetry-swift-core") + .product(name: "OpenTelemetrySdk", package: "opentelemetry-swift-core"), + .product(name: "CoreMetrics", package: "swift-metrics") ] ), .target(name: "Loader"), From d6e5fbdc707c9c36331a9e4f82ff6765d30a165e Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 17:54:14 +0930 Subject: [PATCH 07/20] fix(swift): wait for OTLP endpoint acceptance OpenTelemetry 2.3's HTTP exporter returns before its URLSession request completes, and its flush does not wait for in-flight requests. Wrap its HTTP client with a synchronous transport and only report success after a 2xx response so XCTest cannot move on or exit before the relay owns a completed execution. Keep failed roots in the collector's existing retry queue and prevent the upstream exporter from retaining a duplicate copy. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- .../Core/Telemetry/TelemetryClient+Live.swift | 93 ++++++++++++++++++- 1 file changed, 90 insertions(+), 3 deletions(-) diff --git a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift index 6be1fda8..6938ea41 100644 --- a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift +++ b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift @@ -4,6 +4,10 @@ import OpenTelemetryProtocolExporterCommon import OpenTelemetryProtocolExporterHttp import OpenTelemetrySdk +#if canImport(FoundationNetworking) +import FoundationNetworking +#endif + #if canImport(Darwin) import Darwin #elseif canImport(Glibc) @@ -128,6 +132,90 @@ struct CollectorOTLPConfiguration { } } +private final class SynchronousOtlpTraceExporter: SpanExporter, @unchecked Sendable { + private let exporter: OtlpHttpTraceExporter + private let httpClient: SynchronousHTTPClient + + init(endpoint: URL, configuration: OtlpConfiguration) { + let httpClient = SynchronousHTTPClient(timeout: configuration.timeout) + self.httpClient = httpClient + self.exporter = OtlpHttpTraceExporter( + endpoint: endpoint, + config: configuration, + httpClient: httpClient, + envVarHeaders: nil + ) + } + + func export(spans: [SpanData], explicitTimeout: TimeInterval?) -> SpanExporterResultCode { + self.httpClient.prepare() + _ = self.exporter.export(spans: spans, explicitTimeout: explicitTimeout) + return self.httpClient.result + } + + func flush(explicitTimeout: TimeInterval?) -> SpanExporterResultCode { + self.exporter.flush(explicitTimeout: explicitTimeout) + } + + func shutdown(explicitTimeout: TimeInterval?) { + self.exporter.shutdown(explicitTimeout: explicitTimeout) + } +} + +private final class SynchronousHTTPClient: HTTPClient { + private let lock = NSLock() + private let session: URLSession + private var wasAccepted: Bool? + + init(timeout: TimeInterval) { + let configuration = URLSessionConfiguration.ephemeral + configuration.timeoutIntervalForRequest = timeout + configuration.timeoutIntervalForResource = timeout + configuration.urlCache = nil + self.session = URLSession(configuration: configuration) + } + + func prepare() { + self.lock.lock() + self.wasAccepted = nil + self.lock.unlock() + } + + var result: SpanExporterResultCode { + self.lock.lock() + defer { self.lock.unlock() } + return self.wasAccepted == true ? .success : .failure + } + + func send( + request: URLRequest, + completion: @escaping (Result) -> Void + ) { + let semaphore = DispatchSemaphore(value: 0) + let task = self.session.dataTask(with: request) { [weak self] _, response, error in + let httpResponse = response as? HTTPURLResponse + let accepted = error == nil + && httpResponse.map { (200..<300).contains($0.statusCode) } == true + + self?.lock.lock() + self?.wasAccepted = accepted + self?.lock.unlock() + + // The collector owns failed-span requeueing. Always report transport + // completion to the upstream exporter so it does not queue a second copy. + completion(.success(httpResponse ?? HTTPURLResponse( + url: request.url!, + statusCode: 500, + httpVersion: nil, + headerFields: nil + )!)) + semaphore.signal() + } + task.resume() + semaphore.wait() + } +} + extension TelemetryClient { static func live( environment: EnvironmentValues, @@ -148,10 +236,9 @@ extension TelemetryClient { exportAsJson: false ) let makeExporter = { - OtlpHttpTraceExporter( + SynchronousOtlpTraceExporter( endpoint: configuration.endpoint, - config: exporterConfiguration, - envVarHeaders: nil + configuration: exporterConfiguration ) as any SpanExporter } let live = LiveTelemetryClient( From 302147739a326c12be3060783e12af91fe3d181e Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 18:00:11 +0930 Subject: [PATCH 08/20] Keep Swift 6.0 dependency compatibility Pin OpenTelemetry to the last release before its exact gRPC dependency raised the tools version to Swift 6.1. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- test-collector-swift/Package.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test-collector-swift/Package.swift b/test-collector-swift/Package.swift index b814cfe5..9761f38c 100644 --- a/test-collector-swift/Package.swift +++ b/test-collector-swift/Package.swift @@ -14,8 +14,9 @@ let package = Package( .library(name: "BuildkiteTestCollector", targets: ["BuildkiteTestCollector"]) ], dependencies: [ - .package(url: "https://github.com/open-telemetry/opentelemetry-swift", exact: "2.3.0"), - .package(url: "https://github.com/open-telemetry/opentelemetry-swift-core", exact: "2.3.0"), + // OpenTelemetry 2.2.1 and later require a Swift 6.1-only grpc-swift release. + .package(url: "https://github.com/open-telemetry/opentelemetry-swift", exact: "2.2.0"), + .package(url: "https://github.com/open-telemetry/opentelemetry-swift-core", exact: "2.2.0"), // Keep Swift 6.0 compatibility; swift-metrics 2.9 requires Swift 6.1. .package(url: "https://github.com/apple/swift-metrics", exact: "2.8.0") ], From 67549ecbff5ee895c70c44efc2ab8f5d5947727f Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 18:13:45 +0930 Subject: [PATCH 09/20] Support Swift 5.10 consumers Lower the package manifest and metrics dependency requirements so the collector can be resolved by Xcode 15.4 while continuing to compile its sources in Swift 5 language mode. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- test-collector-swift/Package.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test-collector-swift/Package.swift b/test-collector-swift/Package.swift index 9761f38c..46b0a0e9 100644 --- a/test-collector-swift/Package.swift +++ b/test-collector-swift/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 6.0 +// swift-tools-version: 5.10 import PackageDescription @@ -17,8 +17,8 @@ let package = Package( // OpenTelemetry 2.2.1 and later require a Swift 6.1-only grpc-swift release. .package(url: "https://github.com/open-telemetry/opentelemetry-swift", exact: "2.2.0"), .package(url: "https://github.com/open-telemetry/opentelemetry-swift-core", exact: "2.2.0"), - // Keep Swift 6.0 compatibility; swift-metrics 2.9 requires Swift 6.1. - .package(url: "https://github.com/apple/swift-metrics", exact: "2.8.0") + // Keep Swift 5.10 compatibility; swift-metrics 2.8 requires Swift 6.0. + .package(url: "https://github.com/apple/swift-metrics", exact: "2.7.1") ], targets: [ .target(name: "BuildkiteTestCollector", dependencies: ["Core", "Loader"]), @@ -41,5 +41,5 @@ let package = Package( ] ) ], - swiftLanguageModes: [.v5] + swiftLanguageVersions: [.v5] ) From afe4d3dcabdf859c390676fe18ea1f3d9f7d2630 Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 18:40:56 +0930 Subject: [PATCH 10/20] Harden Swift OTLP integration across platforms Keep application spans attached to XCTest execution roots on Linux by installing an imperative execution-aware context manager, while preserving closure-scoped OpenTelemetry context. Reattach the forwarding processor at test start so application tracer providers registered after collector load are also covered. Do not synthesize Buildkite credentials for standard OTLP endpoints; only the default and collector-specific trusted endpoint receive the suite token and run key. Preserve custom run-environment overrides and execution-name affixes when translating the former upload model into OTLP attributes. Exercise provider replacement, Linux child forwarding, endpoint trust, custom overrides, and name affixes. Correct the documented Swift floor to 5.10. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- test-collector-swift/CHANGELOG.md | 2 +- test-collector-swift/README.md | 6 +- .../Models/Environment/RunEnvironment.swift | 32 +++++ .../Telemetry/ExecutionContextManager.swift | 96 +++++++++++++++ .../Core/Telemetry/TelemetryClient+Live.swift | 113 ++++++++++++++---- .../CoreTests/TelemetryClientTests.swift | 69 ++++++++++- 6 files changed, 287 insertions(+), 31 deletions(-) create mode 100644 test-collector-swift/Sources/Core/Telemetry/ExecutionContextManager.swift diff --git a/test-collector-swift/CHANGELOG.md b/test-collector-swift/CHANGELOG.md index 98fd8331..e4cf9515 100644 --- a/test-collector-swift/CHANGELOG.md +++ b/test-collector-swift/CHANGELOG.md @@ -5,7 +5,7 @@ - Submit XCTest executions as OpenTelemetry root spans instead of proprietary JSON uploads - Export run and execution tags as `buildkite.tag.*` attributes - Support standard OTLP/HTTP exporter configuration and child-span forwarding -- Require Swift 6 and macOS 12 or newer +- Require Swift 5.10 and macOS 12 or newer ## 0.6.0 diff --git a/test-collector-swift/README.md b/test-collector-swift/README.md index de22255a..d64cfbb2 100644 --- a/test-collector-swift/README.md +++ b/test-collector-swift/README.md @@ -64,8 +64,10 @@ export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL="http/protobuf" The signal-specific variables take precedence over `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_HEADERS`, and `OTEL_EXPORTER_OTLP_PROTOCOL`. A generic OTLP endpoint has `/v1/traces` -appended. `BUILDKITE_ANALYTICS_OTLP_ENDPOINT` remains available as a -collector-specific endpoint override. +appended. Standard endpoints receive only the explicitly configured OTLP +headers; the collector does not send the Buildkite suite token to them. +`BUILDKITE_ANALYTICS_OTLP_ENDPOINT` remains available as a trusted, +collector-specific endpoint override that receives the suite token and run key. ### Step 3 diff --git a/test-collector-swift/Sources/Core/Models/Environment/RunEnvironment.swift b/test-collector-swift/Sources/Core/Models/Environment/RunEnvironment.swift index 72885089..301b1a22 100644 --- a/test-collector-swift/Sources/Core/Models/Environment/RunEnvironment.swift +++ b/test-collector-swift/Sources/Core/Models/Environment/RunEnvironment.swift @@ -1,3 +1,5 @@ +import Foundation + /// A type containing information about the environment performing the test run. /// /// The only required property is `key` which represents a unique identifier for a test run. @@ -47,3 +49,33 @@ struct RunEnvironment: Equatable { /// is used, the custom environment value will take precedence. var customEnvironment: [String: AnyCodable]? } + +extension RunEnvironment { + static let customFieldNames = Set([ + "CI", "key", "url", "branch", "commit_sha", "number", "job_id", "message", "debug", + "execution_name_prefix", "execution_name_suffix", "version", "collector", + ]) + + mutating func applyCustomEnvironmentOverrides() { + guard let values = self.customEnvironment else { return } + + func string(_ key: String) -> String? { + guard let value = values[key], !(value.base is NSNull) else { return nil } + return value.base as? String ?? value.description + } + + self.ci = string("CI") ?? self.ci + self.key = string("key") ?? self.key + self.url = string("url") ?? self.url + self.branch = string("branch") ?? self.branch + self.commitSha = string("commit_sha") ?? self.commitSha + self.number = string("number") ?? self.number + self.jobId = string("job_id") ?? self.jobId + self.message = string("message") ?? self.message + self.debug = string("debug") ?? self.debug + self.executionNamePrefix = string("execution_name_prefix") ?? self.executionNamePrefix + self.executionNameSuffix = string("execution_name_suffix") ?? self.executionNameSuffix + self.version = string("version") ?? self.version + self.collector = string("collector") ?? self.collector + } +} diff --git a/test-collector-swift/Sources/Core/Telemetry/ExecutionContextManager.swift b/test-collector-swift/Sources/Core/Telemetry/ExecutionContextManager.swift new file mode 100644 index 00000000..d319a71e --- /dev/null +++ b/test-collector-swift/Sources/Core/Telemetry/ExecutionContextManager.swift @@ -0,0 +1,96 @@ +import Foundation +import OpenTelemetryApi + +#if !canImport(os.activity) +/// Preserves OpenTelemetry's closure-scoped context while allowing XCTest's +/// separate start and finish callbacks to keep an execution span active. +final class ExecutionContextManager: ContextManager { + static let shared = ExecutionContextManager() + + private final class EmptyValue: NSObject {} + + private final class ThreadContext: NSObject { + var values = [String: [AnyObject]]() + } + + private static let emptyValue = EmptyValue() + @TaskLocal private static var taskValues = [String: AnyObject]() + + private let executionSpan = LockIsolated<(any Span)?>(nil) + private let threadContextKey = "com.buildkite.test-collector-swift.otel-context" + + private init() {} + + func getCurrentContextValue(forKey key: OpenTelemetryContextKeys) -> AnyObject? { + if let value = self.threadContext(create: false)?.values[key.rawValue]?.last { + return value === Self.emptyValue ? nil : value + } + if let value = Self.taskValues[key.rawValue] { + return value === Self.emptyValue ? nil : value + } + if key == .span { + return self.executionSpan.withValue { $0 } + } + return nil + } + + func setCurrentContextValue(forKey key: OpenTelemetryContextKeys, value: AnyObject) { + self.threadContext(create: true)?.values[key.rawValue, default: []].append(value) + } + + func removeContextValue(forKey key: OpenTelemetryContextKeys, value: AnyObject) { + guard let context = self.threadContext(create: false), + var values = context.values[key.rawValue], + let index = values.lastIndex(where: { $0 === value }) + else { return } + + values.remove(at: index) + context.values[key.rawValue] = values.isEmpty ? nil : values + } + + func withCurrentContextValue( + forKey key: OpenTelemetryContextKeys, + value: AnyObject?, + _ operation: () throws -> T + ) rethrows -> T { + let scopedValue = value ?? Self.emptyValue + self.setCurrentContextValue(forKey: key, value: scopedValue) + defer { self.removeContextValue(forKey: key, value: scopedValue) } + return try operation() + } + + func withCurrentContextValue( + forKey key: OpenTelemetryContextKeys, + value: AnyObject?, + _ operation: () async throws -> T + ) async rethrows -> T { + var values = Self.taskValues + values[key.rawValue] = value ?? Self.emptyValue + return try await Self.$taskValues.withValue(values, operation: operation) + } + + func setExecutionSpan(_ span: any Span) { + self.executionSpan.withValue { $0 = span } + } + + func removeExecutionSpan(_ span: any Span) { + self.executionSpan.withValue { current in + if current === span { + current = nil + } + } + } + + private func threadContext(create: Bool) -> ThreadContext? { + let dictionary = Thread.current.threadDictionary + if let context = dictionary[self.threadContextKey] as? ThreadContext { + return context + } + guard create else { return nil } + + let context = ThreadContext() + dictionary[self.threadContextKey] = context + return context + } +} +#endif diff --git a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift index 6938ea41..38b42622 100644 --- a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift +++ b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift @@ -50,7 +50,8 @@ struct CollectorOTLPConfiguration { environment: EnvironmentValues, logger: Logger? ) { - let runEnvironment = environment.runEnvironment() + var runEnvironment = environment.runEnvironment() + runEnvironment.applyCustomEnvironmentOverrides() let rawHeaders = environment.otelTracesHeaders ?? environment.otelHeaders let hasExplicitConfiguration = environment.otelTracesEndpoint != nil || environment.otelEndpoint != nil @@ -68,25 +69,39 @@ struct CollectorOTLPConfiguration { } let endpoint: URL - if let value = environment.otelTracesEndpoint ?? environment.analyticsOTLPEndpoint { + let usesBuildkiteCredentials: Bool + if let value = environment.otelTracesEndpoint { guard let url = Self.absoluteURL(value) else { logger?.error("OpenTelemetry traces endpoint is not a valid absolute URL") return nil } endpoint = url + usesBuildkiteCredentials = false + } else if let value = environment.analyticsOTLPEndpoint { + guard let url = Self.absoluteURL(value) else { + logger?.error("OpenTelemetry traces endpoint is not a valid absolute URL") + return nil + } + endpoint = url + usesBuildkiteCredentials = true } else if let value = environment.otelEndpoint { guard let url = Self.absoluteURL(value) else { logger?.error("OpenTelemetry endpoint is not a valid absolute URL") return nil } endpoint = url.appendingPathComponent("v1/traces") + usesBuildkiteCredentials = false } else { endpoint = URL(string: TestCollector.endpoint)! + usesBuildkiteCredentials = true } - var headers = [("Buildkite-Tests-Run-Key", runEnvironment.key)] - if let token = environment.analyticsToken { - headers.append(("Authorization", "Token token=\"\(token)\"")) + var headers = [(String, String)]() + if usesBuildkiteCredentials { + headers.append(("Buildkite-Tests-Run-Key", runEnvironment.key)) + if let token = environment.analyticsToken { + headers.append(("Authorization", "Token token=\"\(token)\"")) + } } if let rawHeaders { @@ -262,13 +277,21 @@ extension TelemetryClient { private final class LiveTelemetryClient { private static let providerEnvironmentLock = NSLock() - private let childForwarder: ExecutionChildSpanProcessor? + private let attachedChildProviders = LockIsolated([TracerProviderSdk]()) + private let childForwarder: ExecutionChildSpanProcessor + private let childProviderResource: Resource private let executionProvider: TracerProviderSdk + private let executionNamePrefix: String? + private let executionNameSuffix: String? private let logger: Logger? private let registry = ExecutionTraceRegistry() + private let reportedUnsupportedChildProvider = LockIsolated(false) private let spans = LockIsolated([UUID: any Span]()) private let tracer: any OpenTelemetryApi.Tracer private let jobSpanContext: SpanContext? + #if !canImport(os.activity) + private let executionContextManager: ExecutionContextManager + #endif init( configuration: CollectorOTLPConfiguration, @@ -278,6 +301,8 @@ private final class LiveTelemetryClient { rootExporter: any SpanExporter, childExporter: any SpanExporter ) { + self.executionNamePrefix = configuration.runEnvironment.executionNamePrefix + self.executionNameSuffix = configuration.runEnvironment.executionNameSuffix self.logger = logger self.jobSpanContext = Self.jobSpanContext(environment: environment) @@ -286,6 +311,7 @@ private final class LiveTelemetryClient { environment: environment, uploadTags: uploadTags ) + self.childProviderResource = resource let rootProcessor = SynchronousExecutionSpanProcessor(exporter: rootExporter, logger: logger) self.executionProvider = Self.makeProvider( resource: resource, @@ -304,23 +330,22 @@ private final class LiveTelemetryClient { maxExportBatchSize: 512 ) let forwarder = ExecutionChildSpanProcessor(processor: childProcessor, registry: self.registry) - let globalProvider = OpenTelemetry.instance.tracerProvider - if globalProvider is DefaultTracerProvider { - let provider = Self.makeProvider(resource: resource, processor: forwarder) - OpenTelemetry.registerTracerProvider(tracerProvider: provider) - self.childForwarder = forwarder - } else if let provider = globalProvider as? TracerProviderSdk { - provider.addSpanProcessor(forwarder) - self.childForwarder = forwarder - } else { - logger?.error( - "OpenTelemetry child span export is disabled because the existing tracer provider cannot accept a span processor" - ) - self.childForwarder = nil - } + self.childForwarder = forwarder + #if !canImport(os.activity) + self.executionContextManager = ExecutionContextManager.shared + OpenTelemetry.registerContextManager(contextManager: self.executionContextManager) + #endif + self.attachChildForwarder() } func startExecution(_ test: TestState) -> UUID { + self.attachChildForwarder() + + let qualifiedTestName = [ + self.executionNamePrefix, + "\(test.className).\(test.testName)", + self.executionNameSuffix, + ].compactMap { $0 }.joined(separator: " ") let builder = self.tracer .spanBuilder(spanName: TelemetryValue.rootSpanName) .setNoParent() @@ -333,7 +358,7 @@ private final class LiveTelemetryClient { ) .setAttribute( key: SemanticConventions.Test.caseName.rawValue, - value: "\(test.className).\(test.testName)" + value: qualifiedTestName ) .setAttribute(key: SemanticConventions.Test.suiteName.rawValue, value: test.className) @@ -344,7 +369,11 @@ private final class LiveTelemetryClient { let span = builder.startSpan() self.registry.insert(span.context.traceId) self.spans.withValue { $0[test.id] = span } + #if canImport(os.activity) OpenTelemetry.instance.contextProvider.setActiveSpan(span) + #else + self.executionContextManager.setExecutionSpan(span) + #endif return test.id } @@ -403,14 +432,50 @@ private final class LiveTelemetryClient { } } + #if canImport(os.activity) OpenTelemetry.instance.contextProvider.removeContextForSpan(span) + #else + self.executionContextManager.removeExecutionSpan(span) + #endif self.registry.remove(span.context.traceId) span.end() } func forceFlush() { self.executionProvider.forceFlush(timeout: 30) - self.childForwarder?.forceFlush(timeout: 30) + self.childForwarder.forceFlush(timeout: 30) + } + + private func attachChildForwarder() { + let globalProvider = OpenTelemetry.instance.tracerProvider + if globalProvider is DefaultTracerProvider { + let provider = Self.makeProvider( + resource: self.childProviderResource, + processor: self.childForwarder + ) + self.attachedChildProviders.withValue { $0.append(provider) } + OpenTelemetry.registerTracerProvider(tracerProvider: provider) + } else if let provider = globalProvider as? TracerProviderSdk { + let needsProcessor = self.attachedChildProviders.withValue { providers in + guard !providers.contains(where: { $0 === provider }) else { return false } + providers.append(provider) + return true + } + if needsProcessor { + provider.addSpanProcessor(self.childForwarder) + } + } else { + let shouldReport = self.reportedUnsupportedChildProvider.withValue { reported in + guard !reported else { return false } + reported = true + return true + } + if shouldReport { + self.logger?.error( + "OpenTelemetry child span export is disabled because the existing tracer provider cannot accept a span processor" + ) + } + } } private static func resultStatus(_ result: TestResult) -> String { @@ -504,7 +569,9 @@ private final class LiveTelemetryClient { for (key, value) in uploadTags { set(BuildkiteTelemetryAttribute.tagPrefix + key, value) } - for (key, value) in run.customEnvironment ?? [:] where Self.validAttributeKey(key) { + for (key, value) in run.customEnvironment ?? [:] + where !RunEnvironment.customFieldNames.contains(key) && Self.validAttributeKey(key) + { attributes[key] = AttributeValue(value.base) ?? .string(value.description) } diff --git a/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift index ef31ca19..58e58660 100644 --- a/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift +++ b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift @@ -159,8 +159,7 @@ final class TelemetryClientTests: XCTestCase { XCTAssertEqual(span.links.only?.context.spanId.hexString, "00f067aa0ba902b7") } - #if canImport(os.activity) - func testExportsSpansCreatedUnderTheActiveExecutionAsChildren() throws { + func testExportsSpansFromAReplacementProviderAsExecutionChildren() throws { let rootExporter = InMemoryExporter() let childExporter = InMemoryExporter() let client = try XCTUnwrap(TelemetryClient.live( @@ -170,6 +169,10 @@ final class TelemetryClientTests: XCTestCase { rootExporter: rootExporter, childExporter: childExporter )) + let replacementProvider = TracerProviderBuilder() + .with(sampler: Samplers.alwaysOn) + .build() + OpenTelemetry.registerTracerProvider(tracerProvider: replacementProvider) var test = TestState(id: UUID(), className: "ChildTests", testName: "testChild") let executionID = client.startExecution(test) @@ -187,9 +190,8 @@ final class TelemetryClientTests: XCTestCase { XCTAssertEqual(exportedChild.traceId, root.traceId) XCTAssertEqual(exportedChild.parentSpanId, root.spanId) } - #endif - func testUsesStandardTraceEndpointAndHeadersOverCollectorToken() throws { + func testUsesOnlyConfiguredHeadersForStandardTraceEndpoint() throws { let configuration = try XCTUnwrap(CollectorOTLPConfiguration( environment: EnvironmentValues( values: [ @@ -205,11 +207,68 @@ final class TelemetryClientTests: XCTestCase { )) XCTAssertEqual(configuration.endpoint.absoluteString, "http://127.0.0.1:1234/v1/traces") - XCTAssertEqual(configuration.header(named: "Buildkite-Tests-Run-Key"), "run-key") + XCTAssertNil(configuration.header(named: "Buildkite-Tests-Run-Key")) XCTAssertEqual(configuration.header(named: "Authorization"), "Bearer relay-token") XCTAssertEqual(configuration.header(named: "x-test"), "value/one") } + func testUsesCollectorCredentialsForTrustedEndpointAndCustomRunKey() throws { + let configuration = try XCTUnwrap(CollectorOTLPConfiguration( + environment: EnvironmentValues( + values: [ + "BUILDKITE_ANALYTICS_KEY": "original-run-key", + "BUILDKITE_ANALYTICS_TOKEN": "collector-token", + "BUILDKITE_ANALYTICS_OTLP_ENDPOINT": "http://127.0.0.1:1234/v1/traces", + "BUILDKITE_ANALYTICS_ENVIRONMENT": #"{"key":"custom-run-key"}"#, + ], + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ), + logger: nil + )) + + XCTAssertEqual(configuration.header(named: "Buildkite-Tests-Run-Key"), "custom-run-key") + XCTAssertEqual( + configuration.header(named: "Authorization"), + #"Token token="collector-token""# + ) + } + + func testAppliesExecutionNameAffixesAndCustomRunEnvironmentOverrides() throws { + let rootExporter = InMemoryExporter() + let client = try XCTUnwrap(TelemetryClient.live( + environment: EnvironmentValues( + values: [ + "BUILDKITE_ANALYTICS_KEY": "original-run-key", + "BUILDKITE_ANALYTICS_TOKEN": "collector-token", + "BUILDKITE_ANALYTICS_BRANCH": "original-branch", + "BUILDKITE_ANALYTICS_EXECUTION_NAME_PREFIX": "[ios]", + "BUILDKITE_ANALYTICS_EXECUTION_NAME_SUFFIX": "[debug]", + "BUILDKITE_ANALYTICS_ENVIRONMENT": #"{"key":"custom-run-key","branch":"custom-branch","custom.flag":true}"#, + ], + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ), + uploadTags: [:], + logger: nil, + rootExporter: rootExporter, + childExporter: InMemoryExporter() + )) + var test = TestState(id: UUID(), className: "PaymentTests", testName: "testChargeCard") + + let executionID = client.startExecution(test) + test.result = .passed + client.finishExecution(executionID, test: test, tags: nil) + + let span = try XCTUnwrap(rootExporter.getFinishedSpanItems().only) + XCTAssertEqual(span.attributes["test.case.name"], .string("[ios] PaymentTests.testChargeCard [debug]")) + XCTAssertEqual(span.resource.attributes[BuildkiteTelemetryAttribute.runKey], .string("custom-run-key")) + XCTAssertEqual(span.resource.attributes["vcs.ref.head.name"], .string("custom-branch")) + XCTAssertEqual(span.resource.attributes["custom.flag"], .bool(true)) + XCTAssertNil(span.resource.attributes["key"]) + XCTAssertNil(span.resource.attributes["branch"]) + } + func testRetriesFailedExecutionWithTheNextExecution() throws { let rootExporter = FailOnceExporter() let client = try XCTUnwrap(TelemetryClient.live( From 78830abea13a5f64177663eba7adfde4a05956e2 Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 18:53:48 +0930 Subject: [PATCH 11/20] Handle late OpenTelemetry provider registration Preserve the platform's existing context manager while observing active-span reads, allowing the collector to attach execution forwarding to providers registered during test setup or the test body before their first span starts. Also correct the documented minimum Swift version to 5.10. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- test-collector-swift/README.md | 2 +- .../Telemetry/ExecutionContextManager.swift | 137 ++++++++++++------ .../Core/Telemetry/TelemetryClient+Live.swift | 19 +-- .../CoreTests/TelemetryClientTests.swift | 8 +- 4 files changed, 106 insertions(+), 60 deletions(-) diff --git a/test-collector-swift/README.md b/test-collector-swift/README.md index d64cfbb2..1d5d3808 100644 --- a/test-collector-swift/README.md +++ b/test-collector-swift/README.md @@ -11,7 +11,7 @@ root span. Sampled OpenTelemetry spans created by the code under test can appear as its children. Test execution data is submitted only through OTLP; the collector does not use the legacy proprietary execution upload API. -The collector requires Swift 6 or newer. Its macOS deployment target is macOS +The collector requires Swift 5.10 or newer. Its macOS deployment target is macOS 12 or newer. ## 👉 Installing diff --git a/test-collector-swift/Sources/Core/Telemetry/ExecutionContextManager.swift b/test-collector-swift/Sources/Core/Telemetry/ExecutionContextManager.swift index d319a71e..51777a3f 100644 --- a/test-collector-swift/Sources/Core/Telemetry/ExecutionContextManager.swift +++ b/test-collector-swift/Sources/Core/Telemetry/ExecutionContextManager.swift @@ -1,51 +1,65 @@ import Foundation import OpenTelemetryApi -#if !canImport(os.activity) /// Preserves OpenTelemetry's closure-scoped context while allowing XCTest's -/// separate start and finish callbacks to keep an execution span active. +/// separate start and finish callbacks to keep an execution span active. It +/// also observes span creation so providers registered during a test can be +/// configured before their first child span starts. final class ExecutionContextManager: ContextManager { - static let shared = ExecutionContextManager() - - private final class EmptyValue: NSObject {} - - private final class ThreadContext: NSObject { - var values = [String: [AnyObject]]() + private final class ThreadSuppression: NSObject { + var counts = [String: Int]() } - private static let emptyValue = EmptyValue() - @TaskLocal private static var taskValues = [String: AnyObject]() + @TaskLocal private static var suppressedKeys = Set() + private let contextReadHandler = LockIsolated<(() -> Void)?>(nil) + private let delegate: OpenTelemetryContextProvider private let executionSpan = LockIsolated<(any Span)?>(nil) - private let threadContextKey = "com.buildkite.test-collector-swift.otel-context" + private let threadSuppressionKey = "com.buildkite.test-collector-swift.otel-context-suppression" - private init() {} + init(delegate: OpenTelemetryContextProvider) { + self.delegate = delegate + } func getCurrentContextValue(forKey key: OpenTelemetryContextKeys) -> AnyObject? { - if let value = self.threadContext(create: false)?.values[key.rawValue]?.last { - return value === Self.emptyValue ? nil : value - } - if let value = Self.taskValues[key.rawValue] { - return value === Self.emptyValue ? nil : value - } if key == .span { - return self.executionSpan.withValue { $0 } + let handler = self.contextReadHandler.withValue { $0 } + handler?() + } + guard !self.isSuppressed(key) else { return nil } + + switch key { + case .span: + return self.delegate.activeSpan ?? self.executionSpan.withValue { $0 } + case .baggage: + return self.delegate.activeBaggage } - return nil } func setCurrentContextValue(forKey key: OpenTelemetryContextKeys, value: AnyObject) { - self.threadContext(create: true)?.values[key.rawValue, default: []].append(value) + switch key { + case .span: + if let span = value as? any Span { + self.delegate.setActiveSpan(span) + } + case .baggage: + if let baggage = value as? any Baggage { + self.delegate.setActiveBaggage(baggage) + } + } } func removeContextValue(forKey key: OpenTelemetryContextKeys, value: AnyObject) { - guard let context = self.threadContext(create: false), - var values = context.values[key.rawValue], - let index = values.lastIndex(where: { $0 === value }) - else { return } - - values.remove(at: index) - context.values[key.rawValue] = values.isEmpty ? nil : values + switch key { + case .span: + if let span = value as? any Span { + self.delegate.removeContextForSpan(span) + } + case .baggage: + if let baggage = value as? any Baggage { + self.delegate.removeContextForBaggage(baggage) + } + } } func withCurrentContextValue( @@ -53,10 +67,16 @@ final class ExecutionContextManager: ContextManager { value: AnyObject?, _ operation: () throws -> T ) rethrows -> T { - let scopedValue = value ?? Self.emptyValue - self.setCurrentContextValue(forKey: key, value: scopedValue) - defer { self.removeContextValue(forKey: key, value: scopedValue) } - return try operation() + switch (key, value) { + case let (.span, span as any SpanBase): + return try self.delegate.withActiveSpan(span, operation) + case let (.baggage, baggage as any Baggage): + return try self.delegate.withActiveBaggage(baggage, operation) + case (_, nil): + return try self.withSuppressedContext(forKey: key, operation) + default: + return try operation() + } } func withCurrentContextValue( @@ -64,16 +84,27 @@ final class ExecutionContextManager: ContextManager { value: AnyObject?, _ operation: () async throws -> T ) async rethrows -> T { - var values = Self.taskValues - values[key.rawValue] = value ?? Self.emptyValue - return try await Self.$taskValues.withValue(values, operation: operation) + switch (key, value) { + case let (.span, span as any SpanBase): + return try await self.delegate.withActiveSpan(span, operation) + case let (.baggage, baggage as any Baggage): + return try await self.delegate.withActiveBaggage(baggage, operation) + case (_, nil): + var keys = Self.suppressedKeys + keys.insert(key.rawValue) + return try await Self.$suppressedKeys.withValue(keys, operation: operation) + default: + return try await operation() + } } func setExecutionSpan(_ span: any Span) { self.executionSpan.withValue { $0 = span } + self.delegate.setActiveSpan(span) } func removeExecutionSpan(_ span: any Span) { + self.delegate.removeContextForSpan(span) self.executionSpan.withValue { current in if current === span { current = nil @@ -81,16 +112,38 @@ final class ExecutionContextManager: ContextManager { } } - private func threadContext(create: Bool) -> ThreadContext? { + func setContextReadHandler(_ handler: @escaping () -> Void) { + self.contextReadHandler.withValue { $0 = handler } + } + + private func isSuppressed(_ key: OpenTelemetryContextKeys) -> Bool { + if Self.suppressedKeys.contains(key.rawValue) { + return true + } + return (self.threadSuppression(create: false)?.counts[key.rawValue] ?? 0) > 0 + } + + private func withSuppressedContext( + forKey key: OpenTelemetryContextKeys, + _ operation: () throws -> T + ) rethrows -> T { + let suppression = self.threadSuppression(create: true)! + suppression.counts[key.rawValue, default: 0] += 1 + defer { + suppression.counts[key.rawValue, default: 0] -= 1 + } + return try operation() + } + + private func threadSuppression(create: Bool) -> ThreadSuppression? { let dictionary = Thread.current.threadDictionary - if let context = dictionary[self.threadContextKey] as? ThreadContext { - return context + if let suppression = dictionary[self.threadSuppressionKey] as? ThreadSuppression { + return suppression } guard create else { return nil } - let context = ThreadContext() - dictionary[self.threadContextKey] = context - return context + let suppression = ThreadSuppression() + dictionary[self.threadSuppressionKey] = suppression + return suppression } } -#endif diff --git a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift index 38b42622..82e1bb90 100644 --- a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift +++ b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift @@ -289,9 +289,7 @@ private final class LiveTelemetryClient { private let spans = LockIsolated([UUID: any Span]()) private let tracer: any OpenTelemetryApi.Tracer private let jobSpanContext: SpanContext? - #if !canImport(os.activity) private let executionContextManager: ExecutionContextManager - #endif init( configuration: CollectorOTLPConfiguration, @@ -331,10 +329,13 @@ private final class LiveTelemetryClient { ) let forwarder = ExecutionChildSpanProcessor(processor: childProcessor, registry: self.registry) self.childForwarder = forwarder - #if !canImport(os.activity) - self.executionContextManager = ExecutionContextManager.shared + self.executionContextManager = ExecutionContextManager( + delegate: OpenTelemetry.instance.contextProvider + ) + self.executionContextManager.setContextReadHandler { [weak self] in + self?.attachChildForwarder() + } OpenTelemetry.registerContextManager(contextManager: self.executionContextManager) - #endif self.attachChildForwarder() } @@ -369,11 +370,7 @@ private final class LiveTelemetryClient { let span = builder.startSpan() self.registry.insert(span.context.traceId) self.spans.withValue { $0[test.id] = span } - #if canImport(os.activity) - OpenTelemetry.instance.contextProvider.setActiveSpan(span) - #else self.executionContextManager.setExecutionSpan(span) - #endif return test.id } @@ -432,11 +429,7 @@ private final class LiveTelemetryClient { } } - #if canImport(os.activity) - OpenTelemetry.instance.contextProvider.removeContextForSpan(span) - #else self.executionContextManager.removeExecutionSpan(span) - #endif self.registry.remove(span.context.traceId) span.end() } diff --git a/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift index 58e58660..9a676213 100644 --- a/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift +++ b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift @@ -159,7 +159,7 @@ final class TelemetryClientTests: XCTestCase { XCTAssertEqual(span.links.only?.context.spanId.hexString, "00f067aa0ba902b7") } - func testExportsSpansFromAReplacementProviderAsExecutionChildren() throws { + func testExportsSpansFromAProviderRegisteredAfterExecutionStarts() throws { let rootExporter = InMemoryExporter() let childExporter = InMemoryExporter() let client = try XCTUnwrap(TelemetryClient.live( @@ -169,13 +169,13 @@ final class TelemetryClientTests: XCTestCase { rootExporter: rootExporter, childExporter: childExporter )) + var test = TestState(id: UUID(), className: "ChildTests", testName: "testChild") + + let executionID = client.startExecution(test) let replacementProvider = TracerProviderBuilder() .with(sampler: Samplers.alwaysOn) .build() OpenTelemetry.registerTracerProvider(tracerProvider: replacementProvider) - var test = TestState(id: UUID(), className: "ChildTests", testName: "testChild") - - let executionID = client.startExecution(test) let child = OpenTelemetry.instance.tracerProvider .get(instrumentationName: "example-app") .spanBuilder(spanName: "database.query") From f0984f49e156b95f10338d7a263709e4c5751bce Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 18:59:54 +0930 Subject: [PATCH 12/20] Test the supported Swift 5.10 toolchain Run collector tests and the release build under both Swift 5.10.1 and 6.0.3 so the documented minimum remains continuously verified. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- test-collector-swift/.buildkite/pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/test-collector-swift/.buildkite/pipeline.yml b/test-collector-swift/.buildkite/pipeline.yml index ad19354b..99c54a59 100644 --- a/test-collector-swift/.buildkite/pipeline.yml +++ b/test-collector-swift/.buildkite/pipeline.yml @@ -23,6 +23,7 @@ steps: - docker#v3.7.0: image: "swift:{{matrix}}" matrix: + - "5.10.1" - "6.0.3" - label: ":swift: Tests (macOS, {{matrix}})" From 5af9ae6cad2aa7d8de587f6d46642c2000a84074 Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 22:02:35 +0930 Subject: [PATCH 13/20] Prepare Swift collector for 2.0.0 beta Set the reported collector version to 2.0.0-beta.1 and record that release target in the changelog and README without changing the currently published installation version. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- test-collector-swift/CHANGELOG.md | 4 ++-- test-collector-swift/README.md | 7 ++++++- test-collector-swift/Sources/Core/TestCollector.swift | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/test-collector-swift/CHANGELOG.md b/test-collector-swift/CHANGELOG.md index e4cf9515..7ffafae7 100644 --- a/test-collector-swift/CHANGELOG.md +++ b/test-collector-swift/CHANGELOG.md @@ -1,8 +1,8 @@ # Changelog -## Unreleased +## Unreleased (targeting 2.0.0-beta.1) -- Submit XCTest executions as OpenTelemetry root spans instead of proprietary JSON uploads +- Replace proprietary execution uploads with XCTest execution root spans submitted through OpenTelemetry - Export run and execution tags as `buildkite.tag.*` attributes - Support standard OTLP/HTTP exporter configuration and child-span forwarding - Require Swift 5.10 and macOS 12 or newer diff --git a/test-collector-swift/README.md b/test-collector-swift/README.md index 1d5d3808..978be93a 100644 --- a/test-collector-swift/README.md +++ b/test-collector-swift/README.md @@ -1,4 +1,4 @@ -# Buildkite Test Collector for Swift (Beta) +# Buildkite Test Collector for Swift Official [Buildkite Test Engine](https://buildkite.com/platform/test-engine/) collector for Swift test frameworks ✨ @@ -14,6 +14,11 @@ collector does not use the legacy proprietary execution upload API. The collector requires Swift 5.10 or newer. Its macOS deployment target is macOS 12 or newer. +> [!NOTE] +> The OpenTelemetry-based collector on this branch is being prepared for +> `2.0.0-beta.1` and has not been released. The installation example below +> continues to reference the latest released version, `0.6.0`. + ## 👉 Installing ### Step 1 diff --git a/test-collector-swift/Sources/Core/TestCollector.swift b/test-collector-swift/Sources/Core/TestCollector.swift index e367b92e..d385496e 100644 --- a/test-collector-swift/Sources/Core/TestCollector.swift +++ b/test-collector-swift/Sources/Core/TestCollector.swift @@ -80,5 +80,5 @@ public struct TestCollector { public static let endpoint = "https://tests-otlp.buildkite.com/v1/traces" static let name = "test-collector-swift" - static let version = "0.6.0" + static let version = "2.0.0-beta.1" } From a76e02b0b248787ae42d5f575e96c99c8f7e7d89 Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 22:04:48 +0930 Subject: [PATCH 14/20] Warn when the legacy analytics endpoint is ignored Report BUILDKITE_ANALYTICS_BASE_URL during collector initialization and direct users to the supported OTLP endpoint settings. Add a warning log level and regression coverage. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- test-collector-swift/CHANGELOG.md | 1 + .../Sources/Core/Logger/Logger.swift | 11 +++++++++++ .../Environment/EnvironmentValues.swift | 1 + .../Sources/Core/TestCollector.swift | 6 ++++++ .../Tests/CoreTests/CollectorTests.swift | 19 +++++++++++++++++++ 5 files changed, 38 insertions(+) diff --git a/test-collector-swift/CHANGELOG.md b/test-collector-swift/CHANGELOG.md index 7ffafae7..55c33ca6 100644 --- a/test-collector-swift/CHANGELOG.md +++ b/test-collector-swift/CHANGELOG.md @@ -5,6 +5,7 @@ - Replace proprietary execution uploads with XCTest execution root spans submitted through OpenTelemetry - Export run and execution tags as `buildkite.tag.*` attributes - Support standard OTLP/HTTP exporter configuration and child-span forwarding +- Warn when the unsupported `BUILDKITE_ANALYTICS_BASE_URL` setting is ignored - Require Swift 5.10 and macOS 12 or newer ## 0.6.0 diff --git a/test-collector-swift/Sources/Core/Logger/Logger.swift b/test-collector-swift/Sources/Core/Logger/Logger.swift index 64e0a86c..66dd52d9 100644 --- a/test-collector-swift/Sources/Core/Logger/Logger.swift +++ b/test-collector-swift/Sources/Core/Logger/Logger.swift @@ -61,6 +61,15 @@ struct Logger { self.log(level: .info, message()) } + /// Log a message with the `warning` log level + /// + /// If the logger's `logLevel` is greater than `warning`, nothing will be logged. + /// + /// - Parameter message: The message to be logged. + func warning(_ message: @autoclosure () -> String) { + self.log(level: .warning, message()) + } + /// Log a message with the `error` log level /// /// If the logger's `logLevel` is greater than `error`, nothing will be logged. @@ -88,6 +97,7 @@ extension Logger { enum Level: String, Comparable { case debug case info + case warning case error // Based on values from https://github.com/apple/swift-log @@ -95,6 +105,7 @@ extension Logger { switch self { case .debug: return 1 case .info: return 2 + case .warning: return 3 case .error: return 5 } } diff --git a/test-collector-swift/Sources/Core/Models/Environment/EnvironmentValues.swift b/test-collector-swift/Sources/Core/Models/Environment/EnvironmentValues.swift index 3437e35f..ed66aff4 100644 --- a/test-collector-swift/Sources/Core/Models/Environment/EnvironmentValues.swift +++ b/test-collector-swift/Sources/Core/Models/Environment/EnvironmentValues.swift @@ -81,6 +81,7 @@ struct EnvironmentValues { extension EnvironmentValues { var isAnalyticsEnabled: Bool { self.bool(for: "BUILDKITE_ANALYTICS_ENABLED") ?? true } var analyticsToken: String? { self.string(for: "BUILDKITE_ANALYTICS_TOKEN", private: true) } + var legacyAnalyticsBaseURL: String? { self.string(for: "BUILDKITE_ANALYTICS_BASE_URL") } var analyticsOTLPEndpoint: String? { self.string(for: "BUILDKITE_ANALYTICS_OTLP_ENDPOINT") } var isAnalyticsDebugEnabled: Bool { self.bool(for: "BUILDKITE_ANALYTICS_DEBUG_ENABLED") ?? false } diff --git a/test-collector-swift/Sources/Core/TestCollector.swift b/test-collector-swift/Sources/Core/TestCollector.swift index d385496e..d20aceca 100644 --- a/test-collector-swift/Sources/Core/TestCollector.swift +++ b/test-collector-swift/Sources/Core/TestCollector.swift @@ -20,6 +20,12 @@ public struct TestCollector { return } + if environment.legacyAnalyticsBaseURL != nil { + logger?.warning( + "BUILDKITE_ANALYTICS_BASE_URL is no longer supported and is ignored; use BUILDKITE_ANALYTICS_OTLP_ENDPOINT or OTEL_EXPORTER_OTLP_TRACES_ENDPOINT instead." + ) + } + let envTags = environment.analyticsTags ?? [:] let tags = uploadTags.merging(envTags) { _, env in env } diff --git a/test-collector-swift/Tests/CoreTests/CollectorTests.swift b/test-collector-swift/Tests/CoreTests/CollectorTests.swift index 72c39d89..2cca05cc 100644 --- a/test-collector-swift/Tests/CoreTests/CollectorTests.swift +++ b/test-collector-swift/Tests/CoreTests/CollectorTests.swift @@ -36,6 +36,25 @@ final class CollectorTests: XCTestCase { XCTAssertNil(collector.observer) } + func testWarnsWhenLegacyAnalyticsBaseURLIsIgnored() { + let messages = LockIsolated([String]()) + let logger = Logger(printer: { message in + messages.withValue { $0.append(message) } + }) + let environment = EnvironmentValues(values: [ + "BUILDKITE_ANALYTICS_BASE_URL": "https://example.com/v1/", + ]) + + _ = TestCollector(environment: environment, logger: logger) + logger.waitForLogs() + + XCTAssertTrue(messages.withValue { messages in + messages.contains( + "[BuildkiteTestCollector] warning: BUILDKITE_ANALYTICS_BASE_URL is no longer supported and is ignored; use BUILDKITE_ANALYTICS_OTLP_ENDPOINT or OTEL_EXPORTER_OTLP_TRACES_ENDPOINT instead." + ) + }) + } + func testUploadTagsEnvVarTakesPrecedence() { let environment = EnvironmentValues(values: [ "BUILDKITE_ANALYTICS_TAGS": #"{"shared":"from-env","env-only":"yes"}"#, From a1331a58d32d8fea9efa2ddc40189e24fc55b8cf Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 22:15:32 +0930 Subject: [PATCH 15/20] Back off failed Swift execution exports Preserve synchronous handoff while the configured endpoint is healthy, then use a capped 10, 20, 30, and 60 second retry schedule after failures. Keep queued executions in memory and let the bundle flush make an immediate final attempt. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- test-collector-swift/CHANGELOG.md | 1 + test-collector-swift/README.md | 13 +-- .../SynchronousExecutionSpanProcessor.swift | 46 +++++++--- ...nchronousExecutionSpanProcessorTests.swift | 86 +++++++++++++++++++ .../CoreTests/TelemetryClientTests.swift | 5 +- 5 files changed, 132 insertions(+), 19 deletions(-) create mode 100644 test-collector-swift/Tests/CoreTests/SynchronousExecutionSpanProcessorTests.swift diff --git a/test-collector-swift/CHANGELOG.md b/test-collector-swift/CHANGELOG.md index 55c33ca6..197169a3 100644 --- a/test-collector-swift/CHANGELOG.md +++ b/test-collector-swift/CHANGELOG.md @@ -6,6 +6,7 @@ - Export run and execution tags as `buildkite.tag.*` attributes - Support standard OTLP/HTTP exporter configuration and child-span forwarding - Warn when the unsupported `BUILDKITE_ANALYTICS_BASE_URL` setting is ignored +- Back off failed root-span exports instead of blocking every subsequent test - Require Swift 5.10 and macOS 12 or newer ## 0.6.0 diff --git a/test-collector-swift/README.md b/test-collector-swift/README.md index 978be93a..b4d8a5f2 100644 --- a/test-collector-swift/README.md +++ b/test-collector-swift/README.md @@ -209,12 +209,13 @@ OpenTelemetry SDK provider, the collector adds a forwarding processor without replacing its sampler or exporters. Otherwise, it installs a provider so spans created under a test can be exported as execution children. -Finishing a test waits for its root span to be accepted by the configured OTLP -endpoint. A failed root export stays in the collector's in-memory queue -and is retried by the next execution or the bundle-end flush. When the endpoint -is the Test Engine Client relay, acceptance transfers the span to the -longer-lived parent process, which protects completed executions from an XCTest -runner restart. +While the endpoint is healthy, finishing a test waits for its root span to be +accepted by the configured OTLP endpoint. After a failure, subsequent roots stay +in the collector's in-memory queue while export attempts back off for 10, 20, +30, and then at most 60 seconds. The bundle-end flush always makes an immediate +final attempt. When the endpoint is the Test Engine Client relay, acceptance +transfers the span to the longer-lived parent process, which protects completed +executions from an XCTest runner restart. This is not disk-backed delivery. A hard exit before a test finishes, a process exit while the endpoint remains unavailable, or a machine restart can still diff --git a/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift b/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift index 669b82d3..1afff365 100644 --- a/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift +++ b/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift @@ -3,17 +3,27 @@ import OpenTelemetryApi import OpenTelemetrySdk final class SynchronousExecutionSpanProcessor: SpanProcessor { + private static let retryDelays: [TimeInterval] = [10, 20, 30, 60] + let isStartRequired = false let isEndRequired = true private let exporter: any SpanExporter private let exporterLock = NSLock() private let logger: Logger? + private let now: () -> TimeInterval private var pending = [SpanData]() + private var retryDelayIndex = 0 + private var retryNotBefore: TimeInterval? - init(exporter: any SpanExporter, logger: Logger?) { + init( + exporter: any SpanExporter, + logger: Logger?, + now: @escaping () -> TimeInterval = { ProcessInfo.processInfo.systemUptime } + ) { self.exporter = exporter self.logger = logger + self.now = now } func onStart(parentContext: SpanContext?, span: any ReadableSpan) {} @@ -24,11 +34,13 @@ final class SynchronousExecutionSpanProcessor: SpanProcessor { let name = span.getAttributes()[BuildkiteTelemetryAttribute.testName]?.description ?? span.name self.pending.append(span.toSpanData()) - switch self.exporter.export(spans: self.pending) { - case .success: - self.pending.removeAll() + if let retryNotBefore = self.retryNotBefore, self.now() < retryNotBefore { + return + } + + if self.exportPending() { self.logger?.debug("Exported OpenTelemetry test execution: \(name)") - case .failure: + } else { self.logger?.error( "OpenTelemetry export failed for \(name); the execution remains queued in this process" ) @@ -49,16 +61,26 @@ final class SynchronousExecutionSpanProcessor: SpanProcessor { } private func flush(timeout: TimeInterval?) { - if !self.pending.isEmpty { - switch self.exporter.export(spans: self.pending, explicitTimeout: timeout) { - case .success: - self.pending.removeAll() - case .failure: - self.logger?.error("OpenTelemetry export failed while retrying pending test executions") - } + if !self.pending.isEmpty, !self.exportPending(timeout: timeout) { + self.logger?.error("OpenTelemetry export failed while retrying pending test executions") } if case .failure = self.exporter.flush(explicitTimeout: timeout) { self.logger?.error("OpenTelemetry export failed while flushing pending test executions") } } + + private func exportPending(timeout: TimeInterval? = nil) -> Bool { + switch self.exporter.export(spans: self.pending, explicitTimeout: timeout) { + case .success: + self.pending.removeAll() + self.retryDelayIndex = 0 + self.retryNotBefore = nil + return true + case .failure: + let delay = Self.retryDelays[self.retryDelayIndex] + self.retryNotBefore = self.now() + delay + self.retryDelayIndex = min(self.retryDelayIndex + 1, Self.retryDelays.count - 1) + return false + } + } } diff --git a/test-collector-swift/Tests/CoreTests/SynchronousExecutionSpanProcessorTests.swift b/test-collector-swift/Tests/CoreTests/SynchronousExecutionSpanProcessorTests.swift new file mode 100644 index 00000000..5544977b --- /dev/null +++ b/test-collector-swift/Tests/CoreTests/SynchronousExecutionSpanProcessorTests.swift @@ -0,0 +1,86 @@ +@testable import Core +import OpenTelemetryApi +import OpenTelemetrySdk +import XCTest + +final class SynchronousExecutionSpanProcessorTests: XCTestCase { + func testBacksOffFailedExportsAndResetsAfterSuccess() { + let clock = TestClock() + let exporter = SequencedExporter(results: [ + .failure, .failure, .failure, .failure, .failure, .success, .success, + ]) + let processor = SynchronousExecutionSpanProcessor( + exporter: exporter, + logger: nil, + now: { clock.now } + ) + let tracer = TracerProviderSdk(spanProcessors: [processor]).get( + instrumentationName: "SynchronousExecutionSpanProcessorTests" + ) + + self.endSpan("one", with: tracer) + self.endSpan("two", with: tracer) + clock.now = 9 + self.endSpan("three", with: tracer) + XCTAssertEqual(exporter.exports.count, 1) + + clock.now = 10 + self.endSpan("four", with: tracer) + clock.now = 29 + self.endSpan("five", with: tracer) + XCTAssertEqual(exporter.exports.count, 2) + + clock.now = 30 + self.endSpan("six", with: tracer) + clock.now = 59 + self.endSpan("seven", with: tracer) + XCTAssertEqual(exporter.exports.count, 3) + + clock.now = 60 + self.endSpan("eight", with: tracer) + clock.now = 119 + self.endSpan("nine", with: tracer) + XCTAssertEqual(exporter.exports.count, 4) + + clock.now = 120 + self.endSpan("ten", with: tracer) + clock.now = 179 + self.endSpan("eleven", with: tracer) + XCTAssertEqual(exporter.exports.count, 5) + + clock.now = 180 + self.endSpan("twelve", with: tracer) + self.endSpan("thirteen", with: tracer) + + XCTAssertEqual(exporter.exports.count, 7) + XCTAssertEqual(exporter.exports.map(\.count), [1, 4, 6, 8, 10, 12, 1]) + } + + private func endSpan(_ name: String, with tracer: any Tracer) { + tracer.spanBuilder(spanName: name).setNoParent().startSpan().end() + } +} + +private final class TestClock { + var now: TimeInterval = 0 +} + +private final class SequencedExporter: SpanExporter, @unchecked Sendable { + private var results: [SpanExporterResultCode] + var exports = [[SpanData]]() + + init(results: [SpanExporterResultCode]) { + self.results = results + } + + func export(spans: [SpanData], explicitTimeout: TimeInterval?) -> SpanExporterResultCode { + self.exports.append(spans) + return self.results.removeFirst() + } + + func flush(explicitTimeout: TimeInterval?) -> SpanExporterResultCode { + .success + } + + func shutdown(explicitTimeout: TimeInterval?) {} +} diff --git a/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift index 9a676213..f19466b3 100644 --- a/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift +++ b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift @@ -269,7 +269,7 @@ final class TelemetryClientTests: XCTestCase { XCTAssertNil(span.resource.attributes["branch"]) } - func testRetriesFailedExecutionWithTheNextExecution() throws { + func testBundleFlushRetriesExecutionsQueuedDuringBackoff() throws { let rootExporter = FailOnceExporter() let client = try XCTUnwrap(TelemetryClient.live( environment: self.configuredEnvironment, @@ -286,6 +286,9 @@ final class TelemetryClientTests: XCTestCase { client.finishExecution(executionID, test: test, tags: nil) } + XCTAssertEqual(rootExporter.exports.count, 1) + client.forceFlush() + XCTAssertEqual(rootExporter.exports.count, 2) XCTAssertEqual(rootExporter.exports[0].map(\.name), ["test.execution"]) XCTAssertEqual( From d9246b863732455e4f13ad7cf652c5cdb30d6268 Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 25 Aug 2026 22:23:14 +0930 Subject: [PATCH 16/20] Improve Swift export backoff diagnostics Log executions queued during an active export backoff and directly assert that a successful export resets the next retry delay to 10 seconds. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- .../SynchronousExecutionSpanProcessor.swift | 3 ++ ...nchronousExecutionSpanProcessorTests.swift | 39 +++++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift b/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift index 1afff365..8ad3f4cd 100644 --- a/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift +++ b/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift @@ -35,6 +35,9 @@ final class SynchronousExecutionSpanProcessor: SpanProcessor { let name = span.getAttributes()[BuildkiteTelemetryAttribute.testName]?.description ?? span.name self.pending.append(span.toSpanData()) if let retryNotBefore = self.retryNotBefore, self.now() < retryNotBefore { + self.logger?.debug( + "Queued OpenTelemetry test execution during export backoff: \(name) (\(self.pending.count) pending)" + ) return } diff --git a/test-collector-swift/Tests/CoreTests/SynchronousExecutionSpanProcessorTests.swift b/test-collector-swift/Tests/CoreTests/SynchronousExecutionSpanProcessorTests.swift index 5544977b..f1616fda 100644 --- a/test-collector-swift/Tests/CoreTests/SynchronousExecutionSpanProcessorTests.swift +++ b/test-collector-swift/Tests/CoreTests/SynchronousExecutionSpanProcessorTests.swift @@ -7,7 +7,7 @@ final class SynchronousExecutionSpanProcessorTests: XCTestCase { func testBacksOffFailedExportsAndResetsAfterSuccess() { let clock = TestClock() let exporter = SequencedExporter(results: [ - .failure, .failure, .failure, .failure, .failure, .success, .success, + .failure, .failure, .failure, .failure, .failure, .success, .failure, .success, ]) let processor = SynchronousExecutionSpanProcessor( exporter: exporter, @@ -51,9 +51,42 @@ final class SynchronousExecutionSpanProcessorTests: XCTestCase { clock.now = 180 self.endSpan("twelve", with: tracer) self.endSpan("thirteen", with: tracer) - + clock.now = 189 + self.endSpan("fourteen", with: tracer) XCTAssertEqual(exporter.exports.count, 7) - XCTAssertEqual(exporter.exports.map(\.count), [1, 4, 6, 8, 10, 12, 1]) + + clock.now = 190 + self.endSpan("fifteen", with: tracer) + + XCTAssertEqual(exporter.exports.count, 8) + XCTAssertEqual(exporter.exports.map(\.count), [1, 4, 6, 8, 10, 12, 1, 3]) + } + + func testLogsExecutionsQueuedDuringBackoff() { + let clock = TestClock() + let exporter = SequencedExporter(results: [.failure]) + let messages = LockIsolated([String]()) + let logger = Logger(logLevel: .debug, printer: { message in + messages.withValue { $0.append(message) } + }) + let processor = SynchronousExecutionSpanProcessor( + exporter: exporter, + logger: logger, + now: { clock.now } + ) + let tracer = TracerProviderSdk(spanProcessors: [processor]).get( + instrumentationName: "SynchronousExecutionSpanProcessorTests" + ) + + self.endSpan("one", with: tracer) + self.endSpan("two", with: tracer) + logger.waitForLogs() + + XCTAssertTrue(messages.withValue { messages in + messages.contains( + "[BuildkiteTestCollector] debug: Queued OpenTelemetry test execution during export backoff: two (2 pending)" + ) + }) } private func endSpan(_ name: String, with tracer: any Tracer) { From 9e0e9d859281c004ed21319ad991cfe38e2d3d1d Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Wed, 26 Aug 2026 11:17:46 +0930 Subject: [PATCH 17/20] Isolate Swift OTLP endpoint credentials Require an explicit standard OTLP endpoint before applying standard exporter headers. Ignore and warn about those headers on endpoints that receive Buildkite credentials so unrelated process-wide credentials cannot be sent to Buildkite or replace suite authentication. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- test-collector-swift/CHANGELOG.md | 1 + test-collector-swift/README.md | 15 ++++-- .../Core/Telemetry/TelemetryClient+Live.swift | 18 +++---- .../CoreTests/TelemetryClientTests.swift | 48 +++++++++++++++++++ 4 files changed, 69 insertions(+), 13 deletions(-) diff --git a/test-collector-swift/CHANGELOG.md b/test-collector-swift/CHANGELOG.md index 197169a3..dd2bcb29 100644 --- a/test-collector-swift/CHANGELOG.md +++ b/test-collector-swift/CHANGELOG.md @@ -5,6 +5,7 @@ - Replace proprietary execution uploads with XCTest execution root spans submitted through OpenTelemetry - Export run and execution tags as `buildkite.tag.*` attributes - Support standard OTLP/HTTP exporter configuration and child-span forwarding +- Keep standard OTLP headers isolated from trusted Buildkite endpoints - Warn when the unsupported `BUILDKITE_ANALYTICS_BASE_URL` setting is ignored - Back off failed root-span exports instead of blocking every subsequent test - Require Swift 5.10 and macOS 12 or newer diff --git a/test-collector-swift/README.md b/test-collector-swift/README.md index b4d8a5f2..d3cdf194 100644 --- a/test-collector-swift/README.md +++ b/test-collector-swift/README.md @@ -69,10 +69,14 @@ export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL="http/protobuf" The signal-specific variables take precedence over `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_HEADERS`, and `OTEL_EXPORTER_OTLP_PROTOCOL`. A generic OTLP endpoint has `/v1/traces` -appended. Standard endpoints receive only the explicitly configured OTLP -headers; the collector does not send the Buildkite suite token to them. +appended. Standard header variables are applied only when a standard endpoint +is explicitly configured; headers alone do not enable the collector or select +an endpoint. Standard endpoints receive only the explicitly configured OTLP +headers, and the collector does not send the Buildkite suite token to them. `BUILDKITE_ANALYTICS_OTLP_ENDPOINT` remains available as a trusted, -collector-specific endpoint override that receives the suite token and run key. +collector-specific endpoint override that receives only the suite token and run +key. To send custom headers to a relay, configure it through the standard +endpoint and header variables instead. ### Step 3 @@ -84,8 +88,9 @@ key: value: `$(BUILDKITE_ANALYTICS_TOKEN)` -If you configure an OTLP endpoint, headers, or protocol, map those -`OTEL_EXPORTER_OTLP_TRACES_*` variables into the test runner in the same way. +If you configure a standard OTLP endpoint, map that endpoint and its associated +`OTEL_EXPORTER_OTLP_TRACES_*` header or protocol variables into the test runner +in the same way. The same key value pair can be specified in your main bundle's `info.plist` file if you would rather specify it there. Note variables in the environment take precedent over those in the `info.plist` file. diff --git a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift index 82e1bb90..ec75c349 100644 --- a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift +++ b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift @@ -53,10 +53,12 @@ struct CollectorOTLPConfiguration { var runEnvironment = environment.runEnvironment() runEnvironment.applyCustomEnvironmentOverrides() let rawHeaders = environment.otelTracesHeaders ?? environment.otelHeaders + // Standard headers alone must not enable collection or select the OTLP + // specification's localhost default. They may be process-wide settings for + // unrelated telemetry and are safe to use only with an explicit standard endpoint. let hasExplicitConfiguration = environment.otelTracesEndpoint != nil || environment.otelEndpoint != nil || environment.analyticsOTLPEndpoint != nil - || rawHeaders != nil guard environment.analyticsToken != nil || hasExplicitConfiguration else { return nil } @@ -102,17 +104,17 @@ struct CollectorOTLPConfiguration { if let token = environment.analyticsToken { headers.append(("Authorization", "Token token=\"\(token)\"")) } - } - - if let rawHeaders { + if rawHeaders != nil { + logger?.warning( + "Standard OpenTelemetry exporter headers are ignored for endpoints that receive Buildkite credentials; use OTEL_EXPORTER_OTLP_TRACES_ENDPOINT or OTEL_EXPORTER_OTLP_ENDPOINT to send custom headers." + ) + } + } else if let rawHeaders { guard let parsedHeaders = Self.parseHeaders(rawHeaders) else { logger?.error("OpenTelemetry exporter headers are invalid") return nil } - for header in parsedHeaders { - headers.removeAll { $0.0.caseInsensitiveCompare(header.0) == .orderedSame } - headers.append(header) - } + headers = parsedHeaders } self.endpoint = endpoint diff --git a/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift index f19466b3..e09a0dbd 100644 --- a/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift +++ b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift @@ -212,6 +212,54 @@ final class TelemetryClientTests: XCTestCase { XCTAssertEqual(configuration.header(named: "x-test"), "value/one") } + func testHeadersAloneDoNotEnableCollector() { + let configuration = CollectorOTLPConfiguration( + environment: EnvironmentValues( + values: [ + "OTEL_EXPORTER_OTLP_HEADERS": "authorization=Bearer%20unrelated-token", + ], + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ), + logger: nil + ) + + XCTAssertNil(configuration) + } + + func testIgnoresStandardHeadersForTrustedEndpoint() throws { + let messages = LockIsolated([String]()) + let logger = Logger(printer: { message in + messages.withValue { $0.append(message) } + }) + let configuration = try XCTUnwrap(CollectorOTLPConfiguration( + environment: EnvironmentValues( + values: [ + "BUILDKITE_ANALYTICS_KEY": "run-key", + "BUILDKITE_ANALYTICS_TOKEN": "collector-token", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS": "authorization=Bearer%20unrelated-token,x-test=value", + ], + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ), + logger: logger + )) + logger.waitForLogs() + + XCTAssertEqual(configuration.endpoint.absoluteString, TestCollector.endpoint) + XCTAssertEqual(configuration.header(named: "Buildkite-Tests-Run-Key"), "run-key") + XCTAssertEqual( + configuration.header(named: "Authorization"), + #"Token token="collector-token""# + ) + XCTAssertNil(configuration.header(named: "x-test")) + XCTAssertTrue(messages.withValue { messages in + messages.contains( + "[BuildkiteTestCollector] warning: Standard OpenTelemetry exporter headers are ignored for endpoints that receive Buildkite credentials; use OTEL_EXPORTER_OTLP_TRACES_ENDPOINT or OTEL_EXPORTER_OTLP_ENDPOINT to send custom headers." + ) + }) + } + func testUsesCollectorCredentialsForTrustedEndpointAndCustomRunKey() throws { let configuration = try XCTUnwrap(CollectorOTLPConfiguration( environment: EnvironmentValues( From 988ca429b9fb9c7b44d3726db9b7bab8ad285a7e Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Wed, 26 Aug 2026 21:26:33 +0930 Subject: [PATCH 18/20] Scope Swift OTLP protocol configuration Validate standard OTLP protocol variables only for explicitly configured standard endpoints. Trusted Buildkite endpoints keep their fixed HTTP/protobuf transport when unrelated process-wide OTLP protocol settings are present. Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 Co-authored-by: Amp --- test-collector-swift/CHANGELOG.md | 2 +- test-collector-swift/README.md | 13 +++++++------ .../Core/Telemetry/TelemetryClient+Live.swift | 17 +++++++++-------- .../Tests/CoreTests/TelemetryClientTests.swift | 3 ++- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/test-collector-swift/CHANGELOG.md b/test-collector-swift/CHANGELOG.md index dd2bcb29..6a089580 100644 --- a/test-collector-swift/CHANGELOG.md +++ b/test-collector-swift/CHANGELOG.md @@ -5,7 +5,7 @@ - Replace proprietary execution uploads with XCTest execution root spans submitted through OpenTelemetry - Export run and execution tags as `buildkite.tag.*` attributes - Support standard OTLP/HTTP exporter configuration and child-span forwarding -- Keep standard OTLP headers isolated from trusted Buildkite endpoints +- Apply standard OTLP headers and protocols only with standard endpoints - Warn when the unsupported `BUILDKITE_ANALYTICS_BASE_URL` setting is ignored - Back off failed root-span exports instead of blocking every subsequent test - Require Swift 5.10 and macOS 12 or newer diff --git a/test-collector-swift/README.md b/test-collector-swift/README.md index d3cdf194..1a698023 100644 --- a/test-collector-swift/README.md +++ b/test-collector-swift/README.md @@ -69,14 +69,15 @@ export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL="http/protobuf" The signal-specific variables take precedence over `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_HEADERS`, and `OTEL_EXPORTER_OTLP_PROTOCOL`. A generic OTLP endpoint has `/v1/traces` -appended. Standard header variables are applied only when a standard endpoint -is explicitly configured; headers alone do not enable the collector or select -an endpoint. Standard endpoints receive only the explicitly configured OTLP -headers, and the collector does not send the Buildkite suite token to them. +appended. Standard header and protocol variables are applied only when a +standard endpoint is explicitly configured; those variables alone do not enable +the collector or select an endpoint. Standard endpoints receive only the +explicitly configured OTLP headers, and the collector does not send the +Buildkite suite token to them. `BUILDKITE_ANALYTICS_OTLP_ENDPOINT` remains available as a trusted, collector-specific endpoint override that receives only the suite token and run -key. To send custom headers to a relay, configure it through the standard -endpoint and header variables instead. +key and always uses HTTP/protobuf. To send custom headers or select a protocol +for a relay, configure it through the standard exporter variables instead. ### Step 3 diff --git a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift index ec75c349..b26cade4 100644 --- a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift +++ b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift @@ -62,14 +62,6 @@ struct CollectorOTLPConfiguration { guard environment.analyticsToken != nil || hasExplicitConfiguration else { return nil } - if let protocolName = environment.otelTracesProtocol ?? environment.otelProtocol, - protocolName.lowercased() != "http/protobuf" { - logger?.error( - "Unsupported OpenTelemetry traces protocol \(protocolName); expected http/protobuf" - ) - return nil - } - let endpoint: URL let usesBuildkiteCredentials: Bool if let value = environment.otelTracesEndpoint { @@ -98,6 +90,15 @@ struct CollectorOTLPConfiguration { usesBuildkiteCredentials = true } + if !usesBuildkiteCredentials, + let protocolName = environment.otelTracesProtocol ?? environment.otelProtocol, + protocolName.lowercased() != "http/protobuf" { + logger?.error( + "Unsupported OpenTelemetry traces protocol \(protocolName); expected http/protobuf" + ) + return nil + } + var headers = [(String, String)]() if usesBuildkiteCredentials { headers.append(("Buildkite-Tests-Run-Key", runEnvironment.key)) diff --git a/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift index e09a0dbd..cd07d4db 100644 --- a/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift +++ b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift @@ -227,7 +227,7 @@ final class TelemetryClientTests: XCTestCase { XCTAssertNil(configuration) } - func testIgnoresStandardHeadersForTrustedEndpoint() throws { + func testIgnoresStandardHeadersAndProtocolForTrustedEndpoint() throws { let messages = LockIsolated([String]()) let logger = Logger(printer: { message in messages.withValue { $0.append(message) } @@ -238,6 +238,7 @@ final class TelemetryClientTests: XCTestCase { "BUILDKITE_ANALYTICS_KEY": "run-key", "BUILDKITE_ANALYTICS_TOKEN": "collector-token", "OTEL_EXPORTER_OTLP_TRACES_HEADERS": "authorization=Bearer%20unrelated-token,x-test=value", + "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc", ], getFromEnvironment: { _ in nil }, getFromInfoDictionary: { _ in nil } From 043cae8e1abf7a2f3b159cda10b3ff8f1f88bd97 Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Thu, 27 Aug 2026 19:27:21 +0930 Subject: [PATCH 19/20] Align Swift OTLP metadata with Ruby Keep stable suite, CI run and worker, and VCS identity on provider resources while moving Test Engine run metadata, framework details, custom metadata, and configured tags to each execution root. Preserve suite-owned child resources and use provider-native CI run identity. Reserve the three attributes required for OTLP execution synthesis at the Swift SDK's minimum useful span attribute limit, accounting for its most-recent-attribute retention behavior. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 --- test-collector-swift/CHANGELOG.md | 2 + test-collector-swift/README.md | 17 +- .../Environment/EnvironmentValues.swift | 1 + .../Core/Telemetry/TelemetryClient+Live.swift | 120 ++++++++++-- .../CoreTests/TelemetryClientTests.swift | 177 +++++++++++++++++- 5 files changed, 289 insertions(+), 28 deletions(-) diff --git a/test-collector-swift/CHANGELOG.md b/test-collector-swift/CHANGELOG.md index 6a089580..da8f6579 100644 --- a/test-collector-swift/CHANGELOG.md +++ b/test-collector-swift/CHANGELOG.md @@ -4,6 +4,8 @@ - Replace proprietary execution uploads with XCTest execution root spans submitted through OpenTelemetry - Export run and execution tags as `buildkite.tag.*` attributes +- Keep stable suite, CI worker/run, and VCS identity on OpenTelemetry resources while keeping Test Engine run metadata, framework details, custom metadata, and tags on each execution root +- Prioritize the three attributes required for execution synthesis when OpenTelemetry span attribute limits are constrained - Support standard OTLP/HTTP exporter configuration and child-span forwarding - Apply standard OTLP headers and protocols only with standard endpoints - Warn when the unsupported `BUILDKITE_ANALYTICS_BASE_URL` setting is ignored diff --git a/test-collector-swift/README.md b/test-collector-swift/README.md index 1a698023..288db2cd 100644 --- a/test-collector-swift/README.md +++ b/test-collector-swift/README.md @@ -105,6 +105,7 @@ again and add the following key-value pairs for your CI platform. **Buildkite** ``` +Key: BUILDKITE_AGENT_ID, Value: $(BUILDKITE_AGENT_ID) Key: BUILDKITE_BUILD_ID, Value: $(BUILDKITE_BUILD_ID) Key: BUILDKITE_BUILD_URL, Value: $(BUILDKITE_BUILD_URL) Key: BUILDKITE_BRANCH, Value: $(BUILDKITE_BRANCH) @@ -185,7 +186,8 @@ Run-level tags can also be set programmatically by passing them to `load`. Envir TestCollector.load(uploadTags: ["host.arch": "arm64"]) ``` -Run tags are exported as `buildkite.tag.` resource attributes. +Run tags are exported as `buildkite.tag.` attributes on every test root. +An execution-level tag takes precedence when it uses the same key. ### Execution-level tags @@ -215,6 +217,19 @@ OpenTelemetry SDK provider, the collector adds a forwarding processor without replacing its sampler or exporters. Otherwise, it installs a provider so spans created under a test can be exported as execution children. +Resources identify producer-wide entities: the suite, provider-native CI run +and URL, Buildkite worker, and checked-out VCS ref. Test Engine run metadata, +collector and framework details, custom run metadata, and run tags are attached +only to each `test.execution` root. They are not duplicated onto child spans. +Collector-managed children share the producer resource; children from a tracer +provider configured by the test suite keep that provider's resource. + +The collector prioritizes `buildkite.execution.via`, `buildkite.run_key`, and +`test.case.result.status` so Test Engine can synthesize an execution when the +OpenTelemetry SDK span attribute limit is constrained. The minimum useful span +attribute limit is three; additional metadata and tags may be dropped at that +limit. + While the endpoint is healthy, finishing a test waits for its root span to be accepted by the configured OTLP endpoint. After a failure, subsequent roots stay in the collector's in-memory queue while export attempts back off for 10, 20, diff --git a/test-collector-swift/Sources/Core/Models/Environment/EnvironmentValues.swift b/test-collector-swift/Sources/Core/Models/Environment/EnvironmentValues.swift index ed66aff4..b703758b 100644 --- a/test-collector-swift/Sources/Core/Models/Environment/EnvironmentValues.swift +++ b/test-collector-swift/Sources/Core/Models/Environment/EnvironmentValues.swift @@ -106,6 +106,7 @@ extension EnvironmentValues { var buildkiteBuildId: String? { self.string(for: "BUILDKITE_BUILD_ID") } var buildkiteBuildUrl: String? { self.string(for: "BUILDKITE_BUILD_URL") } + var buildkiteAgentId: String? { self.string(for: "BUILDKITE_AGENT_ID") } var buildkiteBranch: String? { self.string(for: "BUILDKITE_BRANCH") } var buildkiteCommit: String? { self.string(for: "BUILDKITE_COMMIT") } var buildkiteBuildNumber: String? { self.string(for: "BUILDKITE_BUILD_NUMBER") } diff --git a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift index b26cade4..8fc9a582 100644 --- a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift +++ b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift @@ -16,7 +16,6 @@ import Glibc enum BuildkiteTelemetryAttribute { static let annotation = "buildkite.annotation" - static let buildID = "buildkite.build_id" static let buildNumber = "buildkite.build_number" static let collectorName = "buildkite.collector.name" static let collectorVersion = "buildkite.collector.version" @@ -39,6 +38,7 @@ private enum TelemetryValue { static let frameworkName = "xctest" static let rootSpanName = "test.execution" static let skippedResult = "skipped" + static let unsetResult = "unset" } struct CollectorOTLPConfiguration { @@ -240,7 +240,8 @@ extension TelemetryClient { uploadTags: [String: String], logger: Logger?, rootExporter: (any SpanExporter)? = nil, - childExporter: (any SpanExporter)? = nil + childExporter: (any SpanExporter)? = nil, + rootSpanLimits: SpanLimits = SpanLimits() ) -> TelemetryClient? { guard let configuration = CollectorOTLPConfiguration( environment: environment, @@ -265,7 +266,8 @@ extension TelemetryClient { uploadTags: uploadTags, logger: logger, rootExporter: rootExporter ?? makeExporter(), - childExporter: childExporter ?? makeExporter() + childExporter: childExporter ?? makeExporter(), + rootSpanLimits: rootSpanLimits ) return TelemetryClient( @@ -286,6 +288,8 @@ private final class LiveTelemetryClient { private let executionProvider: TracerProviderSdk private let executionNamePrefix: String? private let executionNameSuffix: String? + private let executionAttributes: [String: AttributeValue] + private let runKey: String private let logger: Logger? private let registry = ExecutionTraceRegistry() private let reportedUnsupportedChildProvider = LockIsolated(false) @@ -300,14 +304,20 @@ private final class LiveTelemetryClient { uploadTags: [String: String], logger: Logger?, rootExporter: any SpanExporter, - childExporter: any SpanExporter + childExporter: any SpanExporter, + rootSpanLimits: SpanLimits ) { self.executionNamePrefix = configuration.runEnvironment.executionNamePrefix self.executionNameSuffix = configuration.runEnvironment.executionNameSuffix + self.runKey = configuration.runEnvironment.key self.logger = logger self.jobSpanContext = Self.jobSpanContext(environment: environment) - let resource = Self.resource( + let resource = Self.providerResource( + configuration: configuration, + environment: environment + ) + self.executionAttributes = Self.executionAttributes( configuration: configuration, environment: environment, uploadTags: uploadTags @@ -316,7 +326,8 @@ private final class LiveTelemetryClient { let rootProcessor = SynchronousExecutionSpanProcessor(exporter: rootExporter, logger: logger) self.executionProvider = Self.makeProvider( resource: resource, - processor: rootProcessor + processor: rootProcessor, + spanLimits: rootSpanLimits ) self.tracer = self.executionProvider.get( instrumentationName: TestCollector.name, @@ -353,7 +364,12 @@ private final class LiveTelemetryClient { let builder = self.tracer .spanBuilder(spanName: TelemetryValue.rootSpanName) .setNoParent() - .setAttribute(key: BuildkiteTelemetryAttribute.executionVia, value: TelemetryValue.executionVia) + + for (key, value) in self.executionAttributes { + builder.setAttribute(key: key, value: value) + } + + builder .setAttribute(key: BuildkiteTelemetryAttribute.executionScope, value: test.className) .setAttribute(key: BuildkiteTelemetryAttribute.testName, value: test.testName) .setAttribute( @@ -365,6 +381,20 @@ private final class LiveTelemetryClient { value: qualifiedTestName ) .setAttribute(key: SemanticConventions.Test.suiteName.rawValue, value: test.className) + // The Swift SDK keeps the most recently set attributes when the span's + // start-time attribute limit is exceeded. Set the three synthesis fields + // last so even a limit of three still produces a usable execution. + .setAttribute(key: BuildkiteTelemetryAttribute.executionVia, value: TelemetryValue.executionVia) + .setAttribute( + key: BuildkiteTelemetryAttribute.runKey, + value: self.runKey + ) + // Reserve the result's place before test code can consume the remaining + // attribute budget. finishExecution replaces this placeholder. + .setAttribute( + key: SemanticConventions.Test.caseResultStatus.rawValue, + value: TelemetryValue.unsetResult + ) if let jobSpanContext = self.jobSpanContext { builder.addLink(spanContext: jobSpanContext) @@ -494,11 +524,13 @@ private final class LiveTelemetryClient { private static func makeProvider( resource: Resource, - processor: any SpanProcessor + processor: any SpanProcessor, + spanLimits: SpanLimits = SpanLimits() ) -> TracerProviderSdk { self.withoutInheritedTraceContext { TracerProviderBuilder() .with(resource: resource) + .with(spanLimits: spanLimits) .with(sampler: Samplers.alwaysOn) .add(spanProcessor: processor) .build() @@ -526,12 +558,14 @@ private final class LiveTelemetryClient { return operation() } - private static func resource( + // A resource identifies producer-wide entities shared by every span from the + // provider. Test Engine run fields and tags belong to each execution root. + private static func providerResource( configuration: CollectorOTLPConfiguration, - environment: EnvironmentValues, - uploadTags: [String: String] + environment: EnvironmentValues ) -> Resource { let run = configuration.runEnvironment + let pipelineRun = Self.ciPipelineRun(run: run, environment: environment) var attributes = [String: AttributeValue]() func set(_ key: String, _ value: String?) { @@ -540,21 +574,48 @@ private final class LiveTelemetryClient { } } - set(SemanticConventions.Service.name.rawValue, environment.testEngineSuiteSlug ?? TestCollector.name) + set(SemanticConventions.Service.name.rawValue, environment.testEngineSuiteSlug) set(SemanticConventions.Service.namespace.rawValue, environment.buildkiteOrganizationSlug) - set(SemanticConventions.Service.instanceId.rawValue, run.jobId) - set(BuildkiteTelemetryAttribute.runKey, run.key) - set(BuildkiteTelemetryAttribute.runURL, run.url) + set(SemanticConventions.Cicd.pipelineRunId.rawValue, pipelineRun.id) + if pipelineRun.id != nil { + set(SemanticConventions.Cicd.pipelineRunUrlFull.rawValue, pipelineRun.url) + } + set(SemanticConventions.Cicd.workerId.rawValue, environment.buildkiteAgentId) set(SemanticConventions.Vcs.refHeadName.rawValue, run.branch) set(SemanticConventions.Vcs.refHeadRevision.rawValue, run.commitSha) if run.branch != nil { set( - SemanticConventions.Vcs.refHeadType.rawValue, + SemanticConventions.Vcs.refType.rawValue, environment.buildkiteTag == nil ? "branch" : "tag" ) } + + return EnvVarResource.get().merging(other: Resource(attributes: attributes)) + } + + // These fields describe each test execution, not the provider that emitted + // its child spans. Configure-level tags are set on every root; per-test tags + // are applied at finish time and override matching configure-level tags. + private static func executionAttributes( + configuration: CollectorOTLPConfiguration, + environment: EnvironmentValues, + uploadTags: [String: String] + ) -> [String: AttributeValue] { + let run = configuration.runEnvironment + let pipelineRun = Self.ciPipelineRun(run: run, environment: environment) + var attributes = [String: AttributeValue]() + + func set(_ key: String, _ value: String?) { + if let value, Self.validAttributeKey(key) { + attributes[key] = .string(value) + } + } + + set(BuildkiteTelemetryAttribute.runKey, run.key) + if run.url != pipelineRun.url { + set(BuildkiteTelemetryAttribute.runURL, run.url) + } set(BuildkiteTelemetryAttribute.buildNumber, run.number) - set(BuildkiteTelemetryAttribute.buildID, environment.buildkiteBuildId) set(BuildkiteTelemetryAttribute.jobID, run.jobId) set(BuildkiteTelemetryAttribute.stepID, environment.buildkiteStepId) set(BuildkiteTelemetryAttribute.message, run.message) @@ -571,7 +632,30 @@ private final class LiveTelemetryClient { attributes[key] = AttributeValue(value.base) ?? .string(value.description) } - return EnvVarResource.get().merging(other: Resource(attributes: attributes)) + return attributes + } + + // Provider-native CI identity is distinct from the Test Engine run key. + private static func ciPipelineRun( + run: RunEnvironment, + environment: EnvironmentValues + ) -> (id: String?, url: String?) { + switch run.ci { + case "buildkite": + return (environment.buildkiteBuildId, environment.buildkiteBuildUrl) + case "github_actions": + let id = environment.gitHubRunId + let url = environment.gitHubRepository.flatMap { repository in + id.map { "https://github.com/\(repository)/actions/runs/\($0)" } + } + return (id, url) + case "circleci": + return (environment.circleWorkflowId, nil) + case "xcodeCloud": + return (environment.xcodeBuildId, nil) + default: + return (nil, nil) + } } private static func validAttributeKey(_ key: String) -> Bool { diff --git a/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift index cd07d4db..93f9f9ad 100644 --- a/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift +++ b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift @@ -28,12 +28,15 @@ final class TelemetryClientTests: XCTestCase { values: [ "BUILDKITE_ANALYTICS_TOKEN": "SECRET", "BUILDKITE_ANALYTICS_KEY": "run-key", + "BUILDKITE_AGENT_ID": "agent-id", "BUILDKITE_BUILD_ID": "build-id", "BUILDKITE_BUILD_NUMBER": "123", "BUILDKITE_BUILD_URL": "https://buildkite.example/builds/123", "BUILDKITE_BRANCH": "main", "BUILDKITE_COMMIT": "abc123", "BUILDKITE_JOB_ID": "job-id", + "BUILDKITE_MESSAGE": "Test resource boundaries", + "BUILDKITE_ORGANIZATION_SLUG": "acme", "BUILDKITE_STEP_ID": "step-id", "BUILDKITE_TEST_ENGINE_SUITE_SLUG": "swift-suite", ], @@ -42,7 +45,7 @@ final class TelemetryClientTests: XCTestCase { ) let client = try XCTUnwrap(TelemetryClient.live( environment: environment, - uploadTags: ["language": "swift"], + uploadTags: ["feature": "configured", "language": "swift"], logger: nil, rootExporter: rootExporter, childExporter: childExporter @@ -54,11 +57,18 @@ final class TelemetryClientTests: XCTestCase { ) let executionID = client.startExecution(test) + let child = OpenTelemetry.instance.tracerProvider + .get(instrumentationName: "example-app") + .spanBuilder(spanName: "database.query") + .startSpan() + child.end() client.annotate("checkpoint", executionID: executionID) test.result = .passed client.finishExecution(executionID, test: test, tags: ["feature": "payments"]) + client.forceFlush() let span = try XCTUnwrap(rootExporter.getFinishedSpanItems().only) + let exportedChild = try XCTUnwrap(childExporter.getFinishedSpanItems().only) XCTAssertEqual(span.name, "test.execution") XCTAssertNil(span.parentSpanId) XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.executionVia], .string("otlp")) @@ -66,9 +76,39 @@ final class TelemetryClientTests: XCTestCase { XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.testName], .string("testChargeCard")) XCTAssertEqual(span.attributes["test.case.result.status"], .string("pass")) XCTAssertEqual(span.attributes["buildkite.tag.feature"], .string("payments")) - XCTAssertEqual(span.resource.attributes[BuildkiteTelemetryAttribute.runKey], .string("run-key")) - XCTAssertEqual(span.resource.attributes[BuildkiteTelemetryAttribute.buildID], .string("build-id")) - XCTAssertEqual(span.resource.attributes["buildkite.tag.language"], .string("swift")) + XCTAssertEqual(span.attributes["buildkite.tag.language"], .string("swift")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.runKey], .string("run-key")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.buildNumber], .string("123")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.jobID], .string("job-id")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.stepID], .string("step-id")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.message], .string("Test resource boundaries")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.collectorName], .string(TestCollector.name)) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.collectorVersion], .string(TestCollector.version)) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.frameworkName], .string("xctest")) + XCTAssertNil(span.attributes[BuildkiteTelemetryAttribute.runURL]) + XCTAssertEqual(span.resource.attributes["service.name"], .string("swift-suite")) + XCTAssertEqual(span.resource.attributes["service.namespace"], .string("acme")) + XCTAssertEqual(span.resource.attributes["cicd.pipeline.run.id"], .string("build-id")) + XCTAssertEqual( + span.resource.attributes["cicd.pipeline.run.url.full"], + .string("https://buildkite.example/builds/123") + ) + XCTAssertEqual(span.resource.attributes["cicd.worker.id"], .string("agent-id")) + XCTAssertEqual(span.resource.attributes["vcs.ref.head.name"], .string("main")) + XCTAssertEqual(span.resource.attributes["vcs.ref.head.revision"], .string("abc123")) + XCTAssertEqual(span.resource.attributes["vcs.ref.type"], .string("branch")) + XCTAssertNil(span.resource.attributes["service.instance.id"]) + XCTAssertNil(span.resource.attributes[BuildkiteTelemetryAttribute.runKey]) + XCTAssertNil(span.resource.attributes[BuildkiteTelemetryAttribute.jobID]) + XCTAssertNil(span.resource.attributes[BuildkiteTelemetryAttribute.frameworkName]) + XCTAssertNil(span.resource.attributes["buildkite.tag.language"]) + XCTAssertEqual(exportedChild.traceId, span.traceId) + XCTAssertEqual(exportedChild.parentSpanId, span.spanId) + XCTAssertEqual(exportedChild.resource.attributes["service.name"], .string("swift-suite")) + XCTAssertEqual(exportedChild.resource.attributes["cicd.pipeline.run.id"], .string("build-id")) + XCTAssertNil(exportedChild.resource.attributes[BuildkiteTelemetryAttribute.runKey]) + XCTAssertNil(exportedChild.resource.attributes[BuildkiteTelemetryAttribute.frameworkName]) + XCTAssertNil(exportedChild.resource.attributes["buildkite.tag.language"]) XCTAssertEqual(span.events.only?.name, "test.annotation") XCTAssertEqual( span.events.only?.attributes[BuildkiteTelemetryAttribute.annotation], @@ -173,6 +213,7 @@ final class TelemetryClientTests: XCTestCase { let executionID = client.startExecution(test) let replacementProvider = TracerProviderBuilder() + .with(resource: Resource(attributes: ["service.name": .string("suite-app")])) .with(sampler: Samplers.alwaysOn) .build() OpenTelemetry.registerTracerProvider(tracerProvider: replacementProvider) @@ -189,6 +230,9 @@ final class TelemetryClientTests: XCTestCase { let exportedChild = try XCTUnwrap(childExporter.getFinishedSpanItems().only) XCTAssertEqual(exportedChild.traceId, root.traceId) XCTAssertEqual(exportedChild.parentSpanId, root.spanId) + XCTAssertEqual(exportedChild.resource.attributes["service.name"], .string("suite-app")) + XCTAssertNil(exportedChild.resource.attributes[BuildkiteTelemetryAttribute.runKey]) + XCTAssertNil(exportedChild.resource.attributes[BuildkiteTelemetryAttribute.frameworkName]) } func testUsesOnlyConfiguredHeadersForStandardTraceEndpoint() throws { @@ -293,7 +337,7 @@ final class TelemetryClientTests: XCTestCase { "BUILDKITE_ANALYTICS_BRANCH": "original-branch", "BUILDKITE_ANALYTICS_EXECUTION_NAME_PREFIX": "[ios]", "BUILDKITE_ANALYTICS_EXECUTION_NAME_SUFFIX": "[debug]", - "BUILDKITE_ANALYTICS_ENVIRONMENT": #"{"key":"custom-run-key","branch":"custom-branch","custom.flag":true}"#, + "BUILDKITE_ANALYTICS_ENVIRONMENT": #"{"key":"custom-run-key","branch":"custom-branch","buildkite.run_key":"wrong-run-key","custom.flag":true}"#, ], getFromEnvironment: { _ in nil }, getFromInfoDictionary: { _ in nil } @@ -311,11 +355,101 @@ final class TelemetryClientTests: XCTestCase { let span = try XCTUnwrap(rootExporter.getFinishedSpanItems().only) XCTAssertEqual(span.attributes["test.case.name"], .string("[ios] PaymentTests.testChargeCard [debug]")) - XCTAssertEqual(span.resource.attributes[BuildkiteTelemetryAttribute.runKey], .string("custom-run-key")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.runKey], .string("custom-run-key")) + XCTAssertEqual(span.attributes["custom.flag"], .bool(true)) XCTAssertEqual(span.resource.attributes["vcs.ref.head.name"], .string("custom-branch")) - XCTAssertEqual(span.resource.attributes["custom.flag"], .bool(true)) - XCTAssertNil(span.resource.attributes["key"]) - XCTAssertNil(span.resource.attributes["branch"]) + XCTAssertNil(span.resource.attributes[BuildkiteTelemetryAttribute.runKey]) + XCTAssertNil(span.resource.attributes["custom.flag"]) + XCTAssertNil(span.attributes["key"]) + XCTAssertNil(span.attributes["branch"]) + } + + func testUsesProviderNativeCIRunIdentityAndKeepsOtherURLsOnExecutionRoot() throws { + let github = try self.exportExecution(environmentValues: [ + "GITHUB_ACTION": "test", + "GITHUB_ACTOR": "octocat", + "GITHUB_REF_NAME": "main", + "GITHUB_REPOSITORY": "acme/payments", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_RUN_ID": "github-123", + "GITHUB_RUN_NUMBER": "42", + "GITHUB_SHA": "abc123", + "GITHUB_WORKFLOW": "Tests", + ]) + XCTAssertEqual(github.resource.attributes["cicd.pipeline.run.id"], .string("github-123")) + XCTAssertEqual( + github.resource.attributes["cicd.pipeline.run.url.full"], + .string("https://github.com/acme/payments/actions/runs/github-123") + ) + XCTAssertNil(github.attributes[BuildkiteTelemetryAttribute.runURL]) + + let circle = try self.exportExecution(environmentValues: [ + "CIRCLE_BRANCH": "main", + "CIRCLE_BUILD_NUM": "42", + "CIRCLE_BUILD_URL": "https://circle.example/jobs/42", + "CIRCLE_SHA1": "abc123", + "CIRCLE_WORKFLOW_ID": "circle-123", + ]) + XCTAssertEqual(circle.resource.attributes["cicd.pipeline.run.id"], .string("circle-123")) + XCTAssertNil(circle.resource.attributes["cicd.pipeline.run.url.full"]) + XCTAssertEqual( + circle.attributes[BuildkiteTelemetryAttribute.runURL], + .string("https://circle.example/jobs/42") + ) + + let xcode = try self.exportExecution(environmentValues: [ + "CI_BRANCH": "main", + "CI_BUILD_ID": "xcode-123", + "CI_BUILD_NUMBER": "42", + "CI_COMMIT": "abc123", + "CI_PULL_REQUEST_HTML_URL": "https://github.com/acme/payments/pull/42", + "CI_WORKFLOW": "Tests", + ]) + XCTAssertEqual(xcode.resource.attributes["cicd.pipeline.run.id"], .string("xcode-123")) + XCTAssertNil(xcode.resource.attributes["cicd.pipeline.run.url.full"]) + XCTAssertEqual( + xcode.attributes[BuildkiteTelemetryAttribute.runURL], + .string("https://github.com/acme/payments/pull/42") + ) + + let buildkite = try self.exportExecution(environmentValues: [ + "BUILDKITE_ANALYTICS_URL": "https://configured.example/run", + "BUILDKITE_BUILD_ID": "buildkite-123", + "BUILDKITE_BUILD_URL": "https://buildkite.example/builds/42", + ]) + XCTAssertEqual(buildkite.resource.attributes["cicd.pipeline.run.id"], .string("buildkite-123")) + XCTAssertEqual( + buildkite.resource.attributes["cicd.pipeline.run.url.full"], + .string("https://buildkite.example/builds/42") + ) + XCTAssertEqual( + buildkite.attributes[BuildkiteTelemetryAttribute.runURL], + .string("https://configured.example/run") + ) + } + + func testRetainsRequiredSynthesisAttributesAtMinimumUsefulSpanLimit() throws { + let rootExporter = InMemoryExporter() + let client = try XCTUnwrap(TelemetryClient.live( + environment: self.configuredEnvironment, + uploadTags: ["configured": "optional"], + logger: nil, + rootExporter: rootExporter, + childExporter: InMemoryExporter(), + rootSpanLimits: SpanLimits().settingAttributeCountLimit(3) + )) + var test = TestState(id: UUID(), className: "LimitedTests", testName: "testRequiredFields") + + let executionID = client.startExecution(test) + test.result = .passed + client.finishExecution(executionID, test: test, tags: ["execution": "optional"]) + + let span = try XCTUnwrap(rootExporter.getFinishedSpanItems().only) + XCTAssertEqual(span.attributes, [ + BuildkiteTelemetryAttribute.executionVia: .string("otlp"), + BuildkiteTelemetryAttribute.runKey: .string("run-key"), + SemanticConventions.Test.caseResultStatus.rawValue: .string("pass"), + ]) } func testBundleFlushRetriesExecutionsQueuedDuringBackoff() throws { @@ -356,6 +490,31 @@ final class TelemetryClientTests: XCTestCase { getFromInfoDictionary: { _ in nil } ) } + + private func exportExecution(environmentValues: [String: String]) throws -> SpanData { + OpenTelemetry.registerTracerProvider(tracerProvider: DefaultTracerProvider.instance) + let rootExporter = InMemoryExporter() + var values = environmentValues + values["BUILDKITE_ANALYTICS_TOKEN"] = "SECRET" + let client = try XCTUnwrap(TelemetryClient.live( + environment: EnvironmentValues( + values: values, + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ), + uploadTags: [:], + logger: nil, + rootExporter: rootExporter, + childExporter: InMemoryExporter() + )) + var test = TestState(id: UUID(), className: "CITests", testName: "testIdentity") + + let executionID = client.startExecution(test) + test.result = .passed + client.finishExecution(executionID, test: test, tags: nil) + + return try XCTUnwrap(rootExporter.getFinishedSpanItems().only) + } } private extension Array { From e600b300d7e4f359861999dda08a2aeb647554f2 Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Thu, 27 Aug 2026 19:36:59 +0930 Subject: [PATCH 20/20] Avoid Swift optimizer stall in span setup Flatten the self-returning SpanBuilder existential calls. Swift 6.2 and 6.3 otherwise spend many minutes in SILCombine after adding the required synthesis attributes, while separate calls produce the same attribute ordering and compile normally. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a037c7-e3e7-70e4-b18f-a43cec542d56 --- .../Core/Telemetry/TelemetryClient+Live.swift | 57 +++++++++---------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift index 8fc9a582..76a74103 100644 --- a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift +++ b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift @@ -361,40 +361,39 @@ private final class LiveTelemetryClient { "\(test.className).\(test.testName)", self.executionNameSuffix, ].compactMap { $0 }.joined(separator: " ") - let builder = self.tracer - .spanBuilder(spanName: TelemetryValue.rootSpanName) - .setNoParent() + let builder = self.tracer.spanBuilder(spanName: TelemetryValue.rootSpanName) + builder.setNoParent() for (key, value) in self.executionAttributes { builder.setAttribute(key: key, value: value) } - builder - .setAttribute(key: BuildkiteTelemetryAttribute.executionScope, value: test.className) - .setAttribute(key: BuildkiteTelemetryAttribute.testName, value: test.testName) - .setAttribute( - key: BuildkiteTelemetryAttribute.executionExternalID, - value: test.id.uuidString - ) - .setAttribute( - key: SemanticConventions.Test.caseName.rawValue, - value: qualifiedTestName - ) - .setAttribute(key: SemanticConventions.Test.suiteName.rawValue, value: test.className) - // The Swift SDK keeps the most recently set attributes when the span's - // start-time attribute limit is exceeded. Set the three synthesis fields - // last so even a limit of three still produces a usable execution. - .setAttribute(key: BuildkiteTelemetryAttribute.executionVia, value: TelemetryValue.executionVia) - .setAttribute( - key: BuildkiteTelemetryAttribute.runKey, - value: self.runKey - ) - // Reserve the result's place before test code can consume the remaining - // attribute budget. finishExecution replaces this placeholder. - .setAttribute( - key: SemanticConventions.Test.caseResultStatus.rawValue, - value: TelemetryValue.unsetResult - ) + builder.setAttribute(key: BuildkiteTelemetryAttribute.executionScope, value: test.className) + builder.setAttribute(key: BuildkiteTelemetryAttribute.testName, value: test.testName) + builder.setAttribute( + key: BuildkiteTelemetryAttribute.executionExternalID, + value: test.id.uuidString + ) + builder.setAttribute( + key: SemanticConventions.Test.caseName.rawValue, + value: qualifiedTestName + ) + builder.setAttribute(key: SemanticConventions.Test.suiteName.rawValue, value: test.className) + + // The Swift SDK keeps the most recently set attributes when the span's + // start-time attribute limit is exceeded. Set the three synthesis fields + // last so even a limit of three still produces a usable execution. + builder.setAttribute( + key: BuildkiteTelemetryAttribute.executionVia, + value: TelemetryValue.executionVia + ) + builder.setAttribute(key: BuildkiteTelemetryAttribute.runKey, value: self.runKey) + // Reserve the result's place before test code can consume the remaining + // attribute budget. finishExecution replaces this placeholder. + builder.setAttribute( + key: SemanticConventions.Test.caseResultStatus.rawValue, + value: TelemetryValue.unsetResult + ) if let jobSpanContext = self.jobSpanContext { builder.addLink(spanContext: jobSpanContext)