diff --git a/test-collector-swift/.buildkite/pipeline.yml b/test-collector-swift/.buildkite/pipeline.yml index 0b6b710c..99c54a59 100644 --- a/test-collector-swift/.buildkite/pipeline.yml +++ b/test-collector-swift/.buildkite/pipeline.yml @@ -23,7 +23,8 @@ steps: - docker#v3.7.0: image: "swift:{{matrix}}" matrix: - - "5.9.2" + - "5.10.1" + - "6.0.3" - label: ":swift: Tests (macOS, {{matrix}})" agents: diff --git a/test-collector-swift/CHANGELOG.md b/test-collector-swift/CHANGELOG.md index 2de16b88..da8f6579 100644 --- a/test-collector-swift/CHANGELOG.md +++ b/test-collector-swift/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## Unreleased (targeting 2.0.0-beta.1) + +- Replace proprietary execution uploads with XCTest execution root spans submitted through OpenTelemetry +- Export run and execution tags as `buildkite.tag.*` attributes +- Keep stable suite, CI worker/run, and VCS identity on OpenTelemetry resources while keeping Test Engine run metadata, framework details, custom metadata, and tags on each execution root +- Prioritize the three attributes required for execution synthesis when OpenTelemetry span attribute limits are constrained +- Support standard OTLP/HTTP exporter configuration and child-span forwarding +- Apply standard OTLP headers and protocols only with standard endpoints +- Warn when the unsupported `BUILDKITE_ANALYTICS_BASE_URL` setting is ignored +- Back off failed root-span exports instead of blocking every subsequent test +- Require Swift 5.10 and macOS 12 or newer + ## 0.6.0 - Add tagging support at upload and execution levels diff --git a/test-collector-swift/Package.swift b/test-collector-swift/Package.swift index b2888da7..46b0a0e9 100644 --- a/test-collector-swift/Package.swift +++ b/test-collector-swift/Package.swift @@ -1,11 +1,11 @@ -// swift-tools-version: 5.7.1 +// swift-tools-version: 5.10 import PackageDescription let package = Package( name: "BuildkiteTestCollector", platforms: [ - .macOS("10.15"), + .macOS("12.0"), .iOS("13.0"), .tvOS("13.0"), .watchOS("6.0") @@ -13,11 +13,33 @@ let package = Package( products: [ .library(name: "BuildkiteTestCollector", targets: ["BuildkiteTestCollector"]) ], + dependencies: [ + // OpenTelemetry 2.2.1 and later require a Swift 6.1-only grpc-swift release. + .package(url: "https://github.com/open-telemetry/opentelemetry-swift", exact: "2.2.0"), + .package(url: "https://github.com/open-telemetry/opentelemetry-swift-core", exact: "2.2.0"), + // Keep Swift 5.10 compatibility; swift-metrics 2.8 requires Swift 6.0. + .package(url: "https://github.com/apple/swift-metrics", exact: "2.7.1") + ], targets: [ .target(name: "BuildkiteTestCollector", dependencies: ["Core", "Loader"]), - .target(name: "Core"), + .target( + name: "Core", + dependencies: [ + .product(name: "OpenTelemetryProtocolExporterHTTP", package: "opentelemetry-swift"), + .product(name: "OpenTelemetrySdk", package: "opentelemetry-swift-core"), + .product(name: "CoreMetrics", package: "swift-metrics") + ] + ), .target(name: "Loader"), .testTarget(name: "BuildkiteTestCollectorTests", dependencies: ["BuildkiteTestCollector"]), - .testTarget(name: "CoreTests", dependencies: ["Core"]) - ] + .testTarget( + name: "CoreTests", + dependencies: [ + "Core", + .product(name: "InMemoryExporter", package: "opentelemetry-swift"), + .product(name: "OpenTelemetrySdk", package: "opentelemetry-swift-core") + ] + ) + ], + swiftLanguageVersions: [.v5] ) diff --git a/test-collector-swift/README.md b/test-collector-swift/README.md index df0187c2..288db2cd 100644 --- a/test-collector-swift/README.md +++ b/test-collector-swift/README.md @@ -1,4 +1,4 @@ -# Buildkite Test Collector for Swift (Beta) +# Buildkite Test Collector for Swift Official [Buildkite Test Engine](https://buildkite.com/platform/test-engine/) collector for Swift test frameworks ✨ @@ -6,6 +6,19 @@ Official [Buildkite Test Engine](https://buildkite.com/platform/test-engine/) co 📦 **Supported CI systems:** Buildkite, GitHub Actions, CircleCI, Xcode Cloud, and others via the `BUILDKITE_ANALYTICS_*` environment variables. +Each test execution is exported as a parentless OpenTelemetry `test.execution` +root span. Sampled OpenTelemetry spans created by the code under test can appear +as its children. Test execution data is submitted only through OTLP; the +collector does not use the legacy proprietary execution upload API. + +The collector requires Swift 5.10 or newer. Its macOS deployment target is macOS +12 or newer. + +> [!NOTE] +> The OpenTelemetry-based collector on this branch is being prepared for +> `2.0.0-beta.1` and has not been released. The installation example below +> continues to reference the latest released version, `0.6.0`. + ## 👉 Installing ### Step 1 @@ -40,6 +53,32 @@ let package = Package( Set the `BUILDKITE_ANALYTICS_TOKEN` secret on your CI to the API token from earlier. +By default, traces are sent using OTLP/HTTP protobuf to +`https://tests-otlp.buildkite.com/v1/traces`. The collector adds the suite token +and run key as request headers. + +To send traces through an OpenTelemetry endpoint such as the Buildkite Test +Engine Client relay, set the standard exporter variables instead: + +```bash +export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="http://127.0.0.1:4318/v1/traces" +export OTEL_EXPORTER_OTLP_TRACES_HEADERS="authorization=Bearer%20local-token" +export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL="http/protobuf" +``` + +The signal-specific variables take precedence over +`OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_HEADERS`, and +`OTEL_EXPORTER_OTLP_PROTOCOL`. A generic OTLP endpoint has `/v1/traces` +appended. Standard header and protocol variables are applied only when a +standard endpoint is explicitly configured; those variables alone do not enable +the collector or select an endpoint. Standard endpoints receive only the +explicitly configured OTLP headers, and the collector does not send the +Buildkite suite token to them. +`BUILDKITE_ANALYTICS_OTLP_ENDPOINT` remains available as a trusted, +collector-specific endpoint override that receives only the suite token and run +key and always uses HTTP/protobuf. To send custom headers or select a protocol +for a relay, configure it through the standard exporter variables instead. + ### Step 3 If you're testing an Xcode project there's an extra step, Xcode doesn't pass environment variables from the process to the test runner so we need to manually map them. Open your test scheme or test plan(whichever you are using) and under the environment variable section add the following entry: @@ -50,22 +89,35 @@ key: value: `$(BUILDKITE_ANALYTICS_TOKEN)` +If you configure a standard OTLP endpoint, map that endpoint and its associated +`OTEL_EXPORTER_OTLP_TRACES_*` header or protocol variables into the test runner +in the same way. + The same key value pair can be specified in your main bundle's `info.plist` file if you would rather specify it there. Note variables in the environment take precedent over those in the `info.plist` file. ### Step 3.5 (Optional) -The only required environment variable is the analytics token but if you're using one of the supported CI platforms they can pass extra information to the test-collector to enrich the reports. Things like commit messages, branch names, build numbers, etc. Open your test scheme or test plan again and add the following key value pairs depending on your CI platform. +When using the default endpoint, only the analytics token is required. Supported +CI platforms can pass extra information to enrich the reports, such as commit +messages, branch names, and build numbers. Open your test scheme or test plan +again and add the following key-value pairs for your CI platform. **Buildkite** ``` +Key: BUILDKITE_AGENT_ID, Value: $(BUILDKITE_AGENT_ID) Key: BUILDKITE_BUILD_ID, Value: $(BUILDKITE_BUILD_ID) Key: BUILDKITE_BUILD_URL, Value: $(BUILDKITE_BUILD_URL) Key: BUILDKITE_BRANCH, Value: $(BUILDKITE_BRANCH) Key: BUILDKITE_COMMIT, Value: $(BUILDKITE_COMMIT) Key: BUILDKITE_BUILD_NUMBER, Value: $(BUILDKITE_BUILD_NUMBER) Key: BUILDKITE_JOB_ID, Value: $(BUILDKITE_JOB_ID) +Key: BUILDKITE_STEP_ID, Value: $(BUILDKITE_STEP_ID) Key: BUILDKITE_MESSAGE, Value: $(BUILDKITE_MESSAGE) +Key: BUILDKITE_ORGANIZATION_SLUG, Value: $(BUILDKITE_ORGANIZATION_SLUG) +Key: BUILDKITE_TEST_ENGINE_SUITE_SLUG, Value: $(BUILDKITE_TEST_ENGINE_SUITE_SLUG) +Key: TRACEPARENT, Value: $(TRACEPARENT) +Key: TRACESTATE, Value: $(TRACESTATE) ``` **Circle CI** @@ -118,9 +170,9 @@ git push origin add-buildkite-test-engine You can tag test executions with key-value pairs to filter and group results in [Test Engine](https://buildkite.com/docs/test-engine/test-suites/tags). -### Upload-level tags +### Run-level tags -Upload-level tags apply to all test executions in a run. Set the `BUILDKITE_ANALYTICS_TAGS` environment variable to a JSON object: +Run-level tags apply to all test executions in a run. Set the `BUILDKITE_ANALYTICS_TAGS` environment variable to a JSON object: ```bash export BUILDKITE_ANALYTICS_TAGS='{"host.arch":"arm64","cloud.region":"us-east-1"}' @@ -128,12 +180,15 @@ export BUILDKITE_ANALYTICS_TAGS='{"host.arch":"arm64","cloud.region":"us-east-1" If you're using an Xcode project, add this to your test scheme or test plan environment variables like the other `BUILDKITE_ANALYTICS_*` variables. -Upload-level tags can also be set programmatically by passing them to `load`. Environment variable tags take precedence over programmatic tags when keys collide: +Run-level tags can also be set programmatically by passing them to `load`. Environment variable tags take precedence over programmatic tags when keys collide: ```swift TestCollector.load(uploadTags: ["host.arch": "arm64"]) ``` +Run tags are exported as `buildkite.tag.` attributes on every test root. +An execution-level tag takes precedence when it uses the same key. + ### Execution-level tags Tag individual tests from within a test method using the `tagExecution` extension on `XCTestCase`: @@ -149,6 +204,45 @@ class PaymentTests: XCTestCase { } ``` +The existing `tagExecution` API is unchanged. Execution tags are exported as +`buildkite.tag.` attributes on that test's root span. + +## OpenTelemetry behavior and delivery + +The collector creates each `test.execution` span through a private AlwaysOn +provider, so it remains a trace root regardless of an application's sampling +configuration. When Buildkite Agent trace context is available, the execution +links to the job span instead of becoming its child. If the test suite has an +OpenTelemetry SDK provider, the collector adds a forwarding processor without +replacing its sampler or exporters. Otherwise, it installs a provider so spans +created under a test can be exported as execution children. + +Resources identify producer-wide entities: the suite, provider-native CI run +and URL, Buildkite worker, and checked-out VCS ref. Test Engine run metadata, +collector and framework details, custom run metadata, and run tags are attached +only to each `test.execution` root. They are not duplicated onto child spans. +Collector-managed children share the producer resource; children from a tracer +provider configured by the test suite keep that provider's resource. + +The collector prioritizes `buildkite.execution.via`, `buildkite.run_key`, and +`test.case.result.status` so Test Engine can synthesize an execution when the +OpenTelemetry SDK span attribute limit is constrained. The minimum useful span +attribute limit is three; additional metadata and tags may be dropped at that +limit. + +While the endpoint is healthy, finishing a test waits for its root span to be +accepted by the configured OTLP endpoint. After a failure, subsequent roots stay +in the collector's in-memory queue while export attempts back off for 10, 20, +30, and then at most 60 seconds. The bundle-end flush always makes an immediate +final attempt. When the endpoint is the Test Engine Client relay, acceptance +transfers the span to the longer-lived parent process, which protects completed +executions from an XCTest runner restart. + +This is not disk-backed delivery. A hard exit before a test finishes, a process +exit while the endpoint remains unavailable, or a machine restart can still +lose spans. An ambiguous network timeout may also result in a retry after the +server accepted the first request. + ## 🔍 Debugging To enable debugging output, set the `BUILDKITE_ANALYTICS_DEBUG_ENABLED` environment variable to `true`. This also needs diff --git a/test-collector-swift/Sources/BuildkiteTestCollector/TestCollector.swift b/test-collector-swift/Sources/BuildkiteTestCollector/TestCollector.swift index 621b80f8..2e8733cf 100644 --- a/test-collector-swift/Sources/BuildkiteTestCollector/TestCollector.swift +++ b/test-collector-swift/Sources/BuildkiteTestCollector/TestCollector.swift @@ -1,8 +1,13 @@ import Core public enum TestCollector { - /// The base URL for the Buildkite Test Engine API. + /// The default OpenTelemetry traces endpoint for Buildkite Test Engine. + public static var endpoint: String { + Core.TestCollector.endpoint + } + + @available(*, deprecated, renamed: "endpoint") public static var baseURL: String { - Core.TestCollector.baseURL + self.endpoint } } diff --git a/test-collector-swift/Sources/Core/ApiClient/ApiClient+Live.swift b/test-collector-swift/Sources/Core/ApiClient/ApiClient+Live.swift deleted file mode 100644 index c2ba2e5c..00000000 --- a/test-collector-swift/Sources/Core/ApiClient/ApiClient+Live.swift +++ /dev/null @@ -1,35 +0,0 @@ -import Foundation - -#if canImport(FoundationNetworking) -import FoundationNetworking -#endif - -extension ApiClient { - static func live( - apiToken: String, - baseURL: URL, - encoder: JSONEncoder = .init(), - decoder: JSONDecoder = .init(), - session: ApiSession = .urlSession(.shared) - ) -> ApiClient { - func makeRequest(from route: ApiRoute) throws -> URLRequest { - var request = URLRequest(url: baseURL) - request.setValue("Token token=\"\(apiToken)\"", forHTTPHeaderField: "Authorization") - - switch route { - case .upload(let testData): - let data = try encoder.encode(testData) - request.url?.appendPathComponent("uploads") - request.setValue("application/json", forHTTPHeaderField: "Content-Type") - request.httpMethod = "POST" - request.httpBody = data - return request - } - } - - return ApiClient(decoder: decoder) { route in - let request = try makeRequest(from: route) - return try await session.data(for: request) - } - } -} diff --git a/test-collector-swift/Sources/Core/ApiClient/ApiClient.swift b/test-collector-swift/Sources/Core/ApiClient/ApiClient.swift deleted file mode 100644 index 59317d4e..00000000 --- a/test-collector-swift/Sources/Core/ApiClient/ApiClient.swift +++ /dev/null @@ -1,35 +0,0 @@ -import Foundation - -#if canImport(FoundationNetworking) -import FoundationNetworking -#endif - -struct ApiClient { - let decoder: JSONDecoder - var request: (ApiRoute) async throws -> (Data, HTTPURLResponse) - - init( - decoder: JSONDecoder = JSONDecoder(), - request: @escaping (ApiRoute) async throws -> (Data, HTTPURLResponse) - ) { - self.request = request - self.decoder = decoder - } - - func data(for route: ApiRoute) async throws -> (value: Data, response: HTTPURLResponse) { - try await self.request(route) - } - - func data( - for route: ApiRoute, - as type: Value.Type - ) async throws -> (value: Value, response: HTTPURLResponse) { - let (data, response) = try await self.data(for: route) - let value = try decode(data, as: type) - return (value, response) - } - - func decode(_ data: Data, as type: Value.Type) throws -> Value { - try self.decoder.decode(type, from: data) - } -} diff --git a/test-collector-swift/Sources/Core/ApiClient/ApiSession.swift b/test-collector-swift/Sources/Core/ApiClient/ApiSession.swift deleted file mode 100644 index 664def17..00000000 --- a/test-collector-swift/Sources/Core/ApiClient/ApiSession.swift +++ /dev/null @@ -1,48 +0,0 @@ -import Foundation - -#if canImport(FoundationNetworking) -import FoundationNetworking -#endif - -struct ApiSession { - var data: (URLRequest) async throws -> (Data, HTTPURLResponse) - - func data(for request: URLRequest) async throws -> (Data, HTTPURLResponse) { - try await self.data(request) - } -} - -extension ApiSession { - static func urlSession(_ session: URLSession) -> ApiSession { - ApiSession { request in - #if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) - if #available(macOS 12, iOS 15, tvOS 15, watchOS 8, *) { - let (data, response) = try await session.data(for: request) - guard let httpResponse = response as? HTTPURLResponse else { - throw URLError(.unsupportedURL) - } - return (data, httpResponse) - } - #endif - - var dataTask: URLSessionDataTask? - let cancel: () -> Void = { dataTask?.cancel() } - - return try await withTaskCancellationHandler( - operation: { - try await withCheckedThrowingContinuation { continuation in - dataTask = session.dataTask(with: request) { data, response, error in - if let data, let response { - continuation.resume(returning: (data, response as! HTTPURLResponse)) - } else { - continuation.resume(throwing: error ?? URLError(.badServerResponse)) - } - } - dataTask?.resume() - } - }, - onCancel: { cancel() } - ) - } - } -} diff --git a/test-collector-swift/Sources/Core/Logger/Logger.swift b/test-collector-swift/Sources/Core/Logger/Logger.swift index 64e0a86c..66dd52d9 100644 --- a/test-collector-swift/Sources/Core/Logger/Logger.swift +++ b/test-collector-swift/Sources/Core/Logger/Logger.swift @@ -61,6 +61,15 @@ struct Logger { self.log(level: .info, message()) } + /// Log a message with the `warning` log level + /// + /// If the logger's `logLevel` is greater than `warning`, nothing will be logged. + /// + /// - Parameter message: The message to be logged. + func warning(_ message: @autoclosure () -> String) { + self.log(level: .warning, message()) + } + /// Log a message with the `error` log level /// /// If the logger's `logLevel` is greater than `error`, nothing will be logged. @@ -88,6 +97,7 @@ extension Logger { enum Level: String, Comparable { case debug case info + case warning case error // Based on values from https://github.com/apple/swift-log @@ -95,6 +105,7 @@ extension Logger { switch self { case .debug: return 1 case .info: return 2 + case .warning: return 3 case .error: return 5 } } diff --git a/test-collector-swift/Sources/Core/Models/Api/ApiRoute.swift b/test-collector-swift/Sources/Core/Models/Api/ApiRoute.swift deleted file mode 100644 index 6b941bd7..00000000 --- a/test-collector-swift/Sources/Core/Models/Api/ApiRoute.swift +++ /dev/null @@ -1,9 +0,0 @@ -import Foundation - -/// Different endpoints for Buildkite Test Engine -/// -/// Currently there is only a single upload endpoint, in the future this could be expanded to include others though. The new route would be handled in -/// `ApiClient`'s `makeRequest` function -enum ApiRoute: Equatable { - case upload(TestResults) -} diff --git a/test-collector-swift/Sources/Core/Models/Api/RunEnvironment.swift b/test-collector-swift/Sources/Core/Models/Api/RunEnvironment.swift deleted file mode 100644 index eb49ca81..00000000 --- a/test-collector-swift/Sources/Core/Models/Api/RunEnvironment.swift +++ /dev/null @@ -1,102 +0,0 @@ -/// A type containing information about the environment performing the test run. -/// -/// The only required property is `key` which represents a unique identifier for a test run. -struct RunEnvironment: Equatable { - /// The continuous integration platform. - var ci: String? - - /// A unique identifier. - var key: String - - /// The URL associated with the test run. - var url: String? - - /// The branch name. - var branch: String? - - /// The commit hash. - var commitSha: String? - - /// The run number. - var number: String? - - /// The job identifier. - var jobId: String? - - /// A message associated with the test run. - var message: String? - - /// A value indicating if the collector ran in debug mode. - var debug: String? - - /// A tag added to the start of the execution name. - var executionNamePrefix: String? - - /// A tag added to the end of the execution name. - var executionNameSuffix: String? - - /// The version of the collector used. - var version: String? - - /// The name of the collector used. - var collector: String? - - /// A dictionary that contains custom values associated with the test run. - /// - /// - Note: Used internally for testing experimental features. If an existing key - /// is used, the custom environment value will take precedence. - var customEnvironment: [String: AnyCodable]? -} - -extension RunEnvironment: Encodable { - struct CodingKey: Swift.CodingKey { - var stringValue: String - var intValue: Int? { Int(self.stringValue) } - - init(_ stringValue: String) { - self.stringValue = stringValue - } - - init(stringValue: String) { - self.stringValue = stringValue - } - - init(intValue: Int) { - self.stringValue = String(intValue) - } - - static var ci = Self("CI") - static var key = Self("key") - static var url = Self("url") - static var branch = Self("branch") - static var commitSha = Self("commit_sha") - static var number = Self("number") - static var jobId = Self("job_id") - static var message = Self("message") - static var debug = Self("debug") - static var executionNamePrefix = Self("execution_name_prefix") - static var executionNameSuffix = Self("execution_name_suffix") - static var version = Self("version") - static var collector = Self("collector") - } - - func encode(to encoder: Encoder) throws { - var container = encoder.container(keyedBy: CodingKey.self) - try container.encodeIfPresent(self.ci, forKey: .ci) - try container.encode(self.key, forKey: .key) - try container.encodeIfPresent(self.url, forKey: .url) - try container.encodeIfPresent(self.branch, forKey: .branch) - try container.encodeIfPresent(self.commitSha, forKey: .commitSha) - try container.encodeIfPresent(self.number, forKey: .number) - try container.encodeIfPresent(self.jobId, forKey: .jobId) - try container.encodeIfPresent(self.message, forKey: .message) - try container.encodeIfPresent(self.debug, forKey: .debug) - try container.encodeIfPresent(self.executionNamePrefix, forKey: .executionNamePrefix) - try container.encodeIfPresent(self.executionNameSuffix, forKey: .executionNameSuffix) - try container.encodeIfPresent(self.version, forKey: .version) - try container.encodeIfPresent(self.collector, forKey: .collector) - for (key, value) in self.customEnvironment ?? [:] { - try container.encode(value, forKey: CodingKey(key)) - } - } -} diff --git a/test-collector-swift/Sources/Core/Models/Api/TestResults.swift b/test-collector-swift/Sources/Core/Models/Api/TestResults.swift deleted file mode 100644 index 9f19f1c2..00000000 --- a/test-collector-swift/Sources/Core/Models/Api/TestResults.swift +++ /dev/null @@ -1,37 +0,0 @@ -/// A type containing a test run's metadata and test results. -/// -/// This is the type expected by the Test Engine upload API. -struct TestResults: Equatable { - /// The format for the included data - var format: String - - /// Information used to identify the test run. - /// Test results with matching run_env[key] will be grouped into a single run by Test Engine - var runEnv: RunEnvironment - - /// Tags to apply to the upload - var tags: [String: String]? - - /// An array of test executions - var data: [Trace] - - /// Constructs test results compatible with the JSON API - /// - /// - Parameters: - /// - runEnv: The run environment for when the data was captured. - /// - tags: Optional tags to apply to the upload. - /// - data: An array of test executions. - /// - Returns: Test results compatible with the JSON API - static func json(runEnv: RunEnvironment, tags: [String: String]? = nil, data: [Trace]) -> TestResults { - TestResults(format: "json", runEnv: runEnv, tags: tags, data: data) - } -} - -extension TestResults: Encodable { - enum CodingKeys: String, CodingKey { - case format - case runEnv = "run_env" - case tags - case data - } -} diff --git a/test-collector-swift/Sources/Core/Models/Api/Trace+FailureExpanded.swift b/test-collector-swift/Sources/Core/Models/Api/Trace+FailureExpanded.swift deleted file mode 100644 index e46aa5da..00000000 --- a/test-collector-swift/Sources/Core/Models/Api/Trace+FailureExpanded.swift +++ /dev/null @@ -1,17 +0,0 @@ -extension Trace { - /// A type containing details related to a test failure that occurred during a trace. - struct FailureExpanded: Encodable, Equatable { - /// An array of strings containing a line separated failure message or additional details. - var expanded: [String] - - /// An array of strings representing frames on the call stack when the failure occurred. - var backtrace: [String] - } -} - -extension Trace.FailureExpanded { - init(issue: TestIssue) { - self.expanded = issue.description.components(separatedBy: "\n") - self.backtrace = issue.sourceCodeContext.callStack.enumerated().map { "\($0) \($1)" } - } -} diff --git a/test-collector-swift/Sources/Core/Models/Api/Trace+Result.swift b/test-collector-swift/Sources/Core/Models/Api/Trace+Result.swift deleted file mode 100644 index 375c9787..00000000 --- a/test-collector-swift/Sources/Core/Models/Api/Trace+Result.swift +++ /dev/null @@ -1,18 +0,0 @@ -extension Trace { - /// The result for the test associated with a trace. - enum Result: String, Encodable, Equatable { - case passed - case failed - case skipped - } -} - -extension Trace.Result { - init(_ result: TestResult) { - switch result { - case .passed: self = .passed - case .skipped: self = .skipped - case .failed: self = .failed - } - } -} diff --git a/test-collector-swift/Sources/Core/Models/Api/Trace+Span.swift b/test-collector-swift/Sources/Core/Models/Api/Trace+Span.swift deleted file mode 100644 index 9ea5c8ca..00000000 --- a/test-collector-swift/Sources/Core/Models/Api/Trace+Span.swift +++ /dev/null @@ -1,35 +0,0 @@ -import Foundation - -extension Trace { - /// A type containing information for a timed block of work. - struct Span: Equatable { - /// The type of span. Used for categorising spans. - var section: String - - /// A monotonic timestamp in seconds for when the span started. - var startAt: TimeInterval? - - /// A monotonic timestamp in seconds for when the span ended. - var endAt: TimeInterval? - - /// The number of seconds that elapsed during this span. - var duration: TimeInterval? - - /// Any information related to this span. - var detail: [String: String] = [:] - - /// Any spans that occurred during this span. - var children: [Span] = [] - } -} - -extension Trace.Span: Encodable { - enum CodingKeys: String, CodingKey { - case section - case startAt = "start_at" - case endAt = "end_at" - case duration - case detail - case children - } -} diff --git a/test-collector-swift/Sources/Core/Models/Api/Trace.swift b/test-collector-swift/Sources/Core/Models/Api/Trace.swift deleted file mode 100644 index b7414e14..00000000 --- a/test-collector-swift/Sources/Core/Models/Api/Trace.swift +++ /dev/null @@ -1,73 +0,0 @@ -/// A type containing the span information and result of a test -struct Trace: Equatable { - /// A unique identifier. - var id: String - - /// The scope of the test. - /// - /// In XCTest this is usually the type name of an XCTestCase. - var scope: String? - - /// The name of the test. - /// - /// In XCTest this is usually a method of an XCTestCase. - var name: String? - - /// The source location of the test. - var location: String? - - /// The name of the file containing the test. - var fileName: String? - - /// The result of the test. - var result: Result? - - /// A short description of the failure. - var failureReason: String? - - /// An array of additional details related failures. - var failureExpanded: [FailureExpanded] = [] - - /// The span for the duration of the test. - var history: Span - - /// Tags associated with this test execution. - var tags: [String: String]? -} - -extension Trace: Encodable { - enum CodingKeys: String, CodingKey { - case id - case scope - case name - case location - case fileName = "file_name" - case result - case failureReason = "failure_reason" - case failureExpanded = "failure_expanded" - case history - case tags - } -} - -extension Trace { - init(test: TestState, span: Span, tags: [String: String]? = nil) { - self.id = test.id.uuidString - self.scope = test.className - self.name = test.testName - self.result = test.result.map(Trace.Result.init) ?? .failed - if test.issues.count > 1 { - let failureReasons = test.issues.map(\.compactDescription).joined(separator: ", ") - self.failureReason = "\(test.issues.count) failures: \(failureReasons)" - } else { - self.failureReason = test.issues.first?.compactDescription - } - if let issueLocation = test.issues.first?.sourceCodeContext.location { - self.fileName = issueLocation.fileName - self.location = "\(issueLocation.fileName):\(issueLocation.line)" - } - self.failureExpanded = test.issues.map(Trace.FailureExpanded.init(issue:)) - self.history = span - self.tags = tags - } -} diff --git a/test-collector-swift/Sources/Core/Models/Api/UploadResponse.swift b/test-collector-swift/Sources/Core/Models/Api/UploadResponse.swift deleted file mode 100644 index 0ce21f7d..00000000 --- a/test-collector-swift/Sources/Core/Models/Api/UploadResponse.swift +++ /dev/null @@ -1,21 +0,0 @@ -import Foundation - -/// Response from Test Engine API after uploading test results. -struct UploadResponse: Equatable { - /// The UUID allocated to this upload - var uploadID: String? - - /// The URL that can be used to view upload details - var uploadURL: String? -} - -extension UploadResponse: Decodable { - enum CodingKeys: String, CodingKey { - case uploadID = "upload_id" - case uploadURL = "upload_url" - } -} - -struct UploadFailureResponse: Equatable, Codable { - var message: String -} diff --git a/test-collector-swift/Sources/Core/Models/Environment/EnvironmentValues.swift b/test-collector-swift/Sources/Core/Models/Environment/EnvironmentValues.swift index 4323a0d9..b703758b 100644 --- a/test-collector-swift/Sources/Core/Models/Environment/EnvironmentValues.swift +++ b/test-collector-swift/Sources/Core/Models/Environment/EnvironmentValues.swift @@ -81,11 +81,21 @@ struct EnvironmentValues { extension EnvironmentValues { var isAnalyticsEnabled: Bool { self.bool(for: "BUILDKITE_ANALYTICS_ENABLED") ?? true } var analyticsToken: String? { self.string(for: "BUILDKITE_ANALYTICS_TOKEN", private: true) } - var analyticsBaseURL: URL? { self.url(for: "BUILDKITE_ANALYTICS_BASE_URL") } + var legacyAnalyticsBaseURL: String? { self.string(for: "BUILDKITE_ANALYTICS_BASE_URL") } + var analyticsOTLPEndpoint: String? { self.string(for: "BUILDKITE_ANALYTICS_OTLP_ENDPOINT") } var isAnalyticsDebugEnabled: Bool { self.bool(for: "BUILDKITE_ANALYTICS_DEBUG_ENABLED") ?? false } var analyticsTags: [String: String]? { self.stringDictionary(for: "BUILDKITE_ANALYTICS_TAGS") } + var otelEndpoint: String? { self.string(for: "OTEL_EXPORTER_OTLP_ENDPOINT") } + var otelHeaders: String? { self.string(for: "OTEL_EXPORTER_OTLP_HEADERS", private: true) } + var otelProtocol: String? { self.string(for: "OTEL_EXPORTER_OTLP_PROTOCOL") } + var otelTracesEndpoint: String? { self.string(for: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") } + var otelTracesHeaders: String? { + self.string(for: "OTEL_EXPORTER_OTLP_TRACES_HEADERS", private: true) + } + var otelTracesProtocol: String? { self.string(for: "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL") } + var analyticsKey: String? { self.string(for: "BUILDKITE_ANALYTICS_KEY") } var analyticsUrl: String? { self.string(for: "BUILDKITE_ANALYTICS_URL") } var analyticsBranch: String? { self.string(for: "BUILDKITE_ANALYTICS_BRANCH") } @@ -96,11 +106,19 @@ extension EnvironmentValues { var buildkiteBuildId: String? { self.string(for: "BUILDKITE_BUILD_ID") } var buildkiteBuildUrl: String? { self.string(for: "BUILDKITE_BUILD_URL") } + var buildkiteAgentId: String? { self.string(for: "BUILDKITE_AGENT_ID") } var buildkiteBranch: String? { self.string(for: "BUILDKITE_BRANCH") } var buildkiteCommit: String? { self.string(for: "BUILDKITE_COMMIT") } var buildkiteBuildNumber: String? { self.string(for: "BUILDKITE_BUILD_NUMBER") } var buildkiteJobId: String? { self.string(for: "BUILDKITE_JOB_ID") } var buildkiteMessage: String? { self.string(for: "BUILDKITE_MESSAGE") } + var buildkiteOrganizationSlug: String? { self.string(for: "BUILDKITE_ORGANIZATION_SLUG") } + var buildkiteStepId: String? { self.string(for: "BUILDKITE_STEP_ID") } + var buildkiteTag: String? { self.string(for: "BUILDKITE_TAG") } + var testEngineSuiteSlug: String? { self.string(for: "BUILDKITE_TEST_ENGINE_SUITE_SLUG") } + + var traceParent: String? { self.string(for: "TRACEPARENT") } + var traceState: String? { self.string(for: "TRACESTATE") } var ci: String? { self.string(for: "CI") } diff --git a/test-collector-swift/Sources/Core/Models/Environment/RunEnvironment.swift b/test-collector-swift/Sources/Core/Models/Environment/RunEnvironment.swift new file mode 100644 index 00000000..301b1a22 --- /dev/null +++ b/test-collector-swift/Sources/Core/Models/Environment/RunEnvironment.swift @@ -0,0 +1,81 @@ +import Foundation + +/// A type containing information about the environment performing the test run. +/// +/// The only required property is `key` which represents a unique identifier for a test run. +struct RunEnvironment: Equatable { + /// The continuous integration platform. + var ci: String? + + /// A unique identifier. + var key: String + + /// The URL associated with the test run. + var url: String? + + /// The branch name. + var branch: String? + + /// The commit hash. + var commitSha: String? + + /// The run number. + var number: String? + + /// The job identifier. + var jobId: String? + + /// A message associated with the test run. + var message: String? + + /// A value indicating if the collector ran in debug mode. + var debug: String? + + /// A tag added to the start of the execution name. + var executionNamePrefix: String? + + /// A tag added to the end of the execution name. + var executionNameSuffix: String? + + /// The version of the collector used. + var version: String? + + /// The name of the collector used. + var collector: String? + + /// A dictionary that contains custom values associated with the test run. + /// + /// - Note: Used internally for testing experimental features. If an existing key + /// is used, the custom environment value will take precedence. + var customEnvironment: [String: AnyCodable]? +} + +extension RunEnvironment { + static let customFieldNames = Set([ + "CI", "key", "url", "branch", "commit_sha", "number", "job_id", "message", "debug", + "execution_name_prefix", "execution_name_suffix", "version", "collector", + ]) + + mutating func applyCustomEnvironmentOverrides() { + guard let values = self.customEnvironment else { return } + + func string(_ key: String) -> String? { + guard let value = values[key], !(value.base is NSNull) else { return nil } + return value.base as? String ?? value.description + } + + self.ci = string("CI") ?? self.ci + self.key = string("key") ?? self.key + self.url = string("url") ?? self.url + self.branch = string("branch") ?? self.branch + self.commitSha = string("commit_sha") ?? self.commitSha + self.number = string("number") ?? self.number + self.jobId = string("job_id") ?? self.jobId + self.message = string("message") ?? self.message + self.debug = string("debug") ?? self.debug + self.executionNamePrefix = string("execution_name_prefix") ?? self.executionNamePrefix + self.executionNameSuffix = string("execution_name_suffix") ?? self.executionNameSuffix + self.version = string("version") ?? self.version + self.collector = string("collector") ?? self.collector + } +} diff --git a/test-collector-swift/Sources/Core/Telemetry/ExecutionChildSpanProcessor.swift b/test-collector-swift/Sources/Core/Telemetry/ExecutionChildSpanProcessor.swift new file mode 100644 index 00000000..e9597e38 --- /dev/null +++ b/test-collector-swift/Sources/Core/Telemetry/ExecutionChildSpanProcessor.swift @@ -0,0 +1,75 @@ +import Foundation +import OpenTelemetryApi +import OpenTelemetrySdk + +final class ExecutionTraceRegistry { + private let traceIDs = LockIsolated(Set()) + + func insert(_ traceID: TraceId) { + _ = self.traceIDs.withValue { $0.insert(traceID) } + } + + func remove(_ traceID: TraceId) { + _ = self.traceIDs.withValue { $0.remove(traceID) } + } + + func contains(_ traceID: TraceId) -> Bool { + self.traceIDs.withValue { $0.contains(traceID) } + } +} + +final class ExecutionChildSpanProcessor: SpanProcessor { + let isStartRequired = true + let isEndRequired = true + + private let registry: ExecutionTraceRegistry + private let state: LockIsolated + + private struct State { + var active = true + var acceptedSpanIDs = Set() + var processor: any SpanProcessor + } + + init(processor: any SpanProcessor, registry: ExecutionTraceRegistry) { + self.registry = registry + self.state = LockIsolated(State(processor: processor)) + } + + func onStart(parentContext: SpanContext?, span: any ReadableSpan) { + guard + let parentContext, + parentContext.traceId == span.context.traceId, + self.registry.contains(span.context.traceId) + else { return } + + self.state.withValue { state in + if state.active { + state.acceptedSpanIDs.insert(span.context.spanId) + } + } + } + + func onEnd(span: any ReadableSpan) { + self.state.withValue { state in + guard state.active, state.acceptedSpanIDs.remove(span.context.spanId) != nil else { return } + state.processor.onEnd(span: span) + } + } + + func forceFlush(timeout: TimeInterval?) { + self.state.withValue { state in + if state.active { + state.processor.forceFlush(timeout: timeout) + } + } + } + + func shutdown(explicitTimeout: TimeInterval?) { + self.state.withValue { state in + state.active = false + state.acceptedSpanIDs.removeAll() + state.processor.shutdown(explicitTimeout: explicitTimeout) + } + } +} diff --git a/test-collector-swift/Sources/Core/Telemetry/ExecutionContextManager.swift b/test-collector-swift/Sources/Core/Telemetry/ExecutionContextManager.swift new file mode 100644 index 00000000..51777a3f --- /dev/null +++ b/test-collector-swift/Sources/Core/Telemetry/ExecutionContextManager.swift @@ -0,0 +1,149 @@ +import Foundation +import OpenTelemetryApi + +/// Preserves OpenTelemetry's closure-scoped context while allowing XCTest's +/// separate start and finish callbacks to keep an execution span active. It +/// also observes span creation so providers registered during a test can be +/// configured before their first child span starts. +final class ExecutionContextManager: ContextManager { + private final class ThreadSuppression: NSObject { + var counts = [String: Int]() + } + + @TaskLocal private static var suppressedKeys = Set() + + private let contextReadHandler = LockIsolated<(() -> Void)?>(nil) + private let delegate: OpenTelemetryContextProvider + private let executionSpan = LockIsolated<(any Span)?>(nil) + private let threadSuppressionKey = "com.buildkite.test-collector-swift.otel-context-suppression" + + init(delegate: OpenTelemetryContextProvider) { + self.delegate = delegate + } + + func getCurrentContextValue(forKey key: OpenTelemetryContextKeys) -> AnyObject? { + if key == .span { + let handler = self.contextReadHandler.withValue { $0 } + handler?() + } + guard !self.isSuppressed(key) else { return nil } + + switch key { + case .span: + return self.delegate.activeSpan ?? self.executionSpan.withValue { $0 } + case .baggage: + return self.delegate.activeBaggage + } + } + + func setCurrentContextValue(forKey key: OpenTelemetryContextKeys, value: AnyObject) { + switch key { + case .span: + if let span = value as? any Span { + self.delegate.setActiveSpan(span) + } + case .baggage: + if let baggage = value as? any Baggage { + self.delegate.setActiveBaggage(baggage) + } + } + } + + func removeContextValue(forKey key: OpenTelemetryContextKeys, value: AnyObject) { + switch key { + case .span: + if let span = value as? any Span { + self.delegate.removeContextForSpan(span) + } + case .baggage: + if let baggage = value as? any Baggage { + self.delegate.removeContextForBaggage(baggage) + } + } + } + + func withCurrentContextValue( + forKey key: OpenTelemetryContextKeys, + value: AnyObject?, + _ operation: () throws -> T + ) rethrows -> T { + switch (key, value) { + case let (.span, span as any SpanBase): + return try self.delegate.withActiveSpan(span, operation) + case let (.baggage, baggage as any Baggage): + return try self.delegate.withActiveBaggage(baggage, operation) + case (_, nil): + return try self.withSuppressedContext(forKey: key, operation) + default: + return try operation() + } + } + + func withCurrentContextValue( + forKey key: OpenTelemetryContextKeys, + value: AnyObject?, + _ operation: () async throws -> T + ) async rethrows -> T { + switch (key, value) { + case let (.span, span as any SpanBase): + return try await self.delegate.withActiveSpan(span, operation) + case let (.baggage, baggage as any Baggage): + return try await self.delegate.withActiveBaggage(baggage, operation) + case (_, nil): + var keys = Self.suppressedKeys + keys.insert(key.rawValue) + return try await Self.$suppressedKeys.withValue(keys, operation: operation) + default: + return try await operation() + } + } + + func setExecutionSpan(_ span: any Span) { + self.executionSpan.withValue { $0 = span } + self.delegate.setActiveSpan(span) + } + + func removeExecutionSpan(_ span: any Span) { + self.delegate.removeContextForSpan(span) + self.executionSpan.withValue { current in + if current === span { + current = nil + } + } + } + + func setContextReadHandler(_ handler: @escaping () -> Void) { + self.contextReadHandler.withValue { $0 = handler } + } + + private func isSuppressed(_ key: OpenTelemetryContextKeys) -> Bool { + if Self.suppressedKeys.contains(key.rawValue) { + return true + } + return (self.threadSuppression(create: false)?.counts[key.rawValue] ?? 0) > 0 + } + + private func withSuppressedContext( + forKey key: OpenTelemetryContextKeys, + _ operation: () throws -> T + ) rethrows -> T { + let suppression = self.threadSuppression(create: true)! + suppression.counts[key.rawValue, default: 0] += 1 + defer { + suppression.counts[key.rawValue, default: 0] -= 1 + } + return try operation() + } + + private func threadSuppression(create: Bool) -> ThreadSuppression? { + let dictionary = Thread.current.threadDictionary + if let suppression = dictionary[self.threadSuppressionKey] as? ThreadSuppression { + return suppression + } + guard create else { return nil } + + let suppression = ThreadSuppression() + dictionary[self.threadSuppressionKey] = suppression + return suppression + } +} diff --git a/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift b/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift new file mode 100644 index 00000000..8ad3f4cd --- /dev/null +++ b/test-collector-swift/Sources/Core/Telemetry/SynchronousExecutionSpanProcessor.swift @@ -0,0 +1,89 @@ +import Foundation +import OpenTelemetryApi +import OpenTelemetrySdk + +final class SynchronousExecutionSpanProcessor: SpanProcessor { + private static let retryDelays: [TimeInterval] = [10, 20, 30, 60] + + let isStartRequired = false + let isEndRequired = true + + private let exporter: any SpanExporter + private let exporterLock = NSLock() + private let logger: Logger? + private let now: () -> TimeInterval + private var pending = [SpanData]() + private var retryDelayIndex = 0 + private var retryNotBefore: TimeInterval? + + init( + exporter: any SpanExporter, + logger: Logger?, + now: @escaping () -> TimeInterval = { ProcessInfo.processInfo.systemUptime } + ) { + self.exporter = exporter + self.logger = logger + self.now = now + } + + func onStart(parentContext: SpanContext?, span: any ReadableSpan) {} + + func onEnd(span: any ReadableSpan) { + self.exporterLock.lock() + defer { self.exporterLock.unlock() } + + let name = span.getAttributes()[BuildkiteTelemetryAttribute.testName]?.description ?? span.name + self.pending.append(span.toSpanData()) + if let retryNotBefore = self.retryNotBefore, self.now() < retryNotBefore { + self.logger?.debug( + "Queued OpenTelemetry test execution during export backoff: \(name) (\(self.pending.count) pending)" + ) + return + } + + if self.exportPending() { + self.logger?.debug("Exported OpenTelemetry test execution: \(name)") + } else { + self.logger?.error( + "OpenTelemetry export failed for \(name); the execution remains queued in this process" + ) + } + } + + func forceFlush(timeout: TimeInterval?) { + self.exporterLock.lock() + defer { self.exporterLock.unlock() } + self.flush(timeout: timeout) + } + + func shutdown(explicitTimeout: TimeInterval?) { + self.exporterLock.lock() + defer { self.exporterLock.unlock() } + self.flush(timeout: explicitTimeout) + self.exporter.shutdown(explicitTimeout: explicitTimeout) + } + + private func flush(timeout: TimeInterval?) { + if !self.pending.isEmpty, !self.exportPending(timeout: timeout) { + self.logger?.error("OpenTelemetry export failed while retrying pending test executions") + } + if case .failure = self.exporter.flush(explicitTimeout: timeout) { + self.logger?.error("OpenTelemetry export failed while flushing pending test executions") + } + } + + private func exportPending(timeout: TimeInterval? = nil) -> Bool { + switch self.exporter.export(spans: self.pending, explicitTimeout: timeout) { + case .success: + self.pending.removeAll() + self.retryDelayIndex = 0 + self.retryNotBefore = nil + return true + case .failure: + let delay = Self.retryDelays[self.retryDelayIndex] + self.retryNotBefore = self.now() + delay + self.retryDelayIndex = min(self.retryDelayIndex + 1, Self.retryDelays.count - 1) + return false + } + } +} diff --git a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift new file mode 100644 index 00000000..76a74103 --- /dev/null +++ b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient+Live.swift @@ -0,0 +1,678 @@ +import Foundation +import OpenTelemetryApi +import OpenTelemetryProtocolExporterCommon +import OpenTelemetryProtocolExporterHttp +import OpenTelemetrySdk + +#if canImport(FoundationNetworking) +import FoundationNetworking +#endif + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +enum BuildkiteTelemetryAttribute { + static let annotation = "buildkite.annotation" + static let buildNumber = "buildkite.build_number" + static let collectorName = "buildkite.collector.name" + static let collectorVersion = "buildkite.collector.version" + static let executionExternalID = "buildkite.test.execution.external_id" + static let executionLocation = "buildkite.test.location" + static let executionScope = "buildkite.test.scope" + static let executionVia = "buildkite.execution.via" + static let frameworkName = "buildkite.test.framework.name" + static let jobID = "buildkite.job_id" + static let message = "buildkite.message" + static let runKey = "buildkite.run_key" + static let runURL = "buildkite.run_url" + static let stepID = "buildkite.step_id" + static let tagPrefix = "buildkite.tag." + static let testName = "buildkite.test.name" +} + +private enum TelemetryValue { + static let executionVia = "otlp" + static let frameworkName = "xctest" + static let rootSpanName = "test.execution" + static let skippedResult = "skipped" + static let unsetResult = "unset" +} + +struct CollectorOTLPConfiguration { + let endpoint: URL + let headers: [(String, String)] + let runEnvironment: RunEnvironment + + init?( + environment: EnvironmentValues, + logger: Logger? + ) { + var runEnvironment = environment.runEnvironment() + runEnvironment.applyCustomEnvironmentOverrides() + let rawHeaders = environment.otelTracesHeaders ?? environment.otelHeaders + // Standard headers alone must not enable collection or select the OTLP + // specification's localhost default. They may be process-wide settings for + // unrelated telemetry and are safe to use only with an explicit standard endpoint. + let hasExplicitConfiguration = environment.otelTracesEndpoint != nil + || environment.otelEndpoint != nil + || environment.analyticsOTLPEndpoint != nil + + guard environment.analyticsToken != nil || hasExplicitConfiguration else { return nil } + + let endpoint: URL + let usesBuildkiteCredentials: Bool + if let value = environment.otelTracesEndpoint { + guard let url = Self.absoluteURL(value) else { + logger?.error("OpenTelemetry traces endpoint is not a valid absolute URL") + return nil + } + endpoint = url + usesBuildkiteCredentials = false + } else if let value = environment.analyticsOTLPEndpoint { + guard let url = Self.absoluteURL(value) else { + logger?.error("OpenTelemetry traces endpoint is not a valid absolute URL") + return nil + } + endpoint = url + usesBuildkiteCredentials = true + } else if let value = environment.otelEndpoint { + guard let url = Self.absoluteURL(value) else { + logger?.error("OpenTelemetry endpoint is not a valid absolute URL") + return nil + } + endpoint = url.appendingPathComponent("v1/traces") + usesBuildkiteCredentials = false + } else { + endpoint = URL(string: TestCollector.endpoint)! + usesBuildkiteCredentials = true + } + + if !usesBuildkiteCredentials, + let protocolName = environment.otelTracesProtocol ?? environment.otelProtocol, + protocolName.lowercased() != "http/protobuf" { + logger?.error( + "Unsupported OpenTelemetry traces protocol \(protocolName); expected http/protobuf" + ) + return nil + } + + var headers = [(String, String)]() + if usesBuildkiteCredentials { + headers.append(("Buildkite-Tests-Run-Key", runEnvironment.key)) + if let token = environment.analyticsToken { + headers.append(("Authorization", "Token token=\"\(token)\"")) + } + if rawHeaders != nil { + logger?.warning( + "Standard OpenTelemetry exporter headers are ignored for endpoints that receive Buildkite credentials; use OTEL_EXPORTER_OTLP_TRACES_ENDPOINT or OTEL_EXPORTER_OTLP_ENDPOINT to send custom headers." + ) + } + } else if let rawHeaders { + guard let parsedHeaders = Self.parseHeaders(rawHeaders) else { + logger?.error("OpenTelemetry exporter headers are invalid") + return nil + } + headers = parsedHeaders + } + + self.endpoint = endpoint + self.headers = headers + self.runEnvironment = runEnvironment + } + + private static func absoluteURL(_ value: String) -> URL? { + guard let url = URL(string: value), url.scheme != nil, url.host != nil else { return nil } + return url + } + + private static func parseHeaders(_ value: String) -> [(String, String)]? { + let entries = value.split(separator: ",", omittingEmptySubsequences: false) + guard !entries.isEmpty else { return nil } + + var headers: [(String, String)] = [] + for entry in entries { + let parts = entry.split(separator: "=", maxSplits: 1, omittingEmptySubsequences: false) + guard parts.count == 2 else { return nil } + + let name = String(parts[0]).removingPercentEncoding? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let value = String(parts[1]).removingPercentEncoding? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard !name.isEmpty, !value.isEmpty else { return nil } + + headers.removeAll { $0.0.caseInsensitiveCompare(name) == .orderedSame } + headers.append((name, value)) + } + return headers + } +} + +private final class SynchronousOtlpTraceExporter: SpanExporter, @unchecked Sendable { + private let exporter: OtlpHttpTraceExporter + private let httpClient: SynchronousHTTPClient + + init(endpoint: URL, configuration: OtlpConfiguration) { + let httpClient = SynchronousHTTPClient(timeout: configuration.timeout) + self.httpClient = httpClient + self.exporter = OtlpHttpTraceExporter( + endpoint: endpoint, + config: configuration, + httpClient: httpClient, + envVarHeaders: nil + ) + } + + func export(spans: [SpanData], explicitTimeout: TimeInterval?) -> SpanExporterResultCode { + self.httpClient.prepare() + _ = self.exporter.export(spans: spans, explicitTimeout: explicitTimeout) + return self.httpClient.result + } + + func flush(explicitTimeout: TimeInterval?) -> SpanExporterResultCode { + self.exporter.flush(explicitTimeout: explicitTimeout) + } + + func shutdown(explicitTimeout: TimeInterval?) { + self.exporter.shutdown(explicitTimeout: explicitTimeout) + } +} + +private final class SynchronousHTTPClient: HTTPClient { + private let lock = NSLock() + private let session: URLSession + private var wasAccepted: Bool? + + init(timeout: TimeInterval) { + let configuration = URLSessionConfiguration.ephemeral + configuration.timeoutIntervalForRequest = timeout + configuration.timeoutIntervalForResource = timeout + configuration.urlCache = nil + self.session = URLSession(configuration: configuration) + } + + func prepare() { + self.lock.lock() + self.wasAccepted = nil + self.lock.unlock() + } + + var result: SpanExporterResultCode { + self.lock.lock() + defer { self.lock.unlock() } + return self.wasAccepted == true ? .success : .failure + } + + func send( + request: URLRequest, + completion: @escaping (Result) -> Void + ) { + let semaphore = DispatchSemaphore(value: 0) + let task = self.session.dataTask(with: request) { [weak self] _, response, error in + let httpResponse = response as? HTTPURLResponse + let accepted = error == nil + && httpResponse.map { (200..<300).contains($0.statusCode) } == true + + self?.lock.lock() + self?.wasAccepted = accepted + self?.lock.unlock() + + // The collector owns failed-span requeueing. Always report transport + // completion to the upstream exporter so it does not queue a second copy. + completion(.success(httpResponse ?? HTTPURLResponse( + url: request.url!, + statusCode: 500, + httpVersion: nil, + headerFields: nil + )!)) + semaphore.signal() + } + task.resume() + semaphore.wait() + } +} + +extension TelemetryClient { + static func live( + environment: EnvironmentValues, + uploadTags: [String: String], + logger: Logger?, + rootExporter: (any SpanExporter)? = nil, + childExporter: (any SpanExporter)? = nil, + rootSpanLimits: SpanLimits = SpanLimits() + ) -> TelemetryClient? { + guard let configuration = CollectorOTLPConfiguration( + environment: environment, + logger: logger + ) else { return nil } + + let exporterConfiguration = OtlpConfiguration( + timeout: 10, + compression: .gzip, + headers: configuration.headers, + exportAsJson: false + ) + let makeExporter = { + SynchronousOtlpTraceExporter( + endpoint: configuration.endpoint, + configuration: exporterConfiguration + ) as any SpanExporter + } + let live = LiveTelemetryClient( + configuration: configuration, + environment: environment, + uploadTags: uploadTags, + logger: logger, + rootExporter: rootExporter ?? makeExporter(), + childExporter: childExporter ?? makeExporter(), + rootSpanLimits: rootSpanLimits + ) + + return TelemetryClient( + start: live.startExecution, + annotate: live.annotate, + finish: live.finishExecution, + flush: live.forceFlush + ) + } +} + +private final class LiveTelemetryClient { + private static let providerEnvironmentLock = NSLock() + + private let attachedChildProviders = LockIsolated([TracerProviderSdk]()) + private let childForwarder: ExecutionChildSpanProcessor + private let childProviderResource: Resource + private let executionProvider: TracerProviderSdk + private let executionNamePrefix: String? + private let executionNameSuffix: String? + private let executionAttributes: [String: AttributeValue] + private let runKey: String + private let logger: Logger? + private let registry = ExecutionTraceRegistry() + private let reportedUnsupportedChildProvider = LockIsolated(false) + private let spans = LockIsolated([UUID: any Span]()) + private let tracer: any OpenTelemetryApi.Tracer + private let jobSpanContext: SpanContext? + private let executionContextManager: ExecutionContextManager + + init( + configuration: CollectorOTLPConfiguration, + environment: EnvironmentValues, + uploadTags: [String: String], + logger: Logger?, + rootExporter: any SpanExporter, + childExporter: any SpanExporter, + rootSpanLimits: SpanLimits + ) { + self.executionNamePrefix = configuration.runEnvironment.executionNamePrefix + self.executionNameSuffix = configuration.runEnvironment.executionNameSuffix + self.runKey = configuration.runEnvironment.key + self.logger = logger + self.jobSpanContext = Self.jobSpanContext(environment: environment) + + let resource = Self.providerResource( + configuration: configuration, + environment: environment + ) + self.executionAttributes = Self.executionAttributes( + configuration: configuration, + environment: environment, + uploadTags: uploadTags + ) + self.childProviderResource = resource + let rootProcessor = SynchronousExecutionSpanProcessor(exporter: rootExporter, logger: logger) + self.executionProvider = Self.makeProvider( + resource: resource, + processor: rootProcessor, + spanLimits: rootSpanLimits + ) + self.tracer = self.executionProvider.get( + instrumentationName: TestCollector.name, + instrumentationVersion: TestCollector.version + ) + + let childProcessor = BatchSpanProcessor( + spanExporter: childExporter, + scheduleDelay: 1, + exportTimeout: 30, + maxQueueSize: 8192, + maxExportBatchSize: 512 + ) + let forwarder = ExecutionChildSpanProcessor(processor: childProcessor, registry: self.registry) + self.childForwarder = forwarder + self.executionContextManager = ExecutionContextManager( + delegate: OpenTelemetry.instance.contextProvider + ) + self.executionContextManager.setContextReadHandler { [weak self] in + self?.attachChildForwarder() + } + OpenTelemetry.registerContextManager(contextManager: self.executionContextManager) + self.attachChildForwarder() + } + + func startExecution(_ test: TestState) -> UUID { + self.attachChildForwarder() + + let qualifiedTestName = [ + self.executionNamePrefix, + "\(test.className).\(test.testName)", + self.executionNameSuffix, + ].compactMap { $0 }.joined(separator: " ") + let builder = self.tracer.spanBuilder(spanName: TelemetryValue.rootSpanName) + builder.setNoParent() + + for (key, value) in self.executionAttributes { + builder.setAttribute(key: key, value: value) + } + + builder.setAttribute(key: BuildkiteTelemetryAttribute.executionScope, value: test.className) + builder.setAttribute(key: BuildkiteTelemetryAttribute.testName, value: test.testName) + builder.setAttribute( + key: BuildkiteTelemetryAttribute.executionExternalID, + value: test.id.uuidString + ) + builder.setAttribute( + key: SemanticConventions.Test.caseName.rawValue, + value: qualifiedTestName + ) + builder.setAttribute(key: SemanticConventions.Test.suiteName.rawValue, value: test.className) + + // The Swift SDK keeps the most recently set attributes when the span's + // start-time attribute limit is exceeded. Set the three synthesis fields + // last so even a limit of three still produces a usable execution. + builder.setAttribute( + key: BuildkiteTelemetryAttribute.executionVia, + value: TelemetryValue.executionVia + ) + builder.setAttribute(key: BuildkiteTelemetryAttribute.runKey, value: self.runKey) + // Reserve the result's place before test code can consume the remaining + // attribute budget. finishExecution replaces this placeholder. + builder.setAttribute( + key: SemanticConventions.Test.caseResultStatus.rawValue, + value: TelemetryValue.unsetResult + ) + + if let jobSpanContext = self.jobSpanContext { + builder.addLink(spanContext: jobSpanContext) + } + + let span = builder.startSpan() + self.registry.insert(span.context.traceId) + self.spans.withValue { $0[test.id] = span } + self.executionContextManager.setExecutionSpan(span) + return test.id + } + + func annotate(executionID: UUID, content: String) { + guard let span = self.spans.withValue({ $0[executionID] }) else { return } + span.addEvent( + name: "test.annotation", + attributes: [BuildkiteTelemetryAttribute.annotation: .string(content)] + ) + } + + func finishExecution(executionID: UUID, test: TestState, tags: [String: String]?) { + guard let span = self.spans.withValue({ $0.removeValue(forKey: executionID) }) else { return } + + for (key, value) in tags ?? [:] { + span.setAttribute(key: BuildkiteTelemetryAttribute.tagPrefix + key, value: value) + } + + let result = test.result ?? .failed + span.setAttribute( + key: SemanticConventions.Test.caseResultStatus.rawValue, + value: Self.resultStatus(result) + ) + + if let location = test.issues.first?.sourceCodeContext.location { + span.setAttribute(key: SemanticConventions.Code.filePath.rawValue, value: location.filePath) + span.setAttribute(key: SemanticConventions.Code.lineNumber.rawValue, value: Int(location.line)) + span.setAttribute( + key: BuildkiteTelemetryAttribute.executionLocation, + value: "\(location.fileName):\(location.line)" + ) + } + + if result == .failed { + let reason = Self.failureReason(test.issues) + span.status = .error(description: reason) + for issue in test.issues { + var attributes: [String: AttributeValue] = [ + SemanticConventions.Exception.message.rawValue: .string(issue.description), + ] + let backtrace = issue.sourceCodeContext.callStack.enumerated() + .map { "\($0.offset) \($0.element)" } + .joined(separator: "\n") + if !backtrace.isEmpty { + attributes[SemanticConventions.Exception.stacktrace.rawValue] = .string(backtrace) + } + if let error = issue.associatedError { + attributes[SemanticConventions.Exception.type.rawValue] = .string( + String(reflecting: type(of: error)) + ) + } + span.addEvent( + name: SemanticConventions.Exception.exception.rawValue, + attributes: attributes + ) + } + } + + self.executionContextManager.removeExecutionSpan(span) + self.registry.remove(span.context.traceId) + span.end() + } + + func forceFlush() { + self.executionProvider.forceFlush(timeout: 30) + self.childForwarder.forceFlush(timeout: 30) + } + + private func attachChildForwarder() { + let globalProvider = OpenTelemetry.instance.tracerProvider + if globalProvider is DefaultTracerProvider { + let provider = Self.makeProvider( + resource: self.childProviderResource, + processor: self.childForwarder + ) + self.attachedChildProviders.withValue { $0.append(provider) } + OpenTelemetry.registerTracerProvider(tracerProvider: provider) + } else if let provider = globalProvider as? TracerProviderSdk { + let needsProcessor = self.attachedChildProviders.withValue { providers in + guard !providers.contains(where: { $0 === provider }) else { return false } + providers.append(provider) + return true + } + if needsProcessor { + provider.addSpanProcessor(self.childForwarder) + } + } else { + let shouldReport = self.reportedUnsupportedChildProvider.withValue { reported in + guard !reported else { return false } + reported = true + return true + } + if shouldReport { + self.logger?.error( + "OpenTelemetry child span export is disabled because the existing tracer provider cannot accept a span processor" + ) + } + } + } + + private static func resultStatus(_ result: TestResult) -> String { + switch result { + case .passed: + return SemanticConventions.Test.CaseResultStatusValues.pass.description + case .failed: + return SemanticConventions.Test.CaseResultStatusValues.fail.description + case .skipped: + return TelemetryValue.skippedResult + } + } + + private static func failureReason(_ issues: [TestIssue]) -> String { + if issues.count > 1 { + return "\(issues.count) failures: \(issues.map(\.compactDescription).joined(separator: ", "))" + } + return issues.first?.compactDescription ?? "Test failed" + } + + private static func makeProvider( + resource: Resource, + processor: any SpanProcessor, + spanLimits: SpanLimits = SpanLimits() + ) -> TracerProviderSdk { + self.withoutInheritedTraceContext { + TracerProviderBuilder() + .with(resource: resource) + .with(spanLimits: spanLimits) + .with(sampler: Samplers.alwaysOn) + .add(spanProcessor: processor) + .build() + } + } + + private static func withoutInheritedTraceContext(_ operation: () -> T) -> T { + self.providerEnvironmentLock.lock() + defer { self.providerEnvironmentLock.unlock() } + + let names = ["TRACEPARENT", "TRACESTATE"] + let previous = Dictionary(uniqueKeysWithValues: names.map { name in + (name, getenv(name).map { String(cString: $0) }) + }) + names.forEach { unsetenv($0) } + defer { + for name in names { + if let value = previous[name] ?? nil { + setenv(name, value, 1) + } else { + unsetenv(name) + } + } + } + return operation() + } + + // A resource identifies producer-wide entities shared by every span from the + // provider. Test Engine run fields and tags belong to each execution root. + private static func providerResource( + configuration: CollectorOTLPConfiguration, + environment: EnvironmentValues + ) -> Resource { + let run = configuration.runEnvironment + let pipelineRun = Self.ciPipelineRun(run: run, environment: environment) + var attributes = [String: AttributeValue]() + + func set(_ key: String, _ value: String?) { + if let value, Self.validAttributeKey(key) { + attributes[key] = .string(value) + } + } + + set(SemanticConventions.Service.name.rawValue, environment.testEngineSuiteSlug) + set(SemanticConventions.Service.namespace.rawValue, environment.buildkiteOrganizationSlug) + set(SemanticConventions.Cicd.pipelineRunId.rawValue, pipelineRun.id) + if pipelineRun.id != nil { + set(SemanticConventions.Cicd.pipelineRunUrlFull.rawValue, pipelineRun.url) + } + set(SemanticConventions.Cicd.workerId.rawValue, environment.buildkiteAgentId) + set(SemanticConventions.Vcs.refHeadName.rawValue, run.branch) + set(SemanticConventions.Vcs.refHeadRevision.rawValue, run.commitSha) + if run.branch != nil { + set( + SemanticConventions.Vcs.refType.rawValue, + environment.buildkiteTag == nil ? "branch" : "tag" + ) + } + + return EnvVarResource.get().merging(other: Resource(attributes: attributes)) + } + + // These fields describe each test execution, not the provider that emitted + // its child spans. Configure-level tags are set on every root; per-test tags + // are applied at finish time and override matching configure-level tags. + private static func executionAttributes( + configuration: CollectorOTLPConfiguration, + environment: EnvironmentValues, + uploadTags: [String: String] + ) -> [String: AttributeValue] { + let run = configuration.runEnvironment + let pipelineRun = Self.ciPipelineRun(run: run, environment: environment) + var attributes = [String: AttributeValue]() + + func set(_ key: String, _ value: String?) { + if let value, Self.validAttributeKey(key) { + attributes[key] = .string(value) + } + } + + set(BuildkiteTelemetryAttribute.runKey, run.key) + if run.url != pipelineRun.url { + set(BuildkiteTelemetryAttribute.runURL, run.url) + } + set(BuildkiteTelemetryAttribute.buildNumber, run.number) + set(BuildkiteTelemetryAttribute.jobID, run.jobId) + set(BuildkiteTelemetryAttribute.stepID, environment.buildkiteStepId) + set(BuildkiteTelemetryAttribute.message, run.message) + set(BuildkiteTelemetryAttribute.collectorName, run.collector) + set(BuildkiteTelemetryAttribute.collectorVersion, run.version) + set(BuildkiteTelemetryAttribute.frameworkName, TelemetryValue.frameworkName) + + for (key, value) in uploadTags { + set(BuildkiteTelemetryAttribute.tagPrefix + key, value) + } + for (key, value) in run.customEnvironment ?? [:] + where !RunEnvironment.customFieldNames.contains(key) && Self.validAttributeKey(key) + { + attributes[key] = AttributeValue(value.base) ?? .string(value.description) + } + + return attributes + } + + // Provider-native CI identity is distinct from the Test Engine run key. + private static func ciPipelineRun( + run: RunEnvironment, + environment: EnvironmentValues + ) -> (id: String?, url: String?) { + switch run.ci { + case "buildkite": + return (environment.buildkiteBuildId, environment.buildkiteBuildUrl) + case "github_actions": + let id = environment.gitHubRunId + let url = environment.gitHubRepository.flatMap { repository in + id.map { "https://github.com/\(repository)/actions/runs/\($0)" } + } + return (id, url) + case "circleci": + return (environment.circleWorkflowId, nil) + case "xcodeCloud": + return (environment.xcodeBuildId, nil) + default: + return (nil, nil) + } + } + + private static func validAttributeKey(_ key: String) -> Bool { + !key.isEmpty && key.count <= 255 && key.unicodeScalars.allSatisfy { (32...126).contains($0.value) } + } + + private static func jobSpanContext(environment: EnvironmentValues) -> SpanContext? { + guard let traceParent = environment.traceParent else { return nil } + let carrier = [ + "traceparent": traceParent, + "tracestate": environment.traceState ?? "", + ] + return W3CTraceContextPropagator().extract(carrier: carrier, getter: DictionaryGetter()) + } + + private struct DictionaryGetter: Getter { + func get(carrier: [String: String], key: String) -> [String]? { + carrier[key].map { [$0] } + } + } +} diff --git a/test-collector-swift/Sources/Core/Telemetry/TelemetryClient.swift b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient.swift new file mode 100644 index 00000000..238935b7 --- /dev/null +++ b/test-collector-swift/Sources/Core/Telemetry/TelemetryClient.swift @@ -0,0 +1,42 @@ +import Foundation + +struct TelemetryClient { + typealias ExecutionID = UUID + + private let start: (TestState) -> ExecutionID + private let annotateExecution: (ExecutionID, String) -> Void + private let finish: (ExecutionID, TestState, [String: String]?) -> Void + private let flushPending: () -> Void + + init( + start: @escaping (TestState) -> ExecutionID, + annotate: @escaping (ExecutionID, String) -> Void, + finish: @escaping (ExecutionID, TestState, [String: String]?) -> Void, + flush: @escaping () -> Void + ) { + self.start = start + self.annotateExecution = annotate + self.finish = finish + self.flushPending = flush + } + + func startExecution(_ test: TestState) -> ExecutionID { + self.start(test) + } + + func annotate(_ content: String, executionID: ExecutionID) { + self.annotateExecution(executionID, content) + } + + func finishExecution( + _ executionID: ExecutionID, + test: TestState, + tags: [String: String]? + ) { + self.finish(executionID, test, tags) + } + + func forceFlush() { + self.flushPending() + } +} diff --git a/test-collector-swift/Sources/Core/TestCollector.swift b/test-collector-swift/Sources/Core/TestCollector.swift index 4ee8af36..d20aceca 100644 --- a/test-collector-swift/Sources/Core/TestCollector.swift +++ b/test-collector-swift/Sources/Core/TestCollector.swift @@ -20,23 +20,27 @@ public struct TestCollector { return } + if environment.legacyAnalyticsBaseURL != nil { + logger?.warning( + "BUILDKITE_ANALYTICS_BASE_URL is no longer supported and is ignored; use BUILDKITE_ANALYTICS_OTLP_ENDPOINT or OTEL_EXPORTER_OTLP_TRACES_ENDPOINT instead." + ) + } + let envTags = environment.analyticsTags ?? [:] let tags = uploadTags.merging(envTags) { _, env in env } - let tracer = Tracer.live() - - let uploader: UploadClient? - if let apiToken = environment.analyticsToken { - let baseURL = environment.analyticsBaseURL ?? URL(string: Self.baseURL)! - let api = ApiClient.live(apiToken: apiToken, baseURL: baseURL) - let runEnvironment = environment.runEnvironment() - uploader = .live(api: api, runEnvironment: runEnvironment, tags: tags.isEmpty ? nil : tags, logger: logger) - } else { - logger?.info("TestCollector unable to locate API key. Test results will not be uploaded.") - uploader = nil + let telemetry = TelemetryClient.live( + environment: environment, + uploadTags: tags, + logger: logger + ) + if telemetry == nil { + logger?.info( + "TestCollector requires an API token or OpenTelemetry exporter configuration. Test results will not be exported." + ) } - self.observer = TestObserver(logger: logger, tracer: tracer, uploader: uploader) + self.observer = TestObserver(logger: logger, telemetry: telemetry) } /// Annotates the current test. @@ -45,7 +49,7 @@ public struct TestCollector { /// /// - Parameter content: The content of this annotation public func annotate(_ content: @autoclosure () -> String) { - self.observer?.tracer.annotate(content()) + self.observer?.annotate(content()) } /// Tags the execution of the given test case with a key-value pair. @@ -80,7 +84,7 @@ public struct TestCollector { public private(set) static var shared: TestCollector? - public static let baseURL = "https://analytics-api.buildkite.com/v1/" + public static let endpoint = "https://tests-otlp.buildkite.com/v1/traces" static let name = "test-collector-swift" - static let version = "0.6.0" + static let version = "2.0.0-beta.1" } diff --git a/test-collector-swift/Sources/Core/TestObserver.swift b/test-collector-swift/Sources/Core/TestObserver.swift index d578a749..df5d8ec4 100644 --- a/test-collector-swift/Sources/Core/TestObserver.swift +++ b/test-collector-swift/Sources/Core/TestObserver.swift @@ -1,17 +1,16 @@ import XCTest -/// An object that captures test data and uploads it in real time. +/// An object that captures test data and exports it in real time. final class TestObserver: NSObject, XCTestObservation { let logger: Logger? - let tracer: Tracer - let uploader: UploadClient? + let telemetry: TelemetryClient? let uuid: () -> UUID /// The state of the current test. var test: TestState? /// The id associated with the root span of the current test. - var spanId: AnyHashable? + var executionID: TelemetryClient.ExecutionID? /// Per-test execution tags, keyed by test case identity. private let executionTags = LockIsolated([ObjectIdentifier: [String: String]]()) @@ -20,18 +19,15 @@ final class TestObserver: NSObject, XCTestObservation { /// /// - Parameters: /// - logger: A logger. - /// - tracer: The tracer for recording span data. - /// - uploader: The upload client for uploading test results + /// - telemetry: The client for exporting OpenTelemetry spans. /// - uuid: A closure that returns a unique id to associate with an executed test case. init( logger: Logger? = .init(), - tracer: Tracer = .live(), - uploader: UploadClient? = nil, + telemetry: TelemetryClient? = nil, uuid: @escaping () -> UUID = UUID.init ) { self.logger = logger - self.tracer = tracer - self.uploader = uploader + self.telemetry = telemetry self.uuid = uuid } @@ -54,12 +50,13 @@ final class TestObserver: NSObject, XCTestObservation { /// Called exactly once per test case. func testCaseWillStart(_ testCase: XCTestCase) { self.executionTags.withValue { $0[ObjectIdentifier(testCase)] = [:] } - self.spanId = self.tracer.startSpan(section: "top") - self.test = TestState( + let test = TestState( id: self.uuid(), className: XCTestCase.className(of: testCase), testName: XCTestCase.testName(of: testCase) ) + self.test = test + self.executionID = self.telemetry?.startExecution(test) } #if canImport(ObjectiveC) @@ -106,19 +103,13 @@ final class TestObserver: NSObject, XCTestObservation { return result?.isEmpty == true ? nil : result } defer { - spanId = nil + executionID = nil test = nil } - guard - let span = self.spanId.map(self.tracer.endSpan(id:)), - var test = self.test - else { return } + guard let executionID = self.executionID, var test = self.test else { return } test.result = testCase.result - - let trace = Trace(test: test, span: span, tags: tags) - - self.uploader?.record(trace: trace) + self.telemetry?.finishExecution(executionID, test: test, tags: tags) } /// Notifies the observer immediately after all tests in a test bundle finish executing. @@ -128,7 +119,12 @@ final class TestObserver: NSObject, XCTestObservation { /// - Note: The test process will generally exit after this method returns, so it must block until all asynchronous /// work is complete. func testBundleDidFinish(_ testBundle: Bundle) { - self.uploader?.waitForUploads() + self.telemetry?.forceFlush() self.logger?.waitForLogs() } + + func annotate(_ content: String) { + guard let executionID = self.executionID else { return } + self.telemetry?.annotate(content, executionID: executionID) + } } diff --git a/test-collector-swift/Sources/Core/Tracer/Tracer+Live.swift b/test-collector-swift/Sources/Core/Tracer/Tracer+Live.swift deleted file mode 100644 index a9ff71fc..00000000 --- a/test-collector-swift/Sources/Core/Tracer/Tracer+Live.swift +++ /dev/null @@ -1,43 +0,0 @@ -import Foundation - -extension Tracer { - /// Constructs a "live" tracer that uses the system uptime for the start and end time of spans. - /// - /// - Parameters: - /// - systemUptime: A closure that returns the system uptime. - /// - spanId: A closure that returns a unique identifier. - /// - Returns: A tracer that use the system uptime for span times. - static func live( - systemUptime: @escaping () -> TimeInterval = { ProcessInfo.processInfo.systemUptime }, - spanId: @escaping () -> AnyHashable = { UUID() } - ) -> Tracer { - var stack: [(id: AnyHashable, span: Trace.Span)] = [] - - return Tracer( - startSpan: { section, detail in - let id = spanId() - let span = Trace.Span(section: section, startAt: systemUptime(), detail: detail) - stack.append((id, span)) - return id - - }, - endSpan: { id in - assert(!stack.isEmpty, "No active spans") - assert(stack.last!.id == id, "Id does not match current span") - - var span = stack.last!.span - let endAt = systemUptime() - - span.endAt = endAt - span.duration = span.startAt.map { endAt - $0 } - stack.removeLast() - - if !stack.isEmpty { - stack[stack.endIndex - 1].span.children.append(span) - } - - return span - } - ) - } -} diff --git a/test-collector-swift/Sources/Core/Tracer/Tracer.swift b/test-collector-swift/Sources/Core/Tracer/Tracer.swift deleted file mode 100644 index dde4ecf2..00000000 --- a/test-collector-swift/Sources/Core/Tracer/Tracer.swift +++ /dev/null @@ -1,41 +0,0 @@ -import Foundation - -/// A type for capturing spans over time. -struct Tracer { - private var startSpan: (String, [String: String]) -> AnyHashable - private var endSpan: (AnyHashable) -> Trace.Span - - init( - startSpan: @escaping (String, [String: String]) -> AnyHashable, - endSpan: @escaping (AnyHashable) -> Trace.Span - ) { - self.startSpan = startSpan - self.endSpan = endSpan - } - - /// Starts recording a span. - /// - /// - Parameters: - /// - section: The section's name. - /// - detail: Information attached to the span. - /// - Returns: The identifier associated with the started span. - func startSpan(section: String, detail: [String: String] = [:]) -> AnyHashable { - self.startSpan(section, detail) - } - - /// Ends the span that corresponds to the specified id. - /// - /// - Parameter id: The span's identifier. - /// - Returns: The completed span. - func endSpan(id: AnyHashable) -> Trace.Span { - self.endSpan(id) - } - - /// Adds an annotation span as a child to the current span . - /// - /// - Parameter content: The annotation's content. - func annotate(_ content: String) { - let id = self.startSpan(section: "annotation", detail: ["content": content]) - _ = self.endSpan(id: id) - } -} diff --git a/test-collector-swift/Sources/Core/UploadClient/UploadClient+Live.swift b/test-collector-swift/Sources/Core/UploadClient/UploadClient+Live.swift deleted file mode 100644 index 6f2f68d6..00000000 --- a/test-collector-swift/Sources/Core/UploadClient/UploadClient+Live.swift +++ /dev/null @@ -1,121 +0,0 @@ -import Dispatch -import Foundation - -#if canImport(FoundationNetworking) -import FoundationNetworking -#endif - -extension UploadClient { - /// Constructs a "live" upload client that uploads traces using an API client. - /// - /// - Parameters: - /// - api: An API client. - /// - logger: A logger. - /// - runEnvironment: The run environment to accompany uploaded traces. - /// - batchSize: The maximum number of traces per upload. - /// - group: A dispatch group to associate with upload tasks. - /// - Returns: A upload client that uses an api client. - static func live( - api: ApiClient, - runEnvironment: RunEnvironment, - tags: [String: String]? = nil, - logger: Logger? = nil, - batchSize: Int = maximumBatchSize, - group: DispatchGroup = DispatchGroup() - ) -> UploadClient { - let client = LiveClient( - api: api, - batchSize: batchSize, - logger: logger, - runEnvironment: runEnvironment, - tags: tags, - taskGroup: group - ) - - return UploadClient( - record: { client.record(trace: $0) }, - waitForUploads: { client.waitForUploads(timeout: $0) } - ) - } - - private struct LiveClient { - let api: ApiClient - let batchSize: Int - let logger: Logger? - let runEnvironment: RunEnvironment - let tags: [String: String]? - let taskGroup: DispatchGroup - - private let traces = LockIsolated([Trace]()) - - func record(trace: Trace) { - self.traces.withValue { traces in - traces.append(trace) - guard traces.count >= self.batchSize else { return } - self.upload(traces: traces) - traces = [] - } - } - - private func upload(traces: [Trace]) { - // NB: Uploads must enter the task group synchronously to ensure they are waited for - self.taskGroup.enter() - Task { - defer { self.taskGroup.leave() } - let testData = TestResults.json(runEnv: runEnvironment, tags: tags, data: traces) - try await self.upload(testData: testData) - } - } - - private func upload(testData: TestResults) async throws { - self.logger?.debug("Uploading \(testData)") - - var data: Data - var response: HTTPURLResponse - do { - (data, response) = try await self.api.data(for: .upload(testData)) - } catch { - self.logger?.error("Error uploading: \(error.localizedDescription)") - throw error - } - - // Ideally “HTTP 200 OK” etc, but maybe actually “HTTP 200 no error” etc. - let statusString = "HTTP \(response.statusCode) \(HTTPURLResponse.localizedString(forStatusCode: response.statusCode))" - - // Currently this should get HTTP 202 Accepted, but let's be a bit permissive to future changes. - guard (201...204).contains(response.statusCode) else { - if let body = try? self.api.decode(data, as: UploadFailureResponse.self) { - self.logger?.error("Unexpected \(statusString), \(body.message)") - } else { - self.logger?.error("Unexpected \(statusString), (no message)") - } - throw URLError(.badServerResponse, userInfo: [ NSLocalizedDescriptionKey: "Unexpected \(statusString)" ]) - } - - do { - let result = try self.api.decode(data, as: UploadResponse.self) - let uploadID = result.uploadID ?? "(missing)" - let uploadURL = result.uploadURL ?? "(missing)" - self.logger?.debug("\(statusString), ID: \(uploadID), URL: \(uploadURL)") - } catch let decodingError as DecodingError { - self.logger?.error("Warning: error decoding body of \(statusString): \(decodingError)") - // proceed anyway, since we got an HTTP 2xx, and decoding the response isn't critical - } - } - - func waitForUploads(timeout: TimeInterval) { - self.traces.withValue { traces in - guard !traces.isEmpty else { return } - self.upload(traces: traces) - traces = [] - } - let result = self.taskGroup.wait(timeout: timeout) - if result == .timedOut { - self.logger?.error("Upload client timed out before completing all uploads") - } - } - } -} - -// The maximum number of traces that can be sent per upload -private let maximumBatchSize = 5000 diff --git a/test-collector-swift/Sources/Core/UploadClient/UploadClient.swift b/test-collector-swift/Sources/Core/UploadClient/UploadClient.swift deleted file mode 100644 index bc4af9ca..00000000 --- a/test-collector-swift/Sources/Core/UploadClient/UploadClient.swift +++ /dev/null @@ -1,32 +0,0 @@ -import Foundation - -/// A type used to upload traces asynchronously during a test run. -struct UploadClient { - private var record: (Trace) -> Void - private var waitForUploads: (TimeInterval) -> Void - - init( - record: @escaping (Trace) -> Void, - waitForUploads: @escaping (TimeInterval) -> Void - ) { - self.record = record - self.waitForUploads = waitForUploads - } - - /// Records a trace to be included in the next upload. - /// - /// - Parameter trace: The trace to record - func record(trace: Trace) { - self.record(trace) - } - - /// Waits synchronously for the previously submitted traces to be uploaded. - /// - /// - Parameter timeout: The maximum duration in seconds to wait for uploads to complete. - func waitForUploads(timeout: TimeInterval = twoMinutes) { - self.waitForUploads(timeout) - } -} - -// Default timeout used by waitForUploads -private let twoMinutes: TimeInterval = 120 diff --git a/test-collector-swift/Tests/CoreTests/ApiClientTests.swift b/test-collector-swift/Tests/CoreTests/ApiClientTests.swift deleted file mode 100644 index fc6bd08f..00000000 --- a/test-collector-swift/Tests/CoreTests/ApiClientTests.swift +++ /dev/null @@ -1,36 +0,0 @@ -@testable import Core -import XCTest - -#if canImport(FoundationNetworking) -import FoundationNetworking -#endif - -final class ApiClientTests: XCTestCase { - func testApiClientMakesUploadRequest() async throws { - struct Response: Decodable { var status: String } - var requests = [URLRequest]() - let session = ApiSession { request in - requests.append(request) - return ("{\"status\":\"OK\"}".data(using: .utf8)!, .stub()) - } - let api = ApiClient.live( - apiToken: "token", - baseURL: URL(string: "http://api.test.com/")!, - session: session - ) - let results = TestResults.json(runEnv: .init(key: "key"), data: []) - - let (value, _) = try await api.data(for: .upload(results), as: Response.self) - - XCTAssertEqual(value.status, "OK") - XCTAssertEqual(requests.count, 1) - let request = try XCTUnwrap(requests.first) - XCTAssertEqual(request.url?.absoluteString, "http://api.test.com/uploads") - XCTAssertEqual(request.authorizationHeader, "Token token=\"token\"") - XCTAssertEqual(request.httpMethod, "POST") - XCTAssertEqual( - request.httpBodyDictionary, - ["format": "json", "run_env": ["key": "key"], "data": NSArray()] - ) - } -} diff --git a/test-collector-swift/Tests/CoreTests/CollectorTests.swift b/test-collector-swift/Tests/CoreTests/CollectorTests.swift index eaf999ec..2cca05cc 100644 --- a/test-collector-swift/Tests/CoreTests/CollectorTests.swift +++ b/test-collector-swift/Tests/CoreTests/CollectorTests.swift @@ -3,17 +3,31 @@ import XCTest final class CollectorTests: XCTestCase { func testDefaultCollector() throws { - let environment = EnvironmentValues(values: [:]) + let environment = EnvironmentValues( + values: [:], + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ) let collector = TestCollector(environment: environment) let observer = try XCTUnwrap(collector.observer, "Observer should be initialised") - XCTAssertNil(observer.uploader, "Uploader should not be initialised without an api key") + XCTAssertNil(observer.telemetry, "Telemetry should not be initialised without export configuration") } - func testDefaultCollectorWithUploader() throws { + func testDefaultCollectorWithTelemetry() throws { let environment = EnvironmentValues(values: ["BUILDKITE_ANALYTICS_TOKEN": "SECRET"]) let collector = TestCollector(environment: environment) let observer = try XCTUnwrap(collector.observer, "Observer should be created by default") - XCTAssertNotNil(observer.uploader, "Uploader should be initialised when provided an api key") + XCTAssertNotNil(observer.telemetry, "Telemetry should be initialised when provided an api key") + } + + func testCollectorWithOTLPConfigurationDoesNotRequireApiToken() throws { + let environment = EnvironmentValues(values: [ + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://127.0.0.1:4318/v1/traces", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS": "Authorization=Bearer%20relay-token", + ]) + let collector = TestCollector(environment: environment) + let observer = try XCTUnwrap(collector.observer, "Observer should be created by default") + XCTAssertNotNil(observer.telemetry) } func testCollectorIsDisabled() { @@ -22,6 +36,25 @@ final class CollectorTests: XCTestCase { XCTAssertNil(collector.observer) } + func testWarnsWhenLegacyAnalyticsBaseURLIsIgnored() { + let messages = LockIsolated([String]()) + let logger = Logger(printer: { message in + messages.withValue { $0.append(message) } + }) + let environment = EnvironmentValues(values: [ + "BUILDKITE_ANALYTICS_BASE_URL": "https://example.com/v1/", + ]) + + _ = TestCollector(environment: environment, logger: logger) + logger.waitForLogs() + + XCTAssertTrue(messages.withValue { messages in + messages.contains( + "[BuildkiteTestCollector] warning: BUILDKITE_ANALYTICS_BASE_URL is no longer supported and is ignored; use BUILDKITE_ANALYTICS_OTLP_ENDPOINT or OTEL_EXPORTER_OTLP_TRACES_ENDPOINT instead." + ) + }) + } + func testUploadTagsEnvVarTakesPrecedence() { let environment = EnvironmentValues(values: [ "BUILDKITE_ANALYTICS_TAGS": #"{"shared":"from-env","env-only":"yes"}"#, diff --git a/test-collector-swift/Tests/CoreTests/RunEnvironmentTests.swift b/test-collector-swift/Tests/CoreTests/RunEnvironmentTests.swift index c2cf9760..14328870 100644 --- a/test-collector-swift/Tests/CoreTests/RunEnvironmentTests.swift +++ b/test-collector-swift/Tests/CoreTests/RunEnvironmentTests.swift @@ -121,65 +121,4 @@ final class RunEnvironmentTests: XCTestCase { ) } - func testJSONEncoding() throws { - let runEnvironment = RunEnvironment( - ci: "ci", - key: "key", - url: "url", - branch: "branch", - commitSha: "commitSha", - number: "number", - jobId: "jobId", - message: "message", - debug: "debug", - executionNamePrefix: "executionNamePrefix", - executionNameSuffix: "executionNameSuffix", - version: "version", - collector: "collector", - customEnvironment: [ - "custom_key": "customKey", - "tags": ["A", "B", "C"], - ] - ) - - let data = try JSONEncoder().encode(runEnvironment) - - let json = try JSONSerialization.jsonObject(with: data) - - XCTAssertEqual( - json as? NSDictionary, - [ - "CI": "ci", - "key": "key", - "url": "url", - "branch": "branch", - "commit_sha": "commitSha", - "number": "number", - "job_id": "jobId", - "message": "message", - "debug": "debug", - "execution_name_prefix": "executionNamePrefix", - "execution_name_suffix": "executionNameSuffix", - "version": "version", - "collector": "collector", - "custom_key": "customKey", - "tags": ["A", "B", "C"], - ] - ) - } - - func testCustomEnvironmentTakesPrecedence() throws { - let runEnvironment = RunEnvironment( - key: "key", - customEnvironment: [ - "key": "customKey", - ] - ) - - let data = try JSONEncoder().encode(runEnvironment) - - let json = try JSONSerialization.jsonObject(with: data) - - XCTAssertEqual(json as? NSDictionary, ["key": "customKey"]) - } } diff --git a/test-collector-swift/Tests/CoreTests/SynchronousExecutionSpanProcessorTests.swift b/test-collector-swift/Tests/CoreTests/SynchronousExecutionSpanProcessorTests.swift new file mode 100644 index 00000000..f1616fda --- /dev/null +++ b/test-collector-swift/Tests/CoreTests/SynchronousExecutionSpanProcessorTests.swift @@ -0,0 +1,119 @@ +@testable import Core +import OpenTelemetryApi +import OpenTelemetrySdk +import XCTest + +final class SynchronousExecutionSpanProcessorTests: XCTestCase { + func testBacksOffFailedExportsAndResetsAfterSuccess() { + let clock = TestClock() + let exporter = SequencedExporter(results: [ + .failure, .failure, .failure, .failure, .failure, .success, .failure, .success, + ]) + let processor = SynchronousExecutionSpanProcessor( + exporter: exporter, + logger: nil, + now: { clock.now } + ) + let tracer = TracerProviderSdk(spanProcessors: [processor]).get( + instrumentationName: "SynchronousExecutionSpanProcessorTests" + ) + + self.endSpan("one", with: tracer) + self.endSpan("two", with: tracer) + clock.now = 9 + self.endSpan("three", with: tracer) + XCTAssertEqual(exporter.exports.count, 1) + + clock.now = 10 + self.endSpan("four", with: tracer) + clock.now = 29 + self.endSpan("five", with: tracer) + XCTAssertEqual(exporter.exports.count, 2) + + clock.now = 30 + self.endSpan("six", with: tracer) + clock.now = 59 + self.endSpan("seven", with: tracer) + XCTAssertEqual(exporter.exports.count, 3) + + clock.now = 60 + self.endSpan("eight", with: tracer) + clock.now = 119 + self.endSpan("nine", with: tracer) + XCTAssertEqual(exporter.exports.count, 4) + + clock.now = 120 + self.endSpan("ten", with: tracer) + clock.now = 179 + self.endSpan("eleven", with: tracer) + XCTAssertEqual(exporter.exports.count, 5) + + clock.now = 180 + self.endSpan("twelve", with: tracer) + self.endSpan("thirteen", with: tracer) + clock.now = 189 + self.endSpan("fourteen", with: tracer) + XCTAssertEqual(exporter.exports.count, 7) + + clock.now = 190 + self.endSpan("fifteen", with: tracer) + + XCTAssertEqual(exporter.exports.count, 8) + XCTAssertEqual(exporter.exports.map(\.count), [1, 4, 6, 8, 10, 12, 1, 3]) + } + + func testLogsExecutionsQueuedDuringBackoff() { + let clock = TestClock() + let exporter = SequencedExporter(results: [.failure]) + let messages = LockIsolated([String]()) + let logger = Logger(logLevel: .debug, printer: { message in + messages.withValue { $0.append(message) } + }) + let processor = SynchronousExecutionSpanProcessor( + exporter: exporter, + logger: logger, + now: { clock.now } + ) + let tracer = TracerProviderSdk(spanProcessors: [processor]).get( + instrumentationName: "SynchronousExecutionSpanProcessorTests" + ) + + self.endSpan("one", with: tracer) + self.endSpan("two", with: tracer) + logger.waitForLogs() + + XCTAssertTrue(messages.withValue { messages in + messages.contains( + "[BuildkiteTestCollector] debug: Queued OpenTelemetry test execution during export backoff: two (2 pending)" + ) + }) + } + + private func endSpan(_ name: String, with tracer: any Tracer) { + tracer.spanBuilder(spanName: name).setNoParent().startSpan().end() + } +} + +private final class TestClock { + var now: TimeInterval = 0 +} + +private final class SequencedExporter: SpanExporter, @unchecked Sendable { + private var results: [SpanExporterResultCode] + var exports = [[SpanData]]() + + init(results: [SpanExporterResultCode]) { + self.results = results + } + + func export(spans: [SpanData], explicitTimeout: TimeInterval?) -> SpanExporterResultCode { + self.exports.append(spans) + return self.results.removeFirst() + } + + func flush(explicitTimeout: TimeInterval?) -> SpanExporterResultCode { + .success + } + + func shutdown(explicitTimeout: TimeInterval?) {} +} diff --git a/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift new file mode 100644 index 00000000..93f9f9ad --- /dev/null +++ b/test-collector-swift/Tests/CoreTests/TelemetryClientTests.swift @@ -0,0 +1,545 @@ +@testable import Core +import InMemoryExporter +import OpenTelemetryApi +import OpenTelemetrySdk +import XCTest + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +final class TelemetryClientTests: XCTestCase { + override func setUp() { + super.setUp() + OpenTelemetry.registerTracerProvider(tracerProvider: DefaultTracerProvider.instance) + } + + override func tearDown() { + OpenTelemetry.registerTracerProvider(tracerProvider: DefaultTracerProvider.instance) + super.tearDown() + } + + func testExportsExecutionRootSynchronouslyWithAttributesAndRunTags() throws { + let rootExporter = InMemoryExporter() + let childExporter = InMemoryExporter() + let environment = EnvironmentValues( + values: [ + "BUILDKITE_ANALYTICS_TOKEN": "SECRET", + "BUILDKITE_ANALYTICS_KEY": "run-key", + "BUILDKITE_AGENT_ID": "agent-id", + "BUILDKITE_BUILD_ID": "build-id", + "BUILDKITE_BUILD_NUMBER": "123", + "BUILDKITE_BUILD_URL": "https://buildkite.example/builds/123", + "BUILDKITE_BRANCH": "main", + "BUILDKITE_COMMIT": "abc123", + "BUILDKITE_JOB_ID": "job-id", + "BUILDKITE_MESSAGE": "Test resource boundaries", + "BUILDKITE_ORGANIZATION_SLUG": "acme", + "BUILDKITE_STEP_ID": "step-id", + "BUILDKITE_TEST_ENGINE_SUITE_SLUG": "swift-suite", + ], + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ) + let client = try XCTUnwrap(TelemetryClient.live( + environment: environment, + uploadTags: ["feature": "configured", "language": "swift"], + logger: nil, + rootExporter: rootExporter, + childExporter: childExporter + )) + var test = TestState( + id: UUID(uuidString: "00000000-0000-0000-0000-000000000001")!, + className: "PaymentTests", + testName: "testChargeCard" + ) + + let executionID = client.startExecution(test) + let child = OpenTelemetry.instance.tracerProvider + .get(instrumentationName: "example-app") + .spanBuilder(spanName: "database.query") + .startSpan() + child.end() + client.annotate("checkpoint", executionID: executionID) + test.result = .passed + client.finishExecution(executionID, test: test, tags: ["feature": "payments"]) + client.forceFlush() + + let span = try XCTUnwrap(rootExporter.getFinishedSpanItems().only) + let exportedChild = try XCTUnwrap(childExporter.getFinishedSpanItems().only) + XCTAssertEqual(span.name, "test.execution") + XCTAssertNil(span.parentSpanId) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.executionVia], .string("otlp")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.executionScope], .string("PaymentTests")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.testName], .string("testChargeCard")) + XCTAssertEqual(span.attributes["test.case.result.status"], .string("pass")) + XCTAssertEqual(span.attributes["buildkite.tag.feature"], .string("payments")) + XCTAssertEqual(span.attributes["buildkite.tag.language"], .string("swift")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.runKey], .string("run-key")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.buildNumber], .string("123")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.jobID], .string("job-id")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.stepID], .string("step-id")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.message], .string("Test resource boundaries")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.collectorName], .string(TestCollector.name)) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.collectorVersion], .string(TestCollector.version)) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.frameworkName], .string("xctest")) + XCTAssertNil(span.attributes[BuildkiteTelemetryAttribute.runURL]) + XCTAssertEqual(span.resource.attributes["service.name"], .string("swift-suite")) + XCTAssertEqual(span.resource.attributes["service.namespace"], .string("acme")) + XCTAssertEqual(span.resource.attributes["cicd.pipeline.run.id"], .string("build-id")) + XCTAssertEqual( + span.resource.attributes["cicd.pipeline.run.url.full"], + .string("https://buildkite.example/builds/123") + ) + XCTAssertEqual(span.resource.attributes["cicd.worker.id"], .string("agent-id")) + XCTAssertEqual(span.resource.attributes["vcs.ref.head.name"], .string("main")) + XCTAssertEqual(span.resource.attributes["vcs.ref.head.revision"], .string("abc123")) + XCTAssertEqual(span.resource.attributes["vcs.ref.type"], .string("branch")) + XCTAssertNil(span.resource.attributes["service.instance.id"]) + XCTAssertNil(span.resource.attributes[BuildkiteTelemetryAttribute.runKey]) + XCTAssertNil(span.resource.attributes[BuildkiteTelemetryAttribute.jobID]) + XCTAssertNil(span.resource.attributes[BuildkiteTelemetryAttribute.frameworkName]) + XCTAssertNil(span.resource.attributes["buildkite.tag.language"]) + XCTAssertEqual(exportedChild.traceId, span.traceId) + XCTAssertEqual(exportedChild.parentSpanId, span.spanId) + XCTAssertEqual(exportedChild.resource.attributes["service.name"], .string("swift-suite")) + XCTAssertEqual(exportedChild.resource.attributes["cicd.pipeline.run.id"], .string("build-id")) + XCTAssertNil(exportedChild.resource.attributes[BuildkiteTelemetryAttribute.runKey]) + XCTAssertNil(exportedChild.resource.attributes[BuildkiteTelemetryAttribute.frameworkName]) + XCTAssertNil(exportedChild.resource.attributes["buildkite.tag.language"]) + XCTAssertEqual(span.events.only?.name, "test.annotation") + XCTAssertEqual( + span.events.only?.attributes[BuildkiteTelemetryAttribute.annotation], + .string("checkpoint") + ) + } + + func testExportsFailureLocationStatusAndException() throws { + let rootExporter = InMemoryExporter() + let client = try XCTUnwrap(TelemetryClient.live( + environment: self.configuredEnvironment, + uploadTags: [:], + logger: nil, + rootExporter: rootExporter, + childExporter: InMemoryExporter() + )) + var test = TestState(id: UUID(), className: "FailureTests", testName: "testFailure") + test.result = .failed + test.issues = [ + TestIssue( + compactDescription: "Expected true but was false", + description: "XCTAssertTrue failed", + sourceCodeContext: SourceCodeContext( + callStack: [SourceCodeFrame(address: 1, symbolInfo: SourceCodeSymbolInfo( + imageName: "Tests", + symbolName: "FailureTests.testFailure()" + ))], + location: SourceCodeLocation( + filePath: "/src/FailureTests.swift", + fileName: "FailureTests.swift", + line: 42 + ) + ) + ), + ] + + let executionID = client.startExecution(test) + client.finishExecution(executionID, test: test, tags: nil) + + let span = try XCTUnwrap(rootExporter.getFinishedSpanItems().only) + XCTAssertEqual(span.attributes["test.case.result.status"], .string("fail")) + XCTAssertEqual(span.attributes["code.file.path"], .string("/src/FailureTests.swift")) + XCTAssertEqual(span.attributes["code.line.number"], .int(42)) + XCTAssertEqual(span.status, .error(description: "Expected true but was false")) + XCTAssertEqual(span.events.only?.name, "exception") + XCTAssertEqual(span.events.only?.attributes["exception.message"], .string("XCTAssertTrue failed")) + XCTAssertEqual( + span.events.only?.attributes["exception.stacktrace"], + .string("0 FailureTests.testFailure()") + ) + } + + func testExecutionRemainsRootAndLinksToInheritedJobContext() throws { + let traceParent = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" + let previousTraceParent = getenv("TRACEPARENT").map { String(cString: $0) } + setenv("TRACEPARENT", traceParent, 1) + defer { + if let previousTraceParent { + setenv("TRACEPARENT", previousTraceParent, 1) + } else { + unsetenv("TRACEPARENT") + } + } + + let rootExporter = InMemoryExporter() + let client = try XCTUnwrap(TelemetryClient.live( + environment: EnvironmentValues( + values: [ + "BUILDKITE_ANALYTICS_KEY": "run-key", + "BUILDKITE_ANALYTICS_TOKEN": "SECRET", + ], + getFromInfoDictionary: { _ in nil } + ), + uploadTags: [:], + logger: nil, + rootExporter: rootExporter, + childExporter: InMemoryExporter() + )) + var test = TestState(id: UUID(), className: "LinkedTests", testName: "testLink") + + let executionID = client.startExecution(test) + test.result = .passed + client.finishExecution(executionID, test: test, tags: nil) + + let span = try XCTUnwrap(rootExporter.getFinishedSpanItems().only) + XCTAssertNil(span.parentSpanId) + XCTAssertEqual(span.links.only?.context.traceId.hexString, "4bf92f3577b34da6a3ce929d0e0e4736") + XCTAssertEqual(span.links.only?.context.spanId.hexString, "00f067aa0ba902b7") + } + + func testExportsSpansFromAProviderRegisteredAfterExecutionStarts() throws { + let rootExporter = InMemoryExporter() + let childExporter = InMemoryExporter() + let client = try XCTUnwrap(TelemetryClient.live( + environment: self.configuredEnvironment, + uploadTags: [:], + logger: nil, + rootExporter: rootExporter, + childExporter: childExporter + )) + var test = TestState(id: UUID(), className: "ChildTests", testName: "testChild") + + let executionID = client.startExecution(test) + let replacementProvider = TracerProviderBuilder() + .with(resource: Resource(attributes: ["service.name": .string("suite-app")])) + .with(sampler: Samplers.alwaysOn) + .build() + OpenTelemetry.registerTracerProvider(tracerProvider: replacementProvider) + let child = OpenTelemetry.instance.tracerProvider + .get(instrumentationName: "example-app") + .spanBuilder(spanName: "database.query") + .startSpan() + child.end() + test.result = .passed + client.finishExecution(executionID, test: test, tags: nil) + client.forceFlush() + + let root = try XCTUnwrap(rootExporter.getFinishedSpanItems().only) + let exportedChild = try XCTUnwrap(childExporter.getFinishedSpanItems().only) + XCTAssertEqual(exportedChild.traceId, root.traceId) + XCTAssertEqual(exportedChild.parentSpanId, root.spanId) + XCTAssertEqual(exportedChild.resource.attributes["service.name"], .string("suite-app")) + XCTAssertNil(exportedChild.resource.attributes[BuildkiteTelemetryAttribute.runKey]) + XCTAssertNil(exportedChild.resource.attributes[BuildkiteTelemetryAttribute.frameworkName]) + } + + func testUsesOnlyConfiguredHeadersForStandardTraceEndpoint() throws { + let configuration = try XCTUnwrap(CollectorOTLPConfiguration( + environment: EnvironmentValues( + values: [ + "BUILDKITE_ANALYTICS_KEY": "run-key", + "BUILDKITE_ANALYTICS_TOKEN": "collector-token", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://127.0.0.1:1234/v1/traces", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS": "authorization=Bearer%20relay-token,x-test=value%2Fone", + ], + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ), + logger: nil + )) + + XCTAssertEqual(configuration.endpoint.absoluteString, "http://127.0.0.1:1234/v1/traces") + XCTAssertNil(configuration.header(named: "Buildkite-Tests-Run-Key")) + XCTAssertEqual(configuration.header(named: "Authorization"), "Bearer relay-token") + XCTAssertEqual(configuration.header(named: "x-test"), "value/one") + } + + func testHeadersAloneDoNotEnableCollector() { + let configuration = CollectorOTLPConfiguration( + environment: EnvironmentValues( + values: [ + "OTEL_EXPORTER_OTLP_HEADERS": "authorization=Bearer%20unrelated-token", + ], + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ), + logger: nil + ) + + XCTAssertNil(configuration) + } + + func testIgnoresStandardHeadersAndProtocolForTrustedEndpoint() throws { + let messages = LockIsolated([String]()) + let logger = Logger(printer: { message in + messages.withValue { $0.append(message) } + }) + let configuration = try XCTUnwrap(CollectorOTLPConfiguration( + environment: EnvironmentValues( + values: [ + "BUILDKITE_ANALYTICS_KEY": "run-key", + "BUILDKITE_ANALYTICS_TOKEN": "collector-token", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS": "authorization=Bearer%20unrelated-token,x-test=value", + "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc", + ], + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ), + logger: logger + )) + logger.waitForLogs() + + XCTAssertEqual(configuration.endpoint.absoluteString, TestCollector.endpoint) + XCTAssertEqual(configuration.header(named: "Buildkite-Tests-Run-Key"), "run-key") + XCTAssertEqual( + configuration.header(named: "Authorization"), + #"Token token="collector-token""# + ) + XCTAssertNil(configuration.header(named: "x-test")) + XCTAssertTrue(messages.withValue { messages in + messages.contains( + "[BuildkiteTestCollector] warning: Standard OpenTelemetry exporter headers are ignored for endpoints that receive Buildkite credentials; use OTEL_EXPORTER_OTLP_TRACES_ENDPOINT or OTEL_EXPORTER_OTLP_ENDPOINT to send custom headers." + ) + }) + } + + func testUsesCollectorCredentialsForTrustedEndpointAndCustomRunKey() throws { + let configuration = try XCTUnwrap(CollectorOTLPConfiguration( + environment: EnvironmentValues( + values: [ + "BUILDKITE_ANALYTICS_KEY": "original-run-key", + "BUILDKITE_ANALYTICS_TOKEN": "collector-token", + "BUILDKITE_ANALYTICS_OTLP_ENDPOINT": "http://127.0.0.1:1234/v1/traces", + "BUILDKITE_ANALYTICS_ENVIRONMENT": #"{"key":"custom-run-key"}"#, + ], + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ), + logger: nil + )) + + XCTAssertEqual(configuration.header(named: "Buildkite-Tests-Run-Key"), "custom-run-key") + XCTAssertEqual( + configuration.header(named: "Authorization"), + #"Token token="collector-token""# + ) + } + + func testAppliesExecutionNameAffixesAndCustomRunEnvironmentOverrides() throws { + let rootExporter = InMemoryExporter() + let client = try XCTUnwrap(TelemetryClient.live( + environment: EnvironmentValues( + values: [ + "BUILDKITE_ANALYTICS_KEY": "original-run-key", + "BUILDKITE_ANALYTICS_TOKEN": "collector-token", + "BUILDKITE_ANALYTICS_BRANCH": "original-branch", + "BUILDKITE_ANALYTICS_EXECUTION_NAME_PREFIX": "[ios]", + "BUILDKITE_ANALYTICS_EXECUTION_NAME_SUFFIX": "[debug]", + "BUILDKITE_ANALYTICS_ENVIRONMENT": #"{"key":"custom-run-key","branch":"custom-branch","buildkite.run_key":"wrong-run-key","custom.flag":true}"#, + ], + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ), + uploadTags: [:], + logger: nil, + rootExporter: rootExporter, + childExporter: InMemoryExporter() + )) + var test = TestState(id: UUID(), className: "PaymentTests", testName: "testChargeCard") + + let executionID = client.startExecution(test) + test.result = .passed + client.finishExecution(executionID, test: test, tags: nil) + + let span = try XCTUnwrap(rootExporter.getFinishedSpanItems().only) + XCTAssertEqual(span.attributes["test.case.name"], .string("[ios] PaymentTests.testChargeCard [debug]")) + XCTAssertEqual(span.attributes[BuildkiteTelemetryAttribute.runKey], .string("custom-run-key")) + XCTAssertEqual(span.attributes["custom.flag"], .bool(true)) + XCTAssertEqual(span.resource.attributes["vcs.ref.head.name"], .string("custom-branch")) + XCTAssertNil(span.resource.attributes[BuildkiteTelemetryAttribute.runKey]) + XCTAssertNil(span.resource.attributes["custom.flag"]) + XCTAssertNil(span.attributes["key"]) + XCTAssertNil(span.attributes["branch"]) + } + + func testUsesProviderNativeCIRunIdentityAndKeepsOtherURLsOnExecutionRoot() throws { + let github = try self.exportExecution(environmentValues: [ + "GITHUB_ACTION": "test", + "GITHUB_ACTOR": "octocat", + "GITHUB_REF_NAME": "main", + "GITHUB_REPOSITORY": "acme/payments", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_RUN_ID": "github-123", + "GITHUB_RUN_NUMBER": "42", + "GITHUB_SHA": "abc123", + "GITHUB_WORKFLOW": "Tests", + ]) + XCTAssertEqual(github.resource.attributes["cicd.pipeline.run.id"], .string("github-123")) + XCTAssertEqual( + github.resource.attributes["cicd.pipeline.run.url.full"], + .string("https://github.com/acme/payments/actions/runs/github-123") + ) + XCTAssertNil(github.attributes[BuildkiteTelemetryAttribute.runURL]) + + let circle = try self.exportExecution(environmentValues: [ + "CIRCLE_BRANCH": "main", + "CIRCLE_BUILD_NUM": "42", + "CIRCLE_BUILD_URL": "https://circle.example/jobs/42", + "CIRCLE_SHA1": "abc123", + "CIRCLE_WORKFLOW_ID": "circle-123", + ]) + XCTAssertEqual(circle.resource.attributes["cicd.pipeline.run.id"], .string("circle-123")) + XCTAssertNil(circle.resource.attributes["cicd.pipeline.run.url.full"]) + XCTAssertEqual( + circle.attributes[BuildkiteTelemetryAttribute.runURL], + .string("https://circle.example/jobs/42") + ) + + let xcode = try self.exportExecution(environmentValues: [ + "CI_BRANCH": "main", + "CI_BUILD_ID": "xcode-123", + "CI_BUILD_NUMBER": "42", + "CI_COMMIT": "abc123", + "CI_PULL_REQUEST_HTML_URL": "https://github.com/acme/payments/pull/42", + "CI_WORKFLOW": "Tests", + ]) + XCTAssertEqual(xcode.resource.attributes["cicd.pipeline.run.id"], .string("xcode-123")) + XCTAssertNil(xcode.resource.attributes["cicd.pipeline.run.url.full"]) + XCTAssertEqual( + xcode.attributes[BuildkiteTelemetryAttribute.runURL], + .string("https://github.com/acme/payments/pull/42") + ) + + let buildkite = try self.exportExecution(environmentValues: [ + "BUILDKITE_ANALYTICS_URL": "https://configured.example/run", + "BUILDKITE_BUILD_ID": "buildkite-123", + "BUILDKITE_BUILD_URL": "https://buildkite.example/builds/42", + ]) + XCTAssertEqual(buildkite.resource.attributes["cicd.pipeline.run.id"], .string("buildkite-123")) + XCTAssertEqual( + buildkite.resource.attributes["cicd.pipeline.run.url.full"], + .string("https://buildkite.example/builds/42") + ) + XCTAssertEqual( + buildkite.attributes[BuildkiteTelemetryAttribute.runURL], + .string("https://configured.example/run") + ) + } + + func testRetainsRequiredSynthesisAttributesAtMinimumUsefulSpanLimit() throws { + let rootExporter = InMemoryExporter() + let client = try XCTUnwrap(TelemetryClient.live( + environment: self.configuredEnvironment, + uploadTags: ["configured": "optional"], + logger: nil, + rootExporter: rootExporter, + childExporter: InMemoryExporter(), + rootSpanLimits: SpanLimits().settingAttributeCountLimit(3) + )) + var test = TestState(id: UUID(), className: "LimitedTests", testName: "testRequiredFields") + + let executionID = client.startExecution(test) + test.result = .passed + client.finishExecution(executionID, test: test, tags: ["execution": "optional"]) + + let span = try XCTUnwrap(rootExporter.getFinishedSpanItems().only) + XCTAssertEqual(span.attributes, [ + BuildkiteTelemetryAttribute.executionVia: .string("otlp"), + BuildkiteTelemetryAttribute.runKey: .string("run-key"), + SemanticConventions.Test.caseResultStatus.rawValue: .string("pass"), + ]) + } + + func testBundleFlushRetriesExecutionsQueuedDuringBackoff() throws { + let rootExporter = FailOnceExporter() + let client = try XCTUnwrap(TelemetryClient.live( + environment: self.configuredEnvironment, + uploadTags: [:], + logger: nil, + rootExporter: rootExporter, + childExporter: InMemoryExporter() + )) + + for name in ["testOne", "testTwo"] { + var test = TestState(id: UUID(), className: "RetryTests", testName: name) + let executionID = client.startExecution(test) + test.result = .passed + client.finishExecution(executionID, test: test, tags: nil) + } + + XCTAssertEqual(rootExporter.exports.count, 1) + client.forceFlush() + + XCTAssertEqual(rootExporter.exports.count, 2) + XCTAssertEqual(rootExporter.exports[0].map(\.name), ["test.execution"]) + XCTAssertEqual( + rootExporter.exports[1].map(\.attributes[BuildkiteTelemetryAttribute.testName]), + [.string("testOne"), .string("testTwo")] + ) + } + + private var configuredEnvironment: EnvironmentValues { + EnvironmentValues( + values: [ + "BUILDKITE_ANALYTICS_KEY": "run-key", + "BUILDKITE_ANALYTICS_TOKEN": "SECRET", + ], + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ) + } + + private func exportExecution(environmentValues: [String: String]) throws -> SpanData { + OpenTelemetry.registerTracerProvider(tracerProvider: DefaultTracerProvider.instance) + let rootExporter = InMemoryExporter() + var values = environmentValues + values["BUILDKITE_ANALYTICS_TOKEN"] = "SECRET" + let client = try XCTUnwrap(TelemetryClient.live( + environment: EnvironmentValues( + values: values, + getFromEnvironment: { _ in nil }, + getFromInfoDictionary: { _ in nil } + ), + uploadTags: [:], + logger: nil, + rootExporter: rootExporter, + childExporter: InMemoryExporter() + )) + var test = TestState(id: UUID(), className: "CITests", testName: "testIdentity") + + let executionID = client.startExecution(test) + test.result = .passed + client.finishExecution(executionID, test: test, tags: nil) + + return try XCTUnwrap(rootExporter.getFinishedSpanItems().only) + } +} + +private extension Array { + var only: Element? { + self.count == 1 ? self[0] : nil + } +} + +private extension CollectorOTLPConfiguration { + func header(named name: String) -> String? { + self.headers.first { $0.0.caseInsensitiveCompare(name) == .orderedSame }?.1 + } +} + +private final class FailOnceExporter: SpanExporter, @unchecked Sendable { + var exports = [[SpanData]]() + + func export(spans: [SpanData], explicitTimeout: TimeInterval?) -> SpanExporterResultCode { + self.exports.append(spans) + return self.exports.count == 1 ? .failure : .success + } + + func flush(explicitTimeout: TimeInterval?) -> SpanExporterResultCode { + .success + } + + func shutdown(explicitTimeout: TimeInterval?) {} +} diff --git a/test-collector-swift/Tests/CoreTests/TestHelpers/ApiClient.swift b/test-collector-swift/Tests/CoreTests/TestHelpers/ApiClient.swift deleted file mode 100644 index 7719b3ba..00000000 --- a/test-collector-swift/Tests/CoreTests/TestHelpers/ApiClient.swift +++ /dev/null @@ -1,23 +0,0 @@ -@testable import Core -import Foundation -import XCTest - -#if canImport(FoundationNetworking) -import FoundationNetworking -#endif - -extension ApiClient { - static func fulfill(_ expectation: XCTestExpectation, after seconds: TimeInterval) -> ApiClient { - ApiClient { _ in - try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) - expectation.fulfill() - return (Data(), .stub()) - } - } -} - -extension HTTPURLResponse { - static func stub(from url: URL = URL(string: "test")!, status: Int = 200) -> HTTPURLResponse { - HTTPURLResponse(url: url, statusCode: status, httpVersion: "1.1", headerFields: nil)! - } -} diff --git a/test-collector-swift/Tests/CoreTests/TestHelpers/URLRequest.swift b/test-collector-swift/Tests/CoreTests/TestHelpers/URLRequest.swift deleted file mode 100644 index a18e35ad..00000000 --- a/test-collector-swift/Tests/CoreTests/TestHelpers/URLRequest.swift +++ /dev/null @@ -1,32 +0,0 @@ -import Foundation -import XCTest - -#if canImport(FoundationNetworking) -import FoundationNetworking -#endif - -extension URLRequest { - var authorizationHeader: String? { - self.value(forHTTPHeaderField: "Authorization") - } - - var httpBodyDictionary: NSDictionary? { - guard let data = self.httpBody ?? self.httpBodyStream?.read() else { return nil } - return try? JSONSerialization.jsonObject(with: data, options: []) as? NSDictionary - } -} - -extension InputStream { - fileprivate func read(maxLength: Int = 1024) -> Data { - var data = Data() - self.open() - let buffer = UnsafeMutablePointer.allocate(capacity: maxLength) - while self.hasBytesAvailable { - let bytesRead = self.read(buffer, maxLength: maxLength) - data.append(buffer, count: bytesRead) - } - buffer.deallocate() - self.close() - return data - } -} diff --git a/test-collector-swift/Tests/CoreTests/TestObserverTests.swift b/test-collector-swift/Tests/CoreTests/TestObserverTests.swift index f07e0e8a..cbd09ec8 100644 --- a/test-collector-swift/Tests/CoreTests/TestObserverTests.swift +++ b/test-collector-swift/Tests/CoreTests/TestObserverTests.swift @@ -2,57 +2,36 @@ import XCTest final class TestObserverTests: XCTestCase { - func testExecutionTagsFlowToTrace() { - let recordedTraces = LockIsolated([Trace]()) - let uploader = UploadClient( - record: { trace in recordedTraces.withValue { $0.append(trace) } }, - waitForUploads: { _ in } - ) - let tracer = Tracer( - startSpan: { section, _ in section as AnyHashable }, - endSpan: { id in Trace.Span(section: id as! String) } - ) - let observer = TestObserver(logger: nil, tracer: tracer, uploader: uploader) + func testExecutionTagsFlowToTelemetry() { + let recordedTags = LockIsolated([[String: String]?]()) + let telemetry = self.telemetry { tags in recordedTags.withValue { $0.append(tags) } } + let observer = TestObserver(logger: nil, telemetry: telemetry) observer.testCaseWillStart(self) observer.setTag(for: self, key: "suite", value: "smoke") observer.setTag(for: self, key: "feature", value: "payments") observer.testCaseDidFinish(self) - XCTAssertEqual(recordedTraces.value.count, 1) - XCTAssertEqual(recordedTraces.value.first?.tags, ["suite": "smoke", "feature": "payments"]) + XCTAssertEqual(recordedTags.value.count, 1) + XCTAssertEqual(recordedTags.value.first!, ["suite": "smoke", "feature": "payments"]) } func testExecutionTagsNilWhenEmpty() { - let recordedTraces = LockIsolated([Trace]()) - let uploader = UploadClient( - record: { trace in recordedTraces.withValue { $0.append(trace) } }, - waitForUploads: { _ in } - ) - let tracer = Tracer( - startSpan: { section, _ in section as AnyHashable }, - endSpan: { id in Trace.Span(section: id as! String) } - ) - let observer = TestObserver(logger: nil, tracer: tracer, uploader: uploader) + let recordedTags = LockIsolated([[String: String]?]()) + let telemetry = self.telemetry { tags in recordedTags.withValue { $0.append(tags) } } + let observer = TestObserver(logger: nil, telemetry: telemetry) observer.testCaseWillStart(self) observer.testCaseDidFinish(self) - XCTAssertEqual(recordedTraces.value.count, 1) - XCTAssertNil(recordedTraces.value.first?.tags) + XCTAssertEqual(recordedTags.value.count, 1) + XCTAssertNil(recordedTags.value.first!) } func testLateTagIsIgnored() { - let recordedTraces = LockIsolated([Trace]()) - let uploader = UploadClient( - record: { trace in recordedTraces.withValue { $0.append(trace) } }, - waitForUploads: { _ in } - ) - let tracer = Tracer( - startSpan: { section, _ in section as AnyHashable }, - endSpan: { id in Trace.Span(section: id as! String) } - ) - let observer = TestObserver(logger: nil, tracer: tracer, uploader: uploader) + let recordedTags = LockIsolated([[String: String]?]()) + let telemetry = self.telemetry { tags in recordedTags.withValue { $0.append(tags) } } + let observer = TestObserver(logger: nil, telemetry: telemetry) observer.testCaseWillStart(self) observer.setTag(for: self, key: "before", value: "yes") @@ -61,7 +40,16 @@ final class TestObserverTests: XCTestCase { // Tag set after finish should be ignored observer.setTag(for: self, key: "after", value: "should-not-appear") - XCTAssertEqual(recordedTraces.value.count, 1) - XCTAssertEqual(recordedTraces.value.first?.tags, ["before": "yes"]) + XCTAssertEqual(recordedTags.value.count, 1) + XCTAssertEqual(recordedTags.value.first!, ["before": "yes"]) + } + + private func telemetry(onFinish: @escaping ([String: String]?) -> Void) -> TelemetryClient { + TelemetryClient( + start: { $0.id }, + annotate: { _, _ in }, + finish: { _, _, tags in onFinish(tags) }, + flush: {} + ) } } diff --git a/test-collector-swift/Tests/CoreTests/TestResultsTests.swift b/test-collector-swift/Tests/CoreTests/TestResultsTests.swift deleted file mode 100644 index 24b28f23..00000000 --- a/test-collector-swift/Tests/CoreTests/TestResultsTests.swift +++ /dev/null @@ -1,177 +0,0 @@ -@testable import Core -import XCTest - -final class TestResultsTests: XCTestCase { - func testJSONEncoding() throws { - let testSuccess = TestState( - id: UUID(uuidString: "00000000-0000-0000-0000-000000000000")!, - className: "TestResultsTests", - testName: "testSuccess", - result: .passed, - issues: [], - expectedFailures: [] - ) - - let testFailure = TestState( - id: UUID(uuidString: "00000000-0000-0000-0000-000000000001")!, - className: "TestResultsTests", - testName: "testFailure", - result: .failed, - issues: [ - .init( - compactDescription: "The test failed", - description: """ - The thing you expected to happen didn't happen. - I wish I could provide you with more information. - """, - sourceCodeContext: .init( - callStack: [ - .init( - address: 1, - symbolInfo: .init( - imageName: "Library", - symbolName: "Foo.bar()" - ) - ), - .init( - address: 0, - symbolInfo: .init( - imageName: "CoreTests", - symbolName: "TestResultsTests.testJSONEncoding()" - ) - ), - ], - location: .init( - filePath: "/foo/bar", - fileName: "bar.swift", - line: 10 - ) - ) - ), - ], - expectedFailures: [] - ) - - let testMultipleFailures = TestState( - id: UUID(uuidString: "00000000-0000-0000-0000-000000000002")!, - className: "TestResultsTests", - testName: "testMultipleFailures", - result: .failed, - issues: [ - .init(compactDescription: "First failure", description: "Failure 1", sourceCodeContext: .init( - location: .init( - filePath: "/foo/first", - fileName: "first.swift", - line: 50 - ) - )), - .init(compactDescription: "Second failure", description: "Failure 2", sourceCodeContext: .init( - location: .init( - filePath: "/foo/second", - fileName: "second.swift", - line: 60 - ) - )), - .init(compactDescription: "Third failure", description: "Failure 3", sourceCodeContext: .init( - location: .init( - filePath: "/foo/third", - fileName: "third.swift", - line: 70 - ) - )), - ], - expectedFailures: [] - ) - - let testResults = TestResults.json( - runEnv: RunEnvironment(key: "test"), - tags: ["host.arch": "arm64"], - data: [ - .init(test: testSuccess, span: .init(section: "span0"), tags: ["suite": "smoke"]), - .init(test: testFailure, span: .init(section: "span1")), - .init(test: testMultipleFailures, span: .init(section: "span2")), - ] - ) - - let data = try JSONEncoder().encode(testResults) - - let json = try JSONSerialization.jsonObject(with: data) - - XCTAssertEqual( - json as? NSDictionary, - [ - "format": "json", - "run_env": ["key": "test"], - "tags": ["host.arch": "arm64"], - "data": [ - NSDictionary( - dictionary: [ - "id": "00000000-0000-0000-0000-000000000000", - "scope": "TestResultsTests", - "name": "testSuccess", - "result": "passed", - "tags": ["suite": "smoke"], - "failure_expanded": NSArray(), - "history": NSDictionary( - dictionary: [ - "section": "span0", - "detail": NSDictionary(), - "children": NSArray(), - ] - ), - ] - ), - [ - "id": "00000000-0000-0000-0000-000000000001", - "scope": "TestResultsTests", - "location": "bar.swift:10", - "file_name": "bar.swift", - "name": "testFailure", - "result": "failed", - "failure_reason": "The test failed", - "failure_expanded": [ - [ - "backtrace": [ - "0 Foo.bar()", - "1 TestResultsTests.testJSONEncoding()", - ], - "expanded": [ - "The thing you expected to happen didn't happen.", - "I wish I could provide you with more information.", - ], - ], - ], - "history": NSDictionary( - dictionary: [ - "section": "span1", - "detail": NSDictionary(), - "children": NSArray(), - ] - ), - ], - [ - "id": "00000000-0000-0000-0000-000000000002", - "scope": "TestResultsTests", - "location": "first.swift:50", - "file_name": "first.swift", - "name": "testMultipleFailures", - "result": "failed", - "failure_reason": "3 failures: First failure, Second failure, Third failure", - "failure_expanded": [ - ["backtrace": NSArray(), "expanded": ["Failure 1"]], - ["backtrace": NSArray(), "expanded": ["Failure 2"]], - ["backtrace": NSArray(), "expanded": ["Failure 3"]], - ], - "history": NSDictionary( - dictionary: [ - "section": "span2", - "detail": NSDictionary(), - "children": NSArray(), - ] - ), - ], - ], - ] - ) - } -} diff --git a/test-collector-swift/Tests/CoreTests/UploadClientTests.swift b/test-collector-swift/Tests/CoreTests/UploadClientTests.swift deleted file mode 100644 index 1841a066..00000000 --- a/test-collector-swift/Tests/CoreTests/UploadClientTests.swift +++ /dev/null @@ -1,140 +0,0 @@ -@testable import Core -import XCTest - -#if canImport(FoundationNetworking) -import FoundationNetworking -#endif - -final class UploadClientTests: XCTestCase { - func testWaitSynchronouslyForUploads() throws { - let uploadCompleted = self.expectation(description: "upload completed") - let uploadClient = UploadClient.live( - api: .fulfill(uploadCompleted, after: 0.5), - runEnvironment: EnvironmentValues().runEnvironment() - ) - - uploadClient.record(trace: .mock()) - uploadClient.waitForUploads() - - self.wait(for: [uploadCompleted], timeout: 0) - } - - func testWaitShouldTimeout() throws { - let uploadCompleted = self.expectation(description: "upload completed") - uploadCompleted.isInverted = true - let uploadClient = UploadClient.live( - api: .fulfill(uploadCompleted, after: 0.5), - runEnvironment: EnvironmentValues().runEnvironment() - ) - - uploadClient.record(trace: .mock()) - uploadClient.waitForUploads(timeout: 0.1) - - self.wait(for: [uploadCompleted], timeout: 0) - } - - func testFailureResponseLogsError() throws { - let errorMessage = LockIsolated("") - let logger = Logger(logLevel: .error) { errorMessage.setValue($0) } - - let data = try JSONEncoder().encode(UploadFailureResponse(message: "Something went wrong")) - let api = ApiClient { _ in (data, .stub(status: 500)) } - - let uploadClient = UploadClient.live( - api: api, - runEnvironment: EnvironmentValues().runEnvironment(), - logger: logger - ) - - uploadClient.record(trace: .mock()) - - uploadClient.waitForUploads() - logger.waitForLogs() - - // this varies e.g. macOS “internal server error” vs linux “Internal Server Error” - let statusName = HTTPURLResponse.localizedString(forStatusCode: 500) - XCTAssertEqual(errorMessage.value, "[BuildkiteTestCollector] error: Unexpected HTTP 500 \(statusName), Something went wrong") - } - - func testUploadsInBatchesOf5000ByDefault() throws { - let testResults = LockIsolated([TestResults]()) - - let api = ApiClient { route in - if case let .upload(results) = route { - testResults.withValue { $0.append(results) } - } - return (Data(), .stub()) - } - - let uploadTasks = DispatchGroup() - - let uploadClient = UploadClient.live( - api: api, - runEnvironment: EnvironmentValues().runEnvironment(), - group: uploadTasks - ) - - // Record 4999 traces - for id in 1...4999 { - uploadClient.record(trace: .mock(id: "\(id)")) - } - - // Wait to make sure no uploads were started - XCTAssertEqual(uploadTasks.wait(timeout: 0.1), .success) - XCTAssertEqual(testResults.count, 0) - - // Record one more trace to trigger the first batch of 5000 - uploadClient.record(trace: .mock(id: "5000")) - - // Wait for upload to complete - XCTAssertEqual(uploadTasks.wait(timeout: 0.1), .success) - XCTAssertEqual(testResults.count, 1) - - // Send the remaining traces - for id in 5001...12345 { - let trace = Trace(id: "\(id)", history: .init(section: "section")) - uploadClient.record(trace: trace) - } - - // A second batch will be sent for ids 5001...10000 - XCTAssertEqual(uploadTasks.wait(timeout: 0.1), .success) - XCTAssertEqual(testResults.count, 2) - - // Uploads any remaining traces regardless of batch size - uploadClient.waitForUploads() - - XCTAssertEqual(testResults.count, 3) - XCTAssertEqual(testResults[0].data.map(\.id), (1...5000).map { "\($0)" }) - XCTAssertEqual(testResults[1].data.map(\.id), (5001...10000).map { "\($0)" }) - XCTAssertEqual(testResults[2].data.map(\.id), (10001...12345).map { "\($0)" }) - } - - func testUploadIncludesTags() throws { - let testResults = LockIsolated([TestResults]()) - - let api = ApiClient { route in - if case let .upload(results) = route { - testResults.withValue { $0.append(results) } - } - return (Data(), .stub()) - } - - let uploadClient = UploadClient.live( - api: api, - runEnvironment: EnvironmentValues().runEnvironment(), - tags: ["host.arch": "arm64", "cloud.region": "us-east-1"] - ) - - uploadClient.record(trace: .mock()) - uploadClient.waitForUploads() - - XCTAssertEqual(testResults.count, 1) - XCTAssertEqual(testResults[0].tags, ["host.arch": "arm64", "cloud.region": "us-east-1"]) - } -} - -extension Trace { - fileprivate static func mock(id: String = "id") -> Self { - Trace(id: id, history: .init(section: "stub")) - } -}