From e07e9a7ade00a96f65f7c5323d82bf239e89df19 Mon Sep 17 00:00:00 2001 From: Matan Kushner Date: Wed, 22 Jul 2026 14:24:50 -0400 Subject: [PATCH 1/8] feat(eve): add Datadog reporter starter Signed-off-by: Matan Kushner --- .changeset/tidy-dogs-observe.md | 5 ++++ apps/fixtures/README.md | 1 + .../fixtures/datadog-eval-reporter/.gitignore | 10 +++++++ apps/fixtures/datadog-eval-reporter/README.md | 22 ++++++++++++++++ .../datadog-eval-reporter/agent/agent.ts | 7 +++++ .../agent/instructions.md | 1 + .../agent/instrumentation.ts | 8 ++++++ .../evals/evals.config.ts | 6 +++++ .../datadog-eval-reporter/evals/smoke.eval.ts | 12 +++++++++ .../datadog-eval-reporter/package.json | 22 ++++++++++++++++ .../datadog-eval-reporter/tsconfig.json | 17 ++++++++++++ docs/evals/reporters.mdx | 15 +++++++++++ packages/eve/src/evals/reporters/index.ts | 1 + .../evals/runner/reporters/datadog.test.ts | 13 ++++++++++ .../eve/src/evals/runner/reporters/datadog.ts | 26 +++++++++++++++++++ pnpm-lock.yaml | 22 ++++++++++++++++ 16 files changed, 188 insertions(+) create mode 100644 .changeset/tidy-dogs-observe.md create mode 100644 apps/fixtures/datadog-eval-reporter/.gitignore create mode 100644 apps/fixtures/datadog-eval-reporter/README.md create mode 100644 apps/fixtures/datadog-eval-reporter/agent/agent.ts create mode 100644 apps/fixtures/datadog-eval-reporter/agent/instructions.md create mode 100644 apps/fixtures/datadog-eval-reporter/agent/instrumentation.ts create mode 100644 apps/fixtures/datadog-eval-reporter/evals/evals.config.ts create mode 100644 apps/fixtures/datadog-eval-reporter/evals/smoke.eval.ts create mode 100644 apps/fixtures/datadog-eval-reporter/package.json create mode 100644 apps/fixtures/datadog-eval-reporter/tsconfig.json create mode 100644 packages/eve/src/evals/runner/reporters/datadog.test.ts create mode 100644 packages/eve/src/evals/runner/reporters/datadog.ts diff --git a/.changeset/tidy-dogs-observe.md b/.changeset/tidy-dogs-observe.md new file mode 100644 index 0000000000..05e1bee9fc --- /dev/null +++ b/.changeset/tidy-dogs-observe.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Expose an experimental Datadog eval reporter lifecycle for provider integration development. diff --git a/apps/fixtures/README.md b/apps/fixtures/README.md index 5bc2c15dde..5bd5705fd8 100644 --- a/apps/fixtures/README.md +++ b/apps/fixtures/README.md @@ -6,5 +6,6 @@ surface. - `weather-agent` backs root `pnpm dev`, manual weather-agent smokes, and bundle analysis. - `agent-tui-client` backs the non-e2e TUI smoke scripts in `packages/eve/test/tui-client`. +- `datadog-eval-reporter` is an isolated, OTel-instrumented starter for developing the Datadog eval reporter. When adding fixture behavior, prefer extending an existing fixture unless the new behavior needs incompatible app-level configuration. diff --git a/apps/fixtures/datadog-eval-reporter/.gitignore b/apps/fixtures/datadog-eval-reporter/.gitignore new file mode 100644 index 0000000000..eb095f0f86 --- /dev/null +++ b/apps/fixtures/datadog-eval-reporter/.gitignore @@ -0,0 +1,10 @@ +node_modules +.env* +.eve +.vercel +.next +.output +.nitro +dist +.DS_Store +*.tsbuildinfo diff --git a/apps/fixtures/datadog-eval-reporter/README.md b/apps/fixtures/datadog-eval-reporter/README.md new file mode 100644 index 0000000000..6c4e232723 --- /dev/null +++ b/apps/fixtures/datadog-eval-reporter/README.md @@ -0,0 +1,22 @@ +# Datadog eval reporter fixture + +This fixture gives the Datadog team a small, deterministic eve app for +developing the eval reporter. Its mock model needs no model-provider +credentials. + +The reporter is currently a no-op skeleton. The app's `instrumentation.ts` +does export the eval's runtime trace to Datadog through the standard OTLP +environment variables: + +```bash +export DATADOG_API_KEY="..." +export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL="http/protobuf" +export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="https://otlp.datadoghq.com/v1/traces" +export OTEL_EXPORTER_OTLP_TRACES_HEADERS="dd-api-key=${DATADOG_API_KEY},dd-otlp-source=llmobs" + +pnpm --filter datadog-eval-reporter-fixture eval +``` + +Use the regional OTLP endpoint for a Datadog site outside US1. Implement the +three lifecycle placeholders in the Datadog reporter to create an Experiment, +submit each completed eval, and finalize the run. diff --git a/apps/fixtures/datadog-eval-reporter/agent/agent.ts b/apps/fixtures/datadog-eval-reporter/agent/agent.ts new file mode 100644 index 0000000000..789a412be3 --- /dev/null +++ b/apps/fixtures/datadog-eval-reporter/agent/agent.ts @@ -0,0 +1,7 @@ +import { defineAgent } from "eve"; +import { mockModel } from "eve/evals"; + +export default defineAgent({ + model: mockModel("Hello from the Datadog eval fixture."), + modelContextWindowTokens: 1_000_000, +}); diff --git a/apps/fixtures/datadog-eval-reporter/agent/instructions.md b/apps/fixtures/datadog-eval-reporter/agent/instructions.md new file mode 100644 index 0000000000..0e2611b5dc --- /dev/null +++ b/apps/fixtures/datadog-eval-reporter/agent/instructions.md @@ -0,0 +1 @@ +You are a deterministic test agent used to develop the Datadog eval reporter. diff --git a/apps/fixtures/datadog-eval-reporter/agent/instrumentation.ts b/apps/fixtures/datadog-eval-reporter/agent/instrumentation.ts new file mode 100644 index 0000000000..c418fa7a13 --- /dev/null +++ b/apps/fixtures/datadog-eval-reporter/agent/instrumentation.ts @@ -0,0 +1,8 @@ +import { registerOTel } from "@vercel/otel"; +import { defineInstrumentation } from "eve/instrumentation"; + +export default defineInstrumentation({ + setup({ agentName }) { + registerOTel({ serviceName: agentName }); + }, +}); diff --git a/apps/fixtures/datadog-eval-reporter/evals/evals.config.ts b/apps/fixtures/datadog-eval-reporter/evals/evals.config.ts new file mode 100644 index 0000000000..b0dfb2f1d2 --- /dev/null +++ b/apps/fixtures/datadog-eval-reporter/evals/evals.config.ts @@ -0,0 +1,6 @@ +import { defineEvalConfig } from "eve/evals"; +import { Datadog } from "eve/evals/reporters"; + +export default defineEvalConfig({ + reporters: [Datadog()], +}); diff --git a/apps/fixtures/datadog-eval-reporter/evals/smoke.eval.ts b/apps/fixtures/datadog-eval-reporter/evals/smoke.eval.ts new file mode 100644 index 0000000000..39bd06b810 --- /dev/null +++ b/apps/fixtures/datadog-eval-reporter/evals/smoke.eval.ts @@ -0,0 +1,12 @@ +import { defineEval } from "eve/evals"; + +export default defineEval({ + description: "Datadog reporter smoke eval.", + + async test(t) { + await t.send("Say hello."); + + t.succeeded(); + t.messageIncludes("Hello from the Datadog eval fixture."); + }, +}); diff --git a/apps/fixtures/datadog-eval-reporter/package.json b/apps/fixtures/datadog-eval-reporter/package.json new file mode 100644 index 0000000000..8251caf32d --- /dev/null +++ b/apps/fixtures/datadog-eval-reporter/package.json @@ -0,0 +1,22 @@ +{ + "name": "datadog-eval-reporter-fixture", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "build": "eve build", + "dev": "eve dev", + "eval": "eve eval --strict", + "typecheck": "eve build && tsc" + }, + "dependencies": { + "@opentelemetry/core": "2.6.1", + "@opentelemetry/sdk-trace-base": "2.6.1", + "@vercel/otel": "2.1.2", + "eve": "workspace:*" + }, + "devDependencies": { + "@types/node": "catalog:", + "typescript": "catalog:" + } +} diff --git a/apps/fixtures/datadog-eval-reporter/tsconfig.json b/apps/fixtures/datadog-eval-reporter/tsconfig.json new file mode 100644 index 0000000000..85e3eceb8e --- /dev/null +++ b/apps/fixtures/datadog-eval-reporter/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "esnext", + "moduleResolution": "bundler", + "outDir": "dist", + "rootDir": ".", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "declaration": true, + "noEmit": true, + "types": ["node"] + }, + "include": ["agent/**/*.ts", "evals/**/*.ts"] +} diff --git a/docs/evals/reporters.mdx b/docs/evals/reporters.mdx index b9ac468a1b..fd3ce8690d 100644 --- a/docs/evals/reporters.mdx +++ b/docs/evals/reporters.mdx @@ -44,6 +44,21 @@ A reporter instance observes the evals that reference it. Share one instance acr Braintrust needs its SDK installed in the app and credentials in the environment: install the `braintrust` package (`npm install braintrust`) and set `BRAINTRUST_API_KEY`. Pass `--skip-report` to run the eval without shipping results, which also suppresses config reporters and is useful locally when iterating. +## Datadog + +`Datadog()` is an experimental, no-op reporter skeleton for developing a Datadog Experiments integration: + +```ts title="evals/evals.config.ts" +import { defineEvalConfig } from "eve/evals"; +import { Datadog } from "eve/evals/reporters"; + +export default defineEvalConfig({ + reporters: [Datadog()], +}); +``` + +Runtime tracing remains an agent concern. Configure `agent/instrumentation.ts` to export OTel spans to Datadog, then implement the reporter lifecycle to associate those spans and eval results with an Experiment. The `datadog-eval-reporter` app under `apps/fixtures` is a runnable starter. + ## JUnit `JUnit({ filePath })` writes JUnit XML for CI annotations. The `--junit ` CLI flag does the same thing without touching the eval file, usually the better fit because CI owns the output path, not the eval: diff --git a/packages/eve/src/evals/reporters/index.ts b/packages/eve/src/evals/reporters/index.ts index 818b704da0..5abb6f03a6 100644 --- a/packages/eve/src/evals/reporters/index.ts +++ b/packages/eve/src/evals/reporters/index.ts @@ -1,5 +1,6 @@ export { Braintrust, type BraintrustReporterConfig } from "#evals/runner/reporters/braintrust.js"; export { Console, type ConsoleReporterConfig } from "#evals/runner/reporters/console.js"; +export { Datadog } from "#evals/runner/reporters/datadog.js"; export { JUnit, type JUnitReporterConfig } from "#evals/runner/reporters/junit.js"; export type { EvalReporter, diff --git a/packages/eve/src/evals/runner/reporters/datadog.test.ts b/packages/eve/src/evals/runner/reporters/datadog.test.ts new file mode 100644 index 0000000000..68b4ee2947 --- /dev/null +++ b/packages/eve/src/evals/runner/reporters/datadog.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; + +import { Datadog } from "#evals/reporters/index.js"; + +describe("Datadog", () => { + it("creates a reporter with the eval lifecycle", () => { + const reporter = Datadog(); + + expect(reporter.onRunStart).toBeTypeOf("function"); + expect(reporter.onEvalComplete).toBeTypeOf("function"); + expect(reporter.onRunComplete).toBeTypeOf("function"); + }); +}); diff --git a/packages/eve/src/evals/runner/reporters/datadog.ts b/packages/eve/src/evals/runner/reporters/datadog.ts new file mode 100644 index 0000000000..959ee5ddb1 --- /dev/null +++ b/packages/eve/src/evals/runner/reporters/datadog.ts @@ -0,0 +1,26 @@ +import type { EvalReporter } from "#evals/runner/reporters/types.js"; +import type { EveEval, EveEvalResult, EveEvalRunSummary, EveEvalTarget } from "#evals/types.js"; + +/** + * Creates an experimental Datadog reporter skeleton. + * + * The reporter is intentionally a no-op while the Datadog integration is + * developed. + */ +export function Datadog(): EvalReporter { + return new DatadogReporter(); +} + +class DatadogReporter implements EvalReporter { + onRunStart(_evaluations: readonly EveEval[], _target: EveEvalTarget): void { + // Create one Datadog Experiment for this eval run. + } + + onEvalComplete(_result: EveEvalResult): void { + // Associate this eval's runtime span with the Experiment and submit its result and scores. + } + + onRunComplete(_summary: EveEvalRunSummary): void { + // Flush pending telemetry and finalize the Datadog Experiment. + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 807afd52b9..ce407ed915 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -399,6 +399,28 @@ importers: specifier: 'catalog:' version: 7.0.2 + apps/fixtures/datadog-eval-reporter: + dependencies: + '@opentelemetry/core': + specifier: 2.6.1 + version: 2.6.1(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': + specifier: 2.6.1 + version: 2.6.1(@opentelemetry/api@1.9.1) + '@vercel/otel': + specifier: 2.1.2 + version: 2.1.2(@opentelemetry/api-logs@0.214.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.214.0(@opentelemetry/api@1.9.1))(@opentelemetry/resources@2.6.1(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.214.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.6.1(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + eve: + specifier: workspace:* + version: link:../../../packages/eve + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 25.9.1 + typescript: + specifier: 'catalog:' + version: 7.0.2 + apps/fixtures/weather-agent: dependencies: eve: From b976760cbc5a3d263fb53b7e8667234ae1ec6404 Mon Sep 17 00:00:00 2001 From: Matan Kushner Date: Wed, 22 Jul 2026 14:36:09 -0400 Subject: [PATCH 2/8] refactor(eve): trim Datadog reporter starter Signed-off-by: Matan Kushner --- .changeset/tidy-dogs-observe.md | 5 ----- apps/fixtures/README.md | 1 - apps/fixtures/datadog-eval-reporter/.gitignore | 10 ---------- apps/fixtures/datadog-eval-reporter/README.md | 12 +++--------- .../datadog-eval-reporter/agent/agent.ts | 2 +- .../agent/instrumentation.ts | 4 +--- .../datadog-eval-reporter/evals/smoke.eval.ts | 2 -- .../fixtures/datadog-eval-reporter/package.json | 10 +--------- .../datadog-eval-reporter/tsconfig.json | 17 ----------------- docs/evals/reporters.mdx | 15 --------------- .../src/evals/runner/reporters/datadog.test.ts | 13 ------------- .../eve/src/evals/runner/reporters/datadog.ts | 7 +------ pnpm-lock.yaml | 10 ---------- 13 files changed, 7 insertions(+), 101 deletions(-) delete mode 100644 .changeset/tidy-dogs-observe.md delete mode 100644 apps/fixtures/datadog-eval-reporter/.gitignore delete mode 100644 apps/fixtures/datadog-eval-reporter/tsconfig.json delete mode 100644 packages/eve/src/evals/runner/reporters/datadog.test.ts diff --git a/.changeset/tidy-dogs-observe.md b/.changeset/tidy-dogs-observe.md deleted file mode 100644 index 05e1bee9fc..0000000000 --- a/.changeset/tidy-dogs-observe.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"eve": patch ---- - -Expose an experimental Datadog eval reporter lifecycle for provider integration development. diff --git a/apps/fixtures/README.md b/apps/fixtures/README.md index 5bd5705fd8..5bc2c15dde 100644 --- a/apps/fixtures/README.md +++ b/apps/fixtures/README.md @@ -6,6 +6,5 @@ surface. - `weather-agent` backs root `pnpm dev`, manual weather-agent smokes, and bundle analysis. - `agent-tui-client` backs the non-e2e TUI smoke scripts in `packages/eve/test/tui-client`. -- `datadog-eval-reporter` is an isolated, OTel-instrumented starter for developing the Datadog eval reporter. When adding fixture behavior, prefer extending an existing fixture unless the new behavior needs incompatible app-level configuration. diff --git a/apps/fixtures/datadog-eval-reporter/.gitignore b/apps/fixtures/datadog-eval-reporter/.gitignore deleted file mode 100644 index eb095f0f86..0000000000 --- a/apps/fixtures/datadog-eval-reporter/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -node_modules -.env* -.eve -.vercel -.next -.output -.nitro -dist -.DS_Store -*.tsbuildinfo diff --git a/apps/fixtures/datadog-eval-reporter/README.md b/apps/fixtures/datadog-eval-reporter/README.md index 6c4e232723..4bbf5e4134 100644 --- a/apps/fixtures/datadog-eval-reporter/README.md +++ b/apps/fixtures/datadog-eval-reporter/README.md @@ -1,12 +1,7 @@ # Datadog eval reporter fixture -This fixture gives the Datadog team a small, deterministic eve app for -developing the eval reporter. Its mock model needs no model-provider -credentials. - -The reporter is currently a no-op skeleton. The app's `instrumentation.ts` -does export the eval's runtime trace to Datadog through the standard OTLP -environment variables: +This deterministic mock-model eval uses the no-op Datadog reporter and exports +its runtime trace through the standard OTLP environment variables: ```bash export DATADOG_API_KEY="..." @@ -17,6 +12,5 @@ export OTEL_EXPORTER_OTLP_TRACES_HEADERS="dd-api-key=${DATADOG_API_KEY},dd-otlp- pnpm --filter datadog-eval-reporter-fixture eval ``` -Use the regional OTLP endpoint for a Datadog site outside US1. Implement the -three lifecycle placeholders in the Datadog reporter to create an Experiment, +Implement the reporter's three lifecycle placeholders to create an Experiment, submit each completed eval, and finalize the run. diff --git a/apps/fixtures/datadog-eval-reporter/agent/agent.ts b/apps/fixtures/datadog-eval-reporter/agent/agent.ts index 789a412be3..0d9836c30c 100644 --- a/apps/fixtures/datadog-eval-reporter/agent/agent.ts +++ b/apps/fixtures/datadog-eval-reporter/agent/agent.ts @@ -2,6 +2,6 @@ import { defineAgent } from "eve"; import { mockModel } from "eve/evals"; export default defineAgent({ - model: mockModel("Hello from the Datadog eval fixture."), + model: mockModel(), modelContextWindowTokens: 1_000_000, }); diff --git a/apps/fixtures/datadog-eval-reporter/agent/instrumentation.ts b/apps/fixtures/datadog-eval-reporter/agent/instrumentation.ts index c418fa7a13..caf5439d57 100644 --- a/apps/fixtures/datadog-eval-reporter/agent/instrumentation.ts +++ b/apps/fixtures/datadog-eval-reporter/agent/instrumentation.ts @@ -2,7 +2,5 @@ import { registerOTel } from "@vercel/otel"; import { defineInstrumentation } from "eve/instrumentation"; export default defineInstrumentation({ - setup({ agentName }) { - registerOTel({ serviceName: agentName }); - }, + setup: ({ agentName }) => registerOTel({ serviceName: agentName }), }); diff --git a/apps/fixtures/datadog-eval-reporter/evals/smoke.eval.ts b/apps/fixtures/datadog-eval-reporter/evals/smoke.eval.ts index 39bd06b810..1896585936 100644 --- a/apps/fixtures/datadog-eval-reporter/evals/smoke.eval.ts +++ b/apps/fixtures/datadog-eval-reporter/evals/smoke.eval.ts @@ -5,8 +5,6 @@ export default defineEval({ async test(t) { await t.send("Say hello."); - t.succeeded(); - t.messageIncludes("Hello from the Datadog eval fixture."); }, }); diff --git a/apps/fixtures/datadog-eval-reporter/package.json b/apps/fixtures/datadog-eval-reporter/package.json index 8251caf32d..229727c54b 100644 --- a/apps/fixtures/datadog-eval-reporter/package.json +++ b/apps/fixtures/datadog-eval-reporter/package.json @@ -4,19 +4,11 @@ "private": true, "type": "module", "scripts": { - "build": "eve build", - "dev": "eve dev", - "eval": "eve eval --strict", - "typecheck": "eve build && tsc" + "eval": "eve eval --strict" }, "dependencies": { - "@opentelemetry/core": "2.6.1", "@opentelemetry/sdk-trace-base": "2.6.1", "@vercel/otel": "2.1.2", "eve": "workspace:*" - }, - "devDependencies": { - "@types/node": "catalog:", - "typescript": "catalog:" } } diff --git a/apps/fixtures/datadog-eval-reporter/tsconfig.json b/apps/fixtures/datadog-eval-reporter/tsconfig.json deleted file mode 100644 index 85e3eceb8e..0000000000 --- a/apps/fixtures/datadog-eval-reporter/tsconfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "esnext", - "moduleResolution": "bundler", - "outDir": "dist", - "rootDir": ".", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true, - "declaration": true, - "noEmit": true, - "types": ["node"] - }, - "include": ["agent/**/*.ts", "evals/**/*.ts"] -} diff --git a/docs/evals/reporters.mdx b/docs/evals/reporters.mdx index fd3ce8690d..b9ac468a1b 100644 --- a/docs/evals/reporters.mdx +++ b/docs/evals/reporters.mdx @@ -44,21 +44,6 @@ A reporter instance observes the evals that reference it. Share one instance acr Braintrust needs its SDK installed in the app and credentials in the environment: install the `braintrust` package (`npm install braintrust`) and set `BRAINTRUST_API_KEY`. Pass `--skip-report` to run the eval without shipping results, which also suppresses config reporters and is useful locally when iterating. -## Datadog - -`Datadog()` is an experimental, no-op reporter skeleton for developing a Datadog Experiments integration: - -```ts title="evals/evals.config.ts" -import { defineEvalConfig } from "eve/evals"; -import { Datadog } from "eve/evals/reporters"; - -export default defineEvalConfig({ - reporters: [Datadog()], -}); -``` - -Runtime tracing remains an agent concern. Configure `agent/instrumentation.ts` to export OTel spans to Datadog, then implement the reporter lifecycle to associate those spans and eval results with an Experiment. The `datadog-eval-reporter` app under `apps/fixtures` is a runnable starter. - ## JUnit `JUnit({ filePath })` writes JUnit XML for CI annotations. The `--junit ` CLI flag does the same thing without touching the eval file, usually the better fit because CI owns the output path, not the eval: diff --git a/packages/eve/src/evals/runner/reporters/datadog.test.ts b/packages/eve/src/evals/runner/reporters/datadog.test.ts deleted file mode 100644 index 68b4ee2947..0000000000 --- a/packages/eve/src/evals/runner/reporters/datadog.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { Datadog } from "#evals/reporters/index.js"; - -describe("Datadog", () => { - it("creates a reporter with the eval lifecycle", () => { - const reporter = Datadog(); - - expect(reporter.onRunStart).toBeTypeOf("function"); - expect(reporter.onEvalComplete).toBeTypeOf("function"); - expect(reporter.onRunComplete).toBeTypeOf("function"); - }); -}); diff --git a/packages/eve/src/evals/runner/reporters/datadog.ts b/packages/eve/src/evals/runner/reporters/datadog.ts index 959ee5ddb1..19cab4b9cc 100644 --- a/packages/eve/src/evals/runner/reporters/datadog.ts +++ b/packages/eve/src/evals/runner/reporters/datadog.ts @@ -1,12 +1,7 @@ import type { EvalReporter } from "#evals/runner/reporters/types.js"; import type { EveEval, EveEvalResult, EveEvalRunSummary, EveEvalTarget } from "#evals/types.js"; -/** - * Creates an experimental Datadog reporter skeleton. - * - * The reporter is intentionally a no-op while the Datadog integration is - * developed. - */ +/** Creates an intentionally no-op Datadog reporter skeleton. */ export function Datadog(): EvalReporter { return new DatadogReporter(); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ce407ed915..4f18f6d778 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -401,9 +401,6 @@ importers: apps/fixtures/datadog-eval-reporter: dependencies: - '@opentelemetry/core': - specifier: 2.6.1 - version: 2.6.1(@opentelemetry/api@1.9.1) '@opentelemetry/sdk-trace-base': specifier: 2.6.1 version: 2.6.1(@opentelemetry/api@1.9.1) @@ -413,13 +410,6 @@ importers: eve: specifier: workspace:* version: link:../../../packages/eve - devDependencies: - '@types/node': - specifier: 'catalog:' - version: 25.9.1 - typescript: - specifier: 'catalog:' - version: 7.0.2 apps/fixtures/weather-agent: dependencies: From cee20e0606ad218dc3091132de93850c633c10b1 Mon Sep 17 00:00:00 2001 From: "mehul.sonowal" Date: Tue, 28 Jul 2026 12:13:50 -0400 Subject: [PATCH 3/8] feat(evals): add Datadog reporter Signed-off-by: mehul.sonowal --- .changeset/datadog-eval-reporter.md | 5 + docs/evals/reporters.mdx | 19 +- packages/eve/src/evals/reporters/index.ts | 2 +- .../evals/runner/reporters/datadog.test.ts | 196 +++++++++ .../eve/src/evals/runner/reporters/datadog.ts | 386 +++++++++++++++++- research/datadog-experiment-reporter.md | 242 +++++++++++ 6 files changed, 838 insertions(+), 12 deletions(-) create mode 100644 .changeset/datadog-eval-reporter.md create mode 100644 packages/eve/src/evals/runner/reporters/datadog.test.ts create mode 100644 research/datadog-experiment-reporter.md diff --git a/.changeset/datadog-eval-reporter.md b/.changeset/datadog-eval-reporter.md new file mode 100644 index 0000000000..68a58591c5 --- /dev/null +++ b/.changeset/datadog-eval-reporter.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Add a Datadog eval reporter that creates one LLM Observability Experiment per eve eval run and submits eval assertion metrics through the optional `dd-trace` package. diff --git a/docs/evals/reporters.mdx b/docs/evals/reporters.mdx index b9ac468a1b..d96618bca4 100644 --- a/docs/evals/reporters.mdx +++ b/docs/evals/reporters.mdx @@ -1,6 +1,6 @@ --- title: "Reporters" -description: "Ship eval results to Braintrust experiments or JUnit XML. eve runs and scores everything itself." +description: "Ship eval results to Braintrust, Datadog, or JUnit XML. eve runs and scores everything itself." --- eve runs and grades everything itself; reporters ship the results out. The CLI prints a console summary by default (one line per eval, with failed assertions and their messages), and reporters from `eve/evals/reporters` add destinations on top. @@ -44,6 +44,23 @@ A reporter instance observes the evals that reference it. Share one instance acr Braintrust needs its SDK installed in the app and credentials in the environment: install the `braintrust` package (`npm install braintrust`) and set `BRAINTRUST_API_KEY`. Pass `--skip-report` to run the eval without shipping results, which also suppresses config reporters and is useful locally when iterating. +## Datadog + +`Datadog(...)` uploads eval assertion scores to a Datadog LLM Observability Experiment. Put one instance in the config so it covers the whole run: + +```ts title="evals/evals.config.ts" +import { defineEvalConfig } from "eve/evals"; +import { Datadog } from "eve/evals/reporters"; + +export default defineEvalConfig({ + reporters: [Datadog({ projectName: "weather-agent" })], +}); +``` + +The reporter creates one Datadog Experiment on run start, creates a placeholder dataset for the experiment, submits one synthetic experiment span per completed eve eval, associates that eval's assertion scores as experiment metrics, and prints the Datadog Experiment URL after the run completes. Metric labels use Datadog-safe names: gate assertions log under a `gate_` prefix, and characters outside letters, numbers, underscores, and hyphens are normalized to underscores. + +Datadog needs `dd-trace` installed in the app and Datadog credentials in the environment: install the `dd-trace` package (`npm install dd-trace`) and set `DD_API_KEY`, `DD_APP_KEY`, and `DD_SITE` as appropriate. By default, the reporter records assertion scores and eval metadata only; pass `recordInputs: true`, `recordOutputs: true`, or `recordExpectedOutputs: true` if your destination is approved for eval prompts, outputs, or authored expectations. Input is read from the eval's first `t.send(...)` message, falling back to the eval description when no message event is available. Expected output is read from eval `metadata.expectedOutput`, `metadata.expected`, or `metadata.expected_output`. + ## JUnit `JUnit({ filePath })` writes JUnit XML for CI annotations. The `--junit ` CLI flag does the same thing without touching the eval file, usually the better fit because CI owns the output path, not the eval: diff --git a/packages/eve/src/evals/reporters/index.ts b/packages/eve/src/evals/reporters/index.ts index 5abb6f03a6..63b968c795 100644 --- a/packages/eve/src/evals/reporters/index.ts +++ b/packages/eve/src/evals/reporters/index.ts @@ -1,6 +1,6 @@ export { Braintrust, type BraintrustReporterConfig } from "#evals/runner/reporters/braintrust.js"; export { Console, type ConsoleReporterConfig } from "#evals/runner/reporters/console.js"; -export { Datadog } from "#evals/runner/reporters/datadog.js"; +export { Datadog, type DatadogReporterConfig } from "#evals/runner/reporters/datadog.js"; export { JUnit, type JUnitReporterConfig } from "#evals/runner/reporters/junit.js"; export type { EvalReporter, diff --git a/packages/eve/src/evals/runner/reporters/datadog.test.ts b/packages/eve/src/evals/runner/reporters/datadog.test.ts new file mode 100644 index 0000000000..66bae45534 --- /dev/null +++ b/packages/eve/src/evals/runner/reporters/datadog.test.ts @@ -0,0 +1,196 @@ +import { describe, expect, it, vi } from "vitest"; + +import { createEmptyDerivedFacts } from "#evals/runner/derive-run-facts.js"; +import { Datadog, type DatadogReporterConfig } from "#evals/reporters/index.js"; +import type { EveEval, EveEvalResult, EveEvalRunSummary, EveEvalTarget } from "#evals/types.js"; + +function makeTarget(kind: "local" | "remote" = "local"): EveEvalTarget { + return { + capabilities: { devRoutes: kind === "local" }, + kind, + url: kind === "local" ? "http://127.0.0.1:3000" : "https://test.vercel.app", + }; +} + +function makeEval(overrides: Partial = {}): EveEval { + return { + _tag: "EveEval", + id: "eval-1", + description: "Say hello", + tags: ["smoke"], + metadata: { suite: "unit", expectedOutput: "helpful onboarding answer" }, + async test() {}, + ...overrides, + }; +} + +function makeEvalResult(overrides: Partial = {}): EveEvalResult { + return { + id: "eval-1", + result: { + output: "actual output", + finalMessage: "actual output", + status: "completed", + events: [ + { + type: "message.received", + data: { + message: "What should I send you?", + sequence: 1, + turnId: "turn-1", + }, + }, + ], + derived: { + ...createEmptyDerivedFacts(), + toolCalls: [ + { + name: "search", + input: { query: "test" }, + output: null, + status: "completed", + turnIndex: 0, + sessionId: "session-123", + }, + ], + toolCallCount: 1, + messageCount: 1, + }, + sessionId: "session-123", + }, + assertions: [ + { name: "succeeded", score: 1, severity: "gate", passed: true }, + { name: "similarity", score: 0.9, severity: "soft", threshold: 0.6, passed: true }, + { name: "judge.autoevals.closedQA", score: 1, severity: "soft", passed: true }, + ], + verdict: "passed", + startedAt: "2026-01-01T00:00:00.000Z", + completedAt: "2026-01-01T00:00:01.000Z", + ...overrides, + }; +} + +function makeSummary(result: EveEvalResult = makeEvalResult()): EveEvalRunSummary { + return { + target: makeTarget(), + results: [result], + startedAt: "2026-01-01T00:00:00.000Z", + completedAt: "2026-01-01T00:00:02.000Z", + passed: result.verdict === "passed" ? 1 : 0, + failed: result.verdict === "failed" ? 1 : 0, + scored: result.verdict === "scored" ? 1 : 0, + skipped: result.verdict === "skipped" ? 1 : 0, + errored: result.error ? 1 : 0, + }; +} + +function makeConfig(overrides: Partial = {}) { + const span = { + experimentId: "exp-1", + spanId: "span-1", + traceId: "trace-1", + url: "https://dd.test/span", + }; + const recorder = { + experimentId: "exp-1", + url: vi.fn(() => "https://dd.test/experiment"), + submitSpan: vi.fn(async () => span), + submitEvaluationMetrics: vi.fn(async () => undefined), + close: vi.fn(async () => undefined), + }; + const client = { + startExperiment: vi.fn(async () => recorder), + }; + const lines: string[] = []; + const config = { + projectName: "test-project", + client, + log: (line: string) => lines.push(line), + ...overrides, + } satisfies DatadogReporterConfig; + + return { client, config, lines, recorder, span }; +} + +describe("Datadog", () => { + it("creates an experiment, submits one span, and attaches assertion metrics", async () => { + const { client, config, recorder, span } = makeConfig({ experimentName: "run-1" }); + const reporter = Datadog(config); + const evaluation = makeEval(); + const result = makeEvalResult(); + + await reporter.onRunStart([evaluation], makeTarget()); + await reporter.onEvalComplete(result); + + expect(client.startExperiment).toHaveBeenCalledWith( + expect.objectContaining({ + name: "run-1", + projectName: "test-project", + dataset: { name: "run-1 dataset" }, + tags: expect.objectContaining({ source: "eve", target_kind: "local" }), + metadata: expect.objectContaining({ eveEvalIds: ["eval-1"], eveTargetKind: "local" }), + }), + ); + expect(recorder.submitSpan).toHaveBeenCalledWith( + expect.objectContaining({ + id: "eval-1", + name: "eval-1", + durationMs: 1000, + metadata: expect.objectContaining({ + suite: "unit", + eveSessionId: "session-123", + eveVerdict: "passed", + eveToolCalls: ["search"], + }), + tags: expect.objectContaining({ eval_id: "eval-1", eval_verdict: "passed" }), + }), + ); + expect(recorder.submitEvaluationMetrics).toHaveBeenCalledWith(span, [ + expect.objectContaining({ label: "gate_succeeded", value: 1 }), + expect.objectContaining({ label: "similarity", value: 0.9 }), + expect.objectContaining({ label: "judge_autoevals_closedQA", value: 1 }), + expect.objectContaining({ label: "eve_tool_call_count", value: 1 }), + expect.objectContaining({ label: "eve_subagent_call_count", value: 0 }), + expect.objectContaining({ label: "eve_message_count", value: 1 }), + expect.objectContaining({ label: "eve_reasoning_block_count", value: 0 }), + ]); + }); + + it("records eval input and output only when enabled", async () => { + const { config, recorder } = makeConfig({ + recordInputs: true, + recordOutputs: true, + recordExpectedOutputs: true, + }); + const reporter = Datadog(config); + + await reporter.onRunStart([makeEval()], makeTarget()); + await reporter.onEvalComplete(makeEvalResult()); + + expect(recorder.submitSpan).toHaveBeenCalledWith( + expect.objectContaining({ + input: "What should I send you?", + output: "actual output", + expectedOutput: "helpful onboarding answer", + }), + ); + }); + + it("closes the experiment and logs its URL", async () => { + const { config, lines, recorder } = makeConfig(); + const reporter = Datadog(config); + + await reporter.onRunStart([makeEval()], makeTarget()); + await reporter.onRunComplete(makeSummary()); + + expect(recorder.close).toHaveBeenCalledWith({ status: "completed", error: undefined }); + expect(lines.join("\n")).toContain("Datadog experiment URL: https://dd.test/experiment"); + }); + + it("is a no-op before the experiment is initialized", async () => { + const reporter = Datadog(makeConfig().config); + + await reporter.onEvalComplete(makeEvalResult()); + await reporter.onRunComplete(makeSummary()); + }); +}); diff --git a/packages/eve/src/evals/runner/reporters/datadog.ts b/packages/eve/src/evals/runner/reporters/datadog.ts index 19cab4b9cc..824d770f87 100644 --- a/packages/eve/src/evals/runner/reporters/datadog.ts +++ b/packages/eve/src/evals/runner/reporters/datadog.ts @@ -1,21 +1,387 @@ -import type { EvalReporter } from "#evals/runner/reporters/types.js"; +import { createRequire } from "node:module"; + import type { EveEval, EveEvalResult, EveEvalRunSummary, EveEvalTarget } from "#evals/types.js"; +import type { EvalReporter } from "#evals/runner/reporters/types.js"; +import { resolveLocalGitMetadata } from "#evals/runner/resolve-git-metadata.js"; + +/** Configuration for the Datadog reporter. */ +export interface DatadogReporterConfig { + /** Datadog LLM Observability project name. Defaults to `DD_LLMOBS_ML_APP`, `DD_SERVICE`, or the first eval id. */ + readonly projectName?: string; + /** Name for the placeholder dataset used by the experiment. Defaults to ` dataset`. */ + readonly datasetName?: string; + /** Name for the created experiment. Defaults to a timestamped eve eval run name. */ + readonly experimentName?: string; + /** Optional experiment description. */ + readonly description?: string; + /** Datadog service name used when the reporter initializes `dd-trace`. */ + readonly service?: string; + /** Datadog env used when the reporter initializes `dd-trace`. */ + readonly env?: string; + /** Datadog site used when the reporter initializes `dd-trace`. */ + readonly site?: string; + /** LLM Observability ml_app used when the reporter initializes `dd-trace`. Defaults to `projectName`. */ + readonly mlApp?: string; + /** Tags attached to the Datadog Experiment. */ + readonly tags?: Readonly>; + /** Metadata attached to the Datadog Experiment. */ + readonly metadata?: Readonly>; + /** Free-form Datadog Experiment config. */ + readonly config?: Readonly>; + /** Include eval descriptions in the synthetic experiment row input. Defaults to false. */ + readonly recordInputs?: boolean; + /** Include eval outputs in the synthetic experiment row output. Defaults to false. */ + readonly recordOutputs?: boolean; + /** Include `metadata.expectedOutput`, `metadata.expected`, or `metadata.expected_output` on experiment rows. Defaults to false. */ + readonly recordExpectedOutputs?: boolean; + /** Console hook used for tests. */ + readonly log?: (line: string) => void; + /** eve-owned client seam for tests and custom Datadog SDK wiring. */ + readonly client?: { + startExperiment(options: DatadogStartExperimentOptions): Promise; + }; +} + +interface DatadogStartExperimentOptions { + name: string; + projectName?: string; + description?: string; + tags?: Readonly>; + metadata?: Readonly>; + config?: Readonly>; + dataset?: { + name?: string; + }; +} + +interface DatadogExperimentRecorder { + readonly experimentId?: string | null; + url(): string | null; + submitSpan(row: DatadogExperimentSpanInput): Promise; + submitEvaluationMetrics( + span: Pick, + metrics: readonly DatadogEvaluationMetricInput[], + ): Promise; + close(options?: { status?: "completed" | "failed"; error?: string }): Promise; +} + +interface DatadogExperimentSpanInput { + id?: string; + name?: string; + input?: unknown; + output?: unknown; + expectedOutput?: unknown; + metadata?: Readonly>; + tags?: Readonly>; + startedAt?: string; + completedAt?: string; + durationMs?: number; + error?: string; +} + +interface DatadogExperimentSpan { + experimentId: string | null; + spanId: string | null; + traceId?: string | null; + url?: string | null; +} + +interface DatadogEvaluationMetricInput { + label: string; + value?: boolean | number | string | Record; + error?: string; + tags?: Readonly>; +} + +interface DatadogTraceModule { + init(options: { + service?: string; + env?: string; + site?: string; + llmobs?: { + mlApp?: string; + agentlessEnabled?: boolean; + }; + }): DatadogTracer; +} -/** Creates an intentionally no-op Datadog reporter skeleton. */ -export function Datadog(): EvalReporter { - return new DatadogReporter(); +interface DatadogTracer { + readonly llmobs?: { + readonly experiments?: { + startExperiment(options: DatadogStartExperimentOptions): Promise; + }; + }; +} + +/** + * Creates an {@link EvalReporter} that uploads eval assertion scores to a + * Datadog LLM Observability Experiment. Requires the optional `dd-trace` + * package and `DD_API_KEY`/`DD_APP_KEY` credentials unless `config.client` is + * provided. + */ +export function Datadog(config: DatadogReporterConfig = {}): EvalReporter { + return new DatadogReporter(config); } class DatadogReporter implements EvalReporter { - onRunStart(_evaluations: readonly EveEval[], _target: EveEvalTarget): void { - // Create one Datadog Experiment for this eval run. + readonly #config: DatadogReporterConfig; + readonly #evaluations = new Map(); + #recorder: DatadogExperimentRecorder | undefined; + + constructor(config: DatadogReporterConfig) { + this.#config = config; + } + + async onRunStart(evaluations: readonly EveEval[], target: EveEvalTarget): Promise { + this.#evaluations.clear(); + for (const evaluation of evaluations) { + this.#evaluations.set(evaluation.id, evaluation); + } + + const client = await resolveDatadogClient(this.#config, evaluations); + const git = resolveLocalGitMetadata(process.cwd()); + + const experimentName = this.#config.experimentName ?? defaultExperimentName(); + this.#recorder = await client.startExperiment({ + name: experimentName, + projectName: resolveProjectName(this.#config, evaluations), + description: this.#config.description, + dataset: { name: this.#config.datasetName ?? `${experimentName} dataset` }, + tags: resolveExperimentTags(this.#config, target), + metadata: { + ...resolveExperimentMetadata(evaluations, target), + ...(git.sha ? { eveGitCommit: git.sha, eveGitBranch: git.branch } : {}), + ...this.#config.metadata, + }, + config: this.#config.config, + }); + } + + async onEvalComplete(result: EveEvalResult): Promise { + if (!this.#recorder) return; + + const evaluation = this.#evaluations.get(result.id); + const span = await this.#recorder.submitSpan({ + id: result.id, + name: result.id, + input: this.#config.recordInputs ? resolveInput(result, evaluation) : undefined, + output: this.#config.recordOutputs ? result.result.output : undefined, + expectedOutput: this.#config.recordExpectedOutputs + ? resolveExpectedOutput(evaluation) + : undefined, + metadata: resolveResultMetadata(result, evaluation), + tags: resolveResultTags(result, evaluation), + startedAt: result.startedAt, + completedAt: result.completedAt, + durationMs: elapsedMs(result.startedAt, result.completedAt), + error: result.error, + }); + + await this.#recorder.submitEvaluationMetrics(span, resolveEvaluationMetrics(result)); + } + + async onRunComplete(summary: EveEvalRunSummary): Promise { + if (!this.#recorder) return; + + try { + const failed = summary.failed > 0 || summary.errored > 0; + await this.#recorder.close({ + status: failed ? "failed" : "completed", + error: failed ? `${summary.failed} failed, ${summary.errored} errored` : undefined, + }); + + const url = this.#recorder.url(); + if (url) { + (this.#config.log ?? console.log)(`Datadog experiment URL: ${url}\n`); + } + } finally { + this.#recorder = undefined; + } + } +} + +const DD_TRACE_PACKAGE = "dd-trace"; + +async function resolveDatadogClient( + config: DatadogReporterConfig, + evaluations: readonly EveEval[], +): Promise<{ + startExperiment(options: DatadogStartExperimentOptions): Promise; +}> { + if (config.client) return config.client; + + const sdk = await loadDatadogSdk(); + const projectName = resolveProjectName(config, evaluations); + const tracer = sdk.init({ + service: config.service ?? process.env.DD_SERVICE ?? projectName, + env: config.env ?? process.env.DD_ENV, + site: config.site ?? process.env.DD_SITE, + llmobs: { + mlApp: config.mlApp ?? projectName, + agentlessEnabled: true, + }, + }); + + const experiments = tracer.llmobs?.experiments; + if (!experiments?.startExperiment) { + throw new Error( + "The installed 'dd-trace' package does not expose tracer.llmobs.experiments.startExperiment().", + ); } + return experiments; +} - onEvalComplete(_result: EveEvalResult): void { - // Associate this eval's runtime span with the Experiment and submit its result and scores. +async function loadDatadogSdk(): Promise { + try { + const requireFromApp = createRequire(`${process.cwd()}/package.json`); + return requireFromApp(DD_TRACE_PACKAGE) as DatadogTraceModule; + } catch { + try { + const mod = (await import(DD_TRACE_PACKAGE)) as { default?: unknown }; + return (mod.default ?? mod) as DatadogTraceModule; + } catch { + throw new Error( + [ + "The 'dd-trace' package is required for Datadog reporting but was not found.", + "", + "Install it with:", + " npm install dd-trace", + ].join("\n"), + ); + } } +} + +function resolveProjectName( + config: DatadogReporterConfig, + evaluations: readonly EveEval[], +): string { + return ( + config.projectName ?? + config.mlApp ?? + process.env.DD_LLMOBS_ML_APP ?? + process.env.DD_SERVICE ?? + evaluations[0]?.id ?? + "eve evals" + ); +} + +function defaultExperimentName(): string { + return `eve evals ${new Date().toISOString()}`; +} + +function resolveExperimentTags( + config: DatadogReporterConfig, + target: EveEvalTarget, +): Record { + return { + source: "eve", + target_kind: target.kind, + ...config.tags, + }; +} + +function resolveExperimentMetadata( + evaluations: readonly EveEval[], + target: EveEvalTarget, +): Record { + return { + eveEvalIds: evaluations.map((evaluation) => evaluation.id), + eveTargetKind: target.kind, + eveTargetUrl: target.url, + eveTimestamp: new Date().toISOString(), + }; +} + +function resolveResultTags( + result: EveEvalResult, + evaluation: EveEval | undefined, +): Record { + return { + eval_id: result.id, + eval_verdict: result.verdict, + eval_status: result.result.status, + ...(evaluation?.tags?.length ? { eval_tags: evaluation.tags.join(",") } : {}), + }; +} - onRunComplete(_summary: EveEvalRunSummary): void { - // Flush pending telemetry and finalize the Datadog Experiment. +function resolveInput(result: EveEvalResult, evaluation: EveEval | undefined): unknown { + for (const event of result.result.events) { + if (event.type !== "message.received") continue; + return event.data.message; } + + return evaluation?.description ?? ""; +} + +function resolveExpectedOutput(evaluation: EveEval | undefined): unknown { + if (!evaluation?.metadata) return undefined; + return ( + evaluation.metadata.expectedOutput ?? + evaluation.metadata.expected ?? + evaluation.metadata.expected_output + ); +} + +function resolveResultMetadata( + result: EveEvalResult, + evaluation: EveEval | undefined, +): Record { + const failedAssertions = result.assertions + .filter((assertion) => !assertion.passed) + .map((assertion) => ({ name: assertion.name, message: assertion.message })); + + return { + ...evaluation?.metadata, + eveSessionId: result.result.sessionId, + eveStatus: result.result.status, + eveVerdict: result.verdict, + eveSkipReason: result.skipReason, + eveToolCalls: result.result.derived.toolCalls.map((call) => call.name), + eveSubagentCalls: result.result.derived.subagentCalls.map((call) => call.name), + eveParked: result.result.derived.parked, + ...(failedAssertions.length > 0 ? { eveFailedAssertions: failedAssertions } : {}), + ...(result.result.derived.failureCode + ? { eveFailureCode: result.result.derived.failureCode } + : {}), + }; +} + +function resolveEvaluationMetrics(result: EveEvalResult): DatadogEvaluationMetricInput[] { + const metrics: DatadogEvaluationMetricInput[] = []; + + for (const assertion of result.assertions) { + const rawLabel = assertion.severity === "gate" ? `gate_${assertion.name}` : assertion.name; + metrics.push({ + label: toDatadogMetricLabel(rawLabel), + value: assertion.score, + tags: { + assertion_name: assertion.name, + assertion_label: rawLabel, + assertion_severity: assertion.severity, + assertion_passed: String(assertion.passed), + }, + }); + } + + metrics.push( + { label: "eve_tool_call_count", value: result.result.derived.toolCallCount }, + { label: "eve_subagent_call_count", value: result.result.derived.subagentCallCount }, + { label: "eve_message_count", value: result.result.derived.messageCount }, + { label: "eve_reasoning_block_count", value: result.result.derived.reasoningBlockCount }, + ); + + return metrics; +} + +function toDatadogMetricLabel(label: string): string { + const normalized = label.replace(/[^a-zA-Z0-9_-]+/g, "_").replace(/^_+|_+$/g, ""); + return normalized || "metric"; +} + +function elapsedMs(startedAt: string, completedAt: string): number | undefined { + const start = Date.parse(startedAt); + const completed = Date.parse(completedAt); + if (!Number.isFinite(start) || !Number.isFinite(completed)) return undefined; + return Math.max(0, completed - start); } diff --git a/research/datadog-experiment-reporter.md b/research/datadog-experiment-reporter.md new file mode 100644 index 0000000000..7831cb6a81 --- /dev/null +++ b/research/datadog-experiment-reporter.md @@ -0,0 +1,242 @@ +--- +issue: "TBD (Datadog Experiments reporter partner request)" +status: proposed +last_updated: "2026-07-27" +--- + +# Datadog Experiments reporter for eve evals + +Datadog Experiments is the right destination for offline CI eval runs: one eve +run should create one Datadog Experiment, and each completed eval case should +become one experiment row with scores, metadata, and eventually a link back to +the agent runtime trace. + +## MVP scope + +Ship a reporter that can publish completed eve eval results to Datadog +Experiments without re-running the task inside the Datadog SDK. + +``` +evals/evals.config.ts ── Datadog(...) ──► one Datadog Experiment + │ + └─ onEvalComplete(result) ───────► one synthetic experiment span + metrics +``` + +The MVP intentionally uses synthetic experiment spans built from the eval +result. Post-hoc linking to an already-ingested agent span is the next step once +Datadog exposes that association capability. + +## Datadog SDK surface needed + +The current Node Experiments shape owns the dataset/task/evaluator loop. eve +already owns that loop, so the reporter only needs hooks for creating an +Experiment, generating/submitting one experiment span per completed row, and +submitting eval metrics that reference that generated span id. + +Proposed public Node contract: + +```ts +type ExperimentMetricValue = boolean | number | string | Record; + +interface StartExperimentOptions { + /** Experiment display name. */ + name: string; + /** Optional override; defaults to DD_LLMOBS_ML_APP or DD_SERVICE. */ + projectName?: string; + description?: string; + tags?: Record; + metadata?: Record; + config?: Record; + /** Optional dataset linkage when the caller has one. Not required for MVP. */ + dataset?: { + id?: string; + name?: string; + version?: string | number; + }; +} + +interface ExperimentSpanInput { + /** Stable caller-owned row id, e.g. eve eval id. */ + id?: string; + name?: string; + input?: unknown; + output?: unknown; + expectedOutput?: unknown; + metadata?: Record; + tags?: Record; + startedAt?: Date | string | number; + completedAt?: Date | string | number; + durationMs?: number; + error?: Error | string | { type?: string; message: string; stack?: string }; + datasetRecordId?: string; +} + +interface ExperimentSpan { + experimentId: string; + spanId: string; + traceId: string; + url: string | null; +} + +interface EvaluationMetricInput { + label: string; + value?: ExperimentMetricValue; + error?: Error | string | { message: string }; + tags?: Record; + timestamp?: Date | string | number; +} + +interface ExperimentRecorder { + readonly experimentId: string; + url(): string | null; + + /** + * Generates an LLM Obs experiment span id/trace id, serializes the row as an + * experiment span, submits it, and returns the identifiers the caller should + * use for eval metrics. + */ + submitSpan(row: ExperimentSpanInput): Promise; + + /** + * Submits eval metrics for an already-submitted experiment span. Each metric + * is associated to the row by `span.spanId` and to the run by + * `span.experimentId`. + */ + submitEvaluationMetrics( + span: Pick, + metrics: readonly EvaluationMetricInput[], + ): Promise; + + /** Best-effort status patch + flush. */ + close(options?: { status?: "completed" | "failed"; error?: Error | string }): Promise; +} + +interface Experiments { + startExperiment(options: StartExperimentOptions): Promise; +} +``` + +Usage from the eve reporter: + +```ts +const experiment = await tracer.llmobs.experiments.startExperiment({ + name: "eve evals", + projectName: "weather-agent", + tags: { source: "eve" }, +}); + +const span = await experiment.submitSpan({ + id: result.id, + input: evaluation.description, + output: result.result.output, + metadata, + startedAt: result.startedAt, + completedAt: result.completedAt, + error: result.error, +}); + +await experiment.submitEvaluationMetrics(span, [ + { label: "gate:succeeded", value: 1 }, + { label: "similarity", value: 0.92 }, +]); + +await experiment.close({ status: summary.failed > 0 ? "failed" : "completed" }); +``` + +This deliberately keeps span generation and eval metric submission separate so +custom eval runners can submit metrics after they know the generated experiment +span id. A convenience `submitRow({ ...row, metrics })` can be added later, but +the lower-level contract should remain public. + +Until that public API exists, keep any direct Datadog HTTP client in one private +adapter so it can be removed without changing the eve reporter API. + +## eve code changes + +1. Replace `src/evals/runner/reporters/datadog.ts` with a real optional-peer + reporter: + - export `Datadog(config?: DatadogReporterConfig)` and its config type from + `eve/evals/reporters`. + - dynamically import `dd-trace` or a Datadog SDK adapter so `eve` does not + gain a hard runtime dependency. + - accept config for `projectName`/`mlApp`, `experimentName`, `datasetName`, + `tags`, `metadata`, `recordInputs`, `recordOutputs`, and an optional + injected client for tests. +2. On `onRunStart`, create/resolve the Datadog project, materialize a dataset + record per observed eve eval, then create a fresh Experiment for the run. +3. On `onEvalComplete`, map an `EveEvalResult` to one Datadog row: + - `id`: eve eval id. + - input/expected: eval description and/or dataset metadata that eve exposes. + - output/error/timestamps: `result.result.output`, `result.error`, + `startedAt`, `completedAt`. + - scores: soft assertions by assertion name; gate assertions as binary + `gate:` metrics. + - metadata: eval metadata, verdict, skip reason, session id, tool/subagent + call counts, failed assertion messages, runtime identity, and git metadata. +4. On `onRunComplete`, flush pending rows, patch Experiment status + (`completed` or `failed`), and print the Datadog Experiment URL. +5. Add docs in `docs/evals/reporters.mdx`, a changeset for `eve`, and keep all + user-facing copy spelling the framework as `eve`. + +## Runtime trace-linking follow-up + +The reporter cannot currently discover the agent turn's real OTel span. The +harness stores `traceId`/`spanId` in internal session state to keep model-call +spans parented, but that state is not exposed through eval results. + +Add a durable, reporter-visible trace link without requiring reporters to +instrument the black-box agent: + +1. Extend the stream protocol with trace context on a turn lifecycle event, e.g. + `turn.started.data.trace = { traceId, spanId }` when telemetry is enabled. +2. Add `EveEvalTraceLink` to `EveEvalTaskResult`/`EveEvalSessionResult`, derived + from captured stream events. +3. Teach `Datadog` to send `{ traceId, spanId }` when present. +4. Once Datadog supports post-hoc association, switch from synthetic spans to + `attachSpan({ traceId, spanId, scores, metadata })` for instrumented agents, + with synthetic spans retained as the fallback. + +## Example project + +Expand `apps/fixtures/datadog-eval-reporter` into the correctness fixture and +manual dogfood app: + +- real dependencies: `eve`, `dd-trace`, `@vercel/otel`, and OTel exporter + packages needed for Datadog trace submission. +- deterministic `mockModel()` agent so CI and local checks do not need model + credentials. +- `agent/instrumentation.ts` that enables OTel/Datadog telemetry for runtime + traces. +- `evals/evals.config.ts` using `Datadog({ projectName: "eve-datadog-fixture" })`. +- `README.md` and `.env.example` showing both modes: + - local verification against a fake Datadog HTTP endpoint; + - manual verification against Datadog with `DD_API_KEY`, `DD_APP_KEY`, + `DD_SITE`, and `DD_LLMOBS_ML_APP`. + +## Test plan + +- Unit tests for the reporter mapping: lifecycle order, missing SDK error, + no-op before `onRunStart`, score/metadata conversion, skipped/error rows, + summary status, URL logging, and `recordInputs`/`recordOutputs` privacy + switches. +- Integration tests with an injected fake Datadog client or mocked `fetch` that + assert exact project/dataset/experiment/events/status payloads without real + network access. +- Runner-level test to prove config reporters still dedupe and the Datadog + reporter receives scoped summaries. +- Scenario test using `apps/fixtures/datadog-eval-reporter` and `eve eval +--strict`; point the reporter at a local fake Datadog server and assert one + experiment plus one row is submitted. +- Manual dogfood test with real Datadog credentials: run the fixture, open the + printed Experiment URL, verify scores, metadata, run summary, and (after the + follow-up) the link to the runtime trace. + +## Open questions + +- What exact public Node SDK API and package version should eve target for the + recorder-style Experiments surface? +- Does Datadog require a persisted dataset for every Experiment, or can external + eval rows create an Experiment without dataset records? +- What is the backend contract for post-hoc linking to an already-ingested span? +- Should eve expose expected output/inputs to reporters beyond description and + metadata, or keep that data entirely user-authored via eval metadata? From e8597de9e60bc77cb811beda492da1817a38e3ce Mon Sep 17 00:00:00 2001 From: "mehul.sonowal" Date: Mon, 3 Aug 2026 15:33:09 -0400 Subject: [PATCH 4/8] fix(evals): align Datadog reporter with external experiment API Signed-off-by: mehul.sonowal --- .../evals/runner/reporters/datadog.test.ts | 22 ++++++------ .../eve/src/evals/runner/reporters/datadog.ts | 36 +++++++++---------- research/datadog-experiment-reporter.md | 12 +++---- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/packages/eve/src/evals/runner/reporters/datadog.test.ts b/packages/eve/src/evals/runner/reporters/datadog.test.ts index 66bae45534..8a13c87232 100644 --- a/packages/eve/src/evals/runner/reporters/datadog.test.ts +++ b/packages/eve/src/evals/runner/reporters/datadog.test.ts @@ -91,15 +91,15 @@ function makeConfig(overrides: Partial = {}) { traceId: "trace-1", url: "https://dd.test/span", }; - const recorder = { - experimentId: "exp-1", + const experiment = { + experimentId: vi.fn(() => "exp-1"), url: vi.fn(() => "https://dd.test/experiment"), submitSpan: vi.fn(async () => span), submitEvaluationMetrics: vi.fn(async () => undefined), close: vi.fn(async () => undefined), }; const client = { - startExperiment: vi.fn(async () => recorder), + startExperiment: vi.fn(async () => experiment), }; const lines: string[] = []; const config = { @@ -109,12 +109,12 @@ function makeConfig(overrides: Partial = {}) { ...overrides, } satisfies DatadogReporterConfig; - return { client, config, lines, recorder, span }; + return { client, config, lines, experiment, span }; } describe("Datadog", () => { it("creates an experiment, submits one span, and attaches assertion metrics", async () => { - const { client, config, recorder, span } = makeConfig({ experimentName: "run-1" }); + const { client, config, experiment, span } = makeConfig({ experimentName: "run-1" }); const reporter = Datadog(config); const evaluation = makeEval(); const result = makeEvalResult(); @@ -131,7 +131,7 @@ describe("Datadog", () => { metadata: expect.objectContaining({ eveEvalIds: ["eval-1"], eveTargetKind: "local" }), }), ); - expect(recorder.submitSpan).toHaveBeenCalledWith( + expect(experiment.submitSpan).toHaveBeenCalledWith( expect.objectContaining({ id: "eval-1", name: "eval-1", @@ -145,7 +145,7 @@ describe("Datadog", () => { tags: expect.objectContaining({ eval_id: "eval-1", eval_verdict: "passed" }), }), ); - expect(recorder.submitEvaluationMetrics).toHaveBeenCalledWith(span, [ + expect(experiment.submitEvaluationMetrics).toHaveBeenCalledWith(span, [ expect.objectContaining({ label: "gate_succeeded", value: 1 }), expect.objectContaining({ label: "similarity", value: 0.9 }), expect.objectContaining({ label: "judge_autoevals_closedQA", value: 1 }), @@ -157,7 +157,7 @@ describe("Datadog", () => { }); it("records eval input and output only when enabled", async () => { - const { config, recorder } = makeConfig({ + const { config, experiment } = makeConfig({ recordInputs: true, recordOutputs: true, recordExpectedOutputs: true, @@ -167,7 +167,7 @@ describe("Datadog", () => { await reporter.onRunStart([makeEval()], makeTarget()); await reporter.onEvalComplete(makeEvalResult()); - expect(recorder.submitSpan).toHaveBeenCalledWith( + expect(experiment.submitSpan).toHaveBeenCalledWith( expect.objectContaining({ input: "What should I send you?", output: "actual output", @@ -177,13 +177,13 @@ describe("Datadog", () => { }); it("closes the experiment and logs its URL", async () => { - const { config, lines, recorder } = makeConfig(); + const { config, lines, experiment } = makeConfig(); const reporter = Datadog(config); await reporter.onRunStart([makeEval()], makeTarget()); await reporter.onRunComplete(makeSummary()); - expect(recorder.close).toHaveBeenCalledWith({ status: "completed", error: undefined }); + expect(experiment.close).toHaveBeenCalledWith({ status: "completed", error: undefined }); expect(lines.join("\n")).toContain("Datadog experiment URL: https://dd.test/experiment"); }); diff --git a/packages/eve/src/evals/runner/reporters/datadog.ts b/packages/eve/src/evals/runner/reporters/datadog.ts index 824d770f87..250ce643cd 100644 --- a/packages/eve/src/evals/runner/reporters/datadog.ts +++ b/packages/eve/src/evals/runner/reporters/datadog.ts @@ -38,7 +38,7 @@ export interface DatadogReporterConfig { readonly log?: (line: string) => void; /** eve-owned client seam for tests and custom Datadog SDK wiring. */ readonly client?: { - startExperiment(options: DatadogStartExperimentOptions): Promise; + startExperiment(options: DatadogStartExperimentOptions): Promise; }; } @@ -54,18 +54,18 @@ interface DatadogStartExperimentOptions { }; } -interface DatadogExperimentRecorder { - readonly experimentId?: string | null; +interface DatadogExternalExperiment { + experimentId(): string | null; url(): string | null; - submitSpan(row: DatadogExperimentSpanInput): Promise; + submitSpan(row: DatadogExternalExperimentSpanInput): Promise; submitEvaluationMetrics( - span: Pick, + span: Pick, metrics: readonly DatadogEvaluationMetricInput[], ): Promise; close(options?: { status?: "completed" | "failed"; error?: string }): Promise; } -interface DatadogExperimentSpanInput { +interface DatadogExternalExperimentSpanInput { id?: string; name?: string; input?: unknown; @@ -79,7 +79,7 @@ interface DatadogExperimentSpanInput { error?: string; } -interface DatadogExperimentSpan { +interface DatadogExternalExperimentSpan { experimentId: string | null; spanId: string | null; traceId?: string | null; @@ -108,7 +108,7 @@ interface DatadogTraceModule { interface DatadogTracer { readonly llmobs?: { readonly experiments?: { - startExperiment(options: DatadogStartExperimentOptions): Promise; + startExperiment(options: DatadogStartExperimentOptions): Promise; }; }; } @@ -126,7 +126,7 @@ export function Datadog(config: DatadogReporterConfig = {}): EvalReporter { class DatadogReporter implements EvalReporter { readonly #config: DatadogReporterConfig; readonly #evaluations = new Map(); - #recorder: DatadogExperimentRecorder | undefined; + #experiment: DatadogExternalExperiment | undefined; constructor(config: DatadogReporterConfig) { this.#config = config; @@ -142,7 +142,7 @@ class DatadogReporter implements EvalReporter { const git = resolveLocalGitMetadata(process.cwd()); const experimentName = this.#config.experimentName ?? defaultExperimentName(); - this.#recorder = await client.startExperiment({ + this.#experiment = await client.startExperiment({ name: experimentName, projectName: resolveProjectName(this.#config, evaluations), description: this.#config.description, @@ -158,10 +158,10 @@ class DatadogReporter implements EvalReporter { } async onEvalComplete(result: EveEvalResult): Promise { - if (!this.#recorder) return; + if (!this.#experiment) return; const evaluation = this.#evaluations.get(result.id); - const span = await this.#recorder.submitSpan({ + const span = await this.#experiment.submitSpan({ id: result.id, name: result.id, input: this.#config.recordInputs ? resolveInput(result, evaluation) : undefined, @@ -177,25 +177,25 @@ class DatadogReporter implements EvalReporter { error: result.error, }); - await this.#recorder.submitEvaluationMetrics(span, resolveEvaluationMetrics(result)); + await this.#experiment.submitEvaluationMetrics(span, resolveEvaluationMetrics(result)); } async onRunComplete(summary: EveEvalRunSummary): Promise { - if (!this.#recorder) return; + if (!this.#experiment) return; try { const failed = summary.failed > 0 || summary.errored > 0; - await this.#recorder.close({ + await this.#experiment.close({ status: failed ? "failed" : "completed", error: failed ? `${summary.failed} failed, ${summary.errored} errored` : undefined, }); - const url = this.#recorder.url(); + const url = this.#experiment.url(); if (url) { (this.#config.log ?? console.log)(`Datadog experiment URL: ${url}\n`); } } finally { - this.#recorder = undefined; + this.#experiment = undefined; } } } @@ -206,7 +206,7 @@ async function resolveDatadogClient( config: DatadogReporterConfig, evaluations: readonly EveEval[], ): Promise<{ - startExperiment(options: DatadogStartExperimentOptions): Promise; + startExperiment(options: DatadogStartExperimentOptions): Promise; }> { if (config.client) return config.client; diff --git a/research/datadog-experiment-reporter.md b/research/datadog-experiment-reporter.md index 7831cb6a81..1e9472acca 100644 --- a/research/datadog-experiment-reporter.md +++ b/research/datadog-experiment-reporter.md @@ -86,8 +86,8 @@ interface EvaluationMetricInput { timestamp?: Date | string | number; } -interface ExperimentRecorder { - readonly experimentId: string; +interface ExternalExperiment { + experimentId(): string | null; url(): string | null; /** @@ -103,7 +103,7 @@ interface ExperimentRecorder { * `span.experimentId`. */ submitEvaluationMetrics( - span: Pick, + span: Pick, metrics: readonly EvaluationMetricInput[], ): Promise; @@ -112,7 +112,7 @@ interface ExperimentRecorder { } interface Experiments { - startExperiment(options: StartExperimentOptions): Promise; + startExperiment(options: StartExperimentOptions): Promise; } ``` @@ -136,7 +136,7 @@ const span = await experiment.submitSpan({ }); await experiment.submitEvaluationMetrics(span, [ - { label: "gate:succeeded", value: 1 }, + { label: "gate_succeeded", value: 1 }, { label: "similarity", value: 0.92 }, ]); @@ -234,7 +234,7 @@ manual dogfood app: ## Open questions - What exact public Node SDK API and package version should eve target for the - recorder-style Experiments surface? + externally-driven Experiments surface? - Does Datadog require a persisted dataset for every Experiment, or can external eval rows create an Experiment without dataset records? - What is the backend contract for post-hoc linking to an already-ingested span? From be1d5274e2906da5d39b4969473e488ac38b94c1 Mon Sep 17 00:00:00 2001 From: "mehul.sonowal" Date: Tue, 1 Sep 2026 16:13:13 -0400 Subject: [PATCH 5/8] refactor(evals): align Datadog reporter with dd-trace 6.13 Signed-off-by: mehul.sonowal --- .changeset/datadog-eval-reporter.md | 2 +- apps/fixtures/datadog-eval-reporter/README.md | 18 +- .../agent/instrumentation.ts | 6 - .../datadog-eval-reporter/package.json | 3 +- docs/evals/reporters.mdx | 4 +- packages/eve/package.json | 5 + .../evals/runner/reporters/datadog.test.ts | 194 ++++++++- .../eve/src/evals/runner/reporters/datadog.ts | 301 +++++++++----- pnpm-lock.yaml | 376 +++++++++++++++++- research/datadog-experiment-reporter.md | 310 ++++----------- 10 files changed, 865 insertions(+), 354 deletions(-) delete mode 100644 apps/fixtures/datadog-eval-reporter/agent/instrumentation.ts diff --git a/.changeset/datadog-eval-reporter.md b/.changeset/datadog-eval-reporter.md index 68a58591c5..49309098ec 100644 --- a/.changeset/datadog-eval-reporter.md +++ b/.changeset/datadog-eval-reporter.md @@ -2,4 +2,4 @@ "eve": patch --- -Add a Datadog eval reporter that creates one LLM Observability Experiment per eve eval run and submits eval assertion metrics through the optional `dd-trace` package. +Add a Datadog eval reporter that creates one LLM Observability Experiment per eve eval run and submits eval assertion metrics through the optional `dd-trace` package. The integration is tested against the public external Experiment API in `dd-trace@6.13.0`. diff --git a/apps/fixtures/datadog-eval-reporter/README.md b/apps/fixtures/datadog-eval-reporter/README.md index 4bbf5e4134..707ac6371f 100644 --- a/apps/fixtures/datadog-eval-reporter/README.md +++ b/apps/fixtures/datadog-eval-reporter/README.md @@ -1,16 +1,18 @@ # Datadog eval reporter fixture -This deterministic mock-model eval uses the no-op Datadog reporter and exports -its runtime trace through the standard OTLP environment variables: +This deterministic mock-model app exercises the Datadog eval reporter against +`dd-trace@6.13.0`. ```bash -export DATADOG_API_KEY="..." -export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL="http/protobuf" -export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="https://otlp.datadoghq.com/v1/traces" -export OTEL_EXPORTER_OTLP_TRACES_HEADERS="dd-api-key=${DATADOG_API_KEY},dd-otlp-source=llmobs" +export DD_API_KEY="..." +export DD_APP_KEY="..." +export DD_SITE="datadoghq.com" pnpm --filter datadog-eval-reporter-fixture eval ``` -Implement the reporter's three lifecycle placeholders to create an Experiment, -submit each completed eval, and finalize the run. +The run creates one Datadog LLM Observability Experiment, submits one synthetic +experiment span for the smoke eval, attaches the assertion metrics, and prints +the Experiment URL. The fixture does not export the agent runtime's OpenTelemetry +spans; Datadog's external Experiment API generates the experiment row's trace +and span identifiers. diff --git a/apps/fixtures/datadog-eval-reporter/agent/instrumentation.ts b/apps/fixtures/datadog-eval-reporter/agent/instrumentation.ts deleted file mode 100644 index caf5439d57..0000000000 --- a/apps/fixtures/datadog-eval-reporter/agent/instrumentation.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { registerOTel } from "@vercel/otel"; -import { defineInstrumentation } from "eve/instrumentation"; - -export default defineInstrumentation({ - setup: ({ agentName }) => registerOTel({ serviceName: agentName }), -}); diff --git a/apps/fixtures/datadog-eval-reporter/package.json b/apps/fixtures/datadog-eval-reporter/package.json index 229727c54b..800f87647b 100644 --- a/apps/fixtures/datadog-eval-reporter/package.json +++ b/apps/fixtures/datadog-eval-reporter/package.json @@ -7,8 +7,7 @@ "eval": "eve eval --strict" }, "dependencies": { - "@opentelemetry/sdk-trace-base": "2.6.1", - "@vercel/otel": "2.1.2", + "dd-trace": "6.13.0", "eve": "workspace:*" } } diff --git a/docs/evals/reporters.mdx b/docs/evals/reporters.mdx index d96618bca4..d51fc51781 100644 --- a/docs/evals/reporters.mdx +++ b/docs/evals/reporters.mdx @@ -57,9 +57,9 @@ export default defineEvalConfig({ }); ``` -The reporter creates one Datadog Experiment on run start, creates a placeholder dataset for the experiment, submits one synthetic experiment span per completed eve eval, associates that eval's assertion scores as experiment metrics, and prints the Datadog Experiment URL after the run completes. Metric labels use Datadog-safe names: gate assertions log under a `gate_` prefix, and characters outside letters, numbers, underscores, and hyphens are normalized to underscores. +The reporter creates one Datadog Experiment on run start, creates a placeholder dataset for the experiment, submits one synthetic experiment span per completed eve eval, associates that eval's assertion scores as experiment metrics, and prints the Datadog Experiment URL after the run completes. `dd-trace` generates the experiment row's trace and span identifiers; the reporter does not mutate or link the agent runtime's OpenTelemetry spans. By default, assertion metrics use position-based labels such as `gate_assertion_1` and `assertion_2`, and omit assertion names and failure messages because those details may contain inputs, outputs, or authored expectations. Pass `recordAssertionDetails: true` to include those details; labels then use the assertion name, normalize characters outside letters, numbers, underscores, and hyphens to underscores, prefix gate labels with `gate_`, and add numeric suffixes when labels collide. -Datadog needs `dd-trace` installed in the app and Datadog credentials in the environment: install the `dd-trace` package (`npm install dd-trace`) and set `DD_API_KEY`, `DD_APP_KEY`, and `DD_SITE` as appropriate. By default, the reporter records assertion scores and eval metadata only; pass `recordInputs: true`, `recordOutputs: true`, or `recordExpectedOutputs: true` if your destination is approved for eval prompts, outputs, or authored expectations. Input is read from the eval's first `t.send(...)` message, falling back to the eval description when no message event is available. Expected output is read from eval `metadata.expectedOutput`, `metadata.expected`, or `metadata.expected_output`. +Datadog needs `dd-trace` installed in the app and Datadog credentials in the environment. The reporter is tested against `dd-trace@6.13.0`; install it with `npm install dd-trace@6.13.0` and set `DD_API_KEY`, `DD_APP_KEY`, and `DD_SITE` as appropriate. By default, the reporter records assertion scores, eval metadata, and the target URL origin only. Pass `recordInputs: true`, `recordOutputs: true`, `recordExpectedOutputs: true`, or `recordErrors: true` if your destination is approved for eval prompts, outputs, authored expectations, or execution error messages. Input is read from the eval's first `t.send(...)` message, falling back to the eval description when no message event is available. Expected output is read from eval `metadata.expectedOutput`, `metadata.expected`, or `metadata.expected_output`; those keys are excluded from the general metadata payload and are sent only through the opt-in expected-output field. Target URL credentials, paths, query parameters, and fragments are never reported. ## JUnit diff --git a/packages/eve/package.json b/packages/eve/package.json index 8b95c054b7..42c3371ea3 100644 --- a/packages/eve/package.json +++ b/packages/eve/package.json @@ -479,6 +479,7 @@ "chat": "4.34.0", "chokidar": "5.0.0", "commander": "14.0.3", + "dd-trace": "6.13.0", "emulate": "0.6.0", "env-runner": "0.1.16", "eventsource-parser": "3.1.0", @@ -508,6 +509,7 @@ "@opentelemetry/api": "^1.0.0", "ai": "catalog:", "braintrust": "^3.0.0", + "dd-trace": "^6.13.0", "just-bash": "^3.1.0", "microsandbox": "^0.5.0" }, @@ -518,6 +520,9 @@ "braintrust": { "optional": true }, + "dd-trace": { + "optional": true + }, "just-bash": { "optional": true }, diff --git a/packages/eve/src/evals/runner/reporters/datadog.test.ts b/packages/eve/src/evals/runner/reporters/datadog.test.ts index 8a13c87232..62de672f53 100644 --- a/packages/eve/src/evals/runner/reporters/datadog.test.ts +++ b/packages/eve/src/evals/runner/reporters/datadog.test.ts @@ -18,7 +18,12 @@ function makeEval(overrides: Partial = {}): EveEval { id: "eval-1", description: "Say hello", tags: ["smoke"], - metadata: { suite: "unit", expectedOutput: "helpful onboarding answer" }, + metadata: { + suite: "unit", + expectedOutput: "helpful onboarding answer", + expected: "legacy expected answer", + expected_output: "legacy snake-case expected answer", + }, async test() {}, ...overrides, }; @@ -39,6 +44,7 @@ function makeEvalResult(overrides: Partial = {}): EveEvalResult { sequence: 1, turnId: "turn-1", }, + meta: { at: "2026-01-01T00:00:00.000Z", id: "event-1" }, }, ], derived: { @@ -57,6 +63,7 @@ function makeEvalResult(overrides: Partial = {}): EveEvalResult { messageCount: 1, }, sessionId: "session-123", + traceContexts: [], }, assertions: [ { name: "succeeded", score: 1, severity: "gate", passed: true }, @@ -94,12 +101,12 @@ function makeConfig(overrides: Partial = {}) { const experiment = { experimentId: vi.fn(() => "exp-1"), url: vi.fn(() => "https://dd.test/experiment"), - submitSpan: vi.fn(async () => span), - submitEvaluationMetrics: vi.fn(async () => undefined), + submitSpan: vi.fn(async (_row: unknown) => span), + submitEvaluationMetrics: vi.fn(async (_span: unknown, _metrics: unknown) => undefined), close: vi.fn(async () => undefined), }; const client = { - startExperiment: vi.fn(async () => experiment), + startExperiment: vi.fn(async (_options: unknown) => experiment), }; const lines: string[] = []; const config = { @@ -128,12 +135,15 @@ describe("Datadog", () => { projectName: "test-project", dataset: { name: "run-1 dataset" }, tags: expect.objectContaining({ source: "eve", target_kind: "local" }), - metadata: expect.objectContaining({ eveEvalIds: ["eval-1"], eveTargetKind: "local" }), + metadata: expect.objectContaining({ + eveEvalIds: ["eval-1"], + eveTargetKind: "local", + eveTargetOrigin: "http://127.0.0.1:3000", + }), }), ); expect(experiment.submitSpan).toHaveBeenCalledWith( expect.objectContaining({ - id: "eval-1", name: "eval-1", durationMs: 1000, metadata: expect.objectContaining({ @@ -145,15 +155,165 @@ describe("Datadog", () => { tags: expect.objectContaining({ eval_id: "eval-1", eval_verdict: "passed" }), }), ); + const submittedSpan = experiment.submitSpan.mock.calls[0]?.[0]; + expect(submittedSpan).not.toHaveProperty("id"); + expect(submittedSpan).not.toHaveProperty("input"); + expect(submittedSpan).not.toHaveProperty("output"); + expect(submittedSpan).not.toHaveProperty("expectedOutput"); + expect(submittedSpan).not.toHaveProperty("metadata.expectedOutput"); + expect(submittedSpan).not.toHaveProperty("metadata.expected"); + expect(submittedSpan).not.toHaveProperty("metadata.expected_output"); expect(experiment.submitEvaluationMetrics).toHaveBeenCalledWith(span, [ - expect.objectContaining({ label: "gate_succeeded", value: 1 }), - expect.objectContaining({ label: "similarity", value: 0.9 }), - expect.objectContaining({ label: "judge_autoevals_closedQA", value: 1 }), + expect.objectContaining({ label: "gate_assertion_1", value: 1 }), + expect.objectContaining({ label: "assertion_2", value: 0.9 }), + expect.objectContaining({ label: "assertion_3", value: 1 }), expect.objectContaining({ label: "eve_tool_call_count", value: 1 }), expect.objectContaining({ label: "eve_subagent_call_count", value: 0 }), expect.objectContaining({ label: "eve_message_count", value: 1 }), expect.objectContaining({ label: "eve_reasoning_block_count", value: 0 }), ]); + expect(experiment.submitEvaluationMetrics.mock.calls[0]?.[1]).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + tags: expect.objectContaining({ assertion_name: expect.anything() }), + }), + ]), + ); + }); + + it("redacts target URL secrets and execution errors by default", async () => { + const { client, config, experiment } = makeConfig(); + const reporter = Datadog(config); + const target: EveEvalTarget = { + ...makeTarget("remote"), + url: "https://user:password@test.vercel.app/agent?token=private#fragment", + }; + + await reporter.onRunStart([makeEval()], target); + await reporter.onEvalComplete(makeEvalResult({ error: "private application error" })); + + expect(client.startExperiment).toHaveBeenCalledWith( + expect.objectContaining({ + metadata: expect.objectContaining({ eveTargetOrigin: "https://test.vercel.app" }), + }), + ); + expect(client.startExperiment.mock.calls[0]?.[0]).not.toHaveProperty("metadata.eveTargetUrl"); + expect(experiment.submitSpan.mock.calls[0]?.[0]).not.toHaveProperty("error"); + }); + + it("records execution errors only when enabled", async () => { + const { config, experiment } = makeConfig({ recordErrors: true }); + const reporter = Datadog(config); + + await reporter.onRunStart([makeEval()], makeTarget()); + await reporter.onEvalComplete(makeEvalResult({ error: "application error" })); + + expect(experiment.submitSpan).toHaveBeenCalledWith( + expect.objectContaining({ error: "application error" }), + ); + }); + + it("redacts failing assertion details by default", async () => { + const { config, experiment } = makeConfig(); + const reporter = Datadog(config); + const result = makeEvalResult({ + assertions: [ + { + name: "messageIncludes(private expectation)", + message: "got private assistant output", + score: 0, + severity: "gate", + passed: false, + }, + ], + verdict: "failed", + }); + + await reporter.onRunStart([makeEval()], makeTarget()); + await reporter.onEvalComplete(result); + + expect(experiment.submitSpan.mock.calls[0]?.[0]).not.toHaveProperty( + "metadata.eveFailedAssertions", + ); + expect(experiment.submitEvaluationMetrics.mock.calls[0]?.[1]).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + label: "gate_assertion_1", + tags: { + assertion_index: "1", + assertion_severity: "gate", + assertion_passed: "false", + }, + }), + ]), + ); + }); + + it("records assertion names and failure messages only when enabled", async () => { + const { config, experiment } = makeConfig({ recordAssertionDetails: true }); + const reporter = Datadog(config); + const result = makeEvalResult({ + assertions: [ + { + name: "messageIncludes(private expectation)", + message: "got private assistant output", + score: 0, + severity: "gate", + passed: false, + }, + ], + verdict: "failed", + }); + + await reporter.onRunStart([makeEval()], makeTarget()); + await reporter.onEvalComplete(result); + + expect(experiment.submitSpan.mock.calls[0]?.[0]).toHaveProperty( + "metadata.eveFailedAssertions", + [ + { + name: "messageIncludes(private expectation)", + message: "got private assistant output", + }, + ], + ); + expect(experiment.submitEvaluationMetrics.mock.calls[0]?.[1]).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + label: "gate_messageIncludes_private_expectation", + tags: expect.objectContaining({ + assertion_name: "messageIncludes(private expectation)", + }), + }), + ]), + ); + }); + + it("deduplicates assertion metric labels and reserves built-in labels", async () => { + const { config, experiment } = makeConfig({ recordAssertionDetails: true }); + const reporter = Datadog(config); + const result = makeEvalResult({ + assertions: [ + { name: "same label", score: 1, severity: "soft", passed: true }, + { name: "same@label", score: 1, severity: "soft", passed: true }, + { name: "same label", score: 1, severity: "soft", passed: true }, + { name: "eve_tool_call_count", score: 1, severity: "soft", passed: true }, + ], + }); + + await reporter.onRunStart([makeEval()], makeTarget()); + await reporter.onEvalComplete(result); + + expect(experiment.submitEvaluationMetrics.mock.calls[0]?.[1]).toEqual([ + expect.objectContaining({ label: "same_label" }), + expect.objectContaining({ label: "same_label_2" }), + expect.objectContaining({ label: "same_label_3" }), + expect.objectContaining({ label: "eve_tool_call_count_2" }), + expect.objectContaining({ label: "eve_tool_call_count" }), + expect.objectContaining({ label: "eve_subagent_call_count" }), + expect.objectContaining({ label: "eve_message_count" }), + expect.objectContaining({ label: "eve_reasoning_block_count" }), + ]); }); it("records eval input and output only when enabled", async () => { @@ -176,6 +336,22 @@ describe("Datadog", () => { ); }); + it("uses the dd-trace project environment variable by default", async () => { + vi.stubEnv("DD_LLMOBS_PROJECT_NAME", "environment-project"); + try { + const { client, config } = makeConfig({ projectName: undefined }); + const reporter = Datadog(config); + + await reporter.onRunStart([makeEval()], makeTarget()); + + expect(client.startExperiment).toHaveBeenCalledWith( + expect.objectContaining({ projectName: "environment-project" }), + ); + } finally { + vi.unstubAllEnvs(); + } + }); + it("closes the experiment and logs its URL", async () => { const { config, lines, experiment } = makeConfig(); const reporter = Datadog(config); diff --git a/packages/eve/src/evals/runner/reporters/datadog.ts b/packages/eve/src/evals/runner/reporters/datadog.ts index 250ce643cd..f64eae77b8 100644 --- a/packages/eve/src/evals/runner/reporters/datadog.ts +++ b/packages/eve/src/evals/runner/reporters/datadog.ts @@ -1,12 +1,15 @@ import { createRequire } from "node:module"; +import type ddTrace from "dd-trace"; + import type { EveEval, EveEvalResult, EveEvalRunSummary, EveEvalTarget } from "#evals/types.js"; import type { EvalReporter } from "#evals/runner/reporters/types.js"; import { resolveLocalGitMetadata } from "#evals/runner/resolve-git-metadata.js"; +import { parseJsonValue, type JsonValue } from "#shared/json.js"; /** Configuration for the Datadog reporter. */ export interface DatadogReporterConfig { - /** Datadog LLM Observability project name. Defaults to `DD_LLMOBS_ML_APP`, `DD_SERVICE`, or the first eval id. */ + /** Datadog LLM Observability project name. Defaults to `DD_LLMOBS_PROJECT_NAME`, the configured ml_app, `DD_SERVICE`, or the first eval id. */ readonly projectName?: string; /** Name for the placeholder dataset used by the experiment. Defaults to ` dataset`. */ readonly datasetName?: string; @@ -24,16 +27,20 @@ export interface DatadogReporterConfig { readonly mlApp?: string; /** Tags attached to the Datadog Experiment. */ readonly tags?: Readonly>; - /** Metadata attached to the Datadog Experiment. */ + /** JSON-serializable metadata attached to the Datadog Experiment. */ readonly metadata?: Readonly>; - /** Free-form Datadog Experiment config. */ + /** JSON-serializable Datadog Experiment config. */ readonly config?: Readonly>; - /** Include eval descriptions in the synthetic experiment row input. Defaults to false. */ + /** Include the first sent message, falling back to the eval description, in the synthetic experiment row input. Defaults to false. */ readonly recordInputs?: boolean; /** Include eval outputs in the synthetic experiment row output. Defaults to false. */ readonly recordOutputs?: boolean; /** Include `metadata.expectedOutput`, `metadata.expected`, or `metadata.expected_output` on experiment rows. Defaults to false. */ readonly recordExpectedOutputs?: boolean; + /** Include assertion names and failure messages, which may contain inputs, outputs, or expectations. Defaults to false. */ + readonly recordAssertionDetails?: boolean; + /** Include execution error messages, which may contain application data. Defaults to false. */ + readonly recordErrors?: boolean; /** Console hook used for tests. */ readonly log?: (line: string) => void; /** eve-owned client seam for tests and custom Datadog SDK wiring. */ @@ -42,76 +49,73 @@ export interface DatadogReporterConfig { }; } +type DatadogJsonValue = + | string + | number + | boolean + | null + | DatadogJsonValue[] + | { [key: string]: DatadogJsonValue }; + interface DatadogStartExperimentOptions { name: string; projectName?: string; description?: string; - tags?: Readonly>; - metadata?: Readonly>; - config?: Readonly>; + tags?: Record; + metadata?: Record; + config?: Record; dataset?: { + id?: string; + version?: number; name?: string; + description?: string; }; } interface DatadogExternalExperiment { - experimentId(): string | null; + experimentId(): string; url(): string | null; submitSpan(row: DatadogExternalExperimentSpanInput): Promise; submitEvaluationMetrics( span: Pick, - metrics: readonly DatadogEvaluationMetricInput[], + metrics: DatadogEvaluationMetricInput[], ): Promise; - close(options?: { status?: "completed" | "failed"; error?: string }): Promise; + close(options?: { status?: string; error?: string | Error }): Promise; } interface DatadogExternalExperimentSpanInput { - id?: string; name?: string; - input?: unknown; - output?: unknown; - expectedOutput?: unknown; - metadata?: Readonly>; - tags?: Readonly>; - startedAt?: string; - completedAt?: string; + input?: DatadogJsonValue; + output?: DatadogJsonValue; + expectedOutput?: DatadogJsonValue; + metadata?: Record; + tags?: Record; + startedAt?: Date | string | number; + completedAt?: Date | string | number; durationMs?: number; - error?: string; + error?: string | Error | { type?: string; name?: string; message?: string; stack?: string }; + datasetRecordId?: string; + runId?: string; + runIteration?: number; } interface DatadogExternalExperimentSpan { - experimentId: string | null; - spanId: string | null; - traceId?: string | null; - url?: string | null; + experimentId: string; + spanId: string; + traceId: string; + url: string | null; } interface DatadogEvaluationMetricInput { label: string; - value?: boolean | number | string | Record; - error?: string; - tags?: Readonly>; + value?: DatadogJsonValue; + error?: string | Error; + timestamp?: Date | string | number; + tags?: Record; + source?: string; } -interface DatadogTraceModule { - init(options: { - service?: string; - env?: string; - site?: string; - llmobs?: { - mlApp?: string; - agentlessEnabled?: boolean; - }; - }): DatadogTracer; -} - -interface DatadogTracer { - readonly llmobs?: { - readonly experiments?: { - startExperiment(options: DatadogStartExperimentOptions): Promise; - }; - }; -} +type DatadogTraceModule = typeof ddTrace; /** * Creates an {@link EvalReporter} that uploads eval assertion scores to a @@ -142,18 +146,21 @@ class DatadogReporter implements EvalReporter { const git = resolveLocalGitMetadata(process.cwd()); const experimentName = this.#config.experimentName ?? defaultExperimentName(); + const metadata = resolveExperimentMetadata(evaluations, target); + if (git.sha) { + metadata.eveGitCommit = git.sha; + metadata.eveGitBranch = git.branch; + } + Object.assign(metadata, this.#config.metadata); + this.#experiment = await client.startExperiment({ name: experimentName, projectName: resolveProjectName(this.#config, evaluations), description: this.#config.description, dataset: { name: this.#config.datasetName ?? `${experimentName} dataset` }, tags: resolveExperimentTags(this.#config, target), - metadata: { - ...resolveExperimentMetadata(evaluations, target), - ...(git.sha ? { eveGitCommit: git.sha, eveGitBranch: git.branch } : {}), - ...this.#config.metadata, - }, - config: this.#config.config, + metadata: toDatadogJsonRecord(metadata), + config: toDatadogJsonRecord(this.#config.config), }); } @@ -161,23 +168,34 @@ class DatadogReporter implements EvalReporter { if (!this.#experiment) return; const evaluation = this.#evaluations.get(result.id); - const span = await this.#experiment.submitSpan({ - id: result.id, + const spanInput: DatadogExternalExperimentSpanInput = { name: result.id, - input: this.#config.recordInputs ? resolveInput(result, evaluation) : undefined, - output: this.#config.recordOutputs ? result.result.output : undefined, - expectedOutput: this.#config.recordExpectedOutputs - ? resolveExpectedOutput(evaluation) - : undefined, - metadata: resolveResultMetadata(result, evaluation), + metadata: toDatadogJsonRecord( + resolveResultMetadata(result, evaluation, this.#config.recordAssertionDetails === true), + ), tags: resolveResultTags(result, evaluation), startedAt: result.startedAt, completedAt: result.completedAt, durationMs: elapsedMs(result.startedAt, result.completedAt), - error: result.error, - }); + }; + if (this.#config.recordInputs) { + spanInput.input = toOptionalDatadogJsonValue(resolveInput(result, evaluation)); + } + if (this.#config.recordOutputs) { + spanInput.output = toOptionalDatadogJsonValue(result.result.output); + } + if (this.#config.recordExpectedOutputs) { + spanInput.expectedOutput = toOptionalDatadogJsonValue(resolveExpectedOutput(evaluation)); + } + if (this.#config.recordErrors && result.error !== undefined) { + spanInput.error = result.error; + } - await this.#experiment.submitEvaluationMetrics(span, resolveEvaluationMetrics(result)); + const span = await this.#experiment.submitSpan(spanInput); + await this.#experiment.submitEvaluationMetrics( + span, + resolveEvaluationMetrics(result, this.#config.recordAssertionDetails === true), + ); } async onRunComplete(summary: EveEvalRunSummary): Promise { @@ -201,6 +219,17 @@ class DatadogReporter implements EvalReporter { } const DD_TRACE_PACKAGE = "dd-trace"; +const EXPECTED_OUTPUT_METADATA_KEYS: ReadonlySet = new Set([ + "expectedOutput", + "expected", + "expected_output", +]); +const BUILT_IN_METRIC_LABELS = [ + "eve_tool_call_count", + "eve_subagent_call_count", + "eve_message_count", + "eve_reasoning_block_count", +] as const; async function resolveDatadogClient( config: DatadogReporterConfig, @@ -217,6 +246,7 @@ async function resolveDatadogClient( env: config.env ?? process.env.DD_ENV, site: config.site ?? process.env.DD_SITE, llmobs: { + projectName, mlApp: config.mlApp ?? projectName, agentlessEnabled: true, }, @@ -225,7 +255,10 @@ async function resolveDatadogClient( const experiments = tracer.llmobs?.experiments; if (!experiments?.startExperiment) { throw new Error( - "The installed 'dd-trace' package does not expose tracer.llmobs.experiments.startExperiment().", + [ + "The installed 'dd-trace' package does not expose tracer.llmobs.experiments.startExperiment().", + "Update to a release compatible with dd-trace@6.13.0.", + ].join("\n"), ); } return experiments; @@ -244,8 +277,8 @@ async function loadDatadogSdk(): Promise { [ "The 'dd-trace' package is required for Datadog reporting but was not found.", "", - "Install it with:", - " npm install dd-trace", + "Install the tested release with:", + " npm install dd-trace@6.13.0", ].join("\n"), ); } @@ -258,6 +291,7 @@ function resolveProjectName( ): string { return ( config.projectName ?? + process.env.DD_LLMOBS_PROJECT_NAME ?? config.mlApp ?? process.env.DD_LLMOBS_ML_APP ?? process.env.DD_SERVICE ?? @@ -285,24 +319,40 @@ function resolveExperimentMetadata( evaluations: readonly EveEval[], target: EveEvalTarget, ): Record { - return { + const metadata: Record = { eveEvalIds: evaluations.map((evaluation) => evaluation.id), eveTargetKind: target.kind, - eveTargetUrl: target.url, eveTimestamp: new Date().toISOString(), }; + const targetOrigin = resolveTargetOrigin(target.url); + if (targetOrigin !== undefined) { + metadata.eveTargetOrigin = targetOrigin; + } + return metadata; +} + +function resolveTargetOrigin(value: string): string | undefined { + try { + const origin = new URL(value).origin; + return origin === "null" ? undefined : origin; + } catch { + return undefined; + } } function resolveResultTags( result: EveEvalResult, evaluation: EveEval | undefined, ): Record { - return { + const tags: Record = { eval_id: result.id, eval_verdict: result.verdict, eval_status: result.result.status, - ...(evaluation?.tags?.length ? { eval_tags: evaluation.tags.join(",") } : {}), }; + if (evaluation?.tags?.length) { + tags.eval_tags = evaluation.tags.join(","); + } + return tags; } function resolveInput(result: EveEvalResult, evaluation: EveEval | undefined): unknown { @@ -326,13 +376,15 @@ function resolveExpectedOutput(evaluation: EveEval | undefined): unknown { function resolveResultMetadata( result: EveEvalResult, evaluation: EveEval | undefined, + recordAssertionDetails: boolean, ): Record { - const failedAssertions = result.assertions - .filter((assertion) => !assertion.passed) - .map((assertion) => ({ name: assertion.name, message: assertion.message })); - - return { - ...evaluation?.metadata, + const metadata: Record = {}; + for (const [key, value] of Object.entries(evaluation?.metadata ?? {})) { + if (!EXPECTED_OUTPUT_METADATA_KEYS.has(key)) { + metadata[key] = value; + } + } + Object.assign(metadata, { eveSessionId: result.result.sessionId, eveStatus: result.result.status, eveVerdict: result.verdict, @@ -340,27 +392,47 @@ function resolveResultMetadata( eveToolCalls: result.result.derived.toolCalls.map((call) => call.name), eveSubagentCalls: result.result.derived.subagentCalls.map((call) => call.name), eveParked: result.result.derived.parked, - ...(failedAssertions.length > 0 ? { eveFailedAssertions: failedAssertions } : {}), - ...(result.result.derived.failureCode - ? { eveFailureCode: result.result.derived.failureCode } - : {}), - }; + }); + if (recordAssertionDetails) { + const failedAssertions = result.assertions + .filter((assertion) => !assertion.passed) + .map((assertion) => ({ name: assertion.name, message: assertion.message })); + if (failedAssertions.length > 0) { + metadata.eveFailedAssertions = failedAssertions; + } + } + if (result.result.derived.failureCode) { + metadata.eveFailureCode = result.result.derived.failureCode; + } + return metadata; } -function resolveEvaluationMetrics(result: EveEvalResult): DatadogEvaluationMetricInput[] { +function resolveEvaluationMetrics( + result: EveEvalResult, + recordAssertionDetails: boolean, +): DatadogEvaluationMetricInput[] { const metrics: DatadogEvaluationMetricInput[] = []; - - for (const assertion of result.assertions) { - const rawLabel = assertion.severity === "gate" ? `gate_${assertion.name}` : assertion.name; + const usedLabels = new Set(BUILT_IN_METRIC_LABELS); + + for (const [index, assertion] of result.assertions.entries()) { + const rawLabel = recordAssertionDetails + ? assertion.severity === "gate" + ? `gate_${assertion.name}` + : assertion.name + : `${assertion.severity === "gate" ? "gate_" : ""}assertion_${index + 1}`; + const tags: Record = { + assertion_index: String(index + 1), + assertion_severity: assertion.severity, + assertion_passed: String(assertion.passed), + }; + if (recordAssertionDetails) { + tags.assertion_name = assertion.name; + tags.assertion_label = rawLabel; + } metrics.push({ - label: toDatadogMetricLabel(rawLabel), + label: reserveDatadogMetricLabel(toDatadogMetricLabel(rawLabel), usedLabels), value: assertion.score, - tags: { - assertion_name: assertion.name, - assertion_label: rawLabel, - assertion_severity: assertion.severity, - assertion_passed: String(assertion.passed), - }, + tags, }); } @@ -379,9 +451,56 @@ function toDatadogMetricLabel(label: string): string { return normalized || "metric"; } +function reserveDatadogMetricLabel(baseLabel: string, usedLabels: Set): string { + let label = baseLabel; + let suffix = 2; + while (usedLabels.has(label)) { + label = `${baseLabel}_${suffix}`; + suffix += 1; + } + usedLabels.add(label); + return label; +} + function elapsedMs(startedAt: string, completedAt: string): number | undefined { const start = Date.parse(startedAt); const completed = Date.parse(completedAt); if (!Number.isFinite(start) || !Number.isFinite(completed)) return undefined; return Math.max(0, completed - start); } + +function toOptionalDatadogJsonValue(value: unknown): DatadogJsonValue | undefined { + return value === undefined ? undefined : toDatadogJsonValue(value); +} + +function toDatadogJsonRecord( + value: Readonly> | undefined, +): Record | undefined { + if (value === undefined) return undefined; + + const output: Record = {}; + for (const [key, entry] of Object.entries(value)) { + if (entry !== undefined) { + output[key] = toDatadogJsonValue(entry); + } + } + return output; +} + +function toDatadogJsonValue(value: unknown): DatadogJsonValue { + return cloneDatadogJsonValue(parseJsonValue(value)); +} + +function cloneDatadogJsonValue(value: JsonValue): DatadogJsonValue { + if (Array.isArray(value)) { + return value.map((entry) => cloneDatadogJsonValue(entry)); + } + if (value !== null && typeof value === "object") { + const output: Record = {}; + for (const [key, entry] of Object.entries(value)) { + output[key] = cloneDatadogJsonValue(entry); + } + return output; + } + return value; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4f18f6d778..84cad3d85d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -401,12 +401,9 @@ importers: apps/fixtures/datadog-eval-reporter: dependencies: - '@opentelemetry/sdk-trace-base': - specifier: 2.6.1 - version: 2.6.1(@opentelemetry/api@1.9.1) - '@vercel/otel': - specifier: 2.1.2 - version: 2.1.2(@opentelemetry/api-logs@0.214.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.214.0(@opentelemetry/api@1.9.1))(@opentelemetry/resources@2.6.1(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.214.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.6.1(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + dd-trace: + specifier: 6.13.0 + version: 6.13.0 eve: specifier: workspace:* version: link:../../../packages/eve @@ -1420,6 +1417,9 @@ importers: commander: specifier: 14.0.3 version: 14.0.3 + dd-trace: + specifier: 6.13.0 + version: 6.13.0 emulate: specifier: 0.6.0 version: 0.6.0 @@ -2183,6 +2183,28 @@ packages: resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} engines: {node: '>=0.1.90'} + '@datadog/libdatadog@0.12.1': + resolution: {integrity: sha512-4cKRaO1mB9npfklJjOizzJaNBdZvw1V62EVbSD6Y32zX92bTBq/vAno/TTN9dMAvzomXYmvADpGo4798E9fMoA==} + + '@datadog/native-appsec@11.0.2': + resolution: {integrity: sha512-Azs5fhwJx/BXHMnz+4PM2d9Is7Ik8uQPSM90nzvTCOcBYhSxuLhCY0EsDik/15CRknV9YoQrP1vmRrCVq3il5w==} + engines: {node: '>=18'} + + '@datadog/native-iast-taint-tracking@4.2.1': + resolution: {integrity: sha512-KRLu3aTXvyAusouAZGTw2w1Xo67cRTw93yG4C9vTMpIgN9FXiqhlcvtNHvUzaA/KRrwtNwbaUELubOtk/kxOFg==} + + '@datadog/native-metrics@4.0.0': + resolution: {integrity: sha512-rS6Qc8WAbOgbrtDYdqK15gi2xbuIEyfuquUH05g+2DOdiNAswNAGrlIuSdJZQF/qoJzjcFvxb9kCHu8BjiPfQQ==} + engines: {node: '>=18'} + + '@datadog/pprof@5.18.1': + resolution: {integrity: sha512-p8RnantBXCrcsT4pHvcdRtQzhqllls/t6yYb2j8hJrnnFcFfWO+u5WSdqLNxGFNjTEhfaFPu6TOoLhNk9iW9Lw==} + engines: {node: '>=16'} + + '@datadog/wasm-js-rewriter@5.0.4': + resolution: {integrity: sha512-tSjbk51dkNzFMM3P/7y82tyrBj/9CWnPzcVb1JB8TUea/PIholdXSPuGC5F975YvvGB8KDujfmJIacN+EmuXMg==} + engines: {node: '>= 10'} + '@dimforge/rapier3d-compat@0.12.0': resolution: {integrity: sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow==} @@ -4168,42 +4190,84 @@ packages: cpu: [arm] os: [android] + '@oxc-parser/binding-android-arm-eabi@0.132.0': + resolution: {integrity: sha512-KrLaPWa5c9Y7LkW+rKkaUE3y7DBDrQtaf7rlsSDfv6KAHUjgzAIRA761Lrrp6//Yd/Rlie/yEOt9YENCoJnOcw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [android] + '@oxc-parser/binding-android-arm64@0.131.0': resolution: {integrity: sha512-nlGIod6gw75x1aEDgLS+srj+JRGY0HHm9MI9YgzE/B64l6d6+H3MSP9NOgp0+HTg8tp4vV9rVfgQGgd+TfVZcA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [android] + '@oxc-parser/binding-android-arm64@0.132.0': + resolution: {integrity: sha512-SThDrSeamB/kG2+NxcJ5/wSLcV6dUqDknrPLqFYQ0ST/55mtBP4M7Q/f3QbubH6aAd11wpzZn/nwbVRSdobOpg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [android] + '@oxc-parser/binding-darwin-arm64@0.131.0': resolution: {integrity: sha512-jukuV6xe5RbQKFo7QD34NDCLDZp4PSOm8rmckhNdH/60ymG5zXbDzGBEyc+nTkuLQNama2aSGCt+CPfpjNTqyw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [darwin] + '@oxc-parser/binding-darwin-arm64@0.132.0': + resolution: {integrity: sha512-Lc0f/TYoKBghE5/2Gsv7bLXk+TJZunx2Tf61X8hG4ARXdc8UYI26dCGccFSd1AyFbK3jfaNXtMnupggDbjPXdQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [darwin] + '@oxc-parser/binding-darwin-x64@0.131.0': resolution: {integrity: sha512-g3JOo4khe9rslHm5WYaVDWb0HS/M1MLR3I9S8560MkKIcC96VQY00QjOlsuRyfSj/JDXj8i9T7ryPO2RidiXVg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [darwin] + '@oxc-parser/binding-darwin-x64@0.132.0': + resolution: {integrity: sha512-RG2eJIpf7C21z9HSSXFw1bTArdpKe7Y4fwcJTwRq1yCSe1vSavaN9GA1sm9KqzemTLAGVktQ+7qBTGp0vQeUZg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [darwin] + '@oxc-parser/binding-freebsd-x64@0.131.0': resolution: {integrity: sha512-1hziITDTxjMePnX+dR9ocVT+EuZkQ8wm4FPAbmbEiKG+Phbo73J1ZnPAA6Y/aGsWF3McOFnQuZIktAFwalkfJQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [freebsd] + '@oxc-parser/binding-freebsd-x64@0.132.0': + resolution: {integrity: sha512-wQIPntPLtJ8NcBpvKPbEv3NqzV6k8eP8tP/jE9Rg8HTg/j7urZGFSsTCPCW5k77Qfw2DM4vRvc9p3I4yq/Shvw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [freebsd] + '@oxc-parser/binding-linux-arm-gnueabihf@0.131.0': resolution: {integrity: sha512-9uRxfXwyKG9+MwmGQBo2ncPNwZH5HTmCETFM2WiuDBNDCW4NC5ttSQkwCAMrTAWgwMzVBH1CP8pM0v7nebCWXQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm] os: [linux] + '@oxc-parser/binding-linux-arm-gnueabihf@0.132.0': + resolution: {integrity: sha512-PixKEpeSe3yxQWqNyOCBALRYc72+Tj7ILDofUl3iXo25cVOzLA6jHUhmOINRtWIPh7dbUie3QNeabwaQpZTw6w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + '@oxc-parser/binding-linux-arm-musleabihf@0.131.0': resolution: {integrity: sha512-mgbLvzRShXOLBdWGInf08Af4q+pfj1xD8hSgLClDZ9of/BXkB6+LIhTH7fihiDUipqB3yoSkKBWaZ3Ejlf5Yag==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm] os: [linux] + '@oxc-parser/binding-linux-arm-musleabihf@0.132.0': + resolution: {integrity: sha512-sCR+DzGHlyHKnbA2z9zWjTUhIo8Sy0enJl4RDsBwPmkxYynPatpwOAWe8W5127SlW0boqUWHGtr1NWn5UwIhXQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + '@oxc-parser/binding-linux-arm64-gnu@0.131.0': resolution: {integrity: sha512-OPT8++4aN6j2GJ8+3IZHS/byXoZP4aSBn+FoG6rgBJ2fKwPKXWF3MqrFMNW7NKHM28FLY579xYLxJSfgobEqPA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -4211,6 +4275,13 @@ packages: os: [linux] libc: [glibc] + '@oxc-parser/binding-linux-arm64-gnu@0.132.0': + resolution: {integrity: sha512-sQBix5P2cW+IpzTcCwYxnh9yALrKSIkKJThspBvMGcygSMnbzkSvhN7SfuX1hvBk8y1XEChsdkU3ET0V5DmzUw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + '@oxc-parser/binding-linux-arm64-musl@0.131.0': resolution: {integrity: sha512-vtPiwmfVTAXzaxDKsOXG+LwgRAA7WEnaeHzhS5z0GE89gAK18KSXnly7Z6saXXq6L3dVMyK44uoTI03zKxrpmw==} engines: {node: ^20.19.0 || >=22.12.0} @@ -4218,6 +4289,13 @@ packages: os: [linux] libc: [musl] + '@oxc-parser/binding-linux-arm64-musl@0.132.0': + resolution: {integrity: sha512-WozHg3Kc//8Sk756HXXgMbEAvqtG+Lzb9JOojwQzIGDtN78Az2dLttkb71akWYUF/8IgYfDSlfKh4Uot8is5Vw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [musl] + '@oxc-parser/binding-linux-ppc64-gnu@0.131.0': resolution: {integrity: sha512-8AW8L7w5cGHSdZPcyZX2yR0+GUODsT15rbRjfdD54rv6DMbtuEB19ysLOpKJlRGfH6UNYNpCHaU1uJWgTWf1/w==} engines: {node: ^20.19.0 || >=22.12.0} @@ -4225,6 +4303,13 @@ packages: os: [linux] libc: [glibc] + '@oxc-parser/binding-linux-ppc64-gnu@0.132.0': + resolution: {integrity: sha512-CmX/ulNBOEwWTyVRmcpYKAcAizW6+OjtLJgo7fXoL9OqQvjF4VER8tPomv44vwzfSCy1BHbsB0ZlZYzYJNj4cA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + '@oxc-parser/binding-linux-riscv64-gnu@0.131.0': resolution: {integrity: sha512-vvpjkjEOUsPcsYf8evE4MO3aGx9+3wodXEBOicGNnOwTuAik8eBONNkgSdhkGsAblQmfVHJyanRnpxglddTXIA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -4232,6 +4317,13 @@ packages: os: [linux] libc: [glibc] + '@oxc-parser/binding-linux-riscv64-gnu@0.132.0': + resolution: {integrity: sha512-j9oQS+hM90SdhviNGWbPgT4+Rlq+ac++q/zjgwPD1mVHgxHzATvoRGtDx0sXGmFOQ9J9YkwAhYGb5MAHL6TAsA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [riscv64] + os: [linux] + libc: [glibc] + '@oxc-parser/binding-linux-riscv64-musl@0.131.0': resolution: {integrity: sha512-AqmcNC3fClXX+fxQ6VGEN1667xVFiRBkY0CZmDMSiaeFUsv1+UkBPYYi48IUKcA9/ivvoKNRzQl2I4//kT9F/w==} engines: {node: ^20.19.0 || >=22.12.0} @@ -4239,6 +4331,13 @@ packages: os: [linux] libc: [musl] + '@oxc-parser/binding-linux-riscv64-musl@0.132.0': + resolution: {integrity: sha512-bLz+Xi+Agnfmd7kWPEsSVwCn2k4EyIalZkNBcQ0OGIv9rqn8VgCPLNd03tM9mKX/5TdlvDXalz0q71BIrOPNqg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [riscv64] + os: [linux] + libc: [musl] + '@oxc-parser/binding-linux-s390x-gnu@0.131.0': resolution: {integrity: sha512-7d3jOMKy7RSQCcDLIci+ySll2FgsOMl/GiRux4q2JNv0zg4EdhFISa9idvrdN/HEUIQQJNg6dmveUeJl2YErGA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -4246,6 +4345,13 @@ packages: os: [linux] libc: [glibc] + '@oxc-parser/binding-linux-s390x-gnu@0.132.0': + resolution: {integrity: sha512-U6t2qbJU0ypTfyj9QV3W1Y6mITDTL8ai/OR6NUn85vyHthOvobKWgXzU4tu0EskSzlpuVFz1g0jFGulDIUKHxQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + '@oxc-parser/binding-linux-x64-gnu@0.131.0': resolution: {integrity: sha512-JHK/h95qVqVQ+ITER837kcTdwBDFpFaNnOTYGCP0zdUSX/mLKC7tXOoyrTb6vG7iRPwGlcgBil3v2IjYw1FqJA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -4253,6 +4359,13 @@ packages: os: [linux] libc: [glibc] + '@oxc-parser/binding-linux-x64-gnu@0.132.0': + resolution: {integrity: sha512-WcEaSNHFk8yz5YFlQQAlhq6jOFmZBB/RKE7uzhyCIf+pF1Lmv9gUH4221mle2Gd9iHyWT3ySNph8yZgb1xYdWg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [glibc] + '@oxc-parser/binding-linux-x64-musl@0.131.0': resolution: {integrity: sha512-b2BO82O8azXAyf7EUgOPKu145nWypbNyk07HbU09fkzhm9lEA5oPvaN/M8Nlo7tOErVTa2WOgS4QbOnxAPXdDQ==} engines: {node: ^20.19.0 || >=22.12.0} @@ -4260,41 +4373,80 @@ packages: os: [linux] libc: [musl] + '@oxc-parser/binding-linux-x64-musl@0.132.0': + resolution: {integrity: sha512-iQrV4iJzQgRwK3BWRmQl1C3C6g3wYpXN2WLdQdyR+efoUnncdShZAVp9OgcojtlD3MDRbuOMGG3SjxF4fL4nlQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [musl] + '@oxc-parser/binding-openharmony-arm64@0.131.0': resolution: {integrity: sha512-GHO9glZaX7LkX/OGfluEPf1yjg+ehiFbUdowbX6uNWOQhmwKWU4m4+nZ9FJkrHNKuxyI1KKertMdGjVKCApKWA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [openharmony] + '@oxc-parser/binding-openharmony-arm64@0.132.0': + resolution: {integrity: sha512-FWzmUGrZ6GUby4U7WIwcCtab6tdmlTO3xTRRKyb5kjIJVEiaUAT8animUG/nK8ZCA8gkRkPOTId4rl6uTqUmJQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [openharmony] + '@oxc-parser/binding-wasm32-wasi@0.131.0': resolution: {integrity: sha512-3SkikPaEFoih1N83qLVEDLRLeY4nYsf6JT9SnWiMCQ5lGQdKup6bEuKCqkRiG9dD1IIaFeYz9RjlciPmYoFIWA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [wasm32] + '@oxc-parser/binding-wasm32-wasi@0.132.0': + resolution: {integrity: sha512-TlbMppxJI5CjWDes0QaP6G3aneVg1yikBu5QYI+DUShF9WDL66ccgKFNNGmi/Wybtszw6hxwAvv76T4DaPKnHw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [wasm32] + '@oxc-parser/binding-win32-arm64-msvc@0.131.0': resolution: {integrity: sha512-Os5bEhryeA2jkH+ZrnZyAC1EP5gs+X4YB1Fjqml7UPD5kU7ecsK1MPEVMfCrdt/GDNpDbavYXiOXOdyJ5b3OPw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [win32] + '@oxc-parser/binding-win32-arm64-msvc@0.132.0': + resolution: {integrity: sha512-RH/NbFjGKqdUAUi7Oh3LQPxUk2hsWFEEQ38HSnbRQT8QjBZFKqL1fMbmsB3N4jy/KPh9iX94+9dmkEMBBbambw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [win32] + '@oxc-parser/binding-win32-ia32-msvc@0.131.0': resolution: {integrity: sha512-m+jNz9EuF0NXoiptc6B9h5yompZQVW/a5MJeOu5zojfH5yWk82tvF2ccrHkfhgtrS9h9DD5l1Qv8dWlfY7Nz8g==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ia32] os: [win32] + '@oxc-parser/binding-win32-ia32-msvc@0.132.0': + resolution: {integrity: sha512-JUr4jQY9jxoIB/YTLXr6XofSi5xikj6p5/Ns1h0VOBDT0j1jKU+kMsv2xxv51RwnETcXpA1Yw/9oUAfcqfaqEA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ia32] + os: [win32] + '@oxc-parser/binding-win32-x64-msvc@0.131.0': resolution: {integrity: sha512-o14Hk8dAyiEUMFEWEgmAwFZvBt1RzAYLM3xeQ+5315JXgVYhoemivgYcbYVRbsFkS71ShMGlAFE0kPnr460rww==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [win32] + '@oxc-parser/binding-win32-x64-msvc@0.132.0': + resolution: {integrity: sha512-2dapgHpA5X8DSXF4AU36hJWYf6zP0tKjMXFRAZFBD62pkevW/uhFDXoFH9Y/3Fd2EtDrw5ByNnR1wVE9X9y0SQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [win32] + '@oxc-project/types@0.110.0': resolution: {integrity: sha512-6Ct21OIlrEnFEJk5LT4e63pk3btsI6/TusD/GStLi7wYlGJNOl1GI9qvXAnRAxQU9zqA2Oz+UwhfTOU2rPZVow==} '@oxc-project/types@0.131.0': resolution: {integrity: sha512-PgnWDfV0h+b16XNKbXU7Daib/BFSt/J2mEzfYIBu6JB/wNdlU+kVYXCkGA1A9fWkTbOgbjh4e6NhPeQOYvFhEA==} + '@oxc-project/types@0.132.0': + resolution: {integrity: sha512-FESMOxil5Se014ui/Eq8fT5uHJo6nIRwH0PfJrZJXs6Gek3ZVFOrpUv3YIZT20m+extU98Hg1Ym72U58rlsxUQ==} + '@oxc-project/types@0.139.0': resolution: {integrity: sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw==} @@ -9533,6 +9685,14 @@ packages: dc-browser@1.0.4: resolution: {integrity: sha512-7oEtnzNlcE+hr4OvO3GR6Gndgw8BhW+wKOEwMqSleyY7N29jbAxzyW5BaJl7qBCw+6OIxfMWtY0T+6dxq8RWLw==} + dc-polyfill@0.1.11: + resolution: {integrity: sha512-TyyeGcjx0YeThAI9fTFtgsvj5qd4R+aGfVmXiUhevbgzWFDr7IK4tv4YjE6jaGzLHQTchk4h7DHdr5q4WGgaZw==} + engines: {node: '>=12.17'} + + dd-trace@6.13.0: + resolution: {integrity: sha512-/bSj1/IEZ592KyrsdBTB0nVyWcUABmayun/aImoCsJa+hVATumr5d+njqdLPR2WrXSJ7kq4KnZjrnkYgRTH7oA==} + engines: {node: '>=22'} + debounce-fn@4.0.0: resolution: {integrity: sha512-8pYCQiL9Xdcg0UPSD3d+0KMlOjp+KGU5EPwYddgzQ7DATsg4fuUDjQtsYLmWjnk2obnNHgV3vE2Y4jejSOJVBQ==} engines: {node: '>=10'} @@ -11072,6 +11232,10 @@ packages: resolution: {integrity: sha512-0rymlHSFLwZ0ixx8DaQkoIyZojJPY2a0K2nEYslhKJ6jIYO/m0IcCb7iQsFPmS7WmKwISZiIrv5Icstrw/CmqA==} engines: {node: '>=18'} + import-in-the-middle@3.3.3: + resolution: {integrity: sha512-AiohS3H80sXO6owEltjGX+glb7qXaDhBoJb9XcQVH4UI207xu/bDLUcadVKp7Qe576reg9yr/PXZjV5qx8gfbA==} + engines: {node: '>=18'} + import-lazy@4.0.0: resolution: {integrity: sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==} engines: {node: '>=8'} @@ -11452,6 +11616,10 @@ packages: js-yaml@4.1.1: resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} + js-yaml@4.3.2: + resolution: {integrity: sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==} + hasBin: true + jsc-safe-url@0.2.4: resolution: {integrity: sha512-0wM3YBWtYePOjfyXQH5MWQ8H7sdk5EXSwZvmSLKk2RboVQ2Bu239jycHDz5J/8Blf3K0Qnoy2b6xD+z10MFB+Q==} @@ -12460,6 +12628,9 @@ packages: xml2js: optional: true + node-addon-api@6.1.0: + resolution: {integrity: sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==} + node-addon-api@7.1.1: resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==} @@ -12510,6 +12681,10 @@ packages: resolution: {integrity: sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==} engines: {node: '>= 6.13.0'} + node-gyp-build@3.9.0: + resolution: {integrity: sha512-zLcTg6P4AbcHPq465ZMFNXx7XpKKJh+7kkN699NiQWisR2uWYOWNWqRHAmbnmKiL4e9aLSlmy5U7rEMUXV59+A==} + hasBin: true + node-gyp-build@4.8.4: resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} @@ -12737,6 +12912,10 @@ packages: zod: optional: true + opentracing@0.14.7: + resolution: {integrity: sha512-vz9iS7MJ5+Bp1URw8Khvdyw1H/hGvzHWlKQ7eRrQojSCDL1/SrWfrY9QebLw97n2deyRtzHRC3MkQfVNUCo91Q==} + engines: {node: '>=0.10'} + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -12761,6 +12940,10 @@ packages: resolution: {integrity: sha512-SJ3/7ZPbgie8dr5Z9BI/M51zZbpXba+hRSG0MDzVwMW5CRQg2fjYE0jHGlLX4eeiibGgC/mzoDFKSDHwVZEHRQ==} engines: {node: ^20.19.0 || >=22.12.0} + oxc-parser@0.132.0: + resolution: {integrity: sha512-+0LAPHaqtfQlvWdpaAa09SmOaZZgP8C552xosEkGJ4+ruEwP1Vgx+sqBgcBCNfR6KDCmagGOZTde8wmAvcI/Hg==} + engines: {node: ^20.19.0 || >=22.12.0} + oxc-transform@0.111.0: resolution: {integrity: sha512-oa5KKSDNLHZGaiqIGAbCWXeN9IJUAz9MElWcQX90epDxdKc9Hrt/BsLj3K4gDqfAYa5dwdH+ZCFJG9hR74fiGg==} engines: {node: ^20.19.0 || >=22.12.0} @@ -13325,6 +13508,9 @@ packages: resolution: {integrity: sha512-dM0jVuXJPsDN6DvRpea484tCUaMiXWjuCn++HGTqUWzGDjv5tZkEZldAJ/UMlqRYGFrD/etByo4/xOuC/snX2A==} engines: {node: '>=20'} + pprof-format@2.3.1: + resolution: {integrity: sha512-y51Z83qG2vEQBACPu6lkGFREVkHwQaCaNDdSFEMLIqSo3bmpADsbP6J3F2SSk7tYB741oTQ9Kt5YAdQsmiCRkA==} + preact-render-to-string@6.5.11: resolution: {integrity: sha512-ubnauqoGczeGISiOh6RjX0/cdaF8v/oDXIjO85XALCQjwQP+SB4RDXXtvZ6yTYSjG+PC1QRP2AhPgCEsM2EvUw==} peerDependencies: @@ -14140,6 +14326,10 @@ packages: resolution: {integrity: sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==} engines: {node: '>= 12'} + source-map@0.8.0: + resolution: {integrity: sha512-d8EqvL+k/SOXCreS/SUzg2ciyHqBBLcN/yuRjFsbvVhHTE2pgei7oAhmPM7kWFbkX6OSMQfUq4KbkF3au9lhYQ==} + engines: {node: '>= 12'} + space-separated-tokens@2.0.2: resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} @@ -16644,6 +16834,40 @@ snapshots: '@colors/colors@1.5.0': optional: true + '@datadog/libdatadog@0.12.1': + optional: true + + '@datadog/native-appsec@11.0.2': + dependencies: + node-gyp-build: 3.9.0 + optional: true + + '@datadog/native-iast-taint-tracking@4.2.1': + dependencies: + node-gyp-build: 3.9.0 + optional: true + + '@datadog/native-metrics@4.0.0': + dependencies: + node-addon-api: 6.1.0 + node-gyp-build: 3.9.0 + optional: true + + '@datadog/pprof@5.18.1': + dependencies: + node-gyp-build: 4.8.4 + pprof-format: 2.3.1 + source-map: 0.8.0 + optional: true + + '@datadog/wasm-js-rewriter@5.0.4': + dependencies: + js-yaml: 4.3.2 + lru-cache: 7.18.3 + module-details-from-path: 1.0.4 + node-gyp-build: 4.8.4 + optional: true + '@dimforge/rapier3d-compat@0.12.0': {} '@dotenvx/dotenvx@1.75.1': @@ -17908,7 +18132,7 @@ snapshots: srvx: 0.11.22 std-env: 4.2.0 tinyclip: 0.1.15 - tinyexec: 1.2.4 + tinyexec: 1.3.0 ufo: 1.6.4 youch: 4.1.1 optionalDependencies: @@ -18819,51 +19043,99 @@ snapshots: '@oxc-parser/binding-android-arm-eabi@0.131.0': optional: true + '@oxc-parser/binding-android-arm-eabi@0.132.0': + optional: true + '@oxc-parser/binding-android-arm64@0.131.0': optional: true + '@oxc-parser/binding-android-arm64@0.132.0': + optional: true + '@oxc-parser/binding-darwin-arm64@0.131.0': optional: true + '@oxc-parser/binding-darwin-arm64@0.132.0': + optional: true + '@oxc-parser/binding-darwin-x64@0.131.0': optional: true + '@oxc-parser/binding-darwin-x64@0.132.0': + optional: true + '@oxc-parser/binding-freebsd-x64@0.131.0': optional: true + '@oxc-parser/binding-freebsd-x64@0.132.0': + optional: true + '@oxc-parser/binding-linux-arm-gnueabihf@0.131.0': optional: true + '@oxc-parser/binding-linux-arm-gnueabihf@0.132.0': + optional: true + '@oxc-parser/binding-linux-arm-musleabihf@0.131.0': optional: true + '@oxc-parser/binding-linux-arm-musleabihf@0.132.0': + optional: true + '@oxc-parser/binding-linux-arm64-gnu@0.131.0': optional: true + '@oxc-parser/binding-linux-arm64-gnu@0.132.0': + optional: true + '@oxc-parser/binding-linux-arm64-musl@0.131.0': optional: true + '@oxc-parser/binding-linux-arm64-musl@0.132.0': + optional: true + '@oxc-parser/binding-linux-ppc64-gnu@0.131.0': optional: true + '@oxc-parser/binding-linux-ppc64-gnu@0.132.0': + optional: true + '@oxc-parser/binding-linux-riscv64-gnu@0.131.0': optional: true + '@oxc-parser/binding-linux-riscv64-gnu@0.132.0': + optional: true + '@oxc-parser/binding-linux-riscv64-musl@0.131.0': optional: true + '@oxc-parser/binding-linux-riscv64-musl@0.132.0': + optional: true + '@oxc-parser/binding-linux-s390x-gnu@0.131.0': optional: true + '@oxc-parser/binding-linux-s390x-gnu@0.132.0': + optional: true + '@oxc-parser/binding-linux-x64-gnu@0.131.0': optional: true + '@oxc-parser/binding-linux-x64-gnu@0.132.0': + optional: true + '@oxc-parser/binding-linux-x64-musl@0.131.0': optional: true + '@oxc-parser/binding-linux-x64-musl@0.132.0': + optional: true + '@oxc-parser/binding-openharmony-arm64@0.131.0': optional: true + '@oxc-parser/binding-openharmony-arm64@0.132.0': + optional: true + '@oxc-parser/binding-wasm32-wasi@0.131.0': dependencies: '@emnapi/core': 1.10.0 @@ -18871,19 +19143,38 @@ snapshots: '@napi-rs/wasm-runtime': 1.1.6(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) optional: true + '@oxc-parser/binding-wasm32-wasi@0.132.0': + dependencies: + '@emnapi/core': 1.10.0 + '@emnapi/runtime': 1.10.0 + '@napi-rs/wasm-runtime': 1.1.6(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) + optional: true + '@oxc-parser/binding-win32-arm64-msvc@0.131.0': optional: true + '@oxc-parser/binding-win32-arm64-msvc@0.132.0': + optional: true + '@oxc-parser/binding-win32-ia32-msvc@0.131.0': optional: true + '@oxc-parser/binding-win32-ia32-msvc@0.132.0': + optional: true + '@oxc-parser/binding-win32-x64-msvc@0.131.0': optional: true + '@oxc-parser/binding-win32-x64-msvc@0.132.0': + optional: true + '@oxc-project/types@0.110.0': {} '@oxc-project/types@0.131.0': {} + '@oxc-project/types@0.132.0': + optional: true + '@oxc-project/types@0.139.0': {} '@oxc-transform/binding-android-arm-eabi@0.111.0': @@ -24384,6 +24675,24 @@ snapshots: dc-browser@1.0.4: {} + dc-polyfill@0.1.11: {} + + dd-trace@6.13.0: + dependencies: + dc-polyfill: 0.1.11 + import-in-the-middle: 3.3.3 + opentracing: 0.14.7 + optionalDependencies: + '@datadog/libdatadog': 0.12.1 + '@datadog/native-appsec': 11.0.2 + '@datadog/native-iast-taint-tracking': 4.2.1 + '@datadog/native-metrics': 4.0.0 + '@datadog/pprof': 5.18.1 + '@datadog/wasm-js-rewriter': 5.0.4 + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.221.0 + oxc-parser: 0.132.0 + debounce-fn@4.0.0: dependencies: mimic-fn: 3.1.0 @@ -26492,6 +26801,12 @@ snapshots: es-module-lexer: 2.3.1 module-details-from-path: 1.0.4 + import-in-the-middle@3.3.3: + dependencies: + cjs-module-lexer: 2.2.0 + es-module-lexer: 2.3.1 + module-details-from-path: 1.0.4 + import-lazy@4.0.0: {} import-meta-resolve@4.2.0: {} @@ -26870,6 +27185,11 @@ snapshots: dependencies: argparse: 2.0.1 + js-yaml@4.3.2: + dependencies: + argparse: 2.0.1 + optional: true + jsc-safe-url@0.2.4: {} jsesc@3.1.0: {} @@ -28577,6 +28897,9 @@ snapshots: - vite - webpack + node-addon-api@6.1.0: + optional: true + node-addon-api@7.1.1: {} node-domexception@1.0.0: {} @@ -28610,6 +28933,9 @@ snapshots: node-forge@1.4.0: {} + node-gyp-build@3.9.0: + optional: true + node-gyp-build@4.8.4: {} node-html-parser@7.1.0: @@ -28935,7 +29261,7 @@ snapshots: dependencies: citty: 0.2.2 pathe: 2.0.3 - tinyexec: 1.2.4 + tinyexec: 1.3.0 oauth4webapi@3.8.6: {} @@ -29126,6 +29452,8 @@ snapshots: ws: 8.21.3(bufferutil@4.1.0) zod: 4.5.4 + opentracing@0.14.7: {} + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -29203,6 +29531,32 @@ snapshots: '@oxc-parser/binding-win32-ia32-msvc': 0.131.0 '@oxc-parser/binding-win32-x64-msvc': 0.131.0 + oxc-parser@0.132.0: + dependencies: + '@oxc-project/types': 0.132.0 + optionalDependencies: + '@oxc-parser/binding-android-arm-eabi': 0.132.0 + '@oxc-parser/binding-android-arm64': 0.132.0 + '@oxc-parser/binding-darwin-arm64': 0.132.0 + '@oxc-parser/binding-darwin-x64': 0.132.0 + '@oxc-parser/binding-freebsd-x64': 0.132.0 + '@oxc-parser/binding-linux-arm-gnueabihf': 0.132.0 + '@oxc-parser/binding-linux-arm-musleabihf': 0.132.0 + '@oxc-parser/binding-linux-arm64-gnu': 0.132.0 + '@oxc-parser/binding-linux-arm64-musl': 0.132.0 + '@oxc-parser/binding-linux-ppc64-gnu': 0.132.0 + '@oxc-parser/binding-linux-riscv64-gnu': 0.132.0 + '@oxc-parser/binding-linux-riscv64-musl': 0.132.0 + '@oxc-parser/binding-linux-s390x-gnu': 0.132.0 + '@oxc-parser/binding-linux-x64-gnu': 0.132.0 + '@oxc-parser/binding-linux-x64-musl': 0.132.0 + '@oxc-parser/binding-openharmony-arm64': 0.132.0 + '@oxc-parser/binding-wasm32-wasi': 0.132.0 + '@oxc-parser/binding-win32-arm64-msvc': 0.132.0 + '@oxc-parser/binding-win32-ia32-msvc': 0.132.0 + '@oxc-parser/binding-win32-x64-msvc': 0.132.0 + optional: true + oxc-transform@0.111.0(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.3): optionalDependencies: '@oxc-transform/binding-android-arm-eabi': 0.111.0 @@ -29830,6 +30184,9 @@ snapshots: powershell-utils@0.1.0: {} + pprof-format@2.3.1: + optional: true + preact-render-to-string@6.5.11(preact@10.24.3): dependencies: preact: 10.24.3 @@ -31088,6 +31445,9 @@ snapshots: source-map@0.7.6: {} + source-map@0.8.0: + optional: true + space-separated-tokens@2.0.2: {} split-ca@1.0.1: {} diff --git a/research/datadog-experiment-reporter.md b/research/datadog-experiment-reporter.md index 1e9472acca..8d2d09ddfd 100644 --- a/research/datadog-experiment-reporter.md +++ b/research/datadog-experiment-reporter.md @@ -1,242 +1,98 @@ --- issue: "TBD (Datadog Experiments reporter partner request)" status: proposed -last_updated: "2026-07-27" +last_updated: "2026-09-01" --- # Datadog Experiments reporter for eve evals -Datadog Experiments is the right destination for offline CI eval runs: one eve -run should create one Datadog Experiment, and each completed eval case should -become one experiment row with scores, metadata, and eventually a link back to -the agent runtime trace. +Add a reporter that publishes an eve eval run to Datadog LLM Observability +Experiments without asking the Datadog SDK to rerun the task. -## MVP scope - -Ship a reporter that can publish completed eve eval results to Datadog -Experiments without re-running the task inside the Datadog SDK. - -``` -evals/evals.config.ts ── Datadog(...) ──► one Datadog Experiment - │ - └─ onEvalComplete(result) ───────► one synthetic experiment span + metrics -``` - -The MVP intentionally uses synthetic experiment spans built from the eval -result. Post-hoc linking to an already-ingested agent span is the next step once -Datadog exposes that association capability. - -## Datadog SDK surface needed - -The current Node Experiments shape owns the dataset/task/evaluator loop. eve -already owns that loop, so the reporter only needs hooks for creating an -Experiment, generating/submitting one experiment span per completed row, and -submitting eval metrics that reference that generated span id. - -Proposed public Node contract: +## Authoring API ```ts -type ExperimentMetricValue = boolean | number | string | Record; - -interface StartExperimentOptions { - /** Experiment display name. */ - name: string; - /** Optional override; defaults to DD_LLMOBS_ML_APP or DD_SERVICE. */ - projectName?: string; - description?: string; - tags?: Record; - metadata?: Record; - config?: Record; - /** Optional dataset linkage when the caller has one. Not required for MVP. */ - dataset?: { - id?: string; - name?: string; - version?: string | number; - }; -} - -interface ExperimentSpanInput { - /** Stable caller-owned row id, e.g. eve eval id. */ - id?: string; - name?: string; - input?: unknown; - output?: unknown; - expectedOutput?: unknown; - metadata?: Record; - tags?: Record; - startedAt?: Date | string | number; - completedAt?: Date | string | number; - durationMs?: number; - error?: Error | string | { type?: string; message: string; stack?: string }; - datasetRecordId?: string; -} - -interface ExperimentSpan { - experimentId: string; - spanId: string; - traceId: string; - url: string | null; -} - -interface EvaluationMetricInput { - label: string; - value?: ExperimentMetricValue; - error?: Error | string | { message: string }; - tags?: Record; - timestamp?: Date | string | number; -} - -interface ExternalExperiment { - experimentId(): string | null; - url(): string | null; - - /** - * Generates an LLM Obs experiment span id/trace id, serializes the row as an - * experiment span, submits it, and returns the identifiers the caller should - * use for eval metrics. - */ - submitSpan(row: ExperimentSpanInput): Promise; - - /** - * Submits eval metrics for an already-submitted experiment span. Each metric - * is associated to the row by `span.spanId` and to the run by - * `span.experimentId`. - */ - submitEvaluationMetrics( - span: Pick, - metrics: readonly EvaluationMetricInput[], - ): Promise; - - /** Best-effort status patch + flush. */ - close(options?: { status?: "completed" | "failed"; error?: Error | string }): Promise; -} - -interface Experiments { - startExperiment(options: StartExperimentOptions): Promise; -} -``` - -Usage from the eve reporter: - -```ts -const experiment = await tracer.llmobs.experiments.startExperiment({ - name: "eve evals", - projectName: "weather-agent", - tags: { source: "eve" }, -}); - -const span = await experiment.submitSpan({ - id: result.id, - input: evaluation.description, - output: result.result.output, - metadata, - startedAt: result.startedAt, - completedAt: result.completedAt, - error: result.error, +import { defineEvalConfig } from "eve/evals"; +import { Datadog } from "eve/evals/reporters"; + +export default defineEvalConfig({ + reporters: [ + Datadog({ + projectName: "weather-agent", + recordInputs: false, + recordOutputs: false, + }), + ], }); +``` -await experiment.submitEvaluationMetrics(span, [ - { label: "gate_succeeded", value: 1 }, - { label: "similarity", value: 0.92 }, -]); +One shared reporter instance creates one Experiment for the run. Each completed +eval becomes one synthetic experiment span, and each assertion becomes an +Experiment metric associated with the span returned by Datadog. -await experiment.close({ status: summary.failed > 0 ? "failed" : "completed" }); +```text +evals.config.ts ── Datadog(...) ──► Experiment + ├─ eval row + assertion metrics + ├─ eval row + assertion metrics + └─ completed/failed status ``` -This deliberately keeps span generation and eval metric submission separate so -custom eval runners can submit metrics after they know the generated experiment -span id. A convenience `submitRow({ ...row, metrics })` can be added later, but -the lower-level contract should remain public. - -Until that public API exists, keep any direct Datadog HTTP client in one private -adapter so it can be removed without changing the eve reporter API. - -## eve code changes - -1. Replace `src/evals/runner/reporters/datadog.ts` with a real optional-peer - reporter: - - export `Datadog(config?: DatadogReporterConfig)` and its config type from - `eve/evals/reporters`. - - dynamically import `dd-trace` or a Datadog SDK adapter so `eve` does not - gain a hard runtime dependency. - - accept config for `projectName`/`mlApp`, `experimentName`, `datasetName`, - `tags`, `metadata`, `recordInputs`, `recordOutputs`, and an optional - injected client for tests. -2. On `onRunStart`, create/resolve the Datadog project, materialize a dataset - record per observed eve eval, then create a fresh Experiment for the run. -3. On `onEvalComplete`, map an `EveEvalResult` to one Datadog row: - - `id`: eve eval id. - - input/expected: eval description and/or dataset metadata that eve exposes. - - output/error/timestamps: `result.result.output`, `result.error`, - `startedAt`, `completedAt`. - - scores: soft assertions by assertion name; gate assertions as binary - `gate:` metrics. - - metadata: eval metadata, verdict, skip reason, session id, tool/subagent - call counts, failed assertion messages, runtime identity, and git metadata. -4. On `onRunComplete`, flush pending rows, patch Experiment status - (`completed` or `failed`), and print the Datadog Experiment URL. -5. Add docs in `docs/evals/reporters.mdx`, a changeset for `eve`, and keep all - user-facing copy spelling the framework as `eve`. - -## Runtime trace-linking follow-up - -The reporter cannot currently discover the agent turn's real OTel span. The -harness stores `traceId`/`spanId` in internal session state to keep model-call -spans parented, but that state is not exposed through eval results. - -Add a durable, reporter-visible trace link without requiring reporters to -instrument the black-box agent: - -1. Extend the stream protocol with trace context on a turn lifecycle event, e.g. - `turn.started.data.trace = { traceId, spanId }` when telemetry is enabled. -2. Add `EveEvalTraceLink` to `EveEvalTaskResult`/`EveEvalSessionResult`, derived - from captured stream events. -3. Teach `Datadog` to send `{ traceId, spanId }` when present. -4. Once Datadog supports post-hoc association, switch from synthetic spans to - `attachSpan({ traceId, spanId, scores, metadata })` for instrumented agents, - with synthetic spans retained as the fallback. - -## Example project - -Expand `apps/fixtures/datadog-eval-reporter` into the correctness fixture and -manual dogfood app: - -- real dependencies: `eve`, `dd-trace`, `@vercel/otel`, and OTel exporter - packages needed for Datadog trace submission. -- deterministic `mockModel()` agent so CI and local checks do not need model - credentials. -- `agent/instrumentation.ts` that enables OTel/Datadog telemetry for runtime - traces. -- `evals/evals.config.ts` using `Datadog({ projectName: "eve-datadog-fixture" })`. -- `README.md` and `.env.example` showing both modes: - - local verification against a fake Datadog HTTP endpoint; - - manual verification against Datadog with `DD_API_KEY`, `DD_APP_KEY`, - `DD_SITE`, and `DD_LLMOBS_ML_APP`. - -## Test plan - -- Unit tests for the reporter mapping: lifecycle order, missing SDK error, - no-op before `onRunStart`, score/metadata conversion, skipped/error rows, - summary status, URL logging, and `recordInputs`/`recordOutputs` privacy - switches. -- Integration tests with an injected fake Datadog client or mocked `fetch` that - assert exact project/dataset/experiment/events/status payloads without real - network access. -- Runner-level test to prove config reporters still dedupe and the Datadog - reporter receives scoped summaries. -- Scenario test using `apps/fixtures/datadog-eval-reporter` and `eve eval ---strict`; point the reporter at a local fake Datadog server and assert one - experiment plus one row is submitted. -- Manual dogfood test with real Datadog credentials: run the fixture, open the - printed Experiment URL, verify scores, metadata, run summary, and (after the - follow-up) the link to the runtime trace. - -## Open questions - -- What exact public Node SDK API and package version should eve target for the - externally-driven Experiments surface? -- Does Datadog require a persisted dataset for every Experiment, or can external - eval rows create an Experiment without dataset records? -- What is the backend contract for post-hoc linking to an already-ingested span? -- Should eve expose expected output/inputs to reporters beyond description and - metadata, or keep that data entirely user-authored via eval metadata? +## SDK boundary + +Target the public external Experiment API in `dd-trace@6.13.0`: + +- `tracer.llmobs.experiments.startExperiment(...)` +- `experiment.submitSpan(...)` +- `experiment.submitEvaluationMetrics(...)` +- `experiment.close(...)` + +`submitSpan` generates and returns the row's `traceId` and `spanId`. Its input +does not accept caller-owned `id`, `traceId`, `spanId`, or `apmTraceId` fields. +The reporter must not fabricate those fields or mutate `dd-trace` internals. +Keep `dd-trace` optional and load it from the app so the published `eve` package +does not gain a runtime dependency. + +Initialize LLM Observability with both `projectName` and `mlApp`. Resolve the +project from explicit reporter config, `DD_LLMOBS_PROJECT_NAME`, ml_app config, +`DD_LLMOBS_ML_APP`, `DD_SERVICE`, or the first eval id, in that order. + +## Row mapping + +- `name`: the path-derived eve eval id. +- `input`, `output`, and expected output: opt-in because they may contain user + data. +- `metadata`: verdict, status, session id, tool/subagent names, authored eval + metadata, sanitized target origin, and local git metadata when available. +- `tags`: eval id, verdict, status, and authored eval tags. +- metrics: assertion scores under position-based labels plus tool, subagent, + message, and reasoning counts. Assertion names and failure messages require + the explicit `recordAssertionDetails` privacy opt-in. +- timestamps: the values already captured by the eval runner. +- error: opt-in because exception messages may contain application data. + +All metadata, config, inputs, and outputs are normalized to the JSON value shape +accepted by `dd-trace` before submission. + +## Trace-linking boundary + +The external Experiment API creates a synthetic row span and does not support +post-hoc association with an already-ingested agent runtime span. Do not add +private propagation headers, expose trace identifiers in stream events, force +global Datadog environment variables, or mutate private `dd-trace` span fields +to simulate that association. + +If Datadog adds a public linking API later, eve can expose an internal, +reporter-visible trace link and submit it through that API. Synthetic spans +remain the fallback for remote or uninstrumented targets. + +## Validation + +- Unit-test experiment lifecycle, row mapping, privacy switches, metric labels, + project resolution, summary status, and URL logging with an injected client. +- Pin `dd-trace@6.13.0` as an eve development dependency and in the deterministic + reporter fixture so TypeScript and runtime API smoke checks exercise the + supported release. +- Run the targeted reporter and runner unit tests, eve typecheck, dependency + checks, invariant guards, docs checks, and `git diff --check`. +- Use real Datadog credentials only for manual dogfood verification; CI tests + remain deterministic and network-free. From 8ad75d492875ffc27555a4da80e55d6ef3e73514 Mon Sep 17 00:00:00 2001 From: "mehul.sonowal" Date: Wed, 2 Sep 2026 13:26:21 -0400 Subject: [PATCH 6/8] feat(evals): link Datadog experiments to dataset records Signed-off-by: mehul.sonowal --- .changeset/datadog-eval-reporter.md | 2 +- apps/fixtures/datadog-eval-reporter/README.md | 11 +- .../evals/evals.config.ts | 2 +- docs/evals/reporters.mdx | 6 +- .../evals/runner/reporters/datadog.test.ts | 110 +++++++++- .../eve/src/evals/runner/reporters/datadog.ts | 194 ++++++++++++++---- research/datadog-experiment-reporter.md | 20 +- 7 files changed, 286 insertions(+), 59 deletions(-) diff --git a/.changeset/datadog-eval-reporter.md b/.changeset/datadog-eval-reporter.md index 49309098ec..fb11f82443 100644 --- a/.changeset/datadog-eval-reporter.md +++ b/.changeset/datadog-eval-reporter.md @@ -2,4 +2,4 @@ "eve": patch --- -Add a Datadog eval reporter that creates one LLM Observability Experiment per eve eval run and submits eval assertion metrics through the optional `dd-trace` package. The integration is tested against the public external Experiment API in `dd-trace@6.13.0`. +Add a Datadog eval reporter that creates one LLM Observability Experiment per eve eval run and submits eval assertion metrics through the optional `dd-trace` package. Opted-in eval inputs are pushed as versioned dataset records and linked to their experiment rows. The integration is tested against the public dataset and external Experiment APIs in `dd-trace@6.13.0`. diff --git a/apps/fixtures/datadog-eval-reporter/README.md b/apps/fixtures/datadog-eval-reporter/README.md index 707ac6371f..9392b3286c 100644 --- a/apps/fixtures/datadog-eval-reporter/README.md +++ b/apps/fixtures/datadog-eval-reporter/README.md @@ -11,8 +11,9 @@ export DD_SITE="datadoghq.com" pnpm --filter datadog-eval-reporter-fixture eval ``` -The run creates one Datadog LLM Observability Experiment, submits one synthetic -experiment span for the smoke eval, attaches the assertion metrics, and prints -the Experiment URL. The fixture does not export the agent runtime's OpenTelemetry -spans; Datadog's external Experiment API generates the experiment row's trace -and span identifiers. +The run creates a Datadog dataset with one record containing `Say hello.`, +creates one LLM Observability Experiment against that dataset version, links the +record to the smoke eval's synthetic experiment span, attaches the assertion +metrics, and prints the Dataset and Experiment URLs. The fixture does not export +the agent runtime's OpenTelemetry spans; Datadog's public dataset and external +Experiment APIs generate the record, trace, and span identifiers. diff --git a/apps/fixtures/datadog-eval-reporter/evals/evals.config.ts b/apps/fixtures/datadog-eval-reporter/evals/evals.config.ts index b0dfb2f1d2..6a2c105064 100644 --- a/apps/fixtures/datadog-eval-reporter/evals/evals.config.ts +++ b/apps/fixtures/datadog-eval-reporter/evals/evals.config.ts @@ -2,5 +2,5 @@ import { defineEvalConfig } from "eve/evals"; import { Datadog } from "eve/evals/reporters"; export default defineEvalConfig({ - reporters: [Datadog()], + reporters: [Datadog({ recordInputs: true })], }); diff --git a/docs/evals/reporters.mdx b/docs/evals/reporters.mdx index d51fc51781..1c39420939 100644 --- a/docs/evals/reporters.mdx +++ b/docs/evals/reporters.mdx @@ -57,9 +57,11 @@ export default defineEvalConfig({ }); ``` -The reporter creates one Datadog Experiment on run start, creates a placeholder dataset for the experiment, submits one synthetic experiment span per completed eve eval, associates that eval's assertion scores as experiment metrics, and prints the Datadog Experiment URL after the run completes. `dd-trace` generates the experiment row's trace and span identifiers; the reporter does not mutate or link the agent runtime's OpenTelemetry spans. By default, assertion metrics use position-based labels such as `gate_assertion_1` and `assertion_2`, and omit assertion names and failure messages because those details may contain inputs, outputs, or authored expectations. Pass `recordAssertionDetails: true` to include those details; labels then use the assertion name, normalize characters outside letters, numbers, underscores, and hyphens to underscores, prefix gate labels with `gate_`, and add numeric suffixes when labels collide. +The reporter creates one Datadog Experiment, submits one synthetic experiment span per completed eve eval, associates that eval's assertion scores as experiment metrics, and prints the Datadog Experiment URL after the run completes. Without input recording, the external Experiment uses a placeholder dataset and spans are submitted as evals finish. With `recordInputs: true`, the reporter instead creates and pushes a versioned Datadog dataset after the evals finish, adds one record per eval, starts the Experiment against that dataset version, passes each dataset record id to its corresponding experiment span, and prints both Dataset and Experiment URLs. Buffering is necessary because an imperative eval's first `t.send(...)` input is only known after that eval executes. `dd-trace` generates dataset record, experiment trace, and experiment span identifiers; the reporter does not mutate or link the agent runtime's OpenTelemetry spans. -Datadog needs `dd-trace` installed in the app and Datadog credentials in the environment. The reporter is tested against `dd-trace@6.13.0`; install it with `npm install dd-trace@6.13.0` and set `DD_API_KEY`, `DD_APP_KEY`, and `DD_SITE` as appropriate. By default, the reporter records assertion scores, eval metadata, and the target URL origin only. Pass `recordInputs: true`, `recordOutputs: true`, `recordExpectedOutputs: true`, or `recordErrors: true` if your destination is approved for eval prompts, outputs, authored expectations, or execution error messages. Input is read from the eval's first `t.send(...)` message, falling back to the eval description when no message event is available. Expected output is read from eval `metadata.expectedOutput`, `metadata.expected`, or `metadata.expected_output`; those keys are excluded from the general metadata payload and are sent only through the opt-in expected-output field. Target URL credentials, paths, query parameters, and fragments are never reported. +Assertion metrics use descriptive assertion names by default: characters outside letters, numbers, underscores, and hyphens are normalized to underscores, gate labels receive a `gate_` prefix, and repeated or reserved labels receive numeric suffixes. For example, `succeeded` becomes `gate_succeeded` and `calledTool(get_stock_quote)` becomes `gate_calledTool_get_stock_quote`. Authors can use `.label("stable name")` on an assertion handle to choose a stable metric name. Because assertion names can contain authored expectations, review them before exporting sensitive evals. Pass `recordAssertionDetails: true` only when the destination is also approved for raw assertion-name tags and failed assertion messages in row metadata. + +Datadog needs `dd-trace` installed in the app and Datadog credentials in the environment. The reporter is tested against `dd-trace@6.13.0`; install it with `npm install dd-trace@6.13.0` and set `DD_API_KEY`, `DD_APP_KEY`, and `DD_SITE` as appropriate. By default, the reporter records assertion scores, eval metadata, and the target URL origin only. Pass `recordInputs: true`, `recordOutputs: true`, `recordExpectedOutputs: true`, or `recordErrors: true` if your destination is approved for eval prompts, outputs, authored expectations, or execution error messages. `recordInputs` stores the eval input in both the linked dataset record and experiment span. Input is read from the eval's first `t.send(...)` message, falling back to the eval description when no message event is available. When `recordExpectedOutputs` is enabled, expected output is read from eval `metadata.expectedOutput`, `metadata.expected`, or `metadata.expected_output` and stored in both the dataset record and experiment span; when no expected output is authored, the input-only dataset record is still valid. Those expected-output keys are excluded from the general metadata payload. Target URL credentials, paths, query parameters, and fragments are never reported. ## JUnit diff --git a/packages/eve/src/evals/runner/reporters/datadog.test.ts b/packages/eve/src/evals/runner/reporters/datadog.test.ts index 62de672f53..2af2af0470 100644 --- a/packages/eve/src/evals/runner/reporters/datadog.test.ts +++ b/packages/eve/src/evals/runner/reporters/datadog.test.ts @@ -105,7 +105,44 @@ function makeConfig(overrides: Partial = {}) { submitEvaluationMetrics: vi.fn(async (_span: unknown, _metrics: unknown) => undefined), close: vi.fn(async () => undefined), }; + let datasetName = "dataset-1"; + let datasetRecords: Array<{ + id: string; + inputData: unknown; + expectedOutput?: unknown; + metadata?: Readonly>; + }> = []; + const dataset = { + id: vi.fn(() => "dataset-1"), + name: vi.fn(() => datasetName), + version: vi.fn(() => 3), + records: vi.fn(() => datasetRecords), + url: vi.fn(() => "https://dd.test/dataset"), + push: vi.fn(async () => ({ + pushedCount: datasetRecords.length, + totalCount: datasetRecords.length, + })), + }; const client = { + createDataset: vi.fn( + ( + name: string, + options?: { + records?: Array<{ + inputData: unknown; + expectedOutput?: unknown; + metadata?: Readonly>; + }>; + }, + ) => { + datasetName = name; + datasetRecords = (options?.records ?? []).map((record, index) => ({ + id: `record-${index + 1}`, + ...record, + })); + return dataset; + }, + ), startExperiment: vi.fn(async (_options: unknown) => experiment), }; const lines: string[] = []; @@ -116,7 +153,7 @@ function makeConfig(overrides: Partial = {}) { ...overrides, } satisfies DatadogReporterConfig; - return { client, config, lines, experiment, span }; + return { client, config, dataset, experiment, lines, span }; } describe("Datadog", () => { @@ -129,6 +166,7 @@ describe("Datadog", () => { await reporter.onRunStart([evaluation], makeTarget()); await reporter.onEvalComplete(result); + expect(client.createDataset).not.toHaveBeenCalled(); expect(client.startExperiment).toHaveBeenCalledWith( expect.objectContaining({ name: "run-1", @@ -164,9 +202,9 @@ describe("Datadog", () => { expect(submittedSpan).not.toHaveProperty("metadata.expected"); expect(submittedSpan).not.toHaveProperty("metadata.expected_output"); expect(experiment.submitEvaluationMetrics).toHaveBeenCalledWith(span, [ - expect.objectContaining({ label: "gate_assertion_1", value: 1 }), - expect.objectContaining({ label: "assertion_2", value: 0.9 }), - expect.objectContaining({ label: "assertion_3", value: 1 }), + expect.objectContaining({ label: "gate_succeeded", value: 1 }), + expect.objectContaining({ label: "similarity", value: 0.9 }), + expect.objectContaining({ label: "judge_autoevals_closedQA", value: 1 }), expect.objectContaining({ label: "eve_tool_call_count", value: 1 }), expect.objectContaining({ label: "eve_subagent_call_count", value: 0 }), expect.objectContaining({ label: "eve_message_count", value: 1 }), @@ -238,7 +276,7 @@ describe("Datadog", () => { expect(experiment.submitEvaluationMetrics.mock.calls[0]?.[1]).toEqual( expect.arrayContaining([ expect.objectContaining({ - label: "gate_assertion_1", + label: "gate_messageIncludes_private_expectation", tags: { assertion_index: "1", assertion_severity: "gate", @@ -249,7 +287,7 @@ describe("Datadog", () => { ); }); - it("records assertion names and failure messages only when enabled", async () => { + it("records assertion name tags and failure messages only when enabled", async () => { const { config, experiment } = makeConfig({ recordAssertionDetails: true }); const reporter = Datadog(config); const result = makeEvalResult({ @@ -316,24 +354,78 @@ describe("Datadog", () => { ]); }); - it("records eval input and output only when enabled", async () => { - const { config, experiment } = makeConfig({ + it("creates dataset records from opted-in eval inputs and links experiment rows", async () => { + const { client, config, dataset, experiment, lines } = makeConfig({ + datasetName: "greeting inputs", + experimentName: "run-1", recordInputs: true, recordOutputs: true, recordExpectedOutputs: true, }); const reporter = Datadog(config); + const result = makeEvalResult(); await reporter.onRunStart([makeEval()], makeTarget()); - await reporter.onEvalComplete(makeEvalResult()); + await reporter.onEvalComplete(result); + + expect(client.startExperiment).not.toHaveBeenCalled(); + expect(experiment.submitSpan).not.toHaveBeenCalled(); + + await reporter.onRunComplete(makeSummary(result)); + expect(client.createDataset).toHaveBeenCalledWith( + "greeting inputs", + expect.objectContaining({ + projectName: "test-project", + records: [ + { + inputData: "What should I send you?", + expectedOutput: "helpful onboarding answer", + metadata: { eveEvalId: "eval-1" }, + }, + ], + }), + ); + expect(dataset.push).toHaveBeenCalledOnce(); + expect(dataset.push.mock.invocationCallOrder[0]).toBeLessThan( + client.startExperiment.mock.invocationCallOrder[0] ?? 0, + ); + expect(client.startExperiment).toHaveBeenCalledWith( + expect.objectContaining({ + name: "run-1", + dataset: { id: "dataset-1", name: "greeting inputs", version: 3 }, + }), + ); expect(experiment.submitSpan).toHaveBeenCalledWith( expect.objectContaining({ input: "What should I send you?", output: "actual output", expectedOutput: "helpful onboarding answer", + datasetRecordId: "record-1", }), ); + expect(lines.join("\n")).toContain("Datadog dataset URL: https://dd.test/dataset"); + }); + + it("creates an input-only dataset record when no expected output is authored", async () => { + const { client, config } = makeConfig({ + datasetName: "input-only dataset", + recordInputs: true, + recordExpectedOutputs: true, + }); + const reporter = Datadog(config); + const evaluation = makeEval({ metadata: { suite: "unit" } }); + const result = makeEvalResult(); + + await reporter.onRunStart([evaluation], makeTarget()); + await reporter.onEvalComplete(result); + await reporter.onRunComplete(makeSummary(result)); + + const record = client.createDataset.mock.calls[0]?.[1]?.records?.[0]; + expect(record).toEqual({ + inputData: "What should I send you?", + metadata: { eveEvalId: "eval-1" }, + }); }); it("uses the dd-trace project environment variable by default", async () => { diff --git a/packages/eve/src/evals/runner/reporters/datadog.ts b/packages/eve/src/evals/runner/reporters/datadog.ts index f64eae77b8..81c5910ee5 100644 --- a/packages/eve/src/evals/runner/reporters/datadog.ts +++ b/packages/eve/src/evals/runner/reporters/datadog.ts @@ -11,7 +11,7 @@ import { parseJsonValue, type JsonValue } from "#shared/json.js"; export interface DatadogReporterConfig { /** Datadog LLM Observability project name. Defaults to `DD_LLMOBS_PROJECT_NAME`, the configured ml_app, `DD_SERVICE`, or the first eval id. */ readonly projectName?: string; - /** Name for the placeholder dataset used by the experiment. Defaults to ` dataset`. */ + /** Name for the dataset used by the experiment. Defaults to ` dataset`. */ readonly datasetName?: string; /** Name for the created experiment. Defaults to a timestamped eve eval run name. */ readonly experimentName?: string; @@ -31,22 +31,20 @@ export interface DatadogReporterConfig { readonly metadata?: Readonly>; /** JSON-serializable Datadog Experiment config. */ readonly config?: Readonly>; - /** Include the first sent message, falling back to the eval description, in the synthetic experiment row input. Defaults to false. */ + /** Include the first sent message, falling back to the eval description, in the experiment row and a linked dataset record. Defaults to false. */ readonly recordInputs?: boolean; /** Include eval outputs in the synthetic experiment row output. Defaults to false. */ readonly recordOutputs?: boolean; /** Include `metadata.expectedOutput`, `metadata.expected`, or `metadata.expected_output` on experiment rows. Defaults to false. */ readonly recordExpectedOutputs?: boolean; - /** Include assertion names and failure messages, which may contain inputs, outputs, or expectations. Defaults to false. */ + /** Include raw assertion names as metric tags and failure messages in row metadata. Defaults to false. */ readonly recordAssertionDetails?: boolean; /** Include execution error messages, which may contain application data. Defaults to false. */ readonly recordErrors?: boolean; /** Console hook used for tests. */ readonly log?: (line: string) => void; /** eve-owned client seam for tests and custom Datadog SDK wiring. */ - readonly client?: { - startExperiment(options: DatadogStartExperimentOptions): Promise; - }; + readonly client?: DatadogExperimentsClient; } type DatadogJsonValue = @@ -57,6 +55,38 @@ type DatadogJsonValue = | DatadogJsonValue[] | { [key: string]: DatadogJsonValue }; +interface DatadogDatasetRecordInput { + inputData: DatadogJsonValue; + expectedOutput?: DatadogJsonValue; + metadata?: Record; + tags?: string[]; +} + +interface DatadogDatasetRecord { + readonly id: string | null; +} + +interface DatadogDataset { + id(): string | null; + name(): string; + version(): number | null; + records(): readonly DatadogDatasetRecord[]; + url(): string | null; + push(): Promise<{ pushedCount: number; totalCount: number }>; +} + +interface DatadogExperimentsClient { + createDataset?( + name: string, + options?: { + projectName?: string; + description?: string; + records?: DatadogDatasetRecordInput[]; + }, + ): DatadogDataset; + startExperiment(options: DatadogStartExperimentOptions): Promise; +} + interface DatadogStartExperimentOptions { name: string; projectName?: string; @@ -130,13 +160,20 @@ export function Datadog(config: DatadogReporterConfig = {}): EvalReporter { class DatadogReporter implements EvalReporter { readonly #config: DatadogReporterConfig; readonly #evaluations = new Map(); + #client: DatadogExperimentsClient | undefined; + #experimentOptions: DatadogStartExperimentOptions | undefined; #experiment: DatadogExternalExperiment | undefined; + #datasetUrl: string | undefined; constructor(config: DatadogReporterConfig) { this.#config = config; } async onRunStart(evaluations: readonly EveEval[], target: EveEvalTarget): Promise { + this.#client = undefined; + this.#experimentOptions = undefined; + this.#experiment = undefined; + this.#datasetUrl = undefined; this.#evaluations.clear(); for (const evaluation of evaluations) { this.#evaluations.set(evaluation.id, evaluation); @@ -153,7 +190,7 @@ class DatadogReporter implements EvalReporter { } Object.assign(metadata, this.#config.metadata); - this.#experiment = await client.startExperiment({ + const experimentOptions: DatadogStartExperimentOptions = { name: experimentName, projectName: resolveProjectName(this.#config, evaluations), description: this.#config.description, @@ -161,10 +198,119 @@ class DatadogReporter implements EvalReporter { tags: resolveExperimentTags(this.#config, target), metadata: toDatadogJsonRecord(metadata), config: toDatadogJsonRecord(this.#config.config), - }); + }; + + if (this.#config.recordInputs) { + if (!client.createDataset) { + throw new Error( + "The installed 'dd-trace' package does not expose tracer.llmobs.experiments.createDataset().", + ); + } + this.#client = client; + this.#experimentOptions = experimentOptions; + return; + } + + this.#experiment = await client.startExperiment(experimentOptions); } async onEvalComplete(result: EveEvalResult): Promise { + if (this.#config.recordInputs || !this.#experiment) return; + await this.#submitResult(result); + } + + async onRunComplete(summary: EveEvalRunSummary): Promise { + try { + if (this.#config.recordInputs) { + await this.#startDatasetBackedExperiment(summary.results); + } + if (!this.#experiment) return; + + const failed = summary.failed > 0 || summary.errored > 0; + await this.#experiment.close({ + status: failed ? "failed" : "completed", + error: failed ? `${summary.failed} failed, ${summary.errored} errored` : undefined, + }); + + const log = this.#config.log ?? console.log; + if (this.#datasetUrl) { + log(`Datadog dataset URL: ${this.#datasetUrl}\n`); + } + const experimentUrl = this.#experiment.url(); + if (experimentUrl) { + log(`Datadog experiment URL: ${experimentUrl}\n`); + } + } finally { + this.#client = undefined; + this.#experimentOptions = undefined; + this.#experiment = undefined; + this.#datasetUrl = undefined; + } + } + + async #startDatasetBackedExperiment(results: readonly EveEvalResult[]): Promise { + const client = this.#client; + const experimentOptions = this.#experimentOptions; + if (!client?.createDataset || !experimentOptions) return; + + const datasetName = experimentOptions.dataset?.name ?? `${experimentOptions.name} dataset`; + const records = results.map((result): DatadogDatasetRecordInput => { + const evaluation = this.#evaluations.get(result.id); + const record: DatadogDatasetRecordInput = { + inputData: toDatadogJsonValue(resolveInput(result, evaluation)), + metadata: { eveEvalId: result.id }, + }; + if (this.#config.recordExpectedOutputs) { + const expectedOutput = toOptionalDatadogJsonValue(resolveExpectedOutput(evaluation)); + if (expectedOutput !== undefined) { + record.expectedOutput = expectedOutput; + } + } + return record; + }); + const dataset = client.createDataset(datasetName, { + projectName: experimentOptions.projectName, + description: + this.#config.description ?? `Eve eval inputs for experiment '${experimentOptions.name}'.`, + records, + }); + + await dataset.push(); + const datasetId = dataset.id(); + if (!datasetId) { + throw new Error(`Datadog dataset '${datasetName}' has no id after push().`); + } + const datasetRecords = dataset.records(); + if (datasetRecords.length !== results.length) { + throw new Error( + `Datadog dataset '${datasetName}' has ${datasetRecords.length} records for ${results.length} eval results.`, + ); + } + + const datasetOptions: NonNullable = { + id: datasetId, + name: dataset.name(), + }; + const datasetVersion = dataset.version(); + if (datasetVersion !== null) { + datasetOptions.version = datasetVersion; + } + this.#datasetUrl = dataset.url() ?? undefined; + this.#experiment = await client.startExperiment({ + ...experimentOptions, + dataset: datasetOptions, + }); + + for (const [index, result] of results.entries()) { + const datasetRecordId = datasetRecords[index]?.id; + if (!datasetRecordId) { + throw new Error(`Datadog dataset record ${index + 1} has no id after push().`); + } + await this.#submitResult(result, datasetRecordId); + } + } + + async #submitResult(result: EveEvalResult, datasetRecordId?: string): Promise { if (!this.#experiment) return; const evaluation = this.#evaluations.get(result.id); @@ -190,6 +336,9 @@ class DatadogReporter implements EvalReporter { if (this.#config.recordErrors && result.error !== undefined) { spanInput.error = result.error; } + if (datasetRecordId !== undefined) { + spanInput.datasetRecordId = datasetRecordId; + } const span = await this.#experiment.submitSpan(spanInput); await this.#experiment.submitEvaluationMetrics( @@ -197,25 +346,6 @@ class DatadogReporter implements EvalReporter { resolveEvaluationMetrics(result, this.#config.recordAssertionDetails === true), ); } - - async onRunComplete(summary: EveEvalRunSummary): Promise { - if (!this.#experiment) return; - - try { - const failed = summary.failed > 0 || summary.errored > 0; - await this.#experiment.close({ - status: failed ? "failed" : "completed", - error: failed ? `${summary.failed} failed, ${summary.errored} errored` : undefined, - }); - - const url = this.#experiment.url(); - if (url) { - (this.#config.log ?? console.log)(`Datadog experiment URL: ${url}\n`); - } - } finally { - this.#experiment = undefined; - } - } } const DD_TRACE_PACKAGE = "dd-trace"; @@ -234,9 +364,7 @@ const BUILT_IN_METRIC_LABELS = [ async function resolveDatadogClient( config: DatadogReporterConfig, evaluations: readonly EveEval[], -): Promise<{ - startExperiment(options: DatadogStartExperimentOptions): Promise; -}> { +): Promise { if (config.client) return config.client; const sdk = await loadDatadogSdk(); @@ -415,11 +543,7 @@ function resolveEvaluationMetrics( const usedLabels = new Set(BUILT_IN_METRIC_LABELS); for (const [index, assertion] of result.assertions.entries()) { - const rawLabel = recordAssertionDetails - ? assertion.severity === "gate" - ? `gate_${assertion.name}` - : assertion.name - : `${assertion.severity === "gate" ? "gate_" : ""}assertion_${index + 1}`; + const rawLabel = assertion.severity === "gate" ? `gate_${assertion.name}` : assertion.name; const tags: Record = { assertion_index: String(index + 1), assertion_severity: assertion.severity, diff --git a/research/datadog-experiment-reporter.md b/research/datadog-experiment-reporter.md index 8d2d09ddfd..a9c1c720dc 100644 --- a/research/datadog-experiment-reporter.md +++ b/research/datadog-experiment-reporter.md @@ -41,6 +41,8 @@ evals.config.ts ── Datadog(...) ──► Experiment Target the public external Experiment API in `dd-trace@6.13.0`: +- `tracer.llmobs.experiments.createDataset(...)` +- `dataset.push()` - `tracer.llmobs.experiments.startExperiment(...)` - `experiment.submitSpan(...)` - `experiment.submitEvaluationMetrics(...)` @@ -60,13 +62,18 @@ project from explicit reporter config, `DD_LLMOBS_PROJECT_NAME`, ml_app config, - `name`: the path-derived eve eval id. - `input`, `output`, and expected output: opt-in because they may contain user - data. + data. When input recording is enabled, completed evals are buffered until run + completion, their inputs are pushed as versioned dataset records, and each + experiment span carries its generated dataset record id. Expected output is + included in the dataset record only when present and explicitly enabled. - `metadata`: verdict, status, session id, tool/subagent names, authored eval metadata, sanitized target origin, and local git metadata when available. - `tags`: eval id, verdict, status, and authored eval tags. -- metrics: assertion scores under position-based labels plus tool, subagent, - message, and reasoning counts. Assertion names and failure messages require - the explicit `recordAssertionDetails` privacy opt-in. +- metrics: assertion scores under normalized descriptive assertion names plus + tool, subagent, message, and reasoning counts. Gate labels receive a `gate_` + prefix, and authors can set stable names with `.label(...)`. Raw assertion-name + tags and failure messages require the explicit `recordAssertionDetails` + privacy opt-in. - timestamps: the values already captured by the eval runner. - error: opt-in because exception messages may contain application data. @@ -87,8 +94,9 @@ remain the fallback for remote or uninstrumented targets. ## Validation -- Unit-test experiment lifecycle, row mapping, privacy switches, metric labels, - project resolution, summary status, and URL logging with an injected client. +- Unit-test experiment lifecycle, dataset creation and record linkage, row + mapping, privacy switches, metric labels, project resolution, summary status, + and URL logging with an injected client. - Pin `dd-trace@6.13.0` as an eve development dependency and in the deterministic reporter fixture so TypeScript and runtime API smoke checks exercise the supported release. From 287f1619379fa29d57fbd9b67529e61dbaddafee Mon Sep 17 00:00:00 2001 From: "mehul.sonowal" Date: Fri, 4 Sep 2026 12:02:18 -0400 Subject: [PATCH 7/8] fix(evals): keep Datadog reporter fixture hermetic Signed-off-by: mehul.sonowal --- apps/fixtures/datadog-eval-reporter/README.md | 5 ++++- apps/fixtures/datadog-eval-reporter/agent/agent.ts | 2 ++ apps/fixtures/datadog-eval-reporter/evals/evals.config.ts | 4 +++- apps/fixtures/datadog-eval-reporter/package.json | 2 ++ pnpm-lock.yaml | 6 ++++++ 5 files changed, 17 insertions(+), 2 deletions(-) diff --git a/apps/fixtures/datadog-eval-reporter/README.md b/apps/fixtures/datadog-eval-reporter/README.md index 9392b3286c..3c7e7225e5 100644 --- a/apps/fixtures/datadog-eval-reporter/README.md +++ b/apps/fixtures/datadog-eval-reporter/README.md @@ -11,7 +11,10 @@ export DD_SITE="datadoghq.com" pnpm --filter datadog-eval-reporter-fixture eval ``` -The run creates a Datadog dataset with one record containing `Say hello.`, +Without both Datadog keys, the fixture omits the reporter so the shared local, +Postgres, and Vercel e2e matrices remain hermetic. + +The credentialed run creates a Datadog dataset with one record containing `Say hello.`, creates one LLM Observability Experiment against that dataset version, links the record to the smoke eval's synthetic experiment span, attaches the assertion metrics, and prints the Dataset and Experiment URLs. The fixture does not export diff --git a/apps/fixtures/datadog-eval-reporter/agent/agent.ts b/apps/fixtures/datadog-eval-reporter/agent/agent.ts index 0d9836c30c..b7b573d87e 100644 --- a/apps/fixtures/datadog-eval-reporter/agent/agent.ts +++ b/apps/fixtures/datadog-eval-reporter/agent/agent.ts @@ -1,7 +1,9 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; import { mockModel } from "eve/evals"; export default defineAgent({ + ...e2eAgentConfig(), model: mockModel(), modelContextWindowTokens: 1_000_000, }); diff --git a/apps/fixtures/datadog-eval-reporter/evals/evals.config.ts b/apps/fixtures/datadog-eval-reporter/evals/evals.config.ts index 6a2c105064..0dafda9b49 100644 --- a/apps/fixtures/datadog-eval-reporter/evals/evals.config.ts +++ b/apps/fixtures/datadog-eval-reporter/evals/evals.config.ts @@ -1,6 +1,8 @@ import { defineEvalConfig } from "eve/evals"; import { Datadog } from "eve/evals/reporters"; +const hasDatadogCredentials = Boolean(process.env.DD_API_KEY && process.env.DD_APP_KEY); + export default defineEvalConfig({ - reporters: [Datadog({ recordInputs: true })], + reporters: hasDatadogCredentials ? [Datadog({ recordInputs: true })] : [], }); diff --git a/apps/fixtures/datadog-eval-reporter/package.json b/apps/fixtures/datadog-eval-reporter/package.json index 800f87647b..1d0fc8a25f 100644 --- a/apps/fixtures/datadog-eval-reporter/package.json +++ b/apps/fixtures/datadog-eval-reporter/package.json @@ -7,6 +7,8 @@ "eval": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", + "@workflow/world-postgres": "catalog:", "dd-trace": "6.13.0", "eve": "workspace:*" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 84cad3d85d..fabf93a84d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -401,6 +401,12 @@ importers: apps/fixtures/datadog-eval-reporter: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../../../e2e/fixtures/e2e-config + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.40(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) dd-trace: specifier: 6.13.0 version: 6.13.0 From 1c777d5fa3ed190b3218b460df9ca56804d04935 Mon Sep 17 00:00:00 2001 From: Matan Kushner Date: Mon, 7 Sep 2026 09:41:49 +0900 Subject: [PATCH 8/8] chore(eve): refresh validation lockfile Signed-off-by: Matan Kushner --- pnpm-lock.yaml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c4f89e3de2..d649e5f40f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -409,7 +409,7 @@ importers: version: link:../../../e2e/fixtures/e2e-config '@workflow/world-postgres': specifier: 'catalog:' - version: 5.0.0-beta.40(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) + version: 5.0.0-beta.40(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.4)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) dd-trace: specifier: 6.13.0 version: 6.13.0 @@ -11380,10 +11380,6 @@ packages: resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==} engines: {node: '>=6'} - import-in-the-middle@3.3.1: - resolution: {integrity: sha512-0rymlHSFLwZ0ixx8DaQkoIyZojJPY2a0K2nEYslhKJ6jIYO/m0IcCb7iQsFPmS7WmKwISZiIrv5Icstrw/CmqA==} - engines: {node: '>=18'} - import-in-the-middle@3.3.3: resolution: {integrity: sha512-AiohS3H80sXO6owEltjGX+glb7qXaDhBoJb9XcQVH4UI207xu/bDLUcadVKp7Qe576reg9yr/PXZjV5qx8gfbA==} engines: {node: '>=18'} @@ -19010,7 +19006,7 @@ snapshots: dependencies: '@opentelemetry/api': 1.9.1 '@opentelemetry/api-logs': 0.219.0 - import-in-the-middle: 3.3.1 + import-in-the-middle: 3.3.3 require-in-the-middle: 8.0.1(supports-color@10.2.2) transitivePeerDependencies: - supports-color @@ -24538,7 +24534,7 @@ snapshots: cosmiconfig@8.3.6(typescript@7.0.2): dependencies: import-fresh: 3.3.1 - js-yaml: 4.1.1 + js-yaml: 4.3.2 parse-json: 5.2.0 path-type: 4.0.0 optionalDependencies: @@ -27062,12 +27058,6 @@ snapshots: parent-module: 1.0.1 resolve-from: 4.0.0 - import-in-the-middle@3.3.1: - dependencies: - cjs-module-lexer: 2.2.0 - es-module-lexer: 2.3.1 - module-details-from-path: 1.0.4 - import-in-the-middle@3.3.3: dependencies: cjs-module-lexer: 2.2.0 @@ -27455,7 +27445,6 @@ snapshots: js-yaml@4.3.2: dependencies: argparse: 2.0.1 - optional: true jsc-safe-url@0.2.4: {}