feat: add --model-provider flag for keyless Bedrock LLM judge evaluators

.gitignore

@@ -73,3 +73,6 @@ ProtocolTesting/
 browser-tests/.browser-test-env
 browser-tests/test-results/
 browser-tests/playwright-report/
+
+# E2E test output
+test-e2e-output/

e2e-tests/third-party-eval-lifecycle.test.ts

@@ -0,0 +1,214 @@
+import { parseJsonOutput, retry } from '../src/test-utils/index.js';
+import {
+  baseCanRun,
+  hasAws,
+  installCdkTarball,
+  runAgentCoreCLI,
+  teardownE2EProject,
+  writeAwsTargets,
+} from './e2e-helper.js';
+import { randomUUID } from 'node:crypto';
+import { mkdir, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+const canRun = baseCanRun && hasAws;
+
+describe.sequential('e2e: third-party evaluator lifecycle (DeepEval + Autoevals)', () => {
+  let testDir: string;
+  let projectPath: string;
+  const agentName = `E2e3pEval${String(Date.now()).slice(-8)}`;
+  const deepevalEvalName = 'answer_relevancy';
+  const autoevalsEvalName = 'exact_match';
+
+  beforeAll(async () => {
+    if (!canRun) return;
+
+    testDir = join(tmpdir(), `agentcore-e2e-3p-eval-${randomUUID()}`);
+    await mkdir(testDir, { recursive: true });
+
+    const result = await runAgentCoreCLI(
+      [
+        'create',
+        '--name',
+        agentName,
+        '--language',
+        'Python',
+        '--framework',
+        'Strands',
+        '--model-provider',
+        'Bedrock',
+        '--memory',
+        'none',
+        '--json',
+      ],
+      testDir
+    );
+    expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0);
+    projectPath = (parseJsonOutput(result.stdout) as { projectPath: string }).projectPath;
+
+    await writeAwsTargets(projectPath);
+    installCdkTarball(projectPath);
+  }, 300000);
+
+  afterAll(async () => {
+    if (projectPath && hasAws) {
+      await teardownE2EProject(projectPath, agentName, 'Bedrock');
+    }
+    if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
+  }, 600000);
+
+  const run = (args: string[]) => runAgentCoreCLI(args, projectPath);
+
+  it.skipIf(!canRun)(
+    'adds a DeepEval 3P evaluator with --3p-template-json',
+    async () => {
+      const templateJson = JSON.stringify({
+        library: 'deepeval',
+        metric: 'AnswerRelevancyMetric',
+        modelProvider: 'bedrock',
+        model: 'us.anthropic.claude-sonnet-4-20250514-v1:0',
+        params: { threshold: 0.5 },
+      });
+      const result = await run([
+        'add',
+        'evaluator',
+        '--name',
+        deepevalEvalName,
+        '--level',
+        'TRACE',
+        '--type',
+        'code-based',
+        '--3p-template-json',
+        templateJson,
+        '--json',
+      ]);
+      expect(result.exitCode, `Add DeepEval evaluator failed: ${result.stdout}`).toBe(0);
+      const json = parseJsonOutput(result.stdout) as { success: boolean; evaluatorName: string; codePath: string };
+      expect(json.success).toBe(true);
+      expect(json.evaluatorName).toBe(deepevalEvalName);
+      expect(json.codePath).toContain(deepevalEvalName);
+    },
+    60000
+  );
+
+  it.skipIf(!canRun)(
+    'adds an Autoevals 3P evaluator with --3p-template-json',
+    async () => {
+      const templateJson = JSON.stringify({
+        library: 'autoevals',
+        metric: 'ExactMatch',
+        modelProvider: 'bedrock',
+        model: 'us.anthropic.claude-sonnet-4-20250514-v1:0',
+      });
+      const result = await run([
+        'add',
+        'evaluator',
+        '--name',
+        autoevalsEvalName,
+        '--level',
+        'TRACE',
+        '--type',
+        'code-based',
+        '--3p-template-json',
+        templateJson,
+        '--json',
+      ]);
+      expect(result.exitCode, `Add Autoevals evaluator failed: ${result.stdout}`).toBe(0);
+      const json = parseJsonOutput(result.stdout) as { success: boolean; evaluatorName: string; codePath: string };
+      expect(json.success).toBe(true);
+      expect(json.evaluatorName).toBe(autoevalsEvalName);
+      expect(json.codePath).toContain(autoevalsEvalName);
+    },
+    60000
+  );
+
+  it.skipIf(!canRun)(
+    'deploys agent with 3P evaluators',
+    async () => {
+      const result = await run(['deploy', '--yes', '--json']);
+      if (result.exitCode !== 0) {
+        console.log('Deploy stdout:', result.stdout);
+        console.log('Deploy stderr:', result.stderr);
+      }
+      expect(result.exitCode, 'Deploy failed').toBe(0);
+      const json = parseJsonOutput(result.stdout) as { success: boolean };
+      expect(json.success).toBe(true);
+    },
+    600000
+  );
+
+  it.skipIf(!canRun)(
+    'invokes the deployed agent to generate traces',
+    async () => {
+      await retry(
+        async () => {
+          const result = await run(['invoke', '--prompt', 'What is 2+2?', '--runtime', agentName, '--json']);
+          expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0);
+          const json = parseJsonOutput(result.stdout) as { success: boolean };
+          expect(json.success).toBe(true);
+        },
+        3,
+        15000
+      );
+    },
+    180000
+  );
+
+  it.skipIf(!canRun)(
+    'runs on-demand evaluation with DeepEval 3P evaluator',
+    async () => {
+      await retry(
+        async () => {
+          const result = await run([
+            'run',
+            'eval',
+            '--runtime',
+            agentName,
+            '--evaluator',
+            deepevalEvalName,
+            '--days',
+            '1',
+            '--json',
+          ]);
+          expect(result.exitCode, `Run eval failed (stdout: ${result.stdout}, stderr: ${result.stderr})`).toBe(0);
+          const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
+          expect(json).toHaveProperty('success', true);
+          expect(json).toHaveProperty('run');
+        },
+        18,
+        10000
+      );
+    },
+    300000
+  );
+
+  it.skipIf(!canRun)(
+    'runs on-demand evaluation with Autoevals 3P evaluator',
+    async () => {
+      await retry(
+        async () => {
+          const result = await run([
+            'run',
+            'eval',
+            '--runtime',
+            agentName,
+            '--evaluator',
+            autoevalsEvalName,
+            '--days',
+            '1',
+            '--json',
+          ]);
+          expect(result.exitCode, `Run eval failed (stdout: ${result.stdout}, stderr: ${result.stderr})`).toBe(0);
+          const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
+          expect(json).toHaveProperty('success', true);
+          expect(json).toHaveProperty('run');
+        },
+        18,
+        10000
+      );
+    },
+    300000
+  );
+});

src/assets/__tests__/__snapshots__/assets.snapshot.test.ts.snap

@@ -742,6 +742,12 @@ exports[`Assets Directory Snapshots > File listing > should match the expected f
   "container/typescript/dockerignore.template",
   "datasets/predefined-v1.jsonl",
   "datasets/simulated-v1.jsonl",
+  "evaluators/autoevals-lambda/execution-role-policy.json",
+  "evaluators/autoevals-lambda/lambda_function.py",
+  "evaluators/autoevals-lambda/pyproject.toml",
+  "evaluators/deepeval-lambda/execution-role-policy.json",
+  "evaluators/deepeval-lambda/lambda_function.py",
+  "evaluators/deepeval-lambda/pyproject.toml",
   "evaluators/python-lambda/execution-role-policy.json",
   "evaluators/python-lambda/lambda_function.py",
   "evaluators/python-lambda/pyproject.toml",

src/assets/evaluators/autoevals-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+      "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["bedrock:InvokeModel"],
+      "Resource": "*"
+    }
+  ]
+}

src/assets/evaluators/autoevals-lambda/lambda_function.py

@@ -0,0 +1,37 @@
+{{#if ModelProviderBedrock}}
+import os
+
+# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION.
+os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2"))
+
+from autoevals import {{ EvaluatorClass }}, init
+from autoevals.litellm import LiteLLMClient
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+client = LiteLLMClient()
+init(client=client, default_model="{{ Model }}")
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{else}}
+from autoevals import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+    return adapter(evaluator_input, context)

src/assets/evaluators/autoevals-lambda/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (Autoevals)"
+requires-python = ">=3.10"
+dependencies = [
+    "bedrock-agentcore[autoevals]",
+    "autoevals>=0.0.80,<1.0.0",
+{{#if ModelProviderBedrock}}
+    # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge
+    "litellm>=1.60,<1.85",
+{{else}}
+    "openai>=1.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/deepeval-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+      "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["bedrock:InvokeModel"],
+      "Resource": "*"
+    }
+  ]
+}

src/assets/evaluators/deepeval-lambda/lambda_function.py

@@ -0,0 +1,29 @@
+import os
+
+os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval")
+os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES")
+os.chdir("/tmp")
+
+{{#if ModelProviderBedrock}}
+from deepeval.models import AmazonBedrockModel
+{{/if}}
+from deepeval.metrics import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
+
+{{#if ModelProviderBedrock}}
+model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2"))
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}))
+{{else}}
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}}))
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+    return adapter(evaluator_input, context)

src/assets/evaluators/deepeval-lambda/pyproject.toml

@@ -0,0 +1,19 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (DeepEval)"
+requires-python = ">=3.10"
+dependencies = [
+    "bedrock-agentcore[deepeval]",
+    "deepeval>=2.0.0,<3.0.0",
+{{#if ModelProviderBedrock}}
+    "aiobotocore>=2.13.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]