From 6fba5c8aefb230a2fd6638d8decfb117b3aac848 Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Tue, 16 Jun 2026 14:12:59 -0700 Subject: [PATCH 01/25] Add deepeval CLI integration: flags, templates, warnings, memory, unit tests --- .../execution-role-policy.json | 15 ++ .../deepeval-lambda/lambda_function.py | 4 + .../evaluators/deepeval-lambda/pyproject.toml | 16 ++ src/cli/primitives/EvaluatorPrimitive.ts | 153 +++++++++++- .../__tests__/EvaluatorPrimitive.test.ts | 231 +++++++++++++++++- src/cli/templates/EvaluatorRenderer.ts | 11 + src/schema/llm-compacted/agentcore.ts | 1 + src/schema/schemas/primitives/evaluator.ts | 1 + 8 files changed, 425 insertions(+), 7 deletions(-) create mode 100644 src/assets/evaluators/deepeval-lambda/execution-role-policy.json create mode 100644 src/assets/evaluators/deepeval-lambda/lambda_function.py create mode 100644 src/assets/evaluators/deepeval-lambda/pyproject.toml diff --git a/src/assets/evaluators/deepeval-lambda/execution-role-policy.json b/src/assets/evaluators/deepeval-lambda/execution-role-policy.json new file mode 100644 index 000000000..6b47af830 --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/execution-role-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" + }, + { + "Effect": "Allow", + "Action": ["bedrock:InvokeModel"], + "Resource": "*" + } + ] +} diff --git a/src/assets/evaluators/deepeval-lambda/lambda_function.py b/src/assets/evaluators/deepeval-lambda/lambda_function.py new file mode 100644 index 000000000..8738f0ea7 --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/lambda_function.py @@ -0,0 +1,4 @@ +from bedrock_agentcore.evaluation.integrations.deepeval import DeepEvalHandler +from deepeval.metrics import {{ MetricClass }} + +handler = DeepEvalHandler(metric={{ MetricClass }}({{{ MetricParams }}})) diff --git a/src/assets/evaluators/deepeval-lambda/pyproject.toml b/src/assets/evaluators/deepeval-lambda/pyproject.toml new file mode 100644 index 000000000..ac974774c --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/pyproject.toml @@ -0,0 +1,16 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "{{ Name }}" +version = "0.1.0" +description = "AgentCore Code-Based Evaluator (DeepEval)" +requires-python = ">=3.10" +dependencies = [ + "bedrock-agentcore[deepeval]>=1.6.0", + "deepeval>=2.0.0", +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index 51f82f512..bf5b056e6 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -6,7 +6,7 @@ import { getErrorMessage } from '../errors'; import type { RemovalPreview, SchemaChange } from '../operations/remove/types'; import { runCliCommand } from '../telemetry/cli-command-run.js'; import { EvaluatorLevel, EvaluatorType, standardize } from '../telemetry/schemas/common-shapes.js'; -import { renderCodeBasedEvaluatorTemplate } from '../templates/EvaluatorRenderer'; +import { renderCodeBasedEvaluatorTemplate, renderDeepEvalEvaluatorTemplate } from '../templates/EvaluatorRenderer'; import { requireTTY } from '../tui/guards/tty'; import { LEVEL_PLACEHOLDERS, @@ -21,18 +21,58 @@ import { existsSync } from 'node:fs'; import { rm } from 'node:fs/promises'; import { dirname, join } from 'node:path'; +export interface ThirdPartyLibraryOptions { + library: 'deepeval'; + metricClass: string; + metricParams?: string; +} + export interface AddEvaluatorOptions { name: string; level: EvaluationLevel; description?: string; config: EvaluatorConfig; kmsKeyArn?: string; + thirdParty?: ThirdPartyLibraryOptions; } export type RemovableEvaluator = RemovableResource; const DEFAULT_CODE_ENTRYPOINT = 'lambda_function.handler'; const DEFAULT_CODE_TIMEOUT = 60; +const DEEPEVAL_DEFAULT_TIMEOUT = 300; +const DEEPEVAL_DEFAULT_MEMORY_MB = 1024; + +const METRICS_REQUIRING_RETRIEVAL_CONTEXT = new Set([ + 'FaithfulnessMetric', + 'HallucinationMetric', + 'ContextualRelevancyMetric', + 'ContextualPrecisionMetric', + 'ContextualRecallMetric', +]); + +const METRICS_REQUIRING_EXPECTED_OUTPUT = new Set(['ContextualPrecisionMetric', 'ContextualRecallMetric']); + +export function jsonToPythonValue(value: unknown): string { + if (value === null) return 'None'; + if (value === true) return 'True'; + if (value === false) return 'False'; + if (typeof value === 'number') return String(value); + if (typeof value === 'string') return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(jsonToPythonValue).join(', ')}]`; + if (typeof value === 'object') { + const entries = Object.entries(value as Record); + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}: ${jsonToPythonValue(v)}`).join(', ')}}`; + } + return String(value); +} + +export function jsonToKwargs(json: string): string { + const obj = JSON.parse(json) as Record; + return Object.entries(obj) + .map(([key, value]) => `${key}=${jsonToPythonValue(value)}`) + .join(', '); +} /** * EvaluatorPrimitive handles all evaluator add/remove operations. @@ -53,7 +93,19 @@ export class EvaluatorPrimitive extends BasePrimitive', `[LLM] Rating scale preset: ${presetIds.join(', ')} (default: 1-5-quality)`) .option('--lambda-arn ', '[Code-based] Existing Lambda function ARN (external)') .option('--timeout ', '[Code-based] Lambda timeout in seconds, 1-300 (default: 60)') + .option( + '--from-3p-library ', + 'Third-party evaluation library to use (currently: deepeval)' + ) + .option('--metric ', '[3P library] Metric class name (e.g. AnswerRelevancyMetric)') + .option('--parameters ', '[3P library] JSON string of metric constructor kwargs') + .option('--memory ', '[3P library] Lambda memory size in MB, 128-10240 (default: 1024 for deepeval)') .option( '--config ', 'Path to evaluator config JSON file (overrides --model, --instructions, --rating-scale) [non-interactive]' @@ -200,6 +259,10 @@ export class EvaluatorPrimitive extends BasePrimitive 10240) { + fail('--memory must be an integer between 128 and 10240'); + } + } + + // Default --type to code-based when --from-3p-library is set + const evalType = cliOptions.type ?? (from3pLibrary ? 'code-based' : 'llm-as-a-judge'); if (evalType !== 'llm-as-a-judge' && evalType !== 'code-based') { fail(`Invalid --type "${evalType}". Must be one of: llm-as-a-judge, code-based`); } @@ -234,15 +327,31 @@ export class EvaluatorPrimitive extends BasePrimitive { const project = await this.readProjectSpec(); diff --git a/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts b/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts index a521e6a78..e5c50f236 100644 --- a/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts +++ b/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts @@ -1,5 +1,5 @@ import type { EvaluatorConfig } from '../../../schema'; -import { EvaluatorPrimitive } from '../EvaluatorPrimitive.js'; +import { EvaluatorPrimitive, jsonToKwargs, jsonToPythonValue } from '../EvaluatorPrimitive.js'; import { afterEach, describe, expect, it, vi } from 'vitest'; const mockReadProjectSpec = vi.fn(); @@ -27,6 +27,14 @@ vi.mock('../../../lib/index.js', () => ({ }, })); +const mockRenderCodeBased = vi.fn().mockResolvedValue(undefined); +const mockRenderDeepEval = vi.fn().mockResolvedValue(undefined); + +vi.mock('../../templates/EvaluatorRenderer', () => ({ + renderCodeBasedEvaluatorTemplate: (...args: unknown[]) => mockRenderCodeBased(...args), + renderDeepEvalEvaluatorTemplate: (...args: unknown[]) => mockRenderDeepEval(...args), +})); + const validConfig: EvaluatorConfig = { llmAsAJudge: { model: 'us.anthropic.claude-sonnet-4-5-20250929-v1:0', @@ -256,4 +264,225 @@ describe('EvaluatorPrimitive', () => { expect(await primitive.getAllNames()).toEqual([]); }); }); + + describe('buildDeepEvalConfig', () => { + it('returns managed code-based config with deepeval defaults', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildDeepEvalConfig']('my_eval'); + + expect(config).toEqual({ + codeBased: { + managed: { + codeLocation: 'app/my_eval/', + entrypoint: 'lambda_function.handler', + timeoutSeconds: 300, + memorySizeMb: 1024, + additionalPolicies: ['execution-role-policy.json'], + }, + }, + }); + }); + + it('respects custom timeout', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildDeepEvalConfig']('my_eval', '120'); + + expect(config.codeBased!.managed!.timeoutSeconds).toBe(120); + }); + + it('respects custom memory', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildDeepEvalConfig']('my_eval', undefined, '2048'); + + expect(config.codeBased!.managed!.memorySizeMb).toBe(2048); + }); + + it('respects both custom timeout and memory', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildDeepEvalConfig']('my_eval', '60', '512'); + + expect(config.codeBased!.managed!.timeoutSeconds).toBe(60); + expect(config.codeBased!.managed!.memorySizeMb).toBe(512); + }); + + it('sets codeLocation based on evaluator name', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildDeepEvalConfig']('relevancy_check'); + + expect(config.codeBased!.managed!.codeLocation).toBe('app/relevancy_check/'); + }); + }); + + describe('add with thirdParty (deepeval)', () => { + const deepEvalConfig: EvaluatorConfig = { + codeBased: { + managed: { + codeLocation: 'app/deep_eval/', + entrypoint: 'lambda_function.handler', + timeoutSeconds: 300, + memorySizeMb: 1024, + additionalPolicies: ['execution-role-policy.json'], + }, + }, + }; + + it('calls renderDeepEvalEvaluatorTemplate when thirdParty.library is deepeval', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + const result = await primitive.add({ + name: 'deep_eval', + level: 'SESSION', + config: deepEvalConfig, + thirdParty: { + library: 'deepeval', + metricClass: 'AnswerRelevancyMetric', + metricParams: 'threshold=0.7', + }, + }); + + expect(result.success).toBe(true); + expect(result).toHaveProperty('codePath', 'app/deep_eval/'); + expect(mockRenderDeepEval).toHaveBeenCalledOnce(); + expect(mockRenderDeepEval).toHaveBeenCalledWith( + { Name: 'deep_eval', MetricClass: 'AnswerRelevancyMetric', MetricParams: 'threshold=0.7' }, + expect.stringContaining('app/deep_eval') + ); + expect(mockRenderCodeBased).not.toHaveBeenCalled(); + }); + + it('passes empty string for MetricParams when not provided', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + await primitive.add({ + name: 'deep_eval', + level: 'SESSION', + config: deepEvalConfig, + thirdParty: { + library: 'deepeval', + metricClass: 'HallucinationMetric', + }, + }); + + expect(mockRenderDeepEval).toHaveBeenCalledWith( + { Name: 'deep_eval', MetricClass: 'HallucinationMetric', MetricParams: '' }, + expect.any(String) + ); + }); + + it('calls renderCodeBasedEvaluatorTemplate when thirdParty is not set', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + const managedConfig: EvaluatorConfig = { + codeBased: { + managed: { + codeLocation: 'app/plain_eval/', + entrypoint: 'lambda_function.handler', + timeoutSeconds: 60, + additionalPolicies: ['execution-role-policy.json'], + }, + }, + }; + + await primitive.add({ + name: 'plain_eval', + level: 'SESSION', + config: managedConfig, + }); + + expect(mockRenderCodeBased).toHaveBeenCalledOnce(); + expect(mockRenderCodeBased).toHaveBeenCalledWith('plain_eval', expect.stringContaining('app/plain_eval')); + expect(mockRenderDeepEval).not.toHaveBeenCalled(); + }); + }); +}); + +describe('jsonToPythonValue', () => { + it('converts null to None', () => { + expect(jsonToPythonValue(null)).toBe('None'); + }); + + it('converts true to True', () => { + expect(jsonToPythonValue(true)).toBe('True'); + }); + + it('converts false to False', () => { + expect(jsonToPythonValue(false)).toBe('False'); + }); + + it('converts integers', () => { + expect(jsonToPythonValue(42)).toBe('42'); + }); + + it('converts floats', () => { + expect(jsonToPythonValue(0.7)).toBe('0.7'); + }); + + it('converts strings with quotes', () => { + expect(jsonToPythonValue('gpt-4')).toBe('"gpt-4"'); + }); + + it('converts arrays', () => { + expect(jsonToPythonValue([1, 'two', true])).toBe('[1, "two", True]'); + }); + + it('converts nested objects to Python dicts', () => { + expect(jsonToPythonValue({ key: 'value', n: 3 })).toBe('{"key": "value", "n": 3}'); + }); + + it('handles empty arrays', () => { + expect(jsonToPythonValue([])).toBe('[]'); + }); + + it('handles empty objects', () => { + expect(jsonToPythonValue({})).toBe('{}'); + }); +}); + +describe('jsonToKwargs', () => { + it('converts simple number parameter', () => { + expect(jsonToKwargs('{"threshold": 0.7}')).toBe('threshold=0.7'); + }); + + it('converts simple string parameter', () => { + expect(jsonToKwargs('{"model": "gpt-4"}')).toBe('model="gpt-4"'); + }); + + it('converts multiple parameters', () => { + const result = jsonToKwargs('{"threshold": 0.7, "model": "gpt-4"}'); + expect(result).toBe('threshold=0.7, model="gpt-4"'); + }); + + it('converts boolean parameters', () => { + expect(jsonToKwargs('{"verbose": true, "strict": false}')).toBe('verbose=True, strict=False'); + }); + + it('converts null parameters', () => { + expect(jsonToKwargs('{"callback": null}')).toBe('callback=None'); + }); + + it('converts array parameters', () => { + expect(jsonToKwargs('{"tools": ["search", "calculate"]}')).toBe('tools=["search", "calculate"]'); + }); + + it('converts nested object parameters', () => { + const result = jsonToKwargs('{"config": {"temperature": 0.5}}'); + expect(result).toBe('config={"temperature": 0.5}'); + }); + + it('handles mixed types', () => { + const input = '{"threshold": 0.7, "model": "gpt-4", "verbose": true, "tags": ["eval"], "fallback": null}'; + const result = jsonToKwargs(input); + expect(result).toBe('threshold=0.7, model="gpt-4", verbose=True, tags=["eval"], fallback=None'); + }); + + it('throws on invalid JSON', () => { + expect(() => jsonToKwargs('not json')).toThrow(); + }); + + it('returns empty string for empty object', () => { + expect(jsonToKwargs('{}')).toBe(''); + }); }); diff --git a/src/cli/templates/EvaluatorRenderer.ts b/src/cli/templates/EvaluatorRenderer.ts index 6b2f22c24..458106ddf 100644 --- a/src/cli/templates/EvaluatorRenderer.ts +++ b/src/cli/templates/EvaluatorRenderer.ts @@ -10,3 +10,14 @@ export async function renderCodeBasedEvaluatorTemplate(evaluatorName: string, ou const templateDir = getTemplatePath('evaluators', 'python-lambda'); await copyAndRenderDir(templateDir, outputDir, { Name: evaluatorName }); } + +export interface DeepEvalTemplateData { + Name: string; + MetricClass: string; + MetricParams: string; +} + +export async function renderDeepEvalEvaluatorTemplate(data: DeepEvalTemplateData, outputDir: string): Promise { + const templateDir = getTemplatePath('evaluators', 'deepeval-lambda'); + await copyAndRenderDir(templateDir, outputDir, data); +} diff --git a/src/schema/llm-compacted/agentcore.ts b/src/schema/llm-compacted/agentcore.ts index 2ab5290cf..2ec7ebc06 100644 --- a/src/schema/llm-compacted/agentcore.ts +++ b/src/schema/llm-compacted/agentcore.ts @@ -233,6 +233,7 @@ interface ManagedCodeBasedConfig { codeLocation: string; entrypoint: string; // default 'lambda_function.handler' timeoutSeconds: number; // @min 1 @max 300 (default 60) + memorySizeMb?: number; // @min 128 @max 10240 additionalPolicies?: string[]; } diff --git a/src/schema/schemas/primitives/evaluator.ts b/src/schema/schemas/primitives/evaluator.ts index 97d772d52..7bb2b6a23 100644 --- a/src/schema/schemas/primitives/evaluator.ts +++ b/src/schema/schemas/primitives/evaluator.ts @@ -81,6 +81,7 @@ export const ManagedCodeBasedConfigSchema = z.object({ codeLocation: z.string().min(1), entrypoint: z.string().min(1).default('lambda_function.handler'), timeoutSeconds: z.number().int().min(1).max(300).default(60), + memorySizeMb: z.number().int().min(128).max(10240).optional(), additionalPolicies: z.array(z.string().min(1)).optional(), }); From d44430a244918cbb04b0ba153cb533c0c57c55a5 Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Thu, 25 Jun 2026 14:35:32 -0700 Subject: [PATCH 02/25] Refactor CLI to registry-driven third-party evaluator framework (DeepEval + Autoevals) --- .../execution-role-policy.json | 15 ++ .../autoevals-lambda/lambda_function.py | 4 + .../autoevals-lambda/pyproject.toml | 16 ++ .../deepeval-lambda/lambda_function.py | 6 +- src/cli/primitives/EvaluatorPrimitive.ts | 177 ++++++++++---- .../__tests__/EvaluatorPrimitive.test.ts | 215 ++++++++++++++---- src/cli/templates/EvaluatorRenderer.ts | 14 +- 7 files changed, 347 insertions(+), 100 deletions(-) create mode 100644 src/assets/evaluators/autoevals-lambda/execution-role-policy.json create mode 100644 src/assets/evaluators/autoevals-lambda/lambda_function.py create mode 100644 src/assets/evaluators/autoevals-lambda/pyproject.toml diff --git a/src/assets/evaluators/autoevals-lambda/execution-role-policy.json b/src/assets/evaluators/autoevals-lambda/execution-role-policy.json new file mode 100644 index 000000000..6b47af830 --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/execution-role-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" + }, + { + "Effect": "Allow", + "Action": ["bedrock:InvokeModel"], + "Resource": "*" + } + ] +} diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py new file mode 100644 index 000000000..7c6a256b9 --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -0,0 +1,4 @@ +from bedrock_agentcore.evaluation.integrations.autoevals import AutoevalsAdapter +from autoevals import {{ EvaluatorClass }} + +handler = AutoevalsAdapter(scorer={{ EvaluatorClass }}({{{ EvaluatorParams }}})) diff --git a/src/assets/evaluators/autoevals-lambda/pyproject.toml b/src/assets/evaluators/autoevals-lambda/pyproject.toml new file mode 100644 index 000000000..b2e920a3f --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/pyproject.toml @@ -0,0 +1,16 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "{{ Name }}" +version = "0.1.0" +description = "AgentCore Code-Based Evaluator (Autoevals)" +requires-python = ">=3.10" +dependencies = [ + "bedrock-agentcore[autoevals]>=1.6.0", + "autoevals>=0.0.80", +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/assets/evaluators/deepeval-lambda/lambda_function.py b/src/assets/evaluators/deepeval-lambda/lambda_function.py index 8738f0ea7..3b1033daf 100644 --- a/src/assets/evaluators/deepeval-lambda/lambda_function.py +++ b/src/assets/evaluators/deepeval-lambda/lambda_function.py @@ -1,4 +1,4 @@ -from bedrock_agentcore.evaluation.integrations.deepeval import DeepEvalHandler -from deepeval.metrics import {{ MetricClass }} +from bedrock_agentcore.evaluation.integrations.deepeval import DeepEvalAdapter +from deepeval.metrics import {{ EvaluatorClass }} -handler = DeepEvalHandler(metric={{ MetricClass }}({{{ MetricParams }}})) +handler = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index bf5b056e6..fe9fadb13 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -6,7 +6,7 @@ import { getErrorMessage } from '../errors'; import type { RemovalPreview, SchemaChange } from '../operations/remove/types'; import { runCliCommand } from '../telemetry/cli-command-run.js'; import { EvaluatorLevel, EvaluatorType, standardize } from '../telemetry/schemas/common-shapes.js'; -import { renderCodeBasedEvaluatorTemplate, renderDeepEvalEvaluatorTemplate } from '../templates/EvaluatorRenderer'; +import { renderCodeBasedEvaluatorTemplate, renderThirdPartyEvaluatorTemplate } from '../templates/EvaluatorRenderer'; import { requireTTY } from '../tui/guards/tty'; import { LEVEL_PLACEHOLDERS, @@ -21,8 +21,83 @@ import { existsSync } from 'node:fs'; import { rm } from 'node:fs/promises'; import { dirname, join } from 'node:path'; +// ============================================================================ +// Third-Party Library Registry +// ============================================================================ + +interface MetricWarning { + metrics: Set; + message: string; +} + +export interface ThirdPartyLibraryConfig { + templateDir: string; + defaultTimeoutSeconds: number; + defaultMemorySizeMb: number; + warnings: MetricWarning[]; +} + +export const THIRD_PARTY_EVALUATOR_LIBRARIES = { + deepeval: { + templateDir: 'deepeval-lambda', + defaultTimeoutSeconds: 300, + defaultMemorySizeMb: 1024, + warnings: [ + { + metrics: new Set([ + 'FaithfulnessMetric', + 'HallucinationMetric', + 'ContextualRelevancyMetric', + 'ContextualPrecisionMetric', + 'ContextualRecallMetric', + ]), + message: + 'requires retrieval_context from tool-role messages. ' + + 'If your agent has no tool calls, the evaluator will return MISSING_REQUIRED_FIELD at runtime.', + }, + { + metrics: new Set(['ContextualPrecisionMetric', 'ContextualRecallMetric']), + message: + 'requires expected_output via evaluationReferenceInputs. ' + + 'Caller must provide referenceInputs when invoking the Evaluate API.', + }, + ], + }, + autoevals: { + templateDir: 'autoevals-lambda', + defaultTimeoutSeconds: 60, + defaultMemorySizeMb: 512, + warnings: [ + { + metrics: new Set(['Factuality', 'ClosedQA']), + message: + 'requires expected_output via evaluationReferenceInputs. ' + + 'Caller must provide referenceInputs when invoking the Evaluate API.', + }, + { + metrics: new Set(['SQL']), + message: + 'requires expected_output (reference SQL) via evaluationReferenceInputs. ' + + 'Caller must provide referenceInputs when invoking the Evaluate API.', + }, + ], + }, +} satisfies Record; + +export type ThirdPartyLibrary = keyof typeof THIRD_PARTY_EVALUATOR_LIBRARIES; + +const SUPPORTED_LIBRARIES: ThirdPartyLibrary[] = Object.keys(THIRD_PARTY_EVALUATOR_LIBRARIES) as ThirdPartyLibrary[]; + +function isSupportedLibrary(value: string): value is ThirdPartyLibrary { + return value in THIRD_PARTY_EVALUATOR_LIBRARIES; +} + +// ============================================================================ +// Types +// ============================================================================ + export interface ThirdPartyLibraryOptions { - library: 'deepeval'; + library: ThirdPartyLibrary; metricClass: string; metricParams?: string; } @@ -38,20 +113,12 @@ export interface AddEvaluatorOptions { export type RemovableEvaluator = RemovableResource; +// ============================================================================ +// Constants & Utilities +// ============================================================================ + const DEFAULT_CODE_ENTRYPOINT = 'lambda_function.handler'; const DEFAULT_CODE_TIMEOUT = 60; -const DEEPEVAL_DEFAULT_TIMEOUT = 300; -const DEEPEVAL_DEFAULT_MEMORY_MB = 1024; - -const METRICS_REQUIRING_RETRIEVAL_CONTEXT = new Set([ - 'FaithfulnessMetric', - 'HallucinationMetric', - 'ContextualRelevancyMetric', - 'ContextualPrecisionMetric', - 'ContextualRecallMetric', -]); - -const METRICS_REQUIRING_EXPECTED_OUTPUT = new Set(['ContextualPrecisionMetric', 'ContextualRecallMetric']); export function jsonToPythonValue(value: unknown): string { if (value === null) return 'None'; @@ -64,7 +131,7 @@ export function jsonToPythonValue(value: unknown): string { const entries = Object.entries(value as Record); return `{${entries.map(([k, v]) => `${JSON.stringify(k)}: ${jsonToPythonValue(v)}`).join(', ')}}`; } - return String(value); + return String(value as string | number | boolean); } export function jsonToKwargs(json: string): string { @@ -74,6 +141,20 @@ export function jsonToKwargs(json: string): string { .join(', '); } +function getWarningsForMetric(libraryConfig: ThirdPartyLibraryConfig, metricClass: string): string[] { + const messages: string[] = []; + for (const warning of libraryConfig.warnings) { + if (warning.metrics.has(metricClass)) { + messages.push(`⚠️ ${metricClass} ${warning.message}`); + } + } + return messages; +} + +// ============================================================================ +// EvaluatorPrimitive +// ============================================================================ + /** * EvaluatorPrimitive handles all evaluator add/remove operations. */ @@ -94,12 +175,14 @@ export class EvaluatorPrimitive extends BasePrimitive', `[LLM] Rating scale preset: ${presetIds.join(', ')} (default: 1-5-quality)`) .option('--lambda-arn ', '[Code-based] Existing Lambda function ARN (external)') .option('--timeout ', '[Code-based] Lambda timeout in seconds, 1-300 (default: 60)') - .option( - '--from-3p-library ', - 'Third-party evaluation library to use (currently: deepeval)' - ) - .option('--metric ', '[3P library] Metric class name (e.g. AnswerRelevancyMetric)') + .option('--from-3p-library ', `Third-party evaluation library (${SUPPORTED_LIBRARIES.join(', ')})`) + .option('--metric ', '[3P library] Metric/evaluator class name (e.g. AnswerRelevancyMetric)') .option('--parameters ', '[3P library] JSON string of metric constructor kwargs') - .option('--memory ', '[3P library] Lambda memory size in MB, 128-10240 (default: 1024 for deepeval)') + .option('--memory ', '[3P library] Lambda memory size in MB, 128-10240') .option( '--config ', 'Path to evaluator config JSON file (overrides --model, --instructions, --rating-scale) [non-interactive]' @@ -288,10 +368,11 @@ export class EvaluatorPrimitive extends BasePrimitive ({ })); const mockRenderCodeBased = vi.fn().mockResolvedValue(undefined); -const mockRenderDeepEval = vi.fn().mockResolvedValue(undefined); +const mockRenderThirdParty = vi.fn().mockResolvedValue(undefined); vi.mock('../../templates/EvaluatorRenderer', () => ({ renderCodeBasedEvaluatorTemplate: (...args: unknown[]) => mockRenderCodeBased(...args), - renderDeepEvalEvaluatorTemplate: (...args: unknown[]) => mockRenderDeepEval(...args), + renderThirdPartyEvaluatorTemplate: (...args: unknown[]) => mockRenderThirdParty(...args), })); const validConfig: EvaluatorConfig = { @@ -265,55 +270,79 @@ describe('EvaluatorPrimitive', () => { }); }); - describe('buildDeepEvalConfig', () => { - it('returns managed code-based config with deepeval defaults', () => { - // eslint-disable-next-line @typescript-eslint/dot-notation - const config = primitive['buildDeepEvalConfig']('my_eval'); - - expect(config).toEqual({ - codeBased: { - managed: { - codeLocation: 'app/my_eval/', - entrypoint: 'lambda_function.handler', - timeoutSeconds: 300, - memorySizeMb: 1024, - additionalPolicies: ['execution-role-policy.json'], + describe('buildThirdPartyConfig', () => { + describe('deepeval', () => { + const deepevalConfig = THIRD_PARTY_EVALUATOR_LIBRARIES.deepeval; + + it('returns config with deepeval defaults (300s, 1024MB)', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildThirdPartyConfig']('my_eval', deepevalConfig); + + expect(config).toEqual({ + codeBased: { + managed: { + codeLocation: 'app/my_eval/', + entrypoint: 'lambda_function.handler', + timeoutSeconds: 300, + memorySizeMb: 1024, + additionalPolicies: ['execution-role-policy.json'], + }, }, - }, + }); }); - }); - it('respects custom timeout', () => { - // eslint-disable-next-line @typescript-eslint/dot-notation - const config = primitive['buildDeepEvalConfig']('my_eval', '120'); + it('respects custom timeout', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildThirdPartyConfig']('my_eval', deepevalConfig, '120'); - expect(config.codeBased!.managed!.timeoutSeconds).toBe(120); - }); + expect(config.codeBased!.managed!.timeoutSeconds).toBe(120); + }); - it('respects custom memory', () => { - // eslint-disable-next-line @typescript-eslint/dot-notation - const config = primitive['buildDeepEvalConfig']('my_eval', undefined, '2048'); + it('respects custom memory', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildThirdPartyConfig']('my_eval', deepevalConfig, undefined, '2048'); - expect(config.codeBased!.managed!.memorySizeMb).toBe(2048); + expect(config.codeBased!.managed!.memorySizeMb).toBe(2048); + }); }); - it('respects both custom timeout and memory', () => { - // eslint-disable-next-line @typescript-eslint/dot-notation - const config = primitive['buildDeepEvalConfig']('my_eval', '60', '512'); + describe('autoevals', () => { + const autoevalsConfig = THIRD_PARTY_EVALUATOR_LIBRARIES.autoevals; + + it('returns config with autoevals defaults (60s, 512MB)', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildThirdPartyConfig']('fact_check', autoevalsConfig); + + expect(config).toEqual({ + codeBased: { + managed: { + codeLocation: 'app/fact_check/', + entrypoint: 'lambda_function.handler', + timeoutSeconds: 60, + memorySizeMb: 512, + additionalPolicies: ['execution-role-policy.json'], + }, + }, + }); + }); - expect(config.codeBased!.managed!.timeoutSeconds).toBe(60); - expect(config.codeBased!.managed!.memorySizeMb).toBe(512); - }); + it('respects custom timeout', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildThirdPartyConfig']('fact_check', autoevalsConfig, '180'); - it('sets codeLocation based on evaluator name', () => { - // eslint-disable-next-line @typescript-eslint/dot-notation - const config = primitive['buildDeepEvalConfig']('relevancy_check'); + expect(config.codeBased!.managed!.timeoutSeconds).toBe(180); + }); - expect(config.codeBased!.managed!.codeLocation).toBe('app/relevancy_check/'); + it('respects custom memory', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildThirdPartyConfig']('fact_check', autoevalsConfig, undefined, '1024'); + + expect(config.codeBased!.managed!.memorySizeMb).toBe(1024); + }); }); }); - describe('add with thirdParty (deepeval)', () => { + describe('add with thirdParty', () => { const deepEvalConfig: EvaluatorConfig = { codeBased: { managed: { @@ -326,7 +355,19 @@ describe('EvaluatorPrimitive', () => { }, }; - it('calls renderDeepEvalEvaluatorTemplate when thirdParty.library is deepeval', async () => { + const autoevalsEvalConfig: EvaluatorConfig = { + codeBased: { + managed: { + codeLocation: 'app/auto_eval/', + entrypoint: 'lambda_function.handler', + timeoutSeconds: 60, + memorySizeMb: 512, + additionalPolicies: ['execution-role-policy.json'], + }, + }, + }; + + it('calls renderThirdPartyEvaluatorTemplate with deepeval templateDir', async () => { mockReadProjectSpec.mockResolvedValue(makeProject()); mockWriteProjectSpec.mockResolvedValue(undefined); @@ -343,15 +384,42 @@ describe('EvaluatorPrimitive', () => { expect(result.success).toBe(true); expect(result).toHaveProperty('codePath', 'app/deep_eval/'); - expect(mockRenderDeepEval).toHaveBeenCalledOnce(); - expect(mockRenderDeepEval).toHaveBeenCalledWith( - { Name: 'deep_eval', MetricClass: 'AnswerRelevancyMetric', MetricParams: 'threshold=0.7' }, + expect(mockRenderThirdParty).toHaveBeenCalledOnce(); + expect(mockRenderThirdParty).toHaveBeenCalledWith( + 'deepeval-lambda', + { Name: 'deep_eval', EvaluatorClass: 'AnswerRelevancyMetric', EvaluatorParams: 'threshold=0.7' }, expect.stringContaining('app/deep_eval') ); expect(mockRenderCodeBased).not.toHaveBeenCalled(); }); - it('passes empty string for MetricParams when not provided', async () => { + it('calls renderThirdPartyEvaluatorTemplate with autoevals templateDir', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + const result = await primitive.add({ + name: 'auto_eval', + level: 'SESSION', + config: autoevalsEvalConfig, + thirdParty: { + library: 'autoevals', + metricClass: 'Factuality', + metricParams: '', + }, + }); + + expect(result.success).toBe(true); + expect(result).toHaveProperty('codePath', 'app/auto_eval/'); + expect(mockRenderThirdParty).toHaveBeenCalledOnce(); + expect(mockRenderThirdParty).toHaveBeenCalledWith( + 'autoevals-lambda', + { Name: 'auto_eval', EvaluatorClass: 'Factuality', EvaluatorParams: '' }, + expect.stringContaining('app/auto_eval') + ); + expect(mockRenderCodeBased).not.toHaveBeenCalled(); + }); + + it('passes empty string for EvaluatorParams when not provided', async () => { mockReadProjectSpec.mockResolvedValue(makeProject()); mockWriteProjectSpec.mockResolvedValue(undefined); @@ -365,8 +433,9 @@ describe('EvaluatorPrimitive', () => { }, }); - expect(mockRenderDeepEval).toHaveBeenCalledWith( - { Name: 'deep_eval', MetricClass: 'HallucinationMetric', MetricParams: '' }, + expect(mockRenderThirdParty).toHaveBeenCalledWith( + 'deepeval-lambda', + { Name: 'deep_eval', EvaluatorClass: 'HallucinationMetric', EvaluatorParams: '' }, expect.any(String) ); }); @@ -394,11 +463,65 @@ describe('EvaluatorPrimitive', () => { expect(mockRenderCodeBased).toHaveBeenCalledOnce(); expect(mockRenderCodeBased).toHaveBeenCalledWith('plain_eval', expect.stringContaining('app/plain_eval')); - expect(mockRenderDeepEval).not.toHaveBeenCalled(); + expect(mockRenderThirdParty).not.toHaveBeenCalled(); }); }); }); +describe('THIRD_PARTY_EVALUATOR_LIBRARIES registry', () => { + it('contains deepeval with expected defaults', () => { + const config = THIRD_PARTY_EVALUATOR_LIBRARIES.deepeval; + expect(config).toBeDefined(); + expect(config.templateDir).toBe('deepeval-lambda'); + expect(config.defaultTimeoutSeconds).toBe(300); + expect(config.defaultMemorySizeMb).toBe(1024); + }); + + it('contains autoevals with expected defaults', () => { + const config = THIRD_PARTY_EVALUATOR_LIBRARIES.autoevals; + expect(config).toBeDefined(); + expect(config.templateDir).toBe('autoevals-lambda'); + expect(config.defaultTimeoutSeconds).toBe(60); + expect(config.defaultMemorySizeMb).toBe(512); + }); + + it('deepeval has warnings for retrieval_context metrics', () => { + const config = THIRD_PARTY_EVALUATOR_LIBRARIES.deepeval; + const retrievalWarning = config.warnings.find(w => w.metrics.has('FaithfulnessMetric')); + expect(retrievalWarning).toBeDefined(); + expect(retrievalWarning!.message).toContain('retrieval_context'); + expect(retrievalWarning!.metrics.has('HallucinationMetric')).toBe(true); + expect(retrievalWarning!.metrics.has('ContextualRelevancyMetric')).toBe(true); + }); + + it('deepeval has warnings for expected_output metrics', () => { + const config = THIRD_PARTY_EVALUATOR_LIBRARIES.deepeval; + const expectedWarning = config.warnings.find(w => w.metrics.has('ContextualPrecisionMetric')); + expect(expectedWarning).toBeDefined(); + expect(expectedWarning!.message).toContain('expected_output'); + }); + + it('autoevals has warnings for reference-input metrics', () => { + const config = THIRD_PARTY_EVALUATOR_LIBRARIES.autoevals; + const factWarning = config.warnings.find(w => w.metrics.has('Factuality')); + expect(factWarning).toBeDefined(); + expect(factWarning!.message).toContain('expected_output'); + expect(factWarning!.metrics.has('ClosedQA')).toBe(true); + }); + + it('autoevals has warnings for SQL metric', () => { + const config = THIRD_PARTY_EVALUATOR_LIBRARIES.autoevals; + const sqlWarning = config.warnings.find(w => w.metrics.has('SQL')); + expect(sqlWarning).toBeDefined(); + expect(sqlWarning!.message).toContain('reference SQL'); + }); + + it('does not contain unsupported libraries', () => { + expect((THIRD_PARTY_EVALUATOR_LIBRARIES as Record).ragas).toBeUndefined(); + expect((THIRD_PARTY_EVALUATOR_LIBRARIES as Record).langsmith).toBeUndefined(); + }); +}); + describe('jsonToPythonValue', () => { it('converts null to None', () => { expect(jsonToPythonValue(null)).toBe('None'); diff --git a/src/cli/templates/EvaluatorRenderer.ts b/src/cli/templates/EvaluatorRenderer.ts index 458106ddf..6b50242e5 100644 --- a/src/cli/templates/EvaluatorRenderer.ts +++ b/src/cli/templates/EvaluatorRenderer.ts @@ -11,13 +11,17 @@ export async function renderCodeBasedEvaluatorTemplate(evaluatorName: string, ou await copyAndRenderDir(templateDir, outputDir, { Name: evaluatorName }); } -export interface DeepEvalTemplateData { +export interface ThirdPartyEvaluatorTemplateData { Name: string; - MetricClass: string; - MetricParams: string; + EvaluatorClass: string; + EvaluatorParams: string; } -export async function renderDeepEvalEvaluatorTemplate(data: DeepEvalTemplateData, outputDir: string): Promise { - const templateDir = getTemplatePath('evaluators', 'deepeval-lambda'); +export async function renderThirdPartyEvaluatorTemplate( + templateDirName: string, + data: ThirdPartyEvaluatorTemplateData, + outputDir: string +): Promise { + const templateDir = getTemplatePath('evaluators', templateDirName); await copyAndRenderDir(templateDir, outputDir, data); } From 485000e5f3e04041371e81f08bb62d46d3319bcd Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Thu, 16 Jul 2026 09:42:34 -0700 Subject: [PATCH 03/25] fix: Update 3P evaluator templates and add --param/--parameters-file support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - DeepEval template: add /tmp workaround + DEEPEVAL_TELEMETRY_OPT_OUT for Lambda - Autoevals template: fix scorer= → metric= to match SDK API - Rename --from-3p-library → --3p-library - Add --param key=value (repeatable) for metric constructor kwargs - Add --parameters-file for JSON file of kwargs - Add parseParamFlags() utility + unit tests - --param and --parameters-file are mutually exclusive --- .gitignore | 3 + .../assets.snapshot.test.ts.snap | 6 + .../autoevals-lambda/lambda_function.py | 15 ++- .../autoevals-lambda/pyproject.toml | 2 +- .../deepeval-lambda/lambda_function.py | 21 +++- .../evaluators/deepeval-lambda/pyproject.toml | 2 +- src/cli/primitives/EvaluatorPrimitive.ts | 110 ++++++++++++------ .../__tests__/EvaluatorPrimitive.test.ts | 43 ++++++- 8 files changed, 161 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 6613a8f02..e544ccddc 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,6 @@ ProtocolTesting/ browser-tests/.browser-test-env browser-tests/test-results/ browser-tests/playwright-report/ + +# E2E test output +test-e2e-output/ diff --git a/src/assets/__tests__/__snapshots__/assets.snapshot.test.ts.snap b/src/assets/__tests__/__snapshots__/assets.snapshot.test.ts.snap index b935e4f72..6bbaad067 100644 --- a/src/assets/__tests__/__snapshots__/assets.snapshot.test.ts.snap +++ b/src/assets/__tests__/__snapshots__/assets.snapshot.test.ts.snap @@ -742,6 +742,12 @@ exports[`Assets Directory Snapshots > File listing > should match the expected f "container/typescript/dockerignore.template", "datasets/predefined-v1.jsonl", "datasets/simulated-v1.jsonl", + "evaluators/autoevals-lambda/execution-role-policy.json", + "evaluators/autoevals-lambda/lambda_function.py", + "evaluators/autoevals-lambda/pyproject.toml", + "evaluators/deepeval-lambda/execution-role-policy.json", + "evaluators/deepeval-lambda/lambda_function.py", + "evaluators/deepeval-lambda/pyproject.toml", "evaluators/python-lambda/execution-role-policy.json", "evaluators/python-lambda/lambda_function.py", "evaluators/python-lambda/pyproject.toml", diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py index 7c6a256b9..3fa199389 100644 --- a/src/assets/evaluators/autoevals-lambda/lambda_function.py +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -1,4 +1,15 @@ -from bedrock_agentcore.evaluation.integrations.autoevals import AutoevalsAdapter from autoevals import {{ EvaluatorClass }} -handler = AutoevalsAdapter(scorer={{ EvaluatorClass }}({{{ EvaluatorParams }}})) +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoevalsAdapter + +adapter = AutoevalsAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) + + +@custom_code_based_evaluator() +def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput: + return adapter(evaluator_input, context) diff --git a/src/assets/evaluators/autoevals-lambda/pyproject.toml b/src/assets/evaluators/autoevals-lambda/pyproject.toml index b2e920a3f..bd1fa659f 100644 --- a/src/assets/evaluators/autoevals-lambda/pyproject.toml +++ b/src/assets/evaluators/autoevals-lambda/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" description = "AgentCore Code-Based Evaluator (Autoevals)" requires-python = ">=3.10" dependencies = [ - "bedrock-agentcore[autoevals]>=1.6.0", + "bedrock-agentcore[autoevals]", "autoevals>=0.0.80", ] diff --git a/src/assets/evaluators/deepeval-lambda/lambda_function.py b/src/assets/evaluators/deepeval-lambda/lambda_function.py index 3b1033daf..970dfe465 100644 --- a/src/assets/evaluators/deepeval-lambda/lambda_function.py +++ b/src/assets/evaluators/deepeval-lambda/lambda_function.py @@ -1,4 +1,21 @@ -from bedrock_agentcore.evaluation.integrations.deepeval import DeepEvalAdapter +import os + +os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval") +os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES") +os.chdir("/tmp") + from deepeval.metrics import {{ EvaluatorClass }} -handler = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter + +adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) + + +@custom_code_based_evaluator() +def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput: + return adapter(evaluator_input, context) diff --git a/src/assets/evaluators/deepeval-lambda/pyproject.toml b/src/assets/evaluators/deepeval-lambda/pyproject.toml index ac974774c..231940432 100644 --- a/src/assets/evaluators/deepeval-lambda/pyproject.toml +++ b/src/assets/evaluators/deepeval-lambda/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" description = "AgentCore Code-Based Evaluator (DeepEval)" requires-python = ">=3.10" dependencies = [ - "bedrock-agentcore[deepeval]>=1.6.0", + "bedrock-agentcore[deepeval]", "deepeval>=2.0.0", ] diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index fe9fadb13..f2d34f4da 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -17,7 +17,7 @@ import { import { BasePrimitive } from './BasePrimitive'; import type { AddResult, AddScreenComponent, RemovableResource } from './types'; import type { Command } from '@commander-js/extra-typings'; -import { existsSync } from 'node:fs'; +import { existsSync, readFileSync } from 'node:fs'; import { rm } from 'node:fs/promises'; import { dirname, join } from 'node:path'; @@ -141,6 +141,26 @@ export function jsonToKwargs(json: string): string { .join(', '); } +export function parseParamFlags(params: string[]): string { + return params + .map(param => { + const eqIndex = param.indexOf('='); + if (eqIndex === -1) { + throw new Error(`"${param}" is not in key=value format`); + } + const key = param.slice(0, eqIndex); + const rawValue = param.slice(eqIndex + 1); + let value: unknown; + try { + value = JSON.parse(rawValue); + } catch { + value = rawValue; + } + return `${key}=${jsonToPythonValue(value)}`; + }) + .join(', '); +} + function getWarningsForMetric(libraryConfig: ThirdPartyLibraryConfig, metricClass: string): string[] { const messages: string[] = []; for (const warning of libraryConfig.warnings) { @@ -319,9 +339,15 @@ export class EvaluatorPrimitive extends BasePrimitive', `[LLM] Rating scale preset: ${presetIds.join(', ')} (default: 1-5-quality)`) .option('--lambda-arn ', '[Code-based] Existing Lambda function ARN (external)') .option('--timeout ', '[Code-based] Lambda timeout in seconds, 1-300 (default: 60)') - .option('--from-3p-library ', `Third-party evaluation library (${SUPPORTED_LIBRARIES.join(', ')})`) + .option('--3p-library ', `Third-party evaluation library (${SUPPORTED_LIBRARIES.join(', ')})`) .option('--metric ', '[3P library] Metric/evaluator class name (e.g. AnswerRelevancyMetric)') - .option('--parameters ', '[3P library] JSON string of metric constructor kwargs') + .option( + '--param ', + '[3P library] Metric parameter as key=value (repeatable)', + (val: string, prev: string[]) => [...prev, val], + [] as string[] + ) + .option('--parameters-file ', '[3P library] JSON file of metric constructor kwargs') .option('--memory ', '[3P library] Lambda memory size in MB, 128-10240') .option( '--config ', @@ -339,9 +365,10 @@ export class EvaluatorPrimitive extends BasePrimitive 0 && !threePLibrary) { + fail('--param requires --3p-library'); } - if (cliOptions.metric && !from3pLibrary) { - fail('--metric requires --from-3p-library'); + if (cliOptions.parametersFile && !threePLibrary) { + fail('--parameters-file requires --3p-library'); } - if (cliOptions.parameters && !from3pLibrary) { - fail('--parameters requires --from-3p-library'); + if (cliOptions.param.length > 0 && cliOptions.parametersFile) { + fail('--param and --parameters-file cannot be used together'); } - if (cliOptions.memory && !from3pLibrary) { - fail('--memory requires --from-3p-library'); + if (cliOptions.memory && !threePLibrary) { + fail('--memory requires --3p-library'); } if (cliOptions.memory) { const memVal = parseInt(cliOptions.memory, 10); @@ -397,8 +430,8 @@ export class EvaluatorPrimitive extends BasePrimitive 0) { + try { + kwargs = parseParamFlags(cliOptions.param); + } catch (e) { + fail(`Invalid --param value: ${getErrorMessage(e)}`); + } + } else if (cliOptions.parametersFile) { + if (!existsSync(cliOptions.parametersFile)) { + fail(`--parameters-file not found: ${cliOptions.parametersFile}`); + } try { - kwargs = jsonToKwargs(cliOptions.parameters); - } catch { - fail('--parameters must be a valid JSON object (e.g. \'{"threshold": 0.7}\')'); + const fileContent = readFileSync(cliOptions.parametersFile, 'utf-8'); + kwargs = jsonToKwargs(fileContent); + } catch (e) { + fail(`Invalid --parameters-file: ${getErrorMessage(e)}`); } } thirdParty = { - library: from3pLibrary, + library: threePLibrary, metricClass: cliOptions.metric!, metricParams: kwargs, }; } else if (cliOptions.config) { - const { readFileSync } = await import('fs'); configJson = JSON.parse(readFileSync(cliOptions.config, 'utf-8')) as EvaluatorConfig; } else if (evalType === 'code-based') { configJson = this.buildCodeBasedConfig(cliOptions.name!, cliOptions.lambdaArn, cliOptions.timeout); diff --git a/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts b/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts index 33d1d19ba..cc2808ac8 100644 --- a/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts +++ b/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts @@ -4,6 +4,7 @@ import { THIRD_PARTY_EVALUATOR_LIBRARIES, jsonToKwargs, jsonToPythonValue, + parseParamFlags, } from '../EvaluatorPrimitive.js'; import { afterEach, describe, expect, it, vi } from 'vitest'; @@ -496,7 +497,9 @@ describe('THIRD_PARTY_EVALUATOR_LIBRARIES registry', () => { it('deepeval has warnings for expected_output metrics', () => { const config = THIRD_PARTY_EVALUATOR_LIBRARIES.deepeval; - const expectedWarning = config.warnings.find(w => w.metrics.has('ContextualPrecisionMetric')); + const expectedWarning = config.warnings.find( + w => w.metrics.has('ContextualPrecisionMetric') && w.message.includes('expected_output') + ); expect(expectedWarning).toBeDefined(); expect(expectedWarning!.message).toContain('expected_output'); }); @@ -609,3 +612,41 @@ describe('jsonToKwargs', () => { expect(jsonToKwargs('{}')).toBe(''); }); }); + +describe('parseParamFlags', () => { + it('parses number value', () => { + expect(parseParamFlags(['threshold=0.7'])).toBe('threshold=0.7'); + }); + + it('parses string value (JSON-quoted)', () => { + expect(parseParamFlags(['model="gpt-4"'])).toBe('model="gpt-4"'); + }); + + it('parses boolean value', () => { + expect(parseParamFlags(['verbose=true'])).toBe('verbose=True'); + }); + + it('parses array value', () => { + expect(parseParamFlags(['items=[1,2,3]'])).toBe('items=[1, 2, 3]'); + }); + + it('treats unquoted non-JSON string as string', () => { + expect(parseParamFlags(['name=hello world'])).toBe('name="hello world"'); + }); + + it('parses multiple params', () => { + expect(parseParamFlags(['threshold=0.7', 'verbose=true'])).toBe('threshold=0.7, verbose=True'); + }); + + it('throws on missing equals sign', () => { + expect(() => parseParamFlags(['noequalssign'])).toThrow('not in key=value format'); + }); + + it('handles value containing equals sign', () => { + expect(parseParamFlags(['formula=a=b'])).toBe('formula="a=b"'); + }); + + it('parses null value', () => { + expect(parseParamFlags(['callback=null'])).toBe('callback=None'); + }); +}); From 0bff7a8f7b35006b2982ead764f84aedc6c4084f Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Thu, 16 Jul 2026 15:55:25 -0700 Subject: [PATCH 04/25] feat: Add 3P evaluator E2E test + fix autoevals template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add e2e-tests/third-party-eval-lifecycle.test.ts: full lifecycle test for DeepEval + Autoevals 3P evaluators (create → add → deploy → invoke → run eval) - Fix autoevals template: params go to AutoevalsAdapter (not metric constructor) - Add openai>=1.0.0 to autoevals pyproject.toml (required by LLM-based scorers) --- e2e-tests/third-party-eval-lifecycle.test.ts | 203 ++++++++++++++++++ .../autoevals-lambda/lambda_function.py | 2 +- .../autoevals-lambda/pyproject.toml | 1 + 3 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 e2e-tests/third-party-eval-lifecycle.test.ts diff --git a/e2e-tests/third-party-eval-lifecycle.test.ts b/e2e-tests/third-party-eval-lifecycle.test.ts new file mode 100644 index 000000000..354dd4fca --- /dev/null +++ b/e2e-tests/third-party-eval-lifecycle.test.ts @@ -0,0 +1,203 @@ +import { parseJsonOutput, retry } from '../src/test-utils/index.js'; +import { + baseCanRun, + hasAws, + installCdkTarball, + runAgentCoreCLI, + teardownE2EProject, + writeAwsTargets, +} from './e2e-helper.js'; +import { randomUUID } from 'node:crypto'; +import { mkdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const canRun = baseCanRun && hasAws; + +describe.sequential('e2e: third-party evaluator lifecycle (DeepEval + Autoevals)', () => { + let testDir: string; + let projectPath: string; + const agentName = `E2e3pEval${String(Date.now()).slice(-8)}`; + const deepevalEvalName = 'answer_relevancy'; + const autoevalsEvalName = 'exact_match'; + + beforeAll(async () => { + if (!canRun) return; + + testDir = join(tmpdir(), `agentcore-e2e-3p-eval-${randomUUID()}`); + await mkdir(testDir, { recursive: true }); + + const result = await runAgentCoreCLI( + [ + 'create', + '--name', + agentName, + '--language', + 'Python', + '--framework', + 'Strands', + '--model-provider', + 'Bedrock', + '--memory', + 'none', + '--json', + ], + testDir + ); + expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0); + projectPath = (parseJsonOutput(result.stdout) as { projectPath: string }).projectPath; + + await writeAwsTargets(projectPath); + installCdkTarball(projectPath); + }, 300000); + + afterAll(async () => { + if (projectPath && hasAws) { + await teardownE2EProject(projectPath, agentName, 'Bedrock'); + } + if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 }); + }, 600000); + + const run = (args: string[]) => runAgentCoreCLI(args, projectPath); + + it.skipIf(!canRun)( + 'adds a DeepEval 3P evaluator with --3p-library and --param', + async () => { + const result = await run([ + 'add', + 'evaluator', + '--name', + deepevalEvalName, + '--level', + 'TRACE', + '--3p-library', + 'deepeval', + '--metric', + 'AnswerRelevancyMetric', + '--param', + 'threshold=0.5', + '--json', + ]); + expect(result.exitCode, `Add DeepEval evaluator failed: ${result.stdout}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean; evaluatorName: string; codePath: string }; + expect(json.success).toBe(true); + expect(json.evaluatorName).toBe(deepevalEvalName); + expect(json.codePath).toContain(deepevalEvalName); + }, + 60000 + ); + + it.skipIf(!canRun)( + 'adds an Autoevals 3P evaluator with --3p-library', + async () => { + const result = await run([ + 'add', + 'evaluator', + '--name', + autoevalsEvalName, + '--level', + 'TRACE', + '--3p-library', + 'autoevals', + '--metric', + 'ExactMatch', + '--json', + ]); + expect(result.exitCode, `Add Autoevals evaluator failed: ${result.stdout}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean; evaluatorName: string; codePath: string }; + expect(json.success).toBe(true); + expect(json.evaluatorName).toBe(autoevalsEvalName); + expect(json.codePath).toContain(autoevalsEvalName); + }, + 60000 + ); + + it.skipIf(!canRun)( + 'deploys agent with 3P evaluators', + async () => { + const result = await run(['deploy', '--yes', '--json']); + if (result.exitCode !== 0) { + console.log('Deploy stdout:', result.stdout); + console.log('Deploy stderr:', result.stderr); + } + expect(result.exitCode, 'Deploy failed').toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success).toBe(true); + }, + 600000 + ); + + it.skipIf(!canRun)( + 'invokes the deployed agent to generate traces', + async () => { + await retry( + async () => { + const result = await run(['invoke', '--prompt', 'What is 2+2?', '--runtime', agentName, '--json']); + expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success).toBe(true); + }, + 3, + 15000 + ); + }, + 180000 + ); + + it.skipIf(!canRun)( + 'runs on-demand evaluation with DeepEval 3P evaluator', + async () => { + await retry( + async () => { + const result = await run([ + 'run', + 'eval', + '--runtime', + agentName, + '--evaluator', + deepevalEvalName, + '--days', + '1', + '--json', + ]); + expect(result.exitCode, `Run eval failed (stdout: ${result.stdout}, stderr: ${result.stderr})`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + expect(json).toHaveProperty('run'); + }, + 18, + 10000 + ); + }, + 300000 + ); + + it.skipIf(!canRun)( + 'runs on-demand evaluation with Autoevals 3P evaluator', + async () => { + await retry( + async () => { + const result = await run([ + 'run', + 'eval', + '--runtime', + agentName, + '--evaluator', + autoevalsEvalName, + '--days', + '1', + '--json', + ]); + expect(result.exitCode, `Run eval failed (stdout: ${result.stdout}, stderr: ${result.stderr})`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + expect(json).toHaveProperty('run'); + }, + 18, + 10000 + ); + }, + 300000 + ); +}); diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py index 3fa199389..215302cbd 100644 --- a/src/assets/evaluators/autoevals-lambda/lambda_function.py +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -7,7 +7,7 @@ ) from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoevalsAdapter -adapter = AutoevalsAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) +adapter = AutoevalsAdapter(metric={{ EvaluatorClass }}(), {{{ EvaluatorParams }}}) @custom_code_based_evaluator() diff --git a/src/assets/evaluators/autoevals-lambda/pyproject.toml b/src/assets/evaluators/autoevals-lambda/pyproject.toml index bd1fa659f..d117c3786 100644 --- a/src/assets/evaluators/autoevals-lambda/pyproject.toml +++ b/src/assets/evaluators/autoevals-lambda/pyproject.toml @@ -10,6 +10,7 @@ requires-python = ">=3.10" dependencies = [ "bedrock-agentcore[autoevals]", "autoevals>=0.0.80", + "openai>=1.0.0", ] [tool.hatch.build.targets.wheel] From 9153b3cd587ea5893a14f1eec3b64cb81b9e9f8b Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Mon, 20 Jul 2026 13:34:58 -0700 Subject: [PATCH 05/25] =?UTF-8?q?fix:=20Rename=20AutoevalsAdapter=20?= =?UTF-8?q?=E2=86=92=20AutoEvalsAdapter=20in=20template=20(match=20SDK=20r?= =?UTF-8?q?ename)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/assets/evaluators/autoevals-lambda/lambda_function.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py index 215302cbd..86c002767 100644 --- a/src/assets/evaluators/autoevals-lambda/lambda_function.py +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -5,9 +5,9 @@ EvaluatorOutput, custom_code_based_evaluator, ) -from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoevalsAdapter +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter -adapter = AutoevalsAdapter(metric={{ EvaluatorClass }}(), {{{ EvaluatorParams }}}) +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(), {{{ EvaluatorParams }}}) @custom_code_based_evaluator() From 7b5348c5b35da050f03af2592715c40371c52648 Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Wed, 22 Jul 2026 11:54:12 -0700 Subject: [PATCH 06/25] fix: Reject --3p-library with --type llm-as-a-judge Add validation guard so that explicitly passing --type llm-as-a-judge with --3p-library fails with a clear error message instead of silently proceeding into an invalid state. --- src/cli/primitives/EvaluatorPrimitive.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index f2d34f4da..064161c92 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -440,6 +440,7 @@ export class EvaluatorPrimitive extends BasePrimitive Date: Fri, 24 Jul 2026 09:06:38 -0700 Subject: [PATCH 07/25] feat: Add --model-provider flag for keyless Bedrock evaluators Adds --model-provider flag to `add evaluator` command. When bedrock is selected: - DeepEval: uses AmazonBedrockModel (native Bedrock via aiobotocore) - Autoevals: uses LiteLLMClient routing to Bedrock Converse API Templates are conditional via Handlebars {{#if ModelProviderBedrock}}. Dependencies are dynamic: aiobotocore for DeepEval, litellm for Autoevals (only included when bedrock is selected). Default remains openai for backward compatibility. Tested end-to-end on Lambda: - DeepEval + Bedrock: 1.0, Pass (AnswerRelevancy, Claude Haiku) - Autoevals + Bedrock: 0.6, Pass (Factuality, Claude Sonnet 4) - No OpenAI API key, IAM-only auth --- .../autoevals-lambda/lambda_function.py | 25 +++++++ .../autoevals-lambda/pyproject.toml | 5 ++ .../deepeval-lambda/lambda_function.py | 11 +++ .../evaluators/deepeval-lambda/pyproject.toml | 3 + src/cli/primitives/EvaluatorPrimitive.ts | 26 +++++++ .../__tests__/EvaluatorPrimitive.test.ts | 75 +++++++++++++++++++ src/cli/templates/EvaluatorRenderer.ts | 2 + 7 files changed, 147 insertions(+) diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py index 86c002767..97bbabf97 100644 --- a/src/assets/evaluators/autoevals-lambda/lambda_function.py +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -1,3 +1,27 @@ +{{#if ModelProviderBedrock}} +import os + +# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION. +os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2")) + +from autoevals import {{ EvaluatorClass }}, init +from autoevals.litellm import LiteLLMClient + +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter + +# Autoevals grades via an OpenAI-compatible client. LiteLLMClient routes to Bedrock; +# litellm auto-routes Anthropic Claude models through the Converse API. Cross-region +# inference profiles (us.*/eu.*/apac.*) are required for on-demand invocation. +JUDGE_MODEL = os.environ.get("BEDROCK_MODEL_ID", "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0") +init(client=LiteLLMClient(), default_model=JUDGE_MODEL) + +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=LiteLLMClient(), model=JUDGE_MODEL), {{{ EvaluatorParams }}}) +{{else}} from autoevals import {{ EvaluatorClass }} from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( @@ -8,6 +32,7 @@ from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(), {{{ EvaluatorParams }}}) +{{/if}} @custom_code_based_evaluator() diff --git a/src/assets/evaluators/autoevals-lambda/pyproject.toml b/src/assets/evaluators/autoevals-lambda/pyproject.toml index d117c3786..9252dee5a 100644 --- a/src/assets/evaluators/autoevals-lambda/pyproject.toml +++ b/src/assets/evaluators/autoevals-lambda/pyproject.toml @@ -10,7 +10,12 @@ requires-python = ">=3.10" dependencies = [ "bedrock-agentcore[autoevals]", "autoevals>=0.0.80", +{{#if ModelProviderBedrock}} + # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge + "litellm>=1.60,<1.85", +{{else}} "openai>=1.0.0", +{{/if}} ] [tool.hatch.build.targets.wheel] diff --git a/src/assets/evaluators/deepeval-lambda/lambda_function.py b/src/assets/evaluators/deepeval-lambda/lambda_function.py index 970dfe465..31d01bee4 100644 --- a/src/assets/evaluators/deepeval-lambda/lambda_function.py +++ b/src/assets/evaluators/deepeval-lambda/lambda_function.py @@ -4,6 +4,9 @@ os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES") os.chdir("/tmp") +{{#if ModelProviderBedrock}} +from deepeval.models import AmazonBedrockModel +{{/if}} from deepeval.metrics import {{ EvaluatorClass }} from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( @@ -13,7 +16,15 @@ ) from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter +{{#if ModelProviderBedrock}} +MODEL_ID = os.environ.get("BEDROCK_MODEL_ID", "anthropic.claude-3-haiku-20240307-v1:0") +REGION = os.environ.get("AWS_REGION", "us-west-2") + +model = AmazonBedrockModel(model=MODEL_ID, region=REGION) +adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model, {{{ EvaluatorParams }}})) +{{else}} adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) +{{/if}} @custom_code_based_evaluator() diff --git a/src/assets/evaluators/deepeval-lambda/pyproject.toml b/src/assets/evaluators/deepeval-lambda/pyproject.toml index 231940432..f32e3cd77 100644 --- a/src/assets/evaluators/deepeval-lambda/pyproject.toml +++ b/src/assets/evaluators/deepeval-lambda/pyproject.toml @@ -10,6 +10,9 @@ requires-python = ">=3.10" dependencies = [ "bedrock-agentcore[deepeval]", "deepeval>=2.0.0", +{{#if ModelProviderBedrock}} + "aiobotocore>=2.13.0", +{{/if}} ] [tool.hatch.build.targets.wheel] diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index 064161c92..a18442229 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -92,6 +92,14 @@ function isSupportedLibrary(value: string): value is ThirdPartyLibrary { return value in THIRD_PARTY_EVALUATOR_LIBRARIES; } +export const MODEL_PROVIDERS = ['openai', 'bedrock'] as const; + +export type ModelProvider = (typeof MODEL_PROVIDERS)[number]; + +function isSupportedModelProvider(value: string): value is ModelProvider { + return (MODEL_PROVIDERS as readonly string[]).includes(value); +} + // ============================================================================ // Types // ============================================================================ @@ -100,6 +108,8 @@ export interface ThirdPartyLibraryOptions { library: ThirdPartyLibrary; metricClass: string; metricParams?: string; + /** LLM judge provider; defaults to the library's built-in default (OpenAI). */ + modelProvider?: ModelProvider; } export interface AddEvaluatorOptions { @@ -203,6 +213,8 @@ export class EvaluatorPrimitive extends BasePrimitive', '[3P library] JSON file of metric constructor kwargs') + .option( + '--model-provider ', + `[3P library] LLM judge provider: ${MODEL_PROVIDERS.join(', ')} (default: openai)` + ) .option('--memory ', '[3P library] Lambda memory size in MB, 128-10240') .option( '--config ', @@ -369,6 +385,7 @@ export class EvaluatorPrimitive extends BasePrimitive 0 && cliOptions.parametersFile) { fail('--param and --parameters-file cannot be used together'); } + if (cliOptions.modelProvider && !threePLibrary) { + fail('--model-provider requires --3p-library'); + } + if (cliOptions.modelProvider && !isSupportedModelProvider(cliOptions.modelProvider)) { + fail( + `Invalid --model-provider "${cliOptions.modelProvider}". Supported: ${MODEL_PROVIDERS.join(', ')}` + ); + } if (cliOptions.memory && !threePLibrary) { fail('--memory requires --3p-library'); } @@ -481,6 +506,7 @@ export class EvaluatorPrimitive extends BasePrimitive { ); }); + it('passes ModelProviderBedrock to the template when modelProvider is bedrock', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + await primitive.add({ + name: 'auto_eval', + level: 'SESSION', + config: autoevalsEvalConfig, + thirdParty: { + library: 'autoevals', + metricClass: 'Factuality', + metricParams: '', + modelProvider: 'bedrock', + }, + }); + + expect(mockRenderThirdParty).toHaveBeenCalledWith( + 'autoevals-lambda', + { Name: 'auto_eval', EvaluatorClass: 'Factuality', EvaluatorParams: '', ModelProviderBedrock: true }, + expect.stringContaining('app/auto_eval') + ); + }); + + it('omits ModelProviderBedrock when modelProvider is openai', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + await primitive.add({ + name: 'deep_eval', + level: 'SESSION', + config: deepEvalConfig, + thirdParty: { + library: 'deepeval', + metricClass: 'AnswerRelevancyMetric', + metricParams: 'threshold=0.7', + modelProvider: 'openai', + }, + }); + + expect(mockRenderThirdParty).toHaveBeenCalledWith( + 'deepeval-lambda', + { Name: 'deep_eval', EvaluatorClass: 'AnswerRelevancyMetric', EvaluatorParams: 'threshold=0.7' }, + expect.stringContaining('app/deep_eval') + ); + }); + + it('omits ModelProviderBedrock when modelProvider is not set (backwards compatible)', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + await primitive.add({ + name: 'auto_eval', + level: 'SESSION', + config: autoevalsEvalConfig, + thirdParty: { + library: 'autoevals', + metricClass: 'Factuality', + metricParams: '', + }, + }); + + expect(mockRenderThirdParty).toHaveBeenCalledWith( + 'autoevals-lambda', + { Name: 'auto_eval', EvaluatorClass: 'Factuality', EvaluatorParams: '' }, + expect.stringContaining('app/auto_eval') + ); + }); + it('calls renderCodeBasedEvaluatorTemplate when thirdParty is not set', async () => { mockReadProjectSpec.mockResolvedValue(makeProject()); mockWriteProjectSpec.mockResolvedValue(undefined); @@ -525,6 +594,12 @@ describe('THIRD_PARTY_EVALUATOR_LIBRARIES registry', () => { }); }); +describe('MODEL_PROVIDERS', () => { + it('supports openai and bedrock only', () => { + expect(MODEL_PROVIDERS).toEqual(['openai', 'bedrock']); + }); +}); + describe('jsonToPythonValue', () => { it('converts null to None', () => { expect(jsonToPythonValue(null)).toBe('None'); diff --git a/src/cli/templates/EvaluatorRenderer.ts b/src/cli/templates/EvaluatorRenderer.ts index 6b50242e5..577ef4a2b 100644 --- a/src/cli/templates/EvaluatorRenderer.ts +++ b/src/cli/templates/EvaluatorRenderer.ts @@ -15,6 +15,8 @@ export interface ThirdPartyEvaluatorTemplateData { Name: string; EvaluatorClass: string; EvaluatorParams: string; + /** True when the LLM judge runs on Bedrock instead of the library's default (OpenAI). */ + ModelProviderBedrock?: boolean; } export async function renderThirdPartyEvaluatorTemplate( From 1a860d925dda5bb1922ec73c3cdc77abdc4a3315 Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Fri, 24 Jul 2026 13:40:46 -0700 Subject: [PATCH 08/25] fix: Require --model with --model-provider bedrock, remove env var defaults Per Irene's review feedback: - Remove hardcoded default model IDs (not available in all regions) - Require --model flag when --model-provider bedrock is selected - Bake model ID into generated template at scaffold time (no env vars) - Reuse existing --model CLI flag for Bedrock model selection --- .../autoevals-lambda/lambda_function.py | 8 +- .../deepeval-lambda/lambda_function.py | 5 +- src/cli/primitives/EvaluatorPrimitive.ts | 21 ++++- .../__tests__/EvaluatorPrimitive.test.ts | 81 +++++++++++++++++++ src/cli/templates/EvaluatorRenderer.ts | 2 + 5 files changed, 105 insertions(+), 12 deletions(-) diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py index 97bbabf97..3fdb0abbd 100644 --- a/src/assets/evaluators/autoevals-lambda/lambda_function.py +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -14,13 +14,9 @@ ) from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter -# Autoevals grades via an OpenAI-compatible client. LiteLLMClient routes to Bedrock; -# litellm auto-routes Anthropic Claude models through the Converse API. Cross-region -# inference profiles (us.*/eu.*/apac.*) are required for on-demand invocation. -JUDGE_MODEL = os.environ.get("BEDROCK_MODEL_ID", "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0") -init(client=LiteLLMClient(), default_model=JUDGE_MODEL) +init(client=LiteLLMClient(), default_model="{{ Model }}") -adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=LiteLLMClient(), model=JUDGE_MODEL), {{{ EvaluatorParams }}}) +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=LiteLLMClient(), model="{{ Model }}"), {{{ EvaluatorParams }}}) {{else}} from autoevals import {{ EvaluatorClass }} diff --git a/src/assets/evaluators/deepeval-lambda/lambda_function.py b/src/assets/evaluators/deepeval-lambda/lambda_function.py index 31d01bee4..3b690a101 100644 --- a/src/assets/evaluators/deepeval-lambda/lambda_function.py +++ b/src/assets/evaluators/deepeval-lambda/lambda_function.py @@ -17,10 +17,7 @@ from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter {{#if ModelProviderBedrock}} -MODEL_ID = os.environ.get("BEDROCK_MODEL_ID", "anthropic.claude-3-haiku-20240307-v1:0") -REGION = os.environ.get("AWS_REGION", "us-west-2") - -model = AmazonBedrockModel(model=MODEL_ID, region=REGION) +model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2")) adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model, {{{ EvaluatorParams }}})) {{else}} adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index a18442229..da6007a8a 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -110,6 +110,8 @@ export interface ThirdPartyLibraryOptions { metricParams?: string; /** LLM judge provider; defaults to the library's built-in default (OpenAI). */ modelProvider?: ModelProvider; + /** Bedrock model ID (required when modelProvider is 'bedrock'). */ + model?: string; } export interface AddEvaluatorOptions { @@ -215,6 +217,7 @@ export class EvaluatorPrimitive extends BasePrimitive', 'Evaluator name') .option('--level ', 'Evaluation level: SESSION, TRACE, TOOL_CALL') .option('--type ', 'Evaluator type: llm-as-a-judge (default) or code-based') - .option('--model ', '[LLM] Bedrock model ID for LLM-as-a-Judge') + .option( + '--model ', + 'Bedrock model ID: [LLM] judge model for LLM-as-a-Judge, or [3P library] judge model with ' + + '--model-provider bedrock (plain model ID or inference profile, e.g. ' + + 'us.anthropic.claude-sonnet-4-20250514-v1:0 — no bedrock/ prefix)' + ) .option( '--instructions ', '[LLM] Evaluation prompt instructions (must include level-appropriate placeholders, e.g. {context})' @@ -419,7 +427,9 @@ export class EvaluatorPrimitive extends BasePrimitive { ); }); + it('passes Model to the template when model is provided with bedrock provider', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + await primitive.add({ + name: 'auto_eval', + level: 'SESSION', + config: autoevalsEvalConfig, + thirdParty: { + library: 'autoevals', + metricClass: 'Factuality', + metricParams: 'threshold=0.5', + modelProvider: 'bedrock', + model: 'us.anthropic.claude-sonnet-4-20250514-v1:0', + }, + }); + + expect(mockRenderThirdParty).toHaveBeenCalledWith( + 'autoevals-lambda', + { + Name: 'auto_eval', + EvaluatorClass: 'Factuality', + EvaluatorParams: 'threshold=0.5', + ModelProviderBedrock: true, + Model: 'us.anthropic.claude-sonnet-4-20250514-v1:0', + }, + expect.stringContaining('app/auto_eval') + ); + }); + + it('passes Model to the deepeval template when model is provided with bedrock provider', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + await primitive.add({ + name: 'deep_eval', + level: 'SESSION', + config: deepEvalConfig, + thirdParty: { + library: 'deepeval', + metricClass: 'AnswerRelevancyMetric', + metricParams: 'threshold=0.7', + modelProvider: 'bedrock', + model: 'anthropic.claude-3-haiku-20240307-v1:0', + }, + }); + + expect(mockRenderThirdParty).toHaveBeenCalledWith( + 'deepeval-lambda', + { + Name: 'deep_eval', + EvaluatorClass: 'AnswerRelevancyMetric', + EvaluatorParams: 'threshold=0.7', + ModelProviderBedrock: true, + Model: 'anthropic.claude-3-haiku-20240307-v1:0', + }, + expect.stringContaining('app/deep_eval') + ); + }); + + it('omits Model from the render context when not provided', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + await primitive.add({ + name: 'auto_eval', + level: 'SESSION', + config: autoevalsEvalConfig, + thirdParty: { + library: 'autoevals', + metricClass: 'Factuality', + metricParams: '', + modelProvider: 'openai', + }, + }); + + const renderData = mockRenderThirdParty.mock.calls[0]![1] as Record; + expect(renderData).not.toHaveProperty('Model'); + expect(renderData).not.toHaveProperty('ModelProviderBedrock'); + }); + it('omits ModelProviderBedrock when modelProvider is openai', async () => { mockReadProjectSpec.mockResolvedValue(makeProject()); mockWriteProjectSpec.mockResolvedValue(undefined); diff --git a/src/cli/templates/EvaluatorRenderer.ts b/src/cli/templates/EvaluatorRenderer.ts index 577ef4a2b..7f239576c 100644 --- a/src/cli/templates/EvaluatorRenderer.ts +++ b/src/cli/templates/EvaluatorRenderer.ts @@ -17,6 +17,8 @@ export interface ThirdPartyEvaluatorTemplateData { EvaluatorParams: string; /** True when the LLM judge runs on Bedrock instead of the library's default (OpenAI). */ ModelProviderBedrock?: boolean; + /** Bedrock model ID (required when ModelProviderBedrock is true). */ + Model?: string; } export async function renderThirdPartyEvaluatorTemplate( From 591760a5b77e6d1eba341b1fe2c40aae366083a2 Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Fri, 24 Jul 2026 14:55:41 -0700 Subject: [PATCH 09/25] fix: Address Copilot review comments - Reuse single LiteLLMClient instance (avoid duplicate instantiation) - Fix dangling comma when EvaluatorParams is empty (invalid Python) - Validate Python identifier format in jsonToKwargs and parseParamFlags - Add --timeout validation (1-300 range check) --- .../autoevals-lambda/lambda_function.py | 5 ++-- .../deepeval-lambda/lambda_function.py | 2 +- src/cli/primitives/EvaluatorPrimitive.ts | 27 +++++++++++++++---- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py index 3fdb0abbd..69bddfc16 100644 --- a/src/assets/evaluators/autoevals-lambda/lambda_function.py +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -14,9 +14,10 @@ ) from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter -init(client=LiteLLMClient(), default_model="{{ Model }}") +client = LiteLLMClient() +init(client=client, default_model="{{ Model }}") -adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=LiteLLMClient(), model="{{ Model }}"), {{{ EvaluatorParams }}}) +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="{{ Model }}"), {{{ EvaluatorParams }}}) {{else}} from autoevals import {{ EvaluatorClass }} diff --git a/src/assets/evaluators/deepeval-lambda/lambda_function.py b/src/assets/evaluators/deepeval-lambda/lambda_function.py index 3b690a101..a89b22051 100644 --- a/src/assets/evaluators/deepeval-lambda/lambda_function.py +++ b/src/assets/evaluators/deepeval-lambda/lambda_function.py @@ -18,7 +18,7 @@ {{#if ModelProviderBedrock}} model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2")) -adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model, {{{ EvaluatorParams }}})) +adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})) {{else}} adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) {{/if}} diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index da6007a8a..96f2e67f4 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -147,9 +147,17 @@ export function jsonToPythonValue(value: unknown): string { } export function jsonToKwargs(json: string): string { - const obj = JSON.parse(json) as Record; - return Object.entries(obj) - .map(([key, value]) => `${key}=${jsonToPythonValue(value)}`) + const obj = JSON.parse(json); + if (obj == null || typeof obj !== 'object' || Array.isArray(obj)) { + throw new Error('Expected a JSON object of keyword arguments'); + } + return Object.entries(obj as Record) + .map(([key, value]) => { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) { + throw new Error(`Invalid Python kwarg name "${key}"`); + } + return `${key}=${jsonToPythonValue(value)}`; + }) .join(', '); } @@ -160,8 +168,11 @@ export function parseParamFlags(params: string[]): string { if (eqIndex === -1) { throw new Error(`"${param}" is not in key=value format`); } - const key = param.slice(0, eqIndex); - const rawValue = param.slice(eqIndex + 1); + const key = param.slice(0, eqIndex).trim(); + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) { + throw new Error(`Invalid Python kwarg name "${key}" in --param`); + } + const rawValue = param.slice(eqIndex + 1).trim(); let value: unknown; try { value = JSON.parse(rawValue); @@ -464,6 +475,12 @@ export class EvaluatorPrimitive extends BasePrimitive 300) { + fail('--timeout must be an integer between 1 and 300'); + } + } if (cliOptions.memory) { const memVal = parseInt(cliOptions.memory, 10); if (isNaN(memVal) || memVal < 128 || memVal > 10240) { From a1002761dd794b2e93f2055353ee706506c500c2 Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Fri, 24 Jul 2026 15:22:17 -0700 Subject: [PATCH 10/25] fix: Update E2E test to use --model-provider bedrock (keyless) E2E test now uses Bedrock as the LLM judge so it runs without manual OPENAI_API_KEY setup. Adds --model-provider bedrock and --model flags to both evaluator add commands. --- e2e-tests/third-party-eval-lifecycle.test.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/e2e-tests/third-party-eval-lifecycle.test.ts b/e2e-tests/third-party-eval-lifecycle.test.ts index 354dd4fca..ad5380521 100644 --- a/e2e-tests/third-party-eval-lifecycle.test.ts +++ b/e2e-tests/third-party-eval-lifecycle.test.ts @@ -62,7 +62,7 @@ describe.sequential('e2e: third-party evaluator lifecycle (DeepEval + Autoevals) const run = (args: string[]) => runAgentCoreCLI(args, projectPath); it.skipIf(!canRun)( - 'adds a DeepEval 3P evaluator with --3p-library and --param', + 'adds a DeepEval 3P evaluator with --3p-library, --model-provider bedrock, and --param', async () => { const result = await run([ 'add', @@ -75,6 +75,10 @@ describe.sequential('e2e: third-party evaluator lifecycle (DeepEval + Autoevals) 'deepeval', '--metric', 'AnswerRelevancyMetric', + '--model-provider', + 'bedrock', + '--model', + 'us.anthropic.claude-sonnet-4-20250514-v1:0', '--param', 'threshold=0.5', '--json', @@ -89,7 +93,7 @@ describe.sequential('e2e: third-party evaluator lifecycle (DeepEval + Autoevals) ); it.skipIf(!canRun)( - 'adds an Autoevals 3P evaluator with --3p-library', + 'adds an Autoevals 3P evaluator with --3p-library and --model-provider bedrock', async () => { const result = await run([ 'add', @@ -102,6 +106,10 @@ describe.sequential('e2e: third-party evaluator lifecycle (DeepEval + Autoevals) 'autoevals', '--metric', 'ExactMatch', + '--model-provider', + 'bedrock', + '--model', + 'us.anthropic.claude-sonnet-4-20250514-v1:0', '--json', ]); expect(result.exitCode, `Add Autoevals evaluator failed: ${result.stdout}`).toBe(0); From f92635cae832de2b0fdc195bb64db0123e1f9c78 Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Fri, 24 Jul 2026 16:26:14 -0700 Subject: [PATCH 11/25] fix: Default --model-provider to bedrock, add OpenAI warning Per Irene's review: - Default model provider is now bedrock (was openai) - --model is required by default when using --3p-library - Warning printed when user explicitly selects openai: reminds them to set OPENAI_API_KEY on the Lambda --- src/cli/primitives/EvaluatorPrimitive.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index 96f2e67f4..b844db051 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -381,7 +381,7 @@ export class EvaluatorPrimitive extends BasePrimitive', '[3P library] JSON file of metric constructor kwargs') .option( '--model-provider ', - `[3P library] LLM judge provider: ${MODEL_PROVIDERS.join(', ')} (default: openai)` + `[3P library] LLM judge provider: ${MODEL_PROVIDERS.join(', ')} (default: bedrock)` ) .option('--memory ', '[3P library] Lambda memory size in MB, 128-10240') .option( @@ -466,9 +466,10 @@ export class EvaluatorPrimitive extends BasePrimitive Date: Fri, 24 Jul 2026 16:38:20 -0700 Subject: [PATCH 12/25] fix: Use hasOwnProperty for library validation (Copilot review) --- src/cli/primitives/EvaluatorPrimitive.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index b844db051..f0d38fe55 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -89,7 +89,7 @@ export type ThirdPartyLibrary = keyof typeof THIRD_PARTY_EVALUATOR_LIBRARIES; const SUPPORTED_LIBRARIES: ThirdPartyLibrary[] = Object.keys(THIRD_PARTY_EVALUATOR_LIBRARIES) as ThirdPartyLibrary[]; function isSupportedLibrary(value: string): value is ThirdPartyLibrary { - return value in THIRD_PARTY_EVALUATOR_LIBRARIES; + return Object.prototype.hasOwnProperty.call(THIRD_PARTY_EVALUATOR_LIBRARIES, value); } export const MODEL_PROVIDERS = ['openai', 'bedrock'] as const; From f1cf6178e72304dd669adfd0b85de6ae31deaab5 Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Sat, 25 Jul 2026 22:25:50 -0700 Subject: [PATCH 13/25] fix: Allow --model without explicit --model-provider (bedrock is default) --- src/cli/primitives/EvaluatorPrimitive.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index f0d38fe55..cde6dc58b 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -438,8 +438,8 @@ export class EvaluatorPrimitive extends BasePrimitive Date: Wed, 29 Jul 2026 11:35:10 -0700 Subject: [PATCH 14/25] fix: Parse Python-style True/False/None in --param values --param include_reason=True was rendered as include_reason="True" (string) instead of include_reason=True (boolean). JSON.parse doesn't handle Python-style literals, so we now detect True/False/None in the catch block and map them to proper JS types before rendering. --- src/cli/primitives/EvaluatorPrimitive.ts | 7 +++++-- .../primitives/__tests__/EvaluatorPrimitive.test.ts | 12 ++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index cde6dc58b..c7e4d0fbd 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -147,7 +147,7 @@ export function jsonToPythonValue(value: unknown): string { } export function jsonToKwargs(json: string): string { - const obj = JSON.parse(json); + const obj: unknown = JSON.parse(json); if (obj == null || typeof obj !== 'object' || Array.isArray(obj)) { throw new Error('Expected a JSON object of keyword arguments'); } @@ -177,7 +177,10 @@ export function parseParamFlags(params: string[]): string { try { value = JSON.parse(rawValue); } catch { - value = rawValue; + if (rawValue === 'True') value = true; + else if (rawValue === 'False') value = false; + else if (rawValue === 'None') value = null; + else value = rawValue; } return `${key}=${jsonToPythonValue(value)}`; }) diff --git a/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts b/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts index 8f934e7f8..35b92c667 100644 --- a/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts +++ b/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts @@ -805,4 +805,16 @@ describe('parseParamFlags', () => { it('parses null value', () => { expect(parseParamFlags(['callback=null'])).toBe('callback=None'); }); + + it('parses Python-style True as boolean', () => { + expect(parseParamFlags(['include_reason=True'])).toBe('include_reason=True'); + }); + + it('parses Python-style False as boolean', () => { + expect(parseParamFlags(['strict=False'])).toBe('strict=False'); + }); + + it('parses Python-style None as null', () => { + expect(parseParamFlags(['fallback=None'])).toBe('fallback=None'); + }); }); From 11537a1a4f2deefcca953a5145f663d7fff6a7c2 Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Wed, 29 Jul 2026 11:41:30 -0700 Subject: [PATCH 15/25] fix: Autoevals dangling comma + clarify inference profile in --model help 1. Wrap EvaluatorParams in {{#if}} guard in autoevals template to prevent dangling comma when no --param is provided. 2. Update --model help text and error message to clarify that an inference profile ID is required (plain model IDs are not supported). --- .../evaluators/autoevals-lambda/lambda_function.py | 4 ++-- src/cli/primitives/EvaluatorPrimitive.ts | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py index 69bddfc16..a232b5768 100644 --- a/src/assets/evaluators/autoevals-lambda/lambda_function.py +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -17,7 +17,7 @@ client = LiteLLMClient() init(client=client, default_model="{{ Model }}") -adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="{{ Model }}"), {{{ EvaluatorParams }}}) +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) {{else}} from autoevals import {{ EvaluatorClass }} @@ -28,7 +28,7 @@ ) from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter -adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(), {{{ EvaluatorParams }}}) +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) {{/if}} diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index c7e4d0fbd..065409285 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -362,9 +362,9 @@ export class EvaluatorPrimitive extends BasePrimitive', 'Evaluator type: llm-as-a-judge (default) or code-based') .option( '--model ', - 'Bedrock model ID: [LLM] judge model for LLM-as-a-Judge, or [3P library] judge model with ' + - '--model-provider bedrock (plain model ID or inference profile, e.g. ' + - 'us.anthropic.claude-sonnet-4-20250514-v1:0 — no bedrock/ prefix)' + 'Bedrock inference profile ID: [LLM] judge model for LLM-as-a-Judge, or [3P library] judge model with ' + + '--model-provider bedrock. Must be an inference profile (e.g. ' + + 'us.anthropic.claude-sonnet-4-20250514-v1:0), not a plain model ID — no bedrock/ prefix' ) .option( '--instructions ', @@ -472,8 +472,8 @@ export class EvaluatorPrimitive extends BasePrimitive Date: Wed, 29 Jul 2026 14:16:16 -0700 Subject: [PATCH 16/25] fix: Remove --memory flag (to be handled via GitHub issue) Per team discussion, Lambda memory configuration should be handled through the CDK constructs layer, not the CLI. Removing the flag and will file a GitHub issue for the proper implementation. Default memory sizes remain: DeepEval 1024MB, Autoevals 512MB. --- src/cli/primitives/EvaluatorPrimitive.ts | 20 +++---------------- .../__tests__/EvaluatorPrimitive.test.ts | 13 ------------ 2 files changed, 3 insertions(+), 30 deletions(-) diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index 065409285..eca6385e3 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -386,7 +386,6 @@ export class EvaluatorPrimitive extends BasePrimitive', `[3P library] LLM judge provider: ${MODEL_PROVIDERS.join(', ')} (default: bedrock)` ) - .option('--memory ', '[3P library] Lambda memory size in MB, 128-10240') .option( '--config ', 'Path to evaluator config JSON file (overrides --model, --instructions, --rating-scale) [non-interactive]' @@ -408,7 +407,6 @@ export class EvaluatorPrimitive extends BasePrimitive 300) { fail('--timeout must be an integer between 1 and 300'); } } - if (cliOptions.memory) { - const memVal = parseInt(cliOptions.memory, 10); - if (isNaN(memVal) || memVal < 128 || memVal > 10240) { - fail('--memory must be an integer between 128 and 10240'); - } - } // Default --type to code-based when --3p-library is set const evalType = cliOptions.type ?? (threePLibrary ? 'code-based' : 'llm-as-a-judge'); @@ -518,8 +507,7 @@ export class EvaluatorPrimitive extends BasePrimitive 0) { @@ -710,18 +698,16 @@ export class EvaluatorPrimitive extends BasePrimitive { expect(config.codeBased!.managed!.timeoutSeconds).toBe(120); }); - it('respects custom memory', () => { - // eslint-disable-next-line @typescript-eslint/dot-notation - const config = primitive['buildThirdPartyConfig']('my_eval', deepevalConfig, undefined, '2048'); - - expect(config.codeBased!.managed!.memorySizeMb).toBe(2048); - }); }); describe('autoevals', () => { @@ -334,13 +328,6 @@ describe('EvaluatorPrimitive', () => { expect(config.codeBased!.managed!.timeoutSeconds).toBe(180); }); - - it('respects custom memory', () => { - // eslint-disable-next-line @typescript-eslint/dot-notation - const config = primitive['buildThirdPartyConfig']('fact_check', autoevalsConfig, undefined, '1024'); - - expect(config.codeBased!.managed!.memorySizeMb).toBe(1024); - }); }); }); From 565eeda9cdb30ede55965a79d99690a791d6660b Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Wed, 29 Jul 2026 14:19:20 -0700 Subject: [PATCH 17/25] feat: Add --3p-template-json and --3p-template-json-file flags Allows passing all 3P library config as a single JSON object instead of individual flags. The JSON format: {"library": "deepeval", "metric": "...", "modelProvider": "bedrock", "model": "...", "params": {"threshold": 0.5, "include_reason": true}} Both inline (--3p-template-json) and file (--3p-template-json-file) paths are supported. Individual flags (--3p-library, --metric, --param, --model-provider) still work for backward compatibility. --- src/cli/primitives/EvaluatorPrimitive.ts | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index eca6385e3..25e1cbca8 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -386,6 +386,14 @@ export class EvaluatorPrimitive extends BasePrimitive', `[3P library] LLM judge provider: ${MODEL_PROVIDERS.join(', ')} (default: bedrock)` ) + .option( + '--3p-template-json ', + '[Code-based] Inline JSON with 3P library config: {"library", "metric", "modelProvider", "model", "params"}' + ) + .option( + '--3p-template-json-file ', + '[Code-based] Path to JSON file with 3P library config (same format as --3p-template-json)' + ) .option( '--config ', 'Path to evaluator config JSON file (overrides --model, --instructions, --rating-scale) [non-interactive]' @@ -407,6 +415,8 @@ export class EvaluatorPrimitive extends BasePrimitive; + try { + templateObj = JSON.parse(templateJsonStr) as Record; + } catch { + fail('--3p-template-json must be valid JSON'); + } + if (!templateObj.library || typeof templateObj.library !== 'string') { + fail('--3p-template-json must include "library" (e.g. "deepeval" or "autoevals")'); + } + if (!templateObj.metric || typeof templateObj.metric !== 'string') { + fail('--3p-template-json must include "metric" (e.g. "AnswerRelevancyMetric")'); + } + // Populate individual options from template JSON + cliOptions['3pLibrary'] = templateObj.library as string; + cliOptions.metric = templateObj.metric as string; + if (templateObj.modelProvider) cliOptions.modelProvider = templateObj.modelProvider as string; + if (templateObj.model) cliOptions.model = templateObj.model as string; + if (templateObj.params && typeof templateObj.params === 'object') { + cliOptions.parametersFile = undefined; + cliOptions.param = Object.entries(templateObj.params as Record).map( + ([k, v]) => `${k}=${typeof v === 'string' ? v : JSON.stringify(v)}` + ); + } + } + // Validate --3p-library const threePLibraryRaw = cliOptions['3pLibrary']; if (threePLibraryRaw && !isSupportedLibrary(threePLibraryRaw)) { From 978a29c4b5947beffda0180163df9fd3a9541aea Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Wed, 29 Jul 2026 17:15:04 -0700 Subject: [PATCH 18/25] fix: Remove individual 3P flags, use --3p-template-json only Per TJ and Irene's feedback, removed --3p-library, --metric, --param, --parameters-file, and --model-provider flags. Users now use: --3p-template-json '{"library": "deepeval", "metric": "...", ...}' --3p-template-json-file evaluator-config.json This consolidates all 3P library configuration into a single JSON input. --- src/cli/primitives/EvaluatorPrimitive.ts | 141 +++++++---------------- 1 file changed, 44 insertions(+), 97 deletions(-) diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index 25e1cbca8..877323b33 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -362,9 +362,7 @@ export class EvaluatorPrimitive extends BasePrimitive', 'Evaluator type: llm-as-a-judge (default) or code-based') .option( '--model ', - 'Bedrock inference profile ID: [LLM] judge model for LLM-as-a-Judge, or [3P library] judge model with ' + - '--model-provider bedrock. Must be an inference profile (e.g. ' + - 'us.anthropic.claude-sonnet-4-20250514-v1:0), not a plain model ID — no bedrock/ prefix' + '[LLM] Bedrock inference profile ID for LLM-as-a-Judge (e.g. us.anthropic.claude-sonnet-4-20250514-v1:0)' ) .option( '--instructions ', @@ -373,19 +371,6 @@ export class EvaluatorPrimitive extends BasePrimitive', `[LLM] Rating scale preset: ${presetIds.join(', ')} (default: 1-5-quality)`) .option('--lambda-arn ', '[Code-based] Existing Lambda function ARN (external)') .option('--timeout ', '[Code-based] Lambda timeout in seconds, 1-300 (default: 60)') - .option('--3p-library ', `Third-party evaluation library (${SUPPORTED_LIBRARIES.join(', ')})`) - .option('--metric ', '[3P library] Metric/evaluator class name (e.g. AnswerRelevancyMetric)') - .option( - '--param ', - '[3P library] Metric parameter as key=value (repeatable)', - (val: string, prev: string[]) => [...prev, val], - [] as string[] - ) - .option('--parameters-file ', '[3P library] JSON file of metric constructor kwargs') - .option( - '--model-provider ', - `[3P library] LLM judge provider: ${MODEL_PROVIDERS.join(', ')} (default: bedrock)` - ) .option( '--3p-template-json ', '[Code-based] Inline JSON with 3P library config: {"library", "metric", "modelProvider", "model", "params"}' @@ -410,11 +395,6 @@ export class EvaluatorPrimitive extends BasePrimitive; try { @@ -467,62 +454,40 @@ export class EvaluatorPrimitive extends BasePrimitive).map( - ([k, v]) => `${k}=${typeof v === 'string' ? v : JSON.stringify(v)}` + try { + threePParams = Object.entries(templateObj.params as Record) + .map(([k, v]) => { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(k)) { + throw new Error(`Invalid Python kwarg name "${k}"`); + } + return `${k}=${jsonToPythonValue(v)}`; + }) + .join(', '); + } catch (e) { + fail(`Invalid params in --3p-template-json: ${getErrorMessage(e)}`); + } + } + if (threePModelProvider === 'bedrock' && !threePModel) { + fail( + '--3p-template-json requires "model" when modelProvider is bedrock. ' + + 'Pass a Bedrock inference profile ID (e.g. us.anthropic.claude-sonnet-4-20250514-v1:0)' ); } } - // Validate --3p-library - const threePLibraryRaw = cliOptions['3pLibrary']; - if (threePLibraryRaw && !isSupportedLibrary(threePLibraryRaw)) { - fail(`Invalid --3p-library "${threePLibraryRaw}". Supported: ${SUPPORTED_LIBRARIES.join(', ')}`); - } - const threePLibrary = threePLibraryRaw as ThirdPartyLibrary | undefined; - if (threePLibrary) { - if (!cliOptions.metric) fail('--metric is required when using --3p-library'); - if (cliOptions.model && cliOptions.modelProvider === 'openai') { - fail('--model cannot be used with --3p-library when --model-provider is openai'); - } - if (cliOptions.instructions) fail('--instructions cannot be used with --3p-library'); - if (cliOptions.ratingScale) fail('--rating-scale cannot be used with --3p-library'); - if (cliOptions.lambdaArn) fail('--lambda-arn cannot be used with --3p-library'); - if (cliOptions.config) fail('--config cannot be used with --3p-library'); - } - if (cliOptions.metric && !threePLibrary) { - fail('--metric requires --3p-library'); - } - if (cliOptions.param.length > 0 && !threePLibrary) { - fail('--param requires --3p-library'); - } - if (cliOptions.parametersFile && !threePLibrary) { - fail('--parameters-file requires --3p-library'); - } - if (cliOptions.param.length > 0 && cliOptions.parametersFile) { - fail('--param and --parameters-file cannot be used together'); - } - if (cliOptions.modelProvider && !threePLibrary) { - fail('--model-provider requires --3p-library'); - } - if (cliOptions.modelProvider && !isSupportedModelProvider(cliOptions.modelProvider)) { - fail( - `Invalid --model-provider "${cliOptions.modelProvider}". Supported: ${MODEL_PROVIDERS.join(', ')}` - ); - } - const resolvedModelProvider = (cliOptions.modelProvider as ModelProvider | undefined) ?? 'bedrock'; - if (resolvedModelProvider === 'bedrock' && !cliOptions.model && threePLibrary) { - fail( - '--model is required when using --model-provider bedrock (the default). Pass a Bedrock inference ' + - 'profile ID (e.g. us.anthropic.claude-sonnet-4-20250514-v1:0) — plain model IDs are not supported' - ); - } if (cliOptions.timeout) { const timeoutVal = parseInt(cliOptions.timeout, 10); if (isNaN(timeoutVal) || timeoutVal < 1 || timeoutVal > 300) { @@ -530,7 +495,7 @@ export class EvaluatorPrimitive extends BasePrimitive 0) { - try { - kwargs = parseParamFlags(cliOptions.param); - } catch (e) { - fail(`Invalid --param value: ${getErrorMessage(e)}`); - } - } else if (cliOptions.parametersFile) { - if (!existsSync(cliOptions.parametersFile)) { - fail(`--parameters-file not found: ${cliOptions.parametersFile}`); - } - try { - const fileContent = readFileSync(cliOptions.parametersFile, 'utf-8'); - kwargs = jsonToKwargs(fileContent); - } catch (e) { - fail(`Invalid --parameters-file: ${getErrorMessage(e)}`); - } - } thirdParty = { library: threePLibrary, - metricClass: cliOptions.metric!, - metricParams: kwargs, - modelProvider: (cliOptions.modelProvider as ModelProvider | undefined) ?? 'bedrock', - model: cliOptions.model, + metricClass: threePMetric!, + metricParams: threePParams, + modelProvider: threePModelProvider, + model: threePModel, }; } else if (cliOptions.config) { configJson = JSON.parse(readFileSync(cliOptions.config, 'utf-8')) as EvaluatorConfig; From 73376a3a857285008a487ac471ab86e40139fb33 Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Thu, 30 Jul 2026 07:33:10 -0700 Subject: [PATCH 19/25] fix: Remove dead parseParamFlags code + update E2E tests to use --3p-template-json - Remove parseParamFlags function (dead code after flag removal) - Update e2e tests to use --3p-template-json instead of removed --3p-library, --metric, --param, --model-provider flags --- e2e-tests/third-party-eval-lifecycle.test.ts | 43 ++++++++-------- src/cli/primitives/EvaluatorPrimitive.ts | 26 +--------- .../__tests__/EvaluatorPrimitive.test.ts | 50 ------------------- 3 files changed, 24 insertions(+), 95 deletions(-) diff --git a/e2e-tests/third-party-eval-lifecycle.test.ts b/e2e-tests/third-party-eval-lifecycle.test.ts index ad5380521..9beaf398c 100644 --- a/e2e-tests/third-party-eval-lifecycle.test.ts +++ b/e2e-tests/third-party-eval-lifecycle.test.ts @@ -62,8 +62,15 @@ describe.sequential('e2e: third-party evaluator lifecycle (DeepEval + Autoevals) const run = (args: string[]) => runAgentCoreCLI(args, projectPath); it.skipIf(!canRun)( - 'adds a DeepEval 3P evaluator with --3p-library, --model-provider bedrock, and --param', + 'adds a DeepEval 3P evaluator with --3p-template-json', async () => { + const templateJson = JSON.stringify({ + library: 'deepeval', + metric: 'AnswerRelevancyMetric', + modelProvider: 'bedrock', + model: 'us.anthropic.claude-sonnet-4-20250514-v1:0', + params: { threshold: 0.5 }, + }); const result = await run([ 'add', 'evaluator', @@ -71,16 +78,10 @@ describe.sequential('e2e: third-party evaluator lifecycle (DeepEval + Autoevals) deepevalEvalName, '--level', 'TRACE', - '--3p-library', - 'deepeval', - '--metric', - 'AnswerRelevancyMetric', - '--model-provider', - 'bedrock', - '--model', - 'us.anthropic.claude-sonnet-4-20250514-v1:0', - '--param', - 'threshold=0.5', + '--type', + 'code-based', + '--3p-template-json', + templateJson, '--json', ]); expect(result.exitCode, `Add DeepEval evaluator failed: ${result.stdout}`).toBe(0); @@ -93,8 +94,14 @@ describe.sequential('e2e: third-party evaluator lifecycle (DeepEval + Autoevals) ); it.skipIf(!canRun)( - 'adds an Autoevals 3P evaluator with --3p-library and --model-provider bedrock', + 'adds an Autoevals 3P evaluator with --3p-template-json', async () => { + const templateJson = JSON.stringify({ + library: 'autoevals', + metric: 'ExactMatch', + modelProvider: 'bedrock', + model: 'us.anthropic.claude-sonnet-4-20250514-v1:0', + }); const result = await run([ 'add', 'evaluator', @@ -102,14 +109,10 @@ describe.sequential('e2e: third-party evaluator lifecycle (DeepEval + Autoevals) autoevalsEvalName, '--level', 'TRACE', - '--3p-library', - 'autoevals', - '--metric', - 'ExactMatch', - '--model-provider', - 'bedrock', - '--model', - 'us.anthropic.claude-sonnet-4-20250514-v1:0', + '--type', + 'code-based', + '--3p-template-json', + templateJson, '--json', ]); expect(result.exitCode, `Add Autoevals evaluator failed: ${result.stdout}`).toBe(0); diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index 877323b33..b5da90cef 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -161,31 +161,7 @@ export function jsonToKwargs(json: string): string { .join(', '); } -export function parseParamFlags(params: string[]): string { - return params - .map(param => { - const eqIndex = param.indexOf('='); - if (eqIndex === -1) { - throw new Error(`"${param}" is not in key=value format`); - } - const key = param.slice(0, eqIndex).trim(); - if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) { - throw new Error(`Invalid Python kwarg name "${key}" in --param`); - } - const rawValue = param.slice(eqIndex + 1).trim(); - let value: unknown; - try { - value = JSON.parse(rawValue); - } catch { - if (rawValue === 'True') value = true; - else if (rawValue === 'False') value = false; - else if (rawValue === 'None') value = null; - else value = rawValue; - } - return `${key}=${jsonToPythonValue(value)}`; - }) - .join(', '); -} + function getWarningsForMetric(libraryConfig: ThirdPartyLibraryConfig, metricClass: string): string[] { const messages: string[] = []; diff --git a/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts b/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts index 999306de6..5e6233aab 100644 --- a/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts +++ b/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts @@ -5,7 +5,6 @@ import { THIRD_PARTY_EVALUATOR_LIBRARIES, jsonToKwargs, jsonToPythonValue, - parseParamFlags, } from '../EvaluatorPrimitive.js'; import { afterEach, describe, expect, it, vi } from 'vitest'; @@ -756,52 +755,3 @@ describe('jsonToKwargs', () => { }); }); -describe('parseParamFlags', () => { - it('parses number value', () => { - expect(parseParamFlags(['threshold=0.7'])).toBe('threshold=0.7'); - }); - - it('parses string value (JSON-quoted)', () => { - expect(parseParamFlags(['model="gpt-4"'])).toBe('model="gpt-4"'); - }); - - it('parses boolean value', () => { - expect(parseParamFlags(['verbose=true'])).toBe('verbose=True'); - }); - - it('parses array value', () => { - expect(parseParamFlags(['items=[1,2,3]'])).toBe('items=[1, 2, 3]'); - }); - - it('treats unquoted non-JSON string as string', () => { - expect(parseParamFlags(['name=hello world'])).toBe('name="hello world"'); - }); - - it('parses multiple params', () => { - expect(parseParamFlags(['threshold=0.7', 'verbose=true'])).toBe('threshold=0.7, verbose=True'); - }); - - it('throws on missing equals sign', () => { - expect(() => parseParamFlags(['noequalssign'])).toThrow('not in key=value format'); - }); - - it('handles value containing equals sign', () => { - expect(parseParamFlags(['formula=a=b'])).toBe('formula="a=b"'); - }); - - it('parses null value', () => { - expect(parseParamFlags(['callback=null'])).toBe('callback=None'); - }); - - it('parses Python-style True as boolean', () => { - expect(parseParamFlags(['include_reason=True'])).toBe('include_reason=True'); - }); - - it('parses Python-style False as boolean', () => { - expect(parseParamFlags(['strict=False'])).toBe('strict=False'); - }); - - it('parses Python-style None as null', () => { - expect(parseParamFlags(['fallback=None'])).toBe('fallback=None'); - }); -}); From de7768b83514e5f6ab66669f1fcf5754540f5fe7 Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Thu, 30 Jul 2026 07:41:24 -0700 Subject: [PATCH 20/25] fix: Add OpenAI warning + pass model to metric in OpenAI path When modelProvider is "openai": 1. Print warning that OPENAI_API_KEY is required as Lambda env var 2. Pass the user's model choice to the metric constructor (was ignored) --- src/assets/evaluators/autoevals-lambda/lambda_function.py | 2 +- src/cli/primitives/EvaluatorPrimitive.ts | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py index a232b5768..410f056a6 100644 --- a/src/assets/evaluators/autoevals-lambda/lambda_function.py +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -28,7 +28,7 @@ ) from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter -adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) {{/if}} diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index b5da90cef..ce57a80ca 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -462,6 +462,12 @@ export class EvaluatorPrimitive extends BasePrimitive Date: Thu, 30 Jul 2026 09:43:20 -0700 Subject: [PATCH 21/25] fix: Make model optional for deterministic metrics in --3p-template-json Deterministic metrics (ExactMatch, Levenshtein, etc.) don't need a model or model provider. If neither "model" nor "modelProvider" is specified in the JSON, the template generates a simple Lambda without LiteLLMClient/AmazonBedrockModel. Model is only required when modelProvider is explicitly "bedrock". --- src/cli/primitives/EvaluatorPrimitive.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index ce57a80ca..f41449671 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -413,7 +413,7 @@ export class EvaluatorPrimitive extends BasePrimitive Date: Thu, 30 Jul 2026 12:09:25 -0700 Subject: [PATCH 22/25] fix: Pin deepeval and autoevals with upper version bounds deepeval>=2.0.0,<3.0.0 and autoevals>=0.0.80,<1.0.0 to prevent breaking changes from major version bumps. Same pattern as strands-agents-evals>=1.0.3,<2.0.0. --- src/assets/evaluators/autoevals-lambda/pyproject.toml | 2 +- src/assets/evaluators/deepeval-lambda/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/assets/evaluators/autoevals-lambda/pyproject.toml b/src/assets/evaluators/autoevals-lambda/pyproject.toml index 9252dee5a..b37442fd1 100644 --- a/src/assets/evaluators/autoevals-lambda/pyproject.toml +++ b/src/assets/evaluators/autoevals-lambda/pyproject.toml @@ -9,7 +9,7 @@ description = "AgentCore Code-Based Evaluator (Autoevals)" requires-python = ">=3.10" dependencies = [ "bedrock-agentcore[autoevals]", - "autoevals>=0.0.80", + "autoevals>=0.0.80,<1.0.0", {{#if ModelProviderBedrock}} # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge "litellm>=1.60,<1.85", diff --git a/src/assets/evaluators/deepeval-lambda/pyproject.toml b/src/assets/evaluators/deepeval-lambda/pyproject.toml index f32e3cd77..7385ccfc2 100644 --- a/src/assets/evaluators/deepeval-lambda/pyproject.toml +++ b/src/assets/evaluators/deepeval-lambda/pyproject.toml @@ -9,7 +9,7 @@ description = "AgentCore Code-Based Evaluator (DeepEval)" requires-python = ">=3.10" dependencies = [ "bedrock-agentcore[deepeval]", - "deepeval>=2.0.0", + "deepeval>=2.0.0,<3.0.0", {{#if ModelProviderBedrock}} "aiobotocore>=2.13.0", {{/if}} From e7408153e94cbb47d89f40dfa7775d2aa7f0876b Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Thu, 30 Jul 2026 15:40:51 -0700 Subject: [PATCH 23/25] fix: Remove memorySizeMb, fix lint errors per TJ review - Remove memorySizeMb from schema, config, and tests (per TJ) - Fix ESLint errors (template literal type safety) - Update test descriptions --- src/cli/primitives/EvaluatorPrimitive.ts | 26 +++++++------------ .../__tests__/EvaluatorPrimitive.test.ts | 8 ++---- src/schema/llm-compacted/agentcore.ts | 1 - src/schema/schemas/primitives/evaluator.ts | 1 - 4 files changed, 11 insertions(+), 25 deletions(-) diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index f41449671..914005710 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -33,7 +33,6 @@ interface MetricWarning { export interface ThirdPartyLibraryConfig { templateDir: string; defaultTimeoutSeconds: number; - defaultMemorySizeMb: number; warnings: MetricWarning[]; } @@ -41,7 +40,6 @@ export const THIRD_PARTY_EVALUATOR_LIBRARIES = { deepeval: { templateDir: 'deepeval-lambda', defaultTimeoutSeconds: 300, - defaultMemorySizeMb: 1024, warnings: [ { metrics: new Set([ @@ -66,7 +64,6 @@ export const THIRD_PARTY_EVALUATOR_LIBRARIES = { autoevals: { templateDir: 'autoevals-lambda', defaultTimeoutSeconds: 60, - defaultMemorySizeMb: 512, warnings: [ { metrics: new Set(['Factuality', 'ClosedQA']), @@ -161,8 +158,6 @@ export function jsonToKwargs(json: string): string { .join(', '); } - - function getWarningsForMetric(libraryConfig: ThirdPartyLibraryConfig, metricClass: string): string[] { const messages: string[] = []; for (const warning of libraryConfig.warnings) { @@ -430,16 +425,18 @@ export class EvaluatorPrimitive extends BasePrimitive { describe('deepeval', () => { const deepevalConfig = THIRD_PARTY_EVALUATOR_LIBRARIES.deepeval; - it('returns config with deepeval defaults (300s, 1024MB)', () => { + it('returns config with deepeval defaults (300s)', () => { // eslint-disable-next-line @typescript-eslint/dot-notation const config = primitive['buildThirdPartyConfig']('my_eval', deepevalConfig); @@ -285,7 +285,6 @@ describe('EvaluatorPrimitive', () => { codeLocation: 'app/my_eval/', entrypoint: 'lambda_function.handler', timeoutSeconds: 300, - memorySizeMb: 1024, additionalPolicies: ['execution-role-policy.json'], }, }, @@ -304,7 +303,7 @@ describe('EvaluatorPrimitive', () => { describe('autoevals', () => { const autoevalsConfig = THIRD_PARTY_EVALUATOR_LIBRARIES.autoevals; - it('returns config with autoevals defaults (60s, 512MB)', () => { + it('returns config with autoevals defaults (60s)', () => { // eslint-disable-next-line @typescript-eslint/dot-notation const config = primitive['buildThirdPartyConfig']('fact_check', autoevalsConfig); @@ -314,7 +313,6 @@ describe('EvaluatorPrimitive', () => { codeLocation: 'app/fact_check/', entrypoint: 'lambda_function.handler', timeoutSeconds: 60, - memorySizeMb: 512, additionalPolicies: ['execution-role-policy.json'], }, }, @@ -611,7 +609,6 @@ describe('THIRD_PARTY_EVALUATOR_LIBRARIES registry', () => { expect(config).toBeDefined(); expect(config.templateDir).toBe('deepeval-lambda'); expect(config.defaultTimeoutSeconds).toBe(300); - expect(config.defaultMemorySizeMb).toBe(1024); }); it('contains autoevals with expected defaults', () => { @@ -619,7 +616,6 @@ describe('THIRD_PARTY_EVALUATOR_LIBRARIES registry', () => { expect(config).toBeDefined(); expect(config.templateDir).toBe('autoevals-lambda'); expect(config.defaultTimeoutSeconds).toBe(60); - expect(config.defaultMemorySizeMb).toBe(512); }); it('deepeval has warnings for retrieval_context metrics', () => { diff --git a/src/schema/llm-compacted/agentcore.ts b/src/schema/llm-compacted/agentcore.ts index 2ec7ebc06..2ab5290cf 100644 --- a/src/schema/llm-compacted/agentcore.ts +++ b/src/schema/llm-compacted/agentcore.ts @@ -233,7 +233,6 @@ interface ManagedCodeBasedConfig { codeLocation: string; entrypoint: string; // default 'lambda_function.handler' timeoutSeconds: number; // @min 1 @max 300 (default 60) - memorySizeMb?: number; // @min 128 @max 10240 additionalPolicies?: string[]; } diff --git a/src/schema/schemas/primitives/evaluator.ts b/src/schema/schemas/primitives/evaluator.ts index 7bb2b6a23..97d772d52 100644 --- a/src/schema/schemas/primitives/evaluator.ts +++ b/src/schema/schemas/primitives/evaluator.ts @@ -81,7 +81,6 @@ export const ManagedCodeBasedConfigSchema = z.object({ codeLocation: z.string().min(1), entrypoint: z.string().min(1).default('lambda_function.handler'), timeoutSeconds: z.number().int().min(1).max(300).default(60), - memorySizeMb: z.number().int().min(128).max(10240).optional(), additionalPolicies: z.array(z.string().min(1)).optional(), }); From 59aa8b8ae41fca24c4876c460d6c8d4848e64a0d Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Fri, 31 Jul 2026 12:36:57 -0700 Subject: [PATCH 24/25] fix: Resolve typecheck errors (TS2454, TS2353) - Fix templateObj 'used before assigned' by using throw directly - Remove remaining memorySizeMb from test config objects --- src/cli/primitives/EvaluatorPrimitive.ts | 2 +- src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index 914005710..97ea6145b 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -417,7 +417,7 @@ export class EvaluatorPrimitive extends BasePrimitive; } catch { - fail('--3p-template-json must be valid JSON'); + throw new Error('--3p-template-json must be valid JSON'); } if (!templateObj.library || typeof templateObj.library !== 'string') { fail('--3p-template-json must include "library" (e.g. "deepeval" or "autoevals")'); diff --git a/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts b/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts index 1518f4074..79bb9e460 100644 --- a/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts +++ b/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts @@ -335,7 +335,6 @@ describe('EvaluatorPrimitive', () => { codeLocation: 'app/deep_eval/', entrypoint: 'lambda_function.handler', timeoutSeconds: 300, - memorySizeMb: 1024, additionalPolicies: ['execution-role-policy.json'], }, }, @@ -347,7 +346,6 @@ describe('EvaluatorPrimitive', () => { codeLocation: 'app/auto_eval/', entrypoint: 'lambda_function.handler', timeoutSeconds: 60, - memorySizeMb: 512, additionalPolicies: ['execution-role-policy.json'], }, }, From ec5c264a28f792b43a69913ab5f096370cc9ece4 Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Fri, 31 Jul 2026 12:37:59 -0700 Subject: [PATCH 25/25] fix: Apply prettier formatting to test file --- src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts b/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts index 79bb9e460..ad0e47d44 100644 --- a/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts +++ b/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts @@ -297,7 +297,6 @@ describe('EvaluatorPrimitive', () => { expect(config.codeBased!.managed!.timeoutSeconds).toBe(120); }); - }); describe('autoevals', () => { @@ -748,4 +747,3 @@ describe('jsonToKwargs', () => { expect(jsonToKwargs('{}')).toBe(''); }); }); -