diff --git a/.changeset/tool-call-accuracy-v5-parts.md b/.changeset/tool-call-accuracy-v5-parts.md new file mode 100644 index 000000000..ad232ebd5 --- /dev/null +++ b/.changeset/tool-call-accuracy-v5-parts.md @@ -0,0 +1,7 @@ +--- +"@voltagent/scorers": patch +--- + +Fix the code-based `toolCallAccuracy` scorer missing tool calls in AI SDK v5 message parts + +The scorer read tool calls from message `parts` only when a part's `type` was exactly `"tool_call"`. AI SDK v5 encodes a tool call in message parts as `type: "tool-"`, so those calls were skipped and the scorer reported the expected tool as not called even when it was. It now recognizes `tool-` parts too, matching how it already reads them from tool-call lists. diff --git a/packages/scorers/src/tool-call-accuracy.spec.ts b/packages/scorers/src/tool-call-accuracy.spec.ts index e0b72fe4e..3d969e92d 100644 --- a/packages/scorers/src/tool-call-accuracy.spec.ts +++ b/packages/scorers/src/tool-call-accuracy.spec.ts @@ -129,6 +129,36 @@ describe("createToolCallAccuracyScorerCode", () => { expect(result.score).toBe(1); }); + it("extracts tool names from AI SDK v5 message parts (type: tool-)", async () => { + const scorer = createToolCallAccuracyScorerCode({ + expectedTool: "searchProducts", + }); + + const result = await scorer.scorer({ + payload: { + messages: [ + { + id: "msg-1", + role: "assistant", + parts: [ + { + type: "tool-searchProducts", + toolCallId: "call-1", + state: "output-available", + input: {}, + output: {}, + }, + ], + }, + ], + } satisfies ToolCallAccuracyPayload, + params: {} as ToolCallAccuracyParams, + }); + + expect(result.status).toBe("success"); + expect(result.score).toBe(1); + }); + it("supports custom buildPayload mappings", async () => { const scorer = createToolCallAccuracyScorerCode<{ events: Array<{ name: string }> }>({ expectedTool: "searchProducts", diff --git a/packages/scorers/src/tool-call-accuracy.ts b/packages/scorers/src/tool-call-accuracy.ts index 790ba663f..26dcea7de 100644 --- a/packages/scorers/src/tool-call-accuracy.ts +++ b/packages/scorers/src/tool-call-accuracy.ts @@ -276,7 +276,11 @@ function extractToolNamesFromParts(parts: unknown[]): string[] { } const partType = normalizeMessageType(part.type); - if (partType !== "tool_call") { + // AI SDK v5 encodes a tool call in message parts as `type: "tool-"`, + // not `"tool_call"`. Accept those too (extractToolName reads the name from + // the type); tool results and streaming events are still excluded because + // shouldExtractToolNameFromType returns false for them. + if (partType !== "tool_call" && !shouldExtractToolNameFromType(partType)) { continue; }