From 6fedaa307f6396e8ea7af834798e1dcccec09f8f Mon Sep 17 00:00:00 2001 From: Kresna <13603341+slaveofcode@users.noreply.github.com> Date: Sat, 29 Aug 2026 06:58:15 +0700 Subject: [PATCH] fix(agent): recover svg/canvas output when the model skips the JSON action Generative tools carry a big SVG/code arg that rarely survives JSON-escaping, so models (e.g. GLM) emit the or js code block directly and parseAction fails -> 'I didn't quite catch that'. Add recoverContentAction: when parseAction fails and a scoped generative tool matches raw output (an or a js fence), run that tool on the raw content (the executor already extracts/sanitizes it). --- src/hooks/useAgentChat.ts | 7 +++++-- src/tools/agent/loop.lib.test.ts | 21 ++++++++++++++++++++- src/tools/agent/loop.lib.ts | 17 +++++++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/hooks/useAgentChat.ts b/src/hooks/useAgentChat.ts index 641ac5d..5d169f6 100644 --- a/src/hooks/useAgentChat.ts +++ b/src/hooks/useAgentChat.ts @@ -1,7 +1,7 @@ import { useRef, useState } from 'react'; import { classifyIntent } from '@/tools/agent/intent'; import { executorFor, AGENT_EXECUTORS } from '@/tools/agent/executors'; -import { buildSystemPrompt, parseAction, type LoopTool } from '@/tools/agent/loop.lib'; +import { buildSystemPrompt, parseAction, recoverContentAction, type LoopTool } from '@/tools/agent/loop.lib'; import { emptySession, recordUser, applyResolution, historyForPrompt } from '@/tools/agent/session.lib'; import { prefillUrl } from '@/tools/agent/router.lib'; import { buildToolChoicePrompt, parseToolChoice } from '@/tools/agent/select.lib'; @@ -140,7 +140,10 @@ export function useAgentChat(provider: AgentProvider | null) { let produced = false; for (let iter = 0; iter < 8; iter++) { const raw = await provider.chat(convo); - const act = parseAction(raw); + // Generative tools (svg/canvas) often emit the artifact itself instead of + // a JSON action — a big SVG/code blob rarely survives JSON-escaping — so + // recover it and run the right tool. + const act = parseAction(raw) ?? recoverContentAction(raw, offered.map(e => e.toolId)); if (!act) { // Unparseable turn (small models emit junk). If we already handed the // user a result, just close out cleanly — never dump raw JSON to chat. diff --git a/src/tools/agent/loop.lib.test.ts b/src/tools/agent/loop.lib.test.ts index 09f5b2c..2f263c9 100644 --- a/src/tools/agent/loop.lib.test.ts +++ b/src/tools/agent/loop.lib.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { buildSystemPrompt, parseAction, type LoopTool } from './loop.lib'; +import { buildSystemPrompt, parseAction, recoverContentAction, type LoopTool } from './loop.lib'; const TOOLS: LoopTool[] = [ { name: 'base64_encode', description: 'Encode text to Base64', args: [{ name: 'text', type: 'string', required: true }] }, @@ -42,3 +42,22 @@ describe('parseAction', () => { expect(parseAction('{"action":"weird"}')).toBeNull(); }); }); + +describe('recoverContentAction', () => { + it('recovers an svg-viewer call when the model outputs raw ', () => { + const r = recoverContentAction('Here you go:\n', ['svg-viewer', 'qr-gen']); + expect(r?.action).toBe('call_tool'); + expect(r?.action === 'call_tool' && r.tool).toBe('svg-viewer'); + expect(r?.action === 'call_tool' && String(r.args.svg)).toContain(' { + const r = recoverContentAction('```js\nctx.fillRect(0,0,5,5)\n```', ['canvas-draw']); + expect(r?.action === 'call_tool' && r.tool).toBe('canvas-draw'); + }); + it('returns null when the matching tool is not offered', () => { + expect(recoverContentAction('', ['qr-gen'])).toBeNull(); + }); + it('returns null when there is no artifact', () => { + expect(recoverContentAction('just chatting here', ['svg-viewer', 'canvas-draw'])).toBeNull(); + }); +}); diff --git a/src/tools/agent/loop.lib.ts b/src/tools/agent/loop.lib.ts index a22b968..d242841 100644 --- a/src/tools/agent/loop.lib.ts +++ b/src/tools/agent/loop.lib.ts @@ -87,3 +87,20 @@ export function parseAction(raw: string): LoopAction | null { } return null; } + +/** + * Recover a content-producing tool call when the model emitted the artifact + * itself — an `` or a ```js code block — instead of a JSON action. A big + * SVG/code blob rarely survives being embedded (and JSON-escaped) inside the + * action protocol, so models tend to just output it; this routes that raw output + * to the right generative tool. Only fires for tools that are actually offered. + */ +export function recoverContentAction(raw: string, offeredToolIds: string[]): LoopAction | null { + if (offeredToolIds.includes('svg-viewer') && /][\s\S]*<\/svg>/i.test(raw)) { + return { action: 'call_tool', tool: 'svg-viewer', args: { svg: raw } }; + } + if (offeredToolIds.includes('canvas-draw') && /```(?:js|javascript)\b/i.test(raw)) { + return { action: 'call_tool', tool: 'canvas-draw', args: { code: raw } }; + } + return null; +}