diff --git a/src/hooks/useAgentChat.ts b/src/hooks/useAgentChat.ts index 641ac5d..0fb2b62 100644 --- a/src/hooks/useAgentChat.ts +++ b/src/hooks/useAgentChat.ts @@ -1,12 +1,12 @@ import { useRef, useState } from 'react'; import { classifyIntent } from '@/tools/agent/intent'; import { executorFor, AGENT_EXECUTORS } from '@/tools/agent/executors'; -import { buildSystemPrompt, parseAction, type LoopTool } from '@/tools/agent/loop.lib'; +import { buildSystemPrompt, parseAction, recoverContentAction, type LoopTool } from '@/tools/agent/loop.lib'; import { emptySession, recordUser, applyResolution, historyForPrompt } from '@/tools/agent/session.lib'; import { prefillUrl } from '@/tools/agent/router.lib'; import { buildToolChoicePrompt, parseToolChoice } from '@/tools/agent/select.lib'; import { getToolById } from '@/registry/tools'; -import type { AgentProvider, ChatMessage } from '@/services/agent/provider'; +import type { AgentProvider, ChatMessage, ToolSpec, ToolMsg } from '@/services/agent/provider'; export interface ChatUiTurn { role: 'user' | 'assistant'; @@ -119,110 +119,128 @@ export function useAgentChat(provider: AgentProvider | null) { // tiny on-device model gets only the keyword-scoped subset so it can't mis-pick. const capable = provider.capable === true; const offered = capable ? AGENT_EXECUTORS : intent.executors; - const loopTools: LoopTool[] = offered.map(e => ({ - name: e.toolId, - description: e.description, - args: [ - ...e.files.map(f => ({ name: f.key, type: 'file' as const, required: true })), - ...e.params.map(p => ({ name: p.key, type: p.type, required: p.default === undefined })), - ], - })); - const systemPrompt = buildSystemPrompt(loopTools) + (capable - ? '\n- You can call SEVERAL tools in sequence to fulfil one request. The output file of each tool automatically becomes the input for the next, so you can chain them (e.g. get audio from a video, then compress that audio). Plan the steps and call one tool per turn.' - : ''); - const convo: ChatMessage[] = [{ role: 'system', content: systemPrompt }, { role: 'user', content: q }]; - // Files available to this task: seeded from a prior turn on a continuation, - // and accumulated within the loop so a repeated call never re-prompts. + const offeredIds = offered.map(e => e.toolId); + // Files available to this task: seeded from a prior turn on a continuation. const loopFiles: Record = intent.continued ? { ...lastFilesRef.current } : {}; - // The most recent tool OUTPUT, offered as the input to the next tool (chaining). - let chainFile: File | null = null; - const doneKeys = new Set(); + let chainFile: File | null = null; // last tool OUTPUT, piped into the next tool let produced = false; - for (let iter = 0; iter < 8; iter++) { - const raw = await provider.chat(convo); - const act = parseAction(raw); - if (!act) { - // Unparseable turn (small models emit junk). If we already handed the - // user a result, just close out cleanly — never dump raw JSON to chat. - push({ role: 'assistant', text: produced ? 'Done — anything else?' : "I didn't quite catch that. Could you rephrase what you'd like to do?" }); - break; - } - if (act.action === 'final') { - const r = act.text || (produced ? 'Done.' : '(done)'); - push({ role: 'assistant', text: r }); - sessionRef.current = applyResolution(sessionRef.current, { toolId: null, params: {}, reply: r }); - break; - } + const wordCount = (s: string) => s.trim().split(/\s+/).filter(Boolean).length; - const key = act.tool + JSON.stringify(act.args); - if (doneKeys.has(key)) { - // Model re-issued a call it already completed — the result is already - // shown. Stop instead of re-running (or re-prompting for the file). - push({ role: 'assistant', text: 'Done — anything else?' }); - break; - } - - const exec = executorFor(act.tool); - if (!exec) { - convo.push({ role: 'assistant', content: raw }, { role: 'user', content: `TOOL_ERROR: unknown tool ${act.tool}` }); - continue; - } + // Run ONE executor call: collect files (chaining/upload) + required params + // (asking the user for hallucinated/empty ones), execute, show the result, + // and pipe its output. Shared by both the native and prompt loops. + const runExecutor = async (exec: (typeof AGENT_EXECUTORS)[number], argsIn: Record): Promise<{ ok: boolean; resultText: string; cancelled?: boolean }> => { push({ role: 'assistant', text: `→ ${exec.toolId}` }); - const files: Record = {}; - let cancelled = false; for (const fs of exec.files) { - // Prefer the previous tool's OUTPUT (chaining), then a file already - // uploaded this task, otherwise ask. Don't cache a chained output as the - // "original upload" — a later "make it smaller" should re-run on the source. - const piped = chainFile; + const piped = chainFile; // prefer the previous tool's output (chaining) const cached = piped ?? loopFiles[fs.key]; let f: File; if (cached) { f = cached; } else { try { f = await requestFile(fs.label); } - catch { cancelled = true; break; } + catch { updateLastText(`✗ ${exec.toolId} — cancelled`); return { ok: false, resultText: 'cancelled', cancelled: true }; } } files[fs.key] = f; - if (!piped) { loopFiles[fs.key] = f; lastFilesRef.current[fs.key] = f; } + if (!piped) { loopFiles[fs.key] = f; lastFilesRef.current[fs.key] = f; } // remember only real uploads } - if (cancelled) { updateLastText(`✗ ${exec.toolId} — cancelled`); break; } - - // Fill any required text param the model left empty — OR hallucinated — - // by asking the user. A tiny model often echoes the command itself as the - // value ("QR" → a QR of the word "QR"); treat as missing when the value is - // empty, equals the whole query, or is a short phrase that itself triggers - // this same tool (a command word, not content). - const params: Record = { ...act.args }; - const wordCount = (s: string) => s.trim().split(/\s+/).filter(Boolean).length; + const params: Record = { ...argsIn }; for (const ps of exec.params) { - if (ps.default !== undefined) continue; // optional / has a default + if (ps.default !== undefined) continue; const raw = params[ps.key] == null ? '' : String(params[ps.key]).trim(); const echoed = raw !== '' && (raw.toLowerCase() === q.trim().toLowerCase() || (wordCount(raw) <= 3 && exec.match(raw))); if (raw === '' || raw.toUpperCase() === 'UPLOAD' || echoed) { try { params[ps.key] = await requestInput(ps.label); } - catch { cancelled = true; break; } + catch { updateLastText(`✗ ${exec.toolId} — cancelled`); return { ok: false, resultText: 'cancelled', cancelled: true }; } } } - if (cancelled) { updateLastText(`✗ ${exec.toolId} — cancelled`); break; } - try { - const result = await exec.execute({ files, params }, p => { - updateLastText(`→ ${exec.toolId} — ${Math.round(p * 100)}%`); - }); + const result = await exec.execute({ files, params }, p => updateLastText(`→ ${exec.toolId} — ${Math.round(p * 100)}%`)); const blobUrl = result.blob ? URL.createObjectURL(result.blob) : undefined; push({ role: 'assistant', text: `✓ ${exec.toolId}: ${result.text ?? 'produced a file'}`, blobUrl, imgUrl: result.dataUrl, filename: result.filename }); - // Pipe this output into the next tool of the chain. if (result.blob) chainFile = new File([result.blob], result.filename ?? 'output', { type: result.blob.type }); sessionRef.current = applyResolution(sessionRef.current, { toolId: exec.toolId, params, reply: result.text ?? 'done' }); - doneKeys.add(key); produced = true; - convo.push({ role: 'assistant', content: raw }, { role: 'user', content: `TOOL_RESULT ${exec.toolId}: ${result.text ? result.text.slice(0, 400) : 'produced a file for the user'}. Respond with a "final" action now unless the user asked for more.` }); + return { ok: true, resultText: result.text ? result.text.slice(0, 400) : 'produced a file for the user' }; } catch (e) { push({ role: 'assistant', text: `✗ ${exec.toolId} error: ${(e as Error).message}` }); - convo.push({ role: 'assistant', content: raw }, { role: 'user', content: `TOOL_ERROR ${exec.toolId}: ${(e as Error).message}` }); + return { ok: false, resultText: `error: ${(e as Error).message}` }; + } + }; + + // --- Native function-calling loop (cloud): the provider's real tools API --- + const chatTools = provider.chatTools; + if (chatTools) { + const tools: ToolSpec[] = offered.map(e => ({ + name: e.toolId, + description: e.description, + parameters: { + type: 'object', + properties: { + ...Object.fromEntries(e.files.map(f => [f.key, { type: 'string', description: `${f.label} — pass the string "UPLOAD" and the app will ask the user for the file` }])), + ...Object.fromEntries(e.params.map(p => [p.key, { type: p.type === 'number' ? 'number' : 'string', description: p.label }])), + }, + required: [...e.files.map(f => f.key), ...e.params.filter(p => p.default === undefined).map(p => p.key)], + }, + })); + const sys = "You are GoodWebTools' agent. Use the tools to fulfil the user's request. You can call several tools in sequence — each tool's output file automatically becomes the next tool's input, so you can chain them. For a file argument pass the string \"UPLOAD\". When the task is done, reply with a short final message and no tool call."; + const msgs: ToolMsg[] = [{ role: 'system', content: sys }, { role: 'user', content: q }]; + const done = new Set(); + for (let iter = 0; iter < 8; iter++) { + const turn = await chatTools(msgs, tools); + if (!turn.calls.length) { + const r = turn.text || (produced ? 'Done.' : 'Okay.'); + push({ role: 'assistant', text: r }); + sessionRef.current = applyResolution(sessionRef.current, { toolId: null, params: {}, reply: r }); + break; + } + msgs.push({ role: 'assistant', content: turn.text, toolCalls: turn.calls }); + let stop = false; + for (const call of turn.calls) { + const exec = executorFor(call.name); + if (!exec) { msgs.push({ role: 'tool', toolCallId: call.id, content: `unknown tool ${call.name}` }); continue; } + const key = call.name + JSON.stringify(call.args); + if (done.has(key)) { msgs.push({ role: 'tool', toolCallId: call.id, content: 'already done — reply with a final message' }); continue; } + const res = await runExecutor(exec, call.args); + if (res.cancelled) { stop = true; break; } + done.add(key); + msgs.push({ role: 'tool', toolCallId: call.id, content: res.resultText }); + } + if (stop) break; + if (iter === 7) push({ role: 'assistant', text: produced ? 'Done — anything else?' : "I couldn't finish that." }); + } + return; + } + + // --- Prompt-based loop (on-device / no tools API): JSON action protocol --- + const loopTools: LoopTool[] = offered.map(e => ({ + name: e.toolId, + description: e.description, + args: [ + ...e.files.map(f => ({ name: f.key, type: 'file' as const, required: true })), + ...e.params.map(p => ({ name: p.key, type: p.type, required: p.default === undefined })), + ], + })); + const convo: ChatMessage[] = [{ role: 'system', content: buildSystemPrompt(loopTools) }, { role: 'user', content: q }]; + const doneKeys = new Set(); + for (let iter = 0; iter < 8; iter++) { + const raw = await provider.chat(convo); + const act = parseAction(raw) ?? recoverContentAction(raw, offeredIds); + if (!act) { push({ role: 'assistant', text: produced ? 'Done — anything else?' : "I didn't quite catch that. Could you rephrase what you'd like to do?" }); break; } + if (act.action === 'final') { + const r = act.text || (produced ? 'Done.' : '(done)'); + push({ role: 'assistant', text: r }); + sessionRef.current = applyResolution(sessionRef.current, { toolId: null, params: {}, reply: r }); + break; } - // Ran out of iterations mid-task: close out rather than leave it hanging. + const key = act.tool + JSON.stringify(act.args); + if (doneKeys.has(key)) { push({ role: 'assistant', text: 'Done — anything else?' }); break; } + const exec = executorFor(act.tool); + if (!exec) { convo.push({ role: 'assistant', content: raw }, { role: 'user', content: `TOOL_ERROR: unknown tool ${act.tool}` }); continue; } + const res = await runExecutor(exec, act.args); + if (res.cancelled) break; + doneKeys.add(key); + convo.push({ role: 'assistant', content: raw }, { role: 'user', content: `TOOL_RESULT ${exec.toolId}: ${res.resultText}. Respond with a "final" action now unless the user asked for more.` }); if (iter === 7) push({ role: 'assistant', text: produced ? 'Done — anything else?' : "I couldn't finish that — try rephrasing, or a bigger model." }); } } catch (e) { diff --git a/src/services/agent/provider.test.ts b/src/services/agent/provider.test.ts index 743739f..8b0a806 100644 --- a/src/services/agent/provider.test.ts +++ b/src/services/agent/provider.test.ts @@ -36,6 +36,27 @@ describe('createCloudProvider', () => { expect(headers.authorization).toBe('Bearer sk-2'); }); + it('parses OpenAI tool_calls into ToolCall[] and sends the tools', async () => { + const f = mockFetchOnce({ choices: [{ message: { content: '', tool_calls: [{ id: 'c1', type: 'function', function: { name: 'qr-gen', arguments: '{"text":"hi"}' } }] } }] }); + const p = createCloudProvider({ kind: 'openai', baseUrl: 'https://api.example.com/v1', model: 'm', apiKey: 'sk-1' }); + const turn = await p.chatTools!([{ role: 'user', content: 'make a qr' }], [{ name: 'qr-gen', description: 'qr', parameters: { type: 'object', properties: { text: { type: 'string' } } } }]); + expect(turn.calls).toEqual([{ id: 'c1', name: 'qr-gen', args: { text: 'hi' } }]); + const body = JSON.parse((f.mock.calls[0][1] as RequestInit).body as string); + expect(body.tools[0].function.name).toBe('qr-gen'); + expect(body.tool_choice).toBe('auto'); + }); + it('returns final text with no calls', async () => { + mockFetchOnce({ choices: [{ message: { content: 'All done!' } }] }); + const p = createCloudProvider({ kind: 'openai', baseUrl: 'x', model: 'm', apiKey: 'k' }); + expect(await p.chatTools!([{ role: 'user', content: 'hi' }], [])).toEqual({ text: 'All done!', calls: [] }); + }); + it('parses Anthropic tool_use blocks', async () => { + mockFetchOnce({ content: [{ type: 'text', text: 'sure' }, { type: 'tool_use', id: 't1', name: 'svg-viewer', input: { svg: '' } }] }); + const p = createCloudProvider({ kind: 'anthropic', baseUrl: 'https://api.anthropic.com', model: 'claude', apiKey: 'k' }); + const turn = await p.chatTools!([{ role: 'user', content: 'draw' }], [{ name: 'svg-viewer', description: 'd', parameters: {} }]); + expect(turn.text).toBe('sure'); + expect(turn.calls).toEqual([{ id: 't1', name: 'svg-viewer', args: { svg: '' } }]); + }); it('proxies the Anthropic messages endpoint with x-llm-target', async () => { const f = mockFetchOnce({ content: [{ text: 'claude reply' }] }); const p = createCloudProvider({ kind: 'anthropic', baseUrl: 'https://api.anthropic.com', model: 'claude', apiKey: 'sk-3', proxy: true }); diff --git a/src/services/agent/provider.ts b/src/services/agent/provider.ts index aed9831..baa1cec 100644 --- a/src/services/agent/provider.ts +++ b/src/services/agent/provider.ts @@ -7,11 +7,41 @@ */ export interface ChatMessage { role: 'system' | 'user' | 'assistant'; content: string } + +// --- Native function-calling --------------------------------------------------- +/** A tool offered to the model (JSON-Schema params), like MCP tools/list. */ +export interface ToolSpec { name: string; description: string; parameters: Record } +/** A tool call the model asked for. */ +export interface ToolCall { id: string; name: string; args: Record } +/** One turn of a native tool loop: assistant text (optional) + tool calls. */ +export interface ToolTurn { text: string; calls: ToolCall[] } +/** Messages for the native loop — richer than ChatMessage (carry calls/results). */ +export interface ToolMsg { + role: 'system' | 'user' | 'assistant' | 'tool'; + content: string; + toolCalls?: ToolCall[]; // on an assistant turn that called tools + toolCallId?: string; // on a tool-result turn +} + export interface AgentProvider { chat(messages: ChatMessage[]): Promise; /** Capable enough to plan over the full tool catalog and chain tools (cloud * models). Tiny on-device models are NOT — they get a keyword-scoped subset. */ capable?: boolean; + /** Native tool-calling step (present on cloud providers). Uses the provider's + * real tools API so big args (SVG/code) are encoded reliably, not model-typed. */ + chatTools?(messages: ToolMsg[], tools: ToolSpec[]): Promise; +} + +/** Coerce a provider's tool-call args object to the scalar shape executors expect. */ +function coerceArgs(input: unknown): Record { + const out: Record = {}; + if (input && typeof input === 'object') { + for (const [k, v] of Object.entries(input as Record)) { + out[k] = typeof v === 'number' ? v : typeof v === 'string' ? v : (typeof v === 'boolean' ? String(v) : JSON.stringify(v)); + } + } + return out; } /** True when the browser exposes WebGPU (required for the on-device model). */ @@ -146,5 +176,53 @@ export function createCloudProvider(cfg: CloudConfig): AgentProvider { if (!r.ok) throw new Error(j.error?.message || 'API error'); return j.choices?.[0]?.message?.content ?? ''; }, + + async chatTools(messages, tools) { + if (cfg.kind === 'anthropic') { + const system = messages.find(m => m.role === 'system')?.content ?? ''; + const conv = messages.filter(m => m.role !== 'system').map(m => { + if (m.role === 'tool') return { role: 'user', content: [{ type: 'tool_result', tool_use_id: m.toolCallId, content: m.content }] }; + if (m.role === 'assistant' && m.toolCalls?.length) { + return { role: 'assistant', content: [ + ...(m.content ? [{ type: 'text', text: m.content }] : []), + ...m.toolCalls.map(c => ({ type: 'tool_use', id: c.id, name: c.name, input: c.args })), + ] }; + } + return { role: m.role === 'assistant' ? 'assistant' : 'user', content: m.content }; + }); + const r = await providerFetch(cfg.proxy, cfg.baseUrl + '/v1/messages', { + 'content-type': 'application/json', 'x-api-key': cfg.apiKey, + 'anthropic-version': '2023-06-01', 'anthropic-dangerous-direct-browser-access': 'true', + }, { model: cfg.model, max_tokens: 1500, system, tools: tools.map(t => ({ name: t.name, description: t.description, input_schema: t.parameters })), messages: conv }); + const j = await r.json(); + if (!r.ok) throw new Error(j.error?.message || 'API error'); + let text = ''; const calls: ToolCall[] = []; + for (const b of j.content ?? []) { + if (b.type === 'text') text += b.text; + else if (b.type === 'tool_use') calls.push({ id: b.id, name: b.name, args: coerceArgs(b.input) }); + } + return { text, calls }; + } + // OpenAI-compatible (GLM, OpenAI, DeepSeek, Groq, Gemini, OpenRouter, OpenCode). + const msgs = messages.map(m => { + if (m.role === 'tool') return { role: 'tool', tool_call_id: m.toolCallId, content: m.content }; + if (m.role === 'assistant' && m.toolCalls?.length) { + return { role: 'assistant', content: m.content || null, tool_calls: m.toolCalls.map(c => ({ id: c.id, type: 'function', function: { name: c.name, arguments: JSON.stringify(c.args) } })) }; + } + return { role: m.role, content: m.content }; + }); + const r = await providerFetch(cfg.proxy, cfg.baseUrl + '/chat/completions', { + 'content-type': 'application/json', authorization: `Bearer ${cfg.apiKey}`, + }, { model: cfg.model, temperature: 0.2, max_tokens: 1500, tool_choice: 'auto', tools: tools.map(t => ({ type: 'function', function: { name: t.name, description: t.description, parameters: t.parameters } })), messages: msgs }); + const j = await r.json(); + if (!r.ok) throw new Error(j.error?.message || 'API error'); + const msg = j.choices?.[0]?.message ?? {}; + const calls: ToolCall[] = (msg.tool_calls ?? []).map((tc: { id: string; function?: { name: string; arguments?: string } }) => { + let parsed: unknown = {}; + try { parsed = JSON.parse(tc.function?.arguments || '{}'); } catch { /* keep {} */ } + return { id: tc.id, name: tc.function?.name ?? '', args: coerceArgs(parsed) }; + }); + return { text: msg.content ?? '', calls }; + }, }; } diff --git a/src/tools/agent/loop.lib.test.ts b/src/tools/agent/loop.lib.test.ts index 09f5b2c..2f263c9 100644 --- a/src/tools/agent/loop.lib.test.ts +++ b/src/tools/agent/loop.lib.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { buildSystemPrompt, parseAction, type LoopTool } from './loop.lib'; +import { buildSystemPrompt, parseAction, recoverContentAction, type LoopTool } from './loop.lib'; const TOOLS: LoopTool[] = [ { name: 'base64_encode', description: 'Encode text to Base64', args: [{ name: 'text', type: 'string', required: true }] }, @@ -42,3 +42,22 @@ describe('parseAction', () => { expect(parseAction('{"action":"weird"}')).toBeNull(); }); }); + +describe('recoverContentAction', () => { + it('recovers an svg-viewer call when the model outputs raw ', () => { + const r = recoverContentAction('Here you go:\n', ['svg-viewer', 'qr-gen']); + expect(r?.action).toBe('call_tool'); + expect(r?.action === 'call_tool' && r.tool).toBe('svg-viewer'); + expect(r?.action === 'call_tool' && String(r.args.svg)).toContain(' { + const r = recoverContentAction('```js\nctx.fillRect(0,0,5,5)\n```', ['canvas-draw']); + expect(r?.action === 'call_tool' && r.tool).toBe('canvas-draw'); + }); + it('returns null when the matching tool is not offered', () => { + expect(recoverContentAction('', ['qr-gen'])).toBeNull(); + }); + it('returns null when there is no artifact', () => { + expect(recoverContentAction('just chatting here', ['svg-viewer', 'canvas-draw'])).toBeNull(); + }); +}); diff --git a/src/tools/agent/loop.lib.ts b/src/tools/agent/loop.lib.ts index a22b968..d242841 100644 --- a/src/tools/agent/loop.lib.ts +++ b/src/tools/agent/loop.lib.ts @@ -87,3 +87,20 @@ export function parseAction(raw: string): LoopAction | null { } return null; } + +/** + * Recover a content-producing tool call when the model emitted the artifact + * itself — an `` or a ```js code block — instead of a JSON action. A big + * SVG/code blob rarely survives being embedded (and JSON-escaped) inside the + * action protocol, so models tend to just output it; this routes that raw output + * to the right generative tool. Only fires for tools that are actually offered. + */ +export function recoverContentAction(raw: string, offeredToolIds: string[]): LoopAction | null { + if (offeredToolIds.includes('svg-viewer') && /][\s\S]*<\/svg>/i.test(raw)) { + return { action: 'call_tool', tool: 'svg-viewer', args: { svg: raw } }; + } + if (offeredToolIds.includes('canvas-draw') && /```(?:js|javascript)\b/i.test(raw)) { + return { action: 'call_tool', tool: 'canvas-draw', args: { code: raw } }; + } + return null; +}