diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..af2a9e74 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,30 @@ +name: E2E · Ask Agent + +on: + pull_request: + push: + branches: [main, develop] + +jobs: + e2e: + name: Playwright · Ask Agent + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + - name: Install dependencies + run: npm ci + - name: Install Playwright browser + run: npx playwright install --with-deps chromium + - name: Run Ask Agent E2E suite + run: npm run test:e2e + - name: Upload report on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: playwright-report/ + retention-days: 7 diff --git a/.gitignore b/.gitignore index 1d701fb2..aeac4a27 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,9 @@ public/sqlite/ # Unbuilt api-client planning docs (kept local until that feature is picked up) docs/superpowers/plans/2026-08-09-api-client.md docs/superpowers/specs/2026-08-09-api-client-design.md + +# Playwright +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ diff --git a/README.md b/README.md index ccbef6df..30649d4a 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,18 @@ [![Tools](https://img.shields.io/badge/tools-193-16a34a)](https://goodwebtools.com) [![PRs welcome](https://img.shields.io/badge/PRs-welcome-ec4899.svg)](#contributing) -GoodWebTools homepage — a searchable grid of 193 privacy-first browser tools across Image, PDF, Dev, Media and more + + + + + + + + + + + +
GoodWebTools on desktop — searchable grid of 193 privacy-first browser tools across Image, PDF, Dev, Media and moreGoodWebTools on mobile — responsive tool grid that works in any phone browserGoodWebTools mobile floating category menu for quick navigation between tool categories
DesktopMobileMobile category menu
⭐ **If GoodWebTools is useful, star the repo — it helps other people find privacy-first tools.** diff --git a/docs/images/mobile-hero.png b/docs/images/mobile-hero.png new file mode 100644 index 00000000..87788c2f Binary files /dev/null and b/docs/images/mobile-hero.png differ diff --git a/docs/images/mobile-menu.png b/docs/images/mobile-menu.png new file mode 100644 index 00000000..577b0ede Binary files /dev/null and b/docs/images/mobile-menu.png differ diff --git a/docs/superpowers/plans/2026-08-29-ask-agent-e2e.md b/docs/superpowers/plans/2026-08-29-ask-agent-e2e.md new file mode 100644 index 00000000..b7c6d94d --- /dev/null +++ b/docs/superpowers/plans/2026-08-29-ask-agent-e2e.md @@ -0,0 +1,622 @@ +# Ask Agent E2E Testing Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A deterministic Playwright E2E suite for `/ask-agent` that drives the real UI and real executors with a scripted (mocked) model provider, plus a CI workflow, a local real-model smoke script, and desktop-vs-mobile README screenshots. + +**Architecture:** The agent talks to an abstract `AgentProvider`. Tests inject a dev-only **scripted provider** (via `window.__E2E_AGENT__`, gated behind `import.meta.env.DEV` so it never exists in prod). Playwright sets the script, navigates to `/ask-agent`, and asserts on the real chat/executor output. Real-model coverage is a separate, manually-run spec (GitHub runners have no WebGPU). + +**Tech Stack:** Playwright (`@playwright/test`), Vitest (existing), Astro dev server, React island `AskAgent`. + +**Spec:** `docs/superpowers/specs/2026-08-29-ask-agent-e2e-design.md` + +## Global Constraints + +- All new logic is client-side; no server code. (GWT core promise.) +- Commit identity is repo-local `Kresna <13603341+slaveofcode@users.noreply.github.com>`; **no AI-attribution trailers**; secret-sweep the staged diff before every commit. +- The scripted-provider path MUST be behind `import.meta.env.DEV` so it is absent from the production bundle. +- E2E runs against `npm run dev` (dev server, `import.meta.env.DEV === true`), baseURL `http://localhost:4321`. +- Fixtures and screenshots must contain only generic/fake content — no real data, no absolute home-directory paths, no employer content. +- `npm install` uses `--legacy-peer-deps` (committed `.npmrc`). +- Deterministic suite uses only headless-safe executors: `base64`, `json-format`, `terbilang`, `csv-dedupe`. Exclude WebGPU/ffmpeg/mupdf-heavy tools from the fast suite. +- Keep the mobile screenshot capture script **throwaway/uncommitted** (scratchpad), per the user's standing instruction. + +--- + +## File Structure + +- `src/services/agent/e2e-provider.ts` — dev-only scripted `AgentProvider` factory. +- `src/services/agent/e2e-provider.test.ts` — Vitest unit tests for the factory. +- `src/islands/agent/AskAgent.tsx` — add dev-only injection effect + a few `data-testid`s. +- `playwright.config.ts` — Playwright config (webServer = dev, chromium project). +- `e2e/helpers.ts` — shared helpers (install script, send a message, selectors). +- `e2e/agent.spec.ts` — the deterministic mock-provider suite (4 coverage areas). +- `e2e/agent-real.spec.ts` — local-only real-model smoke (Layer 3a), skipped unless opted in. +- `e2e/fixtures/` — `sample.csv`, `a.pdf`, `b.pdf` (tiny, generic). +- `.github/workflows/e2e.yml` — CI job running the deterministic suite. +- `package.json` — `@playwright/test` devDep; `test:e2e`, `test:e2e:real` scripts. +- `README.md` + `docs/images/mobile-*.png` — desktop/mobile side-by-side. + +--- + +## Task 1: Scripted provider factory (unit-tested) + +**Files:** +- Create: `src/services/agent/e2e-provider.ts` +- Test: `src/services/agent/e2e-provider.test.ts` + +**Interfaces:** +- Consumes from `@/services/agent/provider`: `AgentProvider`, `ChatMessage`, `ToolSpec`, `ToolMsg`, `ToolTurn` (`ToolTurn = { text: string; calls: { id: string; name: string; args: Record }[] }`). +- Produces: `type ScriptStep`, `function createScriptedProvider(steps: ScriptStep[], opts?: { capable?: boolean }): AgentProvider`. + +- [ ] **Step 1: Write the failing test** + +```ts +// src/services/agent/e2e-provider.test.ts +import { describe, it, expect } from 'vitest'; +import { createScriptedProvider } from './e2e-provider'; + +describe('createScriptedProvider', () => { + it('replays chatTools steps in order, then ends with an empty final turn', async () => { + const p = createScriptedProvider( + [{ calls: [{ name: 'base64', args: { text: 'hi' } }] }], + { capable: true }, + ); + expect(p.capable).toBe(true); + const t1 = await p.chatTools!([], []); + expect(t1.calls.map(c => c.name)).toEqual(['base64']); + expect(t1.calls[0].id).toBeTruthy(); // an id is generated + const t2 = await p.chatTools!([], []); // steps exhausted + expect(t2.calls).toEqual([]); // -> final, loop can stop + }); + + it('replays chat() steps as raw strings', async () => { + const p = createScriptedProvider([{ chat: 'Hello there!' }], { capable: false }); + expect(p.capable).toBe(false); + expect(await p.chat([])).toBe('Hello there!'); + expect(await p.chat([])).toBe(''); // exhausted -> empty + }); + + it('supports a final text-only tools turn', async () => { + const p = createScriptedProvider([{ text: 'done', calls: [] }], { capable: true }); + const t = await p.chatTools!([], []); + expect(t).toEqual({ text: 'done', calls: [] }); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run src/services/agent/e2e-provider.test.ts` +Expected: FAIL — `createScriptedProvider` is not defined. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// src/services/agent/e2e-provider.ts +import type { AgentProvider, ChatMessage, ToolMsg, ToolSpec, ToolTurn } from './provider'; + +export type ScriptStep = + | { chat: string } + | { calls: { name: string; args: Record }[]; text?: string } + | { text: string; calls?: undefined }; + +/** + * A deterministic, dev/test-only AgentProvider that replays a fixed script. + * `chat` steps feed the prompt loop / chat-mode; `calls`/`text` steps feed the + * native tools loop. Once the script is exhausted, chatTools returns an empty + * final turn and chat returns '' so the agent loop terminates cleanly. + */ +export function createScriptedProvider(steps: ScriptStep[], opts: { capable?: boolean } = {}): AgentProvider { + let i = 0; + let callSeq = 0; + return { + capable: opts.capable, + async chat(_messages: ChatMessage[]): Promise { + const step = steps[i++]; + if (step && 'chat' in step) return step.chat; + if (step && 'text' in step) return step.text; + return ''; + }, + async chatTools(_messages: ToolMsg[], _tools: ToolSpec[]): Promise { + const step = steps[i++]; + if (step && 'calls' in step && step.calls) { + return { text: step.text ?? '', calls: step.calls.map(c => ({ id: `e2e-${++callSeq}`, name: c.name, args: c.args })) }; + } + if (step && 'text' in step) return { text: step.text, calls: [] }; + return { text: '', calls: [] }; + }, + }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run src/services/agent/e2e-provider.test.ts` +Expected: PASS (3 tests). + +- [ ] **Step 5: Commit** + +```bash +git add src/services/agent/e2e-provider.ts src/services/agent/e2e-provider.test.ts +git commit -m "test(agent): scripted AgentProvider for deterministic E2E" +``` + +--- + +## Task 2: Dev-only injection hook + stable test ids in AskAgent + +**Files:** +- Modify: `src/islands/agent/AskAgent.tsx` + +**Interfaces:** +- Consumes: `createScriptedProvider` from Task 1; `setProvider` (existing `useState` setter in AskAgent). +- Produces: a global contract — a test sets `window.__E2E_AGENT__ = { steps: ScriptStep[]; capable?: boolean }` before load; AskAgent installs the scripted provider. Adds `data-testid`s: `agent-messages`, `agent-input`, `agent-send`, `agent-attach-input`. + +- [ ] **Step 1: Add the dev-only effect.** After the existing `useState`/`useRef` block and before the return, add (import `useEffect` from React at the top — the file already imports from `'react'`): + +```tsx + // E2E hook: a scripted provider injected by Playwright. Behind import.meta.env.DEV + // so it is tree-shaken from production — a real user can never trigger it. + useEffect(() => { + if (!import.meta.env.DEV) return; + const cfg = (window as unknown as { __E2E_AGENT__?: { steps: unknown[]; capable?: boolean } }).__E2E_AGENT__; + if (!cfg) return; + import('@/services/agent/e2e-provider').then(({ createScriptedProvider }) => + setProvider(createScriptedProvider(cfg.steps as never, { capable: cfg.capable }))); + }, []); +``` + +- [ ] **Step 2: Add `data-testid`s to the chat + input controls.** In the message list container (the element wrapping `{turns.map(...)}`, around line 149) add `data-testid="agent-messages"`. On the main text input (placeholder `"Tell the agent what you want…"`, ~line 205) add `data-testid="agent-input"`. On the Send button (~line 207) add `data-testid="agent-send"`. On the hidden attach input (`ref={attachInputRef}`, ~line 201) add `data-testid="agent-attach-input"`. + +- [ ] **Step 3: Verify the dev build still compiles and prod strips the hook** + +Run: `npm run lint && npm run build` +Expected: 0 lint errors; build succeeds. (Manual check: `grep -r "__E2E_AGENT__" dist/` returns nothing — the effect body is behind `import.meta.env.DEV` and dropped from the prod bundle.) + +- [ ] **Step 4: Commit** + +```bash +git add src/islands/agent/AskAgent.tsx +git commit -m "test(agent): dev-only scripted-provider injection hook + test ids" +``` + +--- + +## Task 3: Playwright setup + smoke spec + +**Files:** +- Create: `playwright.config.ts`, `e2e/helpers.ts`, `e2e/agent.spec.ts` (smoke only for now) +- Modify: `package.json` + +**Interfaces:** +- Consumes: the dev-server URL `http://localhost:4321`; `data-testid`s from Task 2. +- Produces: `installAgent(page, script)` and `send(page, text)` helpers used by later tasks. + +- [ ] **Step 1: Install Playwright as a devDependency** + +Run: `npm install -D @playwright/test --legacy-peer-deps && npx playwright install chromium` +Expected: `@playwright/test` in `package.json` devDependencies; chromium downloaded. + +- [ ] **Step 2: Add scripts to `package.json`.** In `"scripts"` add: + +```json + "test:e2e": "playwright test", + "test:e2e:real": "PLAYWRIGHT_REAL=1 playwright test e2e/agent-real.spec.ts", +``` + +- [ ] **Step 3: Write `playwright.config.ts`** + +```ts +import { defineConfig, devices } from '@playwright/test'; + +export default defineConfig({ + testDir: './e2e', + timeout: 30_000, + fullyParallel: false, + workers: 1, + reporter: process.env.CI ? [['github'], ['html', { open: 'never' }]] : 'list', + use: { baseURL: 'http://localhost:4321', trace: 'on-first-retry' }, + projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + webServer: { + command: 'npm run dev', + url: 'http://localhost:4321', + reuseExistingServer: true, + timeout: 120_000, + }, +}); +``` + +- [ ] **Step 4: Write `e2e/helpers.ts`** + +```ts +import type { Page } from '@playwright/test'; + +export type ScriptStep = + | { chat: string } + | { calls: { name: string; args: Record }[]; text?: string } + | { text: string }; + +/** Inject the scripted provider BEFORE navigation, then open /ask-agent. */ +export async function installAgent(page: Page, script: { steps: ScriptStep[]; capable?: boolean }) { + await page.addInitScript(s => { (window as unknown as { __E2E_AGENT__: unknown }).__E2E_AGENT__ = s; }, script); + await page.goto('/ask-agent'); + // The panel installs the scripted provider on mount; the input is always present. + await page.getByTestId('agent-input').waitFor(); +} + +/** Type a message and send it. */ +export async function send(page: Page, text: string) { + await page.getByTestId('agent-input').fill(text); + await page.getByTestId('agent-send').click(); +} +``` + +- [ ] **Step 5: Write the smoke test in `e2e/agent.spec.ts`** + +```ts +import { test, expect } from '@playwright/test'; +import { installAgent, send } from './helpers'; + +test('chat mode: small talk gets a reply, no tool runs', async ({ page }) => { + await installAgent(page, { steps: [{ chat: 'Hi! How can I help?' }], capable: false }); + await send(page, 'hello there'); + await expect(page.getByTestId('agent-messages')).toContainText('Hi! How can I help?'); + await expect(page.getByTestId('agent-messages')).not.toContainText('→ '); +}); +``` + +- [ ] **Step 6: Run it** + +Run: `npm run test:e2e -- e2e/agent.spec.ts` +Expected: PASS (1 test). If the dev server isn't running, Playwright starts one. + +- [ ] **Step 7: Commit** + +```bash +git add playwright.config.ts e2e/helpers.ts e2e/agent.spec.ts package.json package-lock.json +git commit -m "test(e2e): Playwright setup + agent smoke test" +``` + +--- + +## Task 4: Core execute + download, and the single-candidate shortcut + +**Files:** +- Modify: `e2e/agent.spec.ts` + +**Interfaces:** +- Consumes: `installAgent`, `send` (Task 3). Executors `base64`, `json-format`, `terbilang` (registry). Result lines render as `✓ : `. + +- [ ] **Step 1: Add the tests** + +```ts +test('runs base64 and shows the result line', async ({ page }) => { + await installAgent(page, { + steps: [{ calls: [{ name: 'base64', args: { text: 'hi' } }] }, { text: 'Done.' }], + capable: true, + }); + await send(page, 'base64 encode this'); + await expect(page.getByTestId('agent-messages')).toContainText('✓ base64:'); + await expect(page.getByTestId('agent-messages')).toContainText('aGk='); // base64("hi") +}); + +test('formats JSON via a scripted tool call', async ({ page }) => { + await installAgent(page, { + steps: [{ calls: [{ name: 'json-format', args: { text: '{"a":1}' } }] }, { text: 'Done.' }], + capable: true, + }); + await send(page, 'format this json'); + await expect(page.getByTestId('agent-messages')).toContainText('✓ json-format:'); +}); + +// Weak-model deterministic path: a single scoped tool runs WITHOUT the model +// emitting JSON (the 0.5B "didn't quite catch that" fix). capable:false, and the +// script's chat is intentionally never consumed because the shortcut bypasses it. +test('single-candidate shortcut runs the tool with no model JSON', async ({ page }) => { + await installAgent(page, { steps: [{ chat: 'garbage that would not parse' }], capable: false }); + await send(page, 'terbilang 1500000'); + await expect(page.getByTestId('agent-messages')).toContainText('✓ terbilang:'); + await expect(page.getByTestId('agent-messages')).not.toContainText('didn’t quite catch'); +}); +``` + +- [ ] **Step 2: Run** + +Run: `npm run test:e2e -- e2e/agent.spec.ts` +Expected: PASS (smoke + 3 new). If `terbilang 1500000` scopes to more than one tool, the shortcut won't fire — verify with `node -e "const {scopeExecutors}=require('./src/tools/agent/executors'); console.log(scopeExecutors('terbilang 1500000').map(e=>e.toolId))"` is not runnable directly (TS); instead confirm via the existing unit test `executors.test.ts` which asserts `terbilang` scopes. If more than one, switch the query to `slugify My Title` (also single-candidate) and assert `✓ slugify:`. + +- [ ] **Step 3: Commit** + +```bash +git add e2e/agent.spec.ts +git commit -m "test(e2e): core execute+download + single-candidate shortcut" +``` + +--- + +## Task 5: File attach + multi-file prompt + +**Files:** +- Create: `e2e/fixtures/sample.csv`, `e2e/fixtures/a.pdf`, `e2e/fixtures/b.pdf` +- Modify: `e2e/agent.spec.ts` + +**Interfaces:** +- Consumes: `agent-attach-input` testid (hidden file input). `csv-dedupe` executor reads the attached CSV's text. Multi-file prompt renders an `input[type="file"][multiple]`. + +- [ ] **Step 1: Create the CSV fixture** `e2e/fixtures/sample.csv` (generic, has a duplicate row): + +``` +name,city +Ada,London +Ada,London +Grace,NYC +``` + +- [ ] **Step 2: Create two 1-page PDF fixtures.** Generate them (no binary-in-plan): run +`node -e "const {PDFDocument}=require('pdf-lib');(async()=>{for(const n of ['a','b']){const d=await PDFDocument.create();d.addPage([200,200]).drawText(n.toUpperCase());require('fs').writeFileSync('e2e/fixtures/'+n+'.pdf',await d.save());}})()"` +Expected: `e2e/fixtures/a.pdf` and `b.pdf` exist (~1 KB each, generic content). + +- [ ] **Step 3: Add the tests** + +```ts +import path from 'node:path'; + +test('consumes an attached CSV and dedupes it', async ({ page }) => { + await installAgent(page, { + steps: [{ calls: [{ name: 'csv-dedupe', args: {} }] }, { text: 'Done.' }], + capable: true, + }); + await page.getByTestId('agent-attach-input').setInputFiles(path.join(__dirname, 'fixtures/sample.csv')); + await send(page, 'remove duplicate rows from this csv'); + await expect(page.getByTestId('agent-messages')).toContainText('✓ csv-dedupe:'); + // a Download link is produced for the output blob + await expect(page.getByRole('link', { name: /download/i })).toBeVisible(); +}); + +test('multi-file tool asks for several files', async ({ page }) => { + await installAgent(page, { + steps: [{ calls: [{ name: 'pdf-merge', args: {} }] }, { text: 'Done.' }], + capable: true, + }); + await send(page, 'merge these pdfs into one'); + // With no attachment, the app prompts with a multiple file input. + await expect(page.locator('input[type="file"][multiple]')).toBeVisible(); +}); +``` + +- [ ] **Step 4: Run** + +Run: `npm run test:e2e -- e2e/agent.spec.ts` +Expected: PASS. If `csv-dedupe` requires a param that gets prompted, check `executors.ts` `csv-dedupe` params; the fixture + scripted empty args should let it run using the attached file. If a Download link doesn't appear because the tool returns text only, assert `✓ csv-dedupe:` text only and drop the link assertion. + +- [ ] **Step 5: Commit** + +```bash +git add e2e/fixtures e2e/agent.spec.ts +git commit -m "test(e2e): file attach (csv-dedupe) + multi-file merge prompt" +``` + +--- + +## Task 6: Weak-model regressions + settings persistence + +**Files:** +- Modify: `e2e/agent.spec.ts` + +**Interfaces:** +- Consumes: `humanSize` behaviour (KB for sub-MB), ID routing (`ganti … ke …`), the once-per-tool echo cap, and localStorage keys `gwt-agent-source`, `gwt-agent-cloud-preset`, `gwt-agent-cloud-model`, `gwt-agent-cloud-proxy`. + +- [ ] **Step 1: Add the regression + persistence tests** + +```ts +test('echo cap: a repeated tool call runs only once', async ({ page }) => { + await installAgent(page, { + steps: [ + { calls: [{ name: 'base64', args: { text: 'hi' } }] }, + { calls: [{ name: 'base64', args: { text: 'hi' } }] }, // duplicate — must be ignored + { text: 'Done.' }, + ], + capable: true, + }); + await send(page, 'base64 encode this'); + await expect(page.getByTestId('agent-messages')).toContainText('✓ base64:'); + const runs = await page.getByText('→ base64', { exact: false }).count(); + expect(runs).toBe(1); +}); + +test('settings persist across reload', async ({ page }) => { + await page.goto('/ask-agent'); + await page.getByRole('button', { name: 'Cloud (API key)' }).click(); + await page.getByRole('combobox').first().selectOption({ index: 1 }); // pick a non-default preset + const preset = await page.getByRole('combobox').first().inputValue(); + await page.reload(); + await page.getByRole('button', { name: 'Cloud (API key)' }).click(); + await expect(page.getByRole('combobox').first()).toHaveValue(preset); +}); +``` + +Note: the `humanSize`/ID-routing behaviours are already locked by Vitest unit tests (`executors.test.ts` `humanSize` + `image-convert`/Bahasa scoping) — the E2E layer covers the *flow*, so we don't duplicate pure-function assertions here. The echo cap and persistence are flow-level and belong in E2E. + +- [ ] **Step 2: Run** + +Run: `npm run test:e2e -- e2e/agent.spec.ts` +Expected: PASS. If `getByRole('combobox')` is ambiguous (on-device model select vs cloud preset), scope with `page.locator('select').nth(0)` after clicking the Cloud tab, or add a `data-testid="cloud-preset"` to the ` { const k = e.target.value; setCloudPreset(k); setCloudModel(CLOUD_PRESETS[k].model); setUseProxy(!!CLOUD_PRESETS[k].proxied); save('gwt-agent-cloud-preset', k); save('gwt-agent-cloud-model', CLOUD_PRESETS[k].model); save('gwt-agent-cloud-proxy', CLOUD_PRESETS[k].proxied ? '1' : '0'); setProvider(null); }} className={inputCls}> + { setCloudModel(e.target.value); save('gwt-agent-cloud-model', e.target.value); }} placeholder="model" className={`${inputCls} w-48`} /> @@ -144,7 +154,7 @@ export default function AskAgent({ lang = 'en' }: { lang?: 'en' | 'id' }) { {loaded && ( <> -
+
{turns.length === 0 &&

{tr.hint}

} {turns.map((t, i) => (
@@ -198,13 +208,13 @@ export default function AskAgent({ lang = 'en' }: { lang?: 'en' | 'id' }) {
)}
- { const fs = Array.from(e.target.files ?? []); if (fs.length) setAttached(a => [...a, ...fs]); e.target.value = ''; }} /> - setInput(e.target.value)} onKeyDown={e => { if (e.key === 'Enter') submit(); }} + setInput(e.target.value)} onKeyDown={e => { if (e.key === 'Enter') submit(); }} placeholder="Tell the agent what you want…" className="flex-1 border-2 border-border bg-muted p-3 outline-none focus:shadow-brutal-sm" /> - +
)} diff --git a/src/services/agent/e2e-provider.test.ts b/src/services/agent/e2e-provider.test.ts new file mode 100644 index 00000000..63eb9779 --- /dev/null +++ b/src/services/agent/e2e-provider.test.ts @@ -0,0 +1,30 @@ +import { describe, it, expect } from 'vitest'; +import { createScriptedProvider } from './e2e-provider'; + +describe('createScriptedProvider', () => { + it('replays chatTools steps in order, then ends with an empty final turn', async () => { + const p = createScriptedProvider( + [{ calls: [{ name: 'base64', args: { text: 'hi' } }] }], + { capable: true }, + ); + expect(p.capable).toBe(true); + const t1 = await p.chatTools!([], []); + expect(t1.calls.map(c => c.name)).toEqual(['base64']); + expect(t1.calls[0].id).toBeTruthy(); // an id is generated + const t2 = await p.chatTools!([], []); // steps exhausted + expect(t2.calls).toEqual([]); // -> final, loop can stop + }); + + it('replays chat() steps as raw strings', async () => { + const p = createScriptedProvider([{ chat: 'Hello there!' }], { capable: false }); + expect(p.capable).toBe(false); + expect(await p.chat([])).toBe('Hello there!'); + expect(await p.chat([])).toBe(''); // exhausted -> empty + }); + + it('supports a final text-only tools turn', async () => { + const p = createScriptedProvider([{ text: 'done', calls: [] }], { capable: true }); + const t = await p.chatTools!([], []); + expect(t).toEqual({ text: 'done', calls: [] }); + }); +}); diff --git a/src/services/agent/e2e-provider.ts b/src/services/agent/e2e-provider.ts new file mode 100644 index 00000000..106c2e41 --- /dev/null +++ b/src/services/agent/e2e-provider.ts @@ -0,0 +1,41 @@ +import type { AgentProvider, ChatMessage, ToolMsg, ToolSpec, ToolTurn } from './provider'; + +export type ScriptStep = + | { chat: string } + | { calls: { name: string; args: Record }[]; text?: string } + | { text: string; calls?: undefined }; + +/** + * A deterministic, dev/test-only AgentProvider that replays a fixed script. + * `chat` steps feed the prompt loop / chat-mode; `calls`/`text` steps feed the + * native tools loop. Once the script is exhausted, chatTools returns an empty + * final turn and chat returns '' so the agent loop terminates cleanly. + */ +export function createScriptedProvider(steps: ScriptStep[], opts: { capable?: boolean } = {}): AgentProvider { + let i = 0; + let callSeq = 0; + return { + capable: opts.capable, + async chat(_messages: ChatMessage[]): Promise { + const step = steps[i++]; + if (step && 'chat' in step) return step.chat; + if (step && 'text' in step && typeof step.text === 'string') return step.text; + return ''; + }, + async chatTools(_messages: ToolMsg[], _tools: ToolSpec[]): Promise { + const step = steps[i++]; + if (step && 'calls' in step && step.calls) { + return { + text: step.text ?? '', + calls: step.calls.map(c => ({ + id: `e2e-${++callSeq}`, + name: c.name, + args: c.args as Record, + })), + }; + } + if (step && 'text' in step && typeof step.text === 'string') return { text: step.text, calls: [] }; + return { text: '', calls: [] }; + }, + }; +}