From d9090b8518ce7a06fefd419be2155bb04cee0175 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Fri, 17 Jul 2026 02:39:48 +0300 Subject: [PATCH] perf: improve prompt-prefix caching for the agentic model Planner and Pilot ran at a 9% prompt cache rate while the default-model agents sat at 75%. The gap was three missing properties: a stable byte-identical prefix, an unchanging tool list, and append-only history. - Pilot: system, verdict and reset prompts now open with their static bodies and close with the per-test task block. Extract buildTaskContext() for the dynamic part; drop the request type from the verdict opener since the user message already carries it. - Pilot: always send the same tool set and gate it with toolChoice 'auto'/'none' instead of flipping the tool list between calls, which invalidated the whole conversation's cache. - Pilot: stop rewriting history via cleanupTag. Pilot conversations are light by design; the trimmed KBs cost more re-tokenized than they save. - Planner: order messages system -> static rules -> static output format -> page research -> approach + URL context, so re-plans of the same page reuse the research prefix. - Route the click disambiguator off the agentic model: ~350-token one-shot prompts can never cache and don't need the expensive model. Prompts are reordered, not rewritten. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 3 +++ src/ai/pilot.ts | 67 +++++++++++++++++++++++------------------------ src/ai/planner.ts | 30 +++++++++++---------- src/ai/tools.ts | 2 +- 4 files changed, 53 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a8b3d7..5c420da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ ## 2026-07-17 ### Changes +- [Pilot] Pilot's instructions and its list of available tools now stay identical across every call in a session, with the scenario details moved to the end. Providers can reuse the prompt they already processed instead of re-reading it on each call, which lowers the cost of long test runs. +- [Planner] Planning rules and output format instructions now come before the page research, so planning more tests for the same page reuses an already-processed prompt. +- When a click matches several elements, the follow-up question that picks the right one now uses the default model instead of the more expensive agentic model. - Fixed startup failing with `AI connection failed: Invalid 'max_output_tokens'` on models that reject a one-token response. The check that verifies your AI credentials at startup no longer caps the reply length, so it works with every provider regardless of their minimum. ## 2026-07-10 diff --git a/src/ai/pilot.ts b/src/ai/pilot.ts index d653d10..9983217 100644 --- a/src/ai/pilot.ts +++ b/src/ai/pilot.ts @@ -154,7 +154,7 @@ export class Pilot implements Agent { const messages = [ { role: 'system' as const, - content: this.buildVerdictSystemPrompt(type, task), + content: this.buildVerdictSystemPrompt(task), }, { role: 'user' as const, content: userContent }, ]; @@ -301,10 +301,8 @@ export class Pilot implements Agent { } } - private buildSharedEvidenceRules(task: Test): string { + private buildSharedEvidenceRules(): string { return dedent` - SCENARIO: ${task.scenario} - EVIDENCE PRIORITY (strict): 1) Final observable state proving the scenario goal 2) verify()/see() results in the LAST few actions before stop/finish @@ -337,6 +335,12 @@ export class Pilot implements Agent { Expected results are MILESTONES, not the goal. Never fail because a milestone (toast, icon, styling) didn't match if the scenario goal IS accomplished. + `; + } + + private buildTaskContext(task: Test): string { + return dedent` + SCENARIO: ${task.scenario} ${this.buildDeletionScope(task)} @@ -351,7 +355,7 @@ export class Pilot implements Agent { iteration's work, but server-side side effects (records created, forms submitted) persist. Unnecessary resets create duplicate data and infinite loops. - ${this.buildSharedEvidenceRules(task)} + ${this.buildSharedEvidenceRules()} DECISION: - "allow": current page cannot host the scenario, irrecoverable error, or no path back. @@ -368,17 +372,19 @@ export class Pilot implements Agent { to verify, how to record. Do not suggest repeating actions that already succeeded. If progress is blocked only because the page lacks target data for the scenario, prefer precondition() over repeated UI attempts. + + ${this.buildTaskContext(task)} `; } - private buildVerdictSystemPrompt(type: string, task: Test): string { + private buildVerdictSystemPrompt(task: Test): string { return dedent` - You are Pilot — final decision maker for test pass/fail. Tester requested ${type}. Review the - evidence and commit to a verdict; "continue" only when evidence is genuinely insufficient. + You are Pilot — final decision maker for test pass/fail. Review the evidence and commit to a + verdict; "continue" only when evidence is genuinely insufficient. ${capabilityGroundingRule} - ${this.buildSharedEvidenceRules(task)} + ${this.buildSharedEvidenceRules()} DECISION: - "pass": scenario goal is fully accomplished. Set requestVerification to a one-sentence claim about @@ -397,6 +403,8 @@ export class Pilot implements Agent { reason field: one short sentence, maximum 120 characters. Do NOT restate the decision ("scenario goal achieved/not achieved"). State what happened: what was verified, what failed, or what evidence was found. + + ${this.buildTaskContext(task)} `; } @@ -474,9 +482,6 @@ export class Pilot implements Agent { .slice(-this.stepsToReview); const actionsContext = this.formatActions(toolCalls); - this.conversation.cleanupTag('page_summary', '...trimmed...', 1); - this.conversation.cleanupTag('recent_actions', '...trimmed...', 2); - return this.sendToPilot( dedent` Navigated to new page. @@ -498,7 +503,8 @@ export class Pilot implements Agent { First: evaluate whether this navigation makes sense for the scenario goal. If the page is unrelated, instruct Tester to back() or reset(). Then plan next steps. `, - 'pilot.reviewNewPage' + 'pilot.reviewNewPage', + { task } ); } @@ -515,8 +521,6 @@ export class Pilot implements Agent { const actionsContext = this.formatActions(toolCalls); const stateContext = this.buildStateContext(currentState); - this.conversation.cleanupTag('recent_actions', '...trimmed...', 2); - const hasFailures = toolCalls.length === 0 || toolCalls.some((t) => !t.wasSuccessful); const text = await this.sendToPilot( @@ -554,7 +558,7 @@ export class Pilot implements Agent { return `CHECKED: ${checked.length > 0 ? checked.join(', ') : 'none'}\nREMAINING: ${remaining.length > 0 ? remaining.join(', ') : 'none'}`; } - private async sendToPilot(userText: string, functionId: string, opts: { tools?: boolean; maxToolRoundtrips?: number; task?: Test } = {}): Promise { + private async sendToPilot(userText: string, functionId: string, opts: { tools?: boolean; maxToolRoundtrips?: number; task: Test }): Promise { debugLog(`sendToPilot: ${functionId}, tools: ${!!opts.tools}, roundtrips: ${opts.maxToolRoundtrips ?? 0}`); let finalUserText = userText; @@ -565,19 +569,14 @@ export class Pilot implements Agent { } } this.conversation!.addUserText(finalUserText); - let tools: any; - if (opts.tools) { - tools = this.pickPlanningTools(); - } - if (opts.tools && opts.task) { - tools = { ...tools, ...this.buildPreconditionTool(opts.task) }; - } + const tools = { ...this.pickPlanningTools(), ...this.buildPreconditionTool(opts.task) }; const result = await this.provider.invokeConversation(this.conversation!, tools, { maxToolRoundtrips: opts.maxToolRoundtrips ?? 0, + toolChoice: opts.tools ? 'auto' : 'none', agentName: 'pilot', - stopWhen: opts.task ? () => opts.task!.hasFinished : undefined, + stopWhen: () => opts.task.hasFinished, telemetry: { functionId }, }); const text = result?.response?.text || ''; @@ -975,16 +974,6 @@ export class Pilot implements Agent { return dedent` You are Pilot — a supervisor that detects problems and intervenes only when needed. - SCENARIO: ${task.scenario} - START URL: ${initialState.url} - PAGE: ${initialState.title || ''} | ${initialState.h1 || ''} - - EXPECTED RESULTS: - ${task.expected.map((e) => `- ${e}`).join('\n')} - - PLANNED STEPS: - ${stepsText} - Your job: plan, review new pages, detect stuck patterns, suggest concrete next steps. Track which expectations are checked. When things go well, encourage briefly and let Tester continue. The current page is usually richer than the page summary lists — prefer exploring it before navigating away. @@ -1054,6 +1043,16 @@ export class Pilot implements Agent { NEXT: Keep user-facing reasons concise: one short sentence, maximum 120 characters, evidence only, no repeated verdict wording. + + SCENARIO: ${task.scenario} + START URL: ${initialState.url} + PAGE: ${initialState.title || ''} | ${initialState.h1 || ''} + + EXPECTED RESULTS: + ${task.expected.map((e) => `- ${e}`).join('\n')} + + PLANNED STEPS: + ${stepsText} `; } } diff --git a/src/ai/planner.ts b/src/ai/planner.ts index 823fa2f..07c838b 100644 --- a/src/ai/planner.ts +++ b/src/ai/planner.ts @@ -374,16 +374,10 @@ export class Planner extends PlannerBase implements Agent { ${dataProtectionRules} ${fileUploadRule} - - ${this.buildApproach(style)} - - - URL: ${state.url || 'Unknown'} - Title: ${state.title || 'Unknown'} - `; conversation.addUserText(planningPrompt); + conversation.addUserText(this.getTasksMessage()); const currentState = this.stateManager.getCurrentState(); const research = await this.researcher.research(currentState || state, { deep: true, @@ -417,6 +411,15 @@ export class Planner extends PlannerBase implements Agent { `); + conversation.addUserText(dedent` + ${this.buildApproach(style)} + + + URL: ${state.url || 'Unknown'} + Title: ${state.title || 'Unknown'} + + `); + if (this.fisherman) { await this.fisherman.ensureReady(state.url); if (this.fisherman.isAvailable()) { @@ -568,8 +571,11 @@ export class Planner extends PlannerBase implements Agent { `); } - const hasCurrentPlan = !!this.currentPlan; - const tasksMessage = dedent` + return conversation; + } + + private getTasksMessage(): string { + return dedent` Provide testing scenarios as structured data with the following requirements: 1. Create a short, descriptive plan name that summarizes what will be tested (e.g., "User Authentication Testing", "Product Catalog Navigation", "Form Validation Tests") @@ -615,12 +621,8 @@ export class Planner extends PlannerBase implements Agent { - Do not wrap text in ** or * quotes, ( or ) brackets. - Avoid using emojis or special characters. 7. Only tests that can be tested from web UI should be proposed. - ${hasCurrentPlan ? '8. CRITICAL: Return ONLY NEW scenarios not in the existing tests list. Return empty array if no new tests needed.' : `8. At least ${this.MIN_TASKS} tests should be proposed.`} + 8. For a fresh plan propose at least ${this.MIN_TASKS} tests; when expanding an existing plan return ONLY new scenarios not in the tested list, and an empty array if none remain. `; - - conversation.addUserText(tasksMessage); - - return conversation; } } diff --git a/src/ai/tools.ts b/src/ai/tools.ts index 9161008..a1ffd85 100644 --- a/src/ai/tools.ts +++ b/src/ai/tools.ts @@ -1270,7 +1270,7 @@ async function disambiguateElements(error: Error | null | undefined, explanation }, ], schema, - provider.getAgenticModel(), + provider.getModelForAgent(), { agentName: 'disambiguator', timeout: 15000 } );