perf: improve prompt-prefix caching for the agentic model

CHANGELOG.md

@@ -3,6 +3,9 @@
 ## 2026-07-17
 
 ### Changes
+- [Pilot] Pilot's instructions and its list of available tools now stay identical across every call in a session, with the scenario details moved to the end. Providers can reuse the prompt they already processed instead of re-reading it on each call, which lowers the cost of long test runs.
+- [Planner] Planning rules and output format instructions now come before the page research, so planning more tests for the same page reuses an already-processed prompt.
+- When a click matches several elements, the follow-up question that picks the right one now uses the default model instead of the more expensive agentic model.
 - [Captain] Answers now finish as soon as the request is fulfilled — previously every completed request triggered one extra AI call with the full conversation context that produced nothing, roughly doubling token usage per request.
 - [Captain] No longer attaches the full page HTML to every request. Only the ARIA snapshot is kept fresh in the conversation; the AI fetches HTML on demand via the context() tool when it actually needs it.
 - [Captain] The slash-command tool now lists only command names instead of the full help text of every command, shrinking the prompt sent on each request.

src/ai/pilot.ts

@@ -154,7 +154,7 @@ export class Pilot implements Agent {
     const messages = [
       {
         role: 'system' as const,
-        content: this.buildVerdictSystemPrompt(type, task),
+        content: this.buildVerdictSystemPrompt(task),
       },
       { role: 'user' as const, content: userContent },
     ];
@@ -301,10 +301,8 @@ export class Pilot implements Agent {
     }
   }
 
-  private buildSharedEvidenceRules(task: Test): string {
+  private buildSharedEvidenceRules(): string {
     return dedent`
-      SCENARIO: ${task.scenario}
-
       EVIDENCE PRIORITY (strict):
       1) Final observable state proving the scenario goal
       2) verify()/see() results in the LAST few actions before stop/finish
@@ -337,6 +335,12 @@ export class Pilot implements Agent {
 
       Expected results are MILESTONES, not the goal. Never fail because a milestone (toast, icon, styling)
       didn't match if the scenario goal IS accomplished.
+    `;
+  }
+
+  private buildTaskContext(task: Test): string {
+    return dedent`
+      SCENARIO: ${task.scenario}
 
       ${this.buildDeletionScope(task)}
 
@@ -351,7 +355,7 @@ export class Pilot implements Agent {
       iteration's work, but server-side side effects (records created, forms submitted) persist.
       Unnecessary resets create duplicate data and infinite loops.
 
-      ${this.buildSharedEvidenceRules(task)}
+      ${this.buildSharedEvidenceRules()}
 
       DECISION:
       - "allow": current page cannot host the scenario, irrecoverable error, or no path back.
@@ -368,17 +372,19 @@ export class Pilot implements Agent {
       to verify, how to record. Do not suggest repeating actions that already succeeded.
       If progress is blocked only because the page lacks target data for the scenario, prefer precondition()
       over repeated UI attempts.
+
+      ${this.buildTaskContext(task)}
     `;
   }
 
-  private buildVerdictSystemPrompt(type: string, task: Test): string {
+  private buildVerdictSystemPrompt(task: Test): string {
     return dedent`
-      You are Pilot — final decision maker for test pass/fail. Tester requested ${type}. Review the
-      evidence and commit to a verdict; "continue" only when evidence is genuinely insufficient.
+      You are Pilot — final decision maker for test pass/fail. Review the evidence and commit to a
+      verdict; "continue" only when evidence is genuinely insufficient.
 
       ${capabilityGroundingRule}
 
-      ${this.buildSharedEvidenceRules(task)}
+      ${this.buildSharedEvidenceRules()}
 
       DECISION:
       - "pass": scenario goal is fully accomplished. Set requestVerification to a one-sentence claim about
@@ -397,6 +403,8 @@ export class Pilot implements Agent {
       reason field: one short sentence, maximum 120 characters. Do NOT restate the decision
       ("scenario goal achieved/not achieved"). State what happened: what was verified, what failed,
       or what evidence was found.
+
+      ${this.buildTaskContext(task)}
     `;
   }
 
@@ -474,9 +482,6 @@ export class Pilot implements Agent {
       .slice(-this.stepsToReview);
     const actionsContext = this.formatActions(toolCalls);
 
-    this.conversation.cleanupTag('page_summary', '...trimmed...', 1);
-    this.conversation.cleanupTag('recent_actions', '...trimmed...', 2);
-
     return this.sendToPilot(
       dedent`
         Navigated to new page.
@@ -498,7 +503,8 @@ export class Pilot implements Agent {
 
         First: evaluate whether this navigation makes sense for the scenario goal. If the page is unrelated, instruct Tester to back() or reset(). Then plan next steps.
       `,
-      'pilot.reviewNewPage'
+      'pilot.reviewNewPage',
+      { task }
     );
   }
 
@@ -515,8 +521,6 @@ export class Pilot implements Agent {
     const actionsContext = this.formatActions(toolCalls);
     const stateContext = this.buildStateContext(currentState);
 
-    this.conversation.cleanupTag('recent_actions', '...trimmed...', 2);
-
     const hasFailures = toolCalls.length === 0 || toolCalls.some((t) => !t.wasSuccessful);
 
     const text = await this.sendToPilot(
@@ -554,7 +558,7 @@ export class Pilot implements Agent {
     return `CHECKED: ${checked.length > 0 ? checked.join(', ') : 'none'}\nREMAINING: ${remaining.length > 0 ? remaining.join(', ') : 'none'}`;
   }
 
-  private async sendToPilot(userText: string, functionId: string, opts: { tools?: boolean; maxToolRoundtrips?: number; task?: Test } = {}): Promise<string> {
+  private async sendToPilot(userText: string, functionId: string, opts: { tools?: boolean; maxToolRoundtrips?: number; task: Test }): Promise<string> {
     debugLog(`sendToPilot: ${functionId}, tools: ${!!opts.tools}, roundtrips: ${opts.maxToolRoundtrips ?? 0}`);
 
     let finalUserText = userText;
@@ -565,19 +569,14 @@ export class Pilot implements Agent {
       }
     }
     this.conversation!.addUserText(finalUserText);
-    let tools: any;
-    if (opts.tools) {
-      tools = this.pickPlanningTools();
-    }
 
-    if (opts.tools && opts.task) {
-      tools = { ...tools, ...this.buildPreconditionTool(opts.task) };
-    }
+    const tools = { ...this.pickPlanningTools(), ...this.buildPreconditionTool(opts.task) };
 
     const result = await this.provider.invokeConversation(this.conversation!, tools, {
       maxToolRoundtrips: opts.maxToolRoundtrips ?? 0,
+      toolChoice: opts.tools ? 'auto' : 'none',
       agentName: 'pilot',
-      stopWhen: opts.task ? () => opts.task!.hasFinished : undefined,
+      stopWhen: () => opts.task.hasFinished,
       telemetry: { functionId },
     });
     const text = result?.response?.text || '';
@@ -975,16 +974,6 @@ export class Pilot implements Agent {
     return dedent`
       You are Pilot — a supervisor that detects problems and intervenes only when needed.
 
-      SCENARIO: ${task.scenario}
-      START URL: ${initialState.url}
-      PAGE: ${initialState.title || ''} | ${initialState.h1 || ''}
-
-      EXPECTED RESULTS:
-      ${task.expected.map((e) => `- ${e}`).join('\n')}
-
-      PLANNED STEPS:
-      ${stepsText}
-
       Your job: plan, review new pages, detect stuck patterns, suggest concrete next steps. Track which
       expectations are checked. When things go well, encourage briefly and let Tester continue. The current
       page is usually richer than the page summary lists — prefer exploring it before navigating away.
@@ -1054,6 +1043,16 @@ export class Pilot implements Agent {
       NEXT: <specific actionable instruction for Tester>
 
       Keep user-facing reasons concise: one short sentence, maximum 120 characters, evidence only, no repeated verdict wording.
+
+      SCENARIO: ${task.scenario}
+      START URL: ${initialState.url}
+      PAGE: ${initialState.title || ''} | ${initialState.h1 || ''}
+
+      EXPECTED RESULTS:
+      ${task.expected.map((e) => `- ${e}`).join('\n')}
+
+      PLANNED STEPS:
+      ${stepsText}
     `;
   }
 }

src/ai/planner.ts

@@ -374,16 +374,10 @@ export class Planner extends PlannerBase implements Agent {
       ${dataProtectionRules}
       ${fileUploadRule}
       </rules>
-
-      ${this.buildApproach(style)}
-
-      <context>
-      URL: ${state.url || 'Unknown'}
-      Title: ${state.title || 'Unknown'}
-      </context>
     `;
 
     conversation.addUserText(planningPrompt);
+    conversation.addUserText(this.getTasksMessage());
     const currentState = this.stateManager.getCurrentState();
     const research = await this.researcher.research(currentState || state, {
       deep: true,
@@ -417,6 +411,15 @@ export class Planner extends PlannerBase implements Agent {
       </page_research>
     `);
 
+    conversation.addUserText(dedent`
+      ${this.buildApproach(style)}
+
+      <context>
+      URL: ${state.url || 'Unknown'}
+      Title: ${state.title || 'Unknown'}
+      </context>
+    `);
+
     if (this.fisherman) {
       await this.fisherman.ensureReady(state.url);
       if (this.fisherman.isAvailable()) {
@@ -568,8 +571,11 @@ export class Planner extends PlannerBase implements Agent {
       `);
     }
 
-    const hasCurrentPlan = !!this.currentPlan;
-    const tasksMessage = dedent`
+    return conversation;
+  }
+
+  private getTasksMessage(): string {
+    return dedent`
     <task>
       Provide testing scenarios as structured data with the following requirements:
       1. Create a short, descriptive plan name that summarizes what will be tested (e.g., "User Authentication Testing", "Product Catalog Navigation", "Form Validation Tests")
@@ -615,12 +621,8 @@ export class Planner extends PlannerBase implements Agent {
          - Do not wrap text in ** or * quotes, ( or ) brackets.
          - Avoid using emojis or special characters.
       7. Only tests that can be tested from web UI should be proposed.
-      ${hasCurrentPlan ? '8. CRITICAL: Return ONLY NEW scenarios not in the existing tests list. Return empty array if no new tests needed.' : `8. At least ${this.MIN_TASKS} tests should be proposed.`}
+      8. For a fresh plan propose at least ${this.MIN_TASKS} tests; when expanding an existing plan return ONLY new scenarios not in the tested list, and an empty array if none remain.
     </task>
     `;
-
-    conversation.addUserText(tasksMessage);
-
-    return conversation;
   }
 }

src/ai/tools.ts

@@ -1270,7 +1270,7 @@ async function disambiguateElements(error: Error | null | undefined, explanation
         },
       ],
       schema,
-      provider.getAgenticModel(),
+      provider.getModelForAgent(),
       { agentName: 'disambiguator', timeout: 15000 }
     );