From 4071568ae428e69b5f1ef7183c6501e4e21deaac Mon Sep 17 00:00:00 2001 From: DavertMik Date: Fri, 17 Jul 2026 23:40:24 +0300 Subject: [PATCH 1/4] refactor: redesign Explorer API to a minimal, recovery-ambient interface Explorer's public surface shrinks from ~42 members to 13. The noise had three sources: four inconsistent error policies, a service-locator getter per collaborator, and the playwrightHelper leak that forced the recovery machinery to be public. - withPage(fn) is the single recovery-wrapped page seam; the page getter gives a raw null-safe peek for background consumers; playwrightHelper is private - one recover(error?) replaces runWithBrowserRecovery, recoverFromBrowserError, restartBrowser, ensurePageAvailable, and handleExecutionError; the recover-then-restart escalation is internal - action() returns a recovery-aware Action, fixing bare createAction() silently bypassing recovery - visit(url) returns the observed ActionResult; beginTest(test) returns a TestRun handle so a startTest/stopTest mismatch is unrepresentable - agents receive services via AgentDeps (TaskAgent owns the shared fields) instead of Explorer getters; ExplorBot now owns StateManager, Reporter, RequestStore, and PlaywrightRecorder - element annotation and eidx helpers move to utils/web-annotate.ts and utils/web-eidx.ts as page-taking functions - other-tabs info becomes observed state on StateManager Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 2 + bin/explorbot-cli.ts | 4 +- boat/doc-collector/src/ai/documentarian.ts | 7 +- boat/doc-collector/src/ai/tools.ts | 23 +- boat/doc-collector/src/docbot.ts | 6 +- boat/doc-collector/src/screenshots.ts | 4 +- src/action.ts | 22 +- src/ai/agent.ts | 20 + src/ai/captain.ts | 17 +- src/ai/captain/idle-mode.ts | 2 +- src/ai/captain/test-mode.ts | 2 +- src/ai/captain/web-mode.ts | 35 +- src/ai/driller.ts | 117 +- src/ai/historian.ts | 5 +- src/ai/historian/codeceptjs.ts | 2 +- src/ai/historian/playwright.ts | 7 +- src/ai/historian/screencast.ts | 7 +- src/ai/navigator.ts | 55 +- src/ai/pilot.ts | 35 +- src/ai/planner.ts | 12 +- src/ai/quartermaster.ts | 9 +- src/ai/rerunner.ts | 39 +- src/ai/researcher.ts | 74 +- src/ai/researcher/coordinates.ts | 7 +- src/ai/researcher/deep-analysis.ts | 14 +- src/ai/researcher/locators.ts | 14 +- src/ai/researcher/sections.ts | 6 +- src/ai/task-agent.ts | 56 +- src/ai/tester.ts | 95 +- src/ai/tools.ts | 81 +- src/command-handler.ts | 2 +- src/commands/context-aria-command.ts | 2 +- src/commands/context-command.ts | 12 +- src/commands/context-data-command.ts | 4 +- src/commands/context-experience-command.ts | 4 +- src/commands/context-html-command.ts | 4 +- src/commands/context-knowledge-command.ts | 2 +- src/commands/drill-command.ts | 2 +- src/commands/explore-command.ts | 7 +- src/commands/freesail-command.ts | 2 +- src/commands/learn-command.ts | 8 +- src/commands/path-command.ts | 2 +- src/commands/research-command.ts | 2 +- src/commands/test-command.ts | 2 +- src/components/App.tsx | 2 +- src/explorbot.ts | 101 +- src/explorer.ts | 1021 +++++++---------- src/state-manager.ts | 8 +- src/utils/hooks-runner.ts | 6 +- src/utils/web-annotate.ts | 64 ++ src/utils/web-eidx.ts | 21 + tests/integration/annotate-elements.test.ts | 2 +- tests/integration/planner.test.ts | 15 +- tests/integration/researcher-browser.test.ts | 35 +- tests/integration/researcher-sections.test.ts | 40 +- tests/integration/researcher.test.ts | 36 +- tests/unit/agent-tools.test.ts | 12 +- tests/unit/annotate-elements.test.ts | 2 +- tests/unit/captain-mode.test.ts | 20 +- tests/unit/doc-collector.test.ts | 116 +- tests/unit/explore-command.test.ts | 60 +- tests/unit/explorer-recovery-url.test.ts | 41 +- tests/unit/explorer-step-listeners.test.ts | 8 +- tests/unit/explorer.test.ts | 29 +- tests/unit/navigator-origin-guard.test.ts | 12 +- tests/unit/pilot-state-context.test.ts | 15 +- tests/unit/rerunner-healing-teardown.test.ts | 2 +- tests/unit/tester-error-page.test.ts | 98 +- tests/unit/tester-focus-scope.test.ts | 64 +- tests/unit/tools-back.test.ts | 6 +- tests/unit/tools.test.ts | 14 +- 71 files changed, 1314 insertions(+), 1370 deletions(-) create mode 100644 src/utils/web-annotate.ts create mode 100644 src/utils/web-eidx.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c420da..ff14015 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## 2026-07-17 ### Changes +- Browser crash recovery now applies to every browser operation. Previously some interactions and page lookups could fail permanently when the page or browser crashed mid-session; now every action, capture, and element query automatically reattaches the page or restarts the browser and retries once before giving up. +- [Captain] The `browser` tool's `restart` action was merged into `recover` — recovering a page now escalates to a full browser restart automatically when needed, so there is no separate action to choose. - [Pilot] Pilot's instructions and its list of available tools now stay identical across every call in a session, with the scenario details moved to the end. Providers can reuse the prompt they already processed instead of re-reading it on each call, which lowers the cost of long test runs. - [Planner] Planning rules and output format instructions now come before the page research, so planning more tests for the same page reuses an already-processed prompt. - When a click matches several elements, the follow-up question that picks the right one now uses the default model instead of the more expensive agentic model. diff --git a/bin/explorbot-cli.ts b/bin/explorbot-cli.ts index 4a4d701..89cacf8 100755 --- a/bin/explorbot-cli.ts +++ b/bin/explorbot-cli.ts @@ -720,11 +720,11 @@ addCommonOptions(program.command('shell ').description('Execute a await explorBot.start(); await explorBot.agentNavigator().visit(url); - const action = explorBot.getExplorer().createAction(); + const action = explorBot.getExplorer().action(); await action.execute(command); log('Command executed successfully'); - const state = explorBot.getExplorer().getStateManager().getCurrentState(); + const state = explorBot.stateManager().getCurrentState(); if (state) log(`URL: ${state.url}`); await explorBot.stop(); diff --git a/boat/doc-collector/src/ai/documentarian.ts b/boat/doc-collector/src/ai/documentarian.ts index 4a6b779..72555f6 100644 --- a/boat/doc-collector/src/ai/documentarian.ts +++ b/boat/doc-collector/src/ai/documentarian.ts @@ -2,6 +2,7 @@ import dedent from 'dedent'; import { z } from 'zod'; import type { AIProvider } from '../../../../src/ai/provider.ts'; import type Explorer from '../../../../src/explorer.ts'; +import type { StateManager } from '../../../../src/state-manager.ts'; import type { WebPageState } from '../../../../src/state-manager.ts'; import { tag } from '../../../../src/utils/logger.ts'; import type { DocbotConfig } from '../config.ts'; @@ -11,11 +12,13 @@ class Documentarian { private provider: AIProvider; private config: DocbotConfig; private explorer?: Explorer; + private stateManager?: StateManager; - constructor(provider: AIProvider, config: DocbotConfig = {}, explorer?: Explorer) { + constructor(provider: AIProvider, config: DocbotConfig = {}, explorer?: Explorer, stateManager?: StateManager) { this.provider = provider; this.config = config; this.explorer = explorer; + this.stateManager = stateManager; } async document(state: WebPageState, research: string, captureState?: CaptureInteractionState): Promise { @@ -46,7 +49,7 @@ class Documentarian { try { tag('info').log('Starting interactive exploration...'); - const deterministicInteractions = await collectDocInteractions(this.explorer!, state, research, this.config, captureState); + const deterministicInteractions = await collectDocInteractions(this.explorer!, this.stateManager!, state, research, this.config, captureState); meaningfulInteractions = this.getMeaningfulInteractions(deterministicInteractions); if (meaningfulInteractions.length > 0) { tag('success').log(`Collected ${meaningfulInteractions.length} deterministic interactions`); diff --git a/boat/doc-collector/src/ai/tools.ts b/boat/doc-collector/src/ai/tools.ts index 2f72b05..9b1ccc8 100644 --- a/boat/doc-collector/src/ai/tools.ts +++ b/boat/doc-collector/src/ai/tools.ts @@ -1,6 +1,6 @@ import { type ResearchElement, parseResearchSections } from '../../../../src/ai/researcher/parser.ts'; import type Explorer from '../../../../src/explorer.ts'; -import type { WebPageState } from '../../../../src/state-manager.ts'; +import type { StateManager, WebPageState } from '../../../../src/state-manager.ts'; import { detectFocusArea } from '../../../../src/utils/aria.ts'; import type { DocbotConfig } from '../config.ts'; @@ -58,14 +58,14 @@ const DEFAULT_WAIT_MS = 700; const TAB_WAIT_MS = 500; const DEFAULT_DENIED_ACTION_LABELS = ['delete', 'remove', 'destroy', 'archive', 'discard', 'logout', 'sign out', 'signout', 'sign_out', 'erase', 'drop']; -export async function collectDocInteractions(explorer: Explorer, state: WebPageState, research: string, config: DocbotConfig = {}, captureState?: CaptureInteractionState): Promise { +export async function collectDocInteractions(explorer: Explorer, stateManager: StateManager, state: WebPageState, research: string, config: DocbotConfig = {}, captureState?: CaptureInteractionState): Promise { const sections = parseResearchSections(research); const transitions: DocStateTransition[] = []; const maxInteractions = getPositiveConfigNumber(config.docs?.maxInteractions, DEFAULT_MAX_INTERACTIONS); const tabGroup = findTabGroup(sections); if (tabGroup) { - transitions.push(...(await exploreTabGroup(explorer, tabGroup, state.url, maxInteractions, captureState))); + transitions.push(...(await exploreTabGroup(explorer, stateManager, tabGroup, state.url, maxInteractions, captureState))); } for (const candidate of findActionCandidates(sections, config)) { @@ -73,7 +73,7 @@ export async function collectDocInteractions(explorer: Explorer, state: WebPageS break; } - const transition = await executeInteraction(explorer, candidate, state.url, DEFAULT_WAIT_MS, captureState); + const transition = await executeInteraction(explorer, stateManager, candidate, state.url, DEFAULT_WAIT_MS, captureState); if (!transition) { continue; } @@ -92,7 +92,7 @@ export function pickDocActionCandidates(research: string, config: DocbotConfig = })); } -async function exploreTabGroup(explorer: Explorer, tabGroup: { elements: ResearchElement[]; container?: string; sectionName: string }, restoreUrl: string, maxInteractions: number, captureState?: CaptureInteractionState): Promise { +async function exploreTabGroup(explorer: Explorer, stateManager: StateManager, tabGroup: { elements: ResearchElement[]; container?: string; sectionName: string }, restoreUrl: string, maxInteractions: number, captureState?: CaptureInteractionState): Promise { const transitions: DocStateTransition[] = []; for (const element of tabGroup.elements) { @@ -102,6 +102,7 @@ async function exploreTabGroup(explorer: Explorer, tabGroup: { elements: Researc const transition = await executeInteraction( explorer, + stateManager, { element, container: tabGroup.container, @@ -123,8 +124,8 @@ async function exploreTabGroup(explorer: Explorer, tabGroup: { elements: Researc return transitions; } -async function executeInteraction(explorer: Explorer, candidate: InteractionCandidate, restoreUrl: string, waitMs: number, captureState?: CaptureInteractionState): Promise { - const beforeState = explorer.getStateManager().getCurrentState(); +async function executeInteraction(explorer: Explorer, stateManager: StateManager, candidate: InteractionCandidate, restoreUrl: string, waitMs: number, captureState?: CaptureInteractionState): Promise { + const beforeState = stateManager.getCurrentState(); if (!beforeState) { return null; } @@ -136,7 +137,7 @@ async function executeInteraction(explorer: Explorer, candidate: InteractionCand await wait(waitMs); - const afterState = explorer.getStateManager().getCurrentState(); + const afterState = stateManager.getCurrentState(); if (!afterState) { return null; } @@ -164,7 +165,7 @@ async function executeInteraction(explorer: Explorer, candidate: InteractionCand } async function attemptInteraction(explorer: Explorer, candidate: InteractionCandidate): Promise { - const action = explorer.createAction(); + const action = explorer.action(); for (const command of buildClickCommands(candidate.element, candidate.container)) { const success = await action.attempt(command, buildPurpose(candidate)); @@ -178,7 +179,7 @@ async function attemptInteraction(explorer: Explorer, candidate: InteractionCand async function restoreInteractionState(explorer: Explorer, restoreUrl: string, primaryCommand?: string | null): Promise { if (primaryCommand) { - const action = explorer.createAction(); + const action = explorer.action(); const restored = await action.attempt(primaryCommand, `Restore initial state on ${restoreUrl}`); if (restored) { await wait(TAB_WAIT_MS); @@ -186,7 +187,7 @@ async function restoreInteractionState(explorer: Explorer, restoreUrl: string, p } } - const action = explorer.createAction(); + const action = explorer.action(); await action.attempt(`I.amOnPage(${JSON.stringify(restoreUrl)})`, `Restore page ${restoreUrl}`); } diff --git a/boat/doc-collector/src/docbot.ts b/boat/doc-collector/src/docbot.ts index ed00560..956629a 100644 --- a/boat/doc-collector/src/docbot.ts +++ b/boat/doc-collector/src/docbot.ts @@ -8,10 +8,10 @@ import { sanitizeFilename } from '../../../src/utils/strings.ts'; import { Documentarian, type PageDocumentation } from './ai/documentarian.ts'; import { type DocbotConfig, DocbotConfigParser } from './config.ts'; import { type DocumentedPage, type SkippedPage, renderPageDocumentation, renderSpecIndex } from './docs-renderer.ts'; -import { renderMermaidBody } from './state-diagram.ts'; import { getDocPageKey, shouldCrawlDocPath } from './path-filter.ts'; import { extractResearchNavigationTargets } from './research-navigation.ts'; import { type DocumentationScreenshot, captureDocumentationScreenshots, captureInteractionScreenshot } from './screenshots.ts'; +import { renderMermaidBody } from './state-diagram.ts'; class DocBot { private explorBot: ExplorBot; @@ -43,7 +43,7 @@ class DocBot { config: this.options.docsConfig, path: this.options.path, }); - this.documentarian = new Documentarian(this.explorBot.getProvider(), this.config, this.explorBot.getExplorer()); + this.documentarian = new Documentarian(this.explorBot.getProvider(), this.config, this.explorBot.getExplorer(), this.explorBot.stateManager()); this.ensureDirectory(this.configParser.getOutputDir()); this.ensureDirectory(this.getPagesDir()); } @@ -76,7 +76,7 @@ class DocBot { continue; } - const stateManager = this.explorBot.getExplorer().getStateManager(); + const stateManager = this.explorBot.stateManager(); if (stateManager.hasVisitedState(target)) { continue; } diff --git a/boat/doc-collector/src/screenshots.ts b/boat/doc-collector/src/screenshots.ts index 4e70d54..81c5ec8 100644 --- a/boat/doc-collector/src/screenshots.ts +++ b/boat/doc-collector/src/screenshots.ts @@ -11,7 +11,7 @@ import type { DocbotConfig } from './config.ts'; const DEFAULT_MAX_SECTION_SCREENSHOTS = 8; export async function captureDocumentationScreenshots(explorer: Explorer, state: WebPageState, research: string, options: DocumentationScreenshotOptions): Promise { - const page = explorer.playwrightHelper?.page; + const page = explorer.page; if (!page) { return []; } @@ -62,7 +62,7 @@ export function getScreenshotSections(research: string): ScreenshotSection[] { } export async function captureInteractionScreenshot(explorer: Explorer, state: WebPageState, transition: DocStateTransition, options: DocumentationScreenshotOptions): Promise { - const page = explorer.playwrightHelper?.page; + const page = explorer.page; if (!page) { return null; } diff --git a/src/action.ts b/src/action.ts index df5fd23..140d7ef 100644 --- a/src/action.ts +++ b/src/action.ts @@ -14,8 +14,8 @@ import { browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } import { captureHtmlForSnapshot, htmlCombinedSnapshot, minifyHtml } from './utils/html.js'; import { createDebug, setStepSpanParent, tag } from './utils/logger.js'; import { sleep, waitForPageReadiness } from './utils/page-readiness.ts'; -import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts'; import { safeFilename } from './utils/strings.ts'; +import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts'; const debugLog = createDebug('explorbot:action'); const CAPTURE_NAVIGATION_TRANSITION_ATTEMPTS = 3; @@ -33,14 +33,16 @@ class Action { public playwrightGroupId: string | null = null; public assertionSteps: Array<{ name: string; args: any[] }> = []; private recorder?: PlaywrightRecorder; + private recovery: RecoveryRunner; private mainDocumentStatus: number | undefined = undefined; - constructor(actor: CodeceptJS.I, stateManager: StateManager, recorder?: PlaywrightRecorder) { + constructor(actor: CodeceptJS.I, stateManager: StateManager, recorder?: PlaywrightRecorder, recovery?: RecoveryRunner) { this.actor = actor; this.stateManager = stateManager; this.config = ConfigParser.getInstance().getConfig(); this.playwrightHelper = container.helpers('Playwright'); this.recorder = recorder; + this.recovery = recovery || ((fn) => fn()); } async saveScreenshot(): Promise { @@ -59,7 +61,15 @@ class Action { } } - async capturePageState({ includeScreenshot = false, codeBlock }: { includeScreenshot?: boolean; codeBlock?: string } = {}): Promise { + async capturePageState(opts: { includeScreenshot?: boolean; codeBlock?: string } = {}): Promise { + return this.recovery(() => this.captureOnce(opts)); + } + + async execute(code: string): Promise { + return this.recovery(() => this.executeOnce(code)); + } + + private async captureOnce({ includeScreenshot = false, codeBlock }: { includeScreenshot?: boolean; codeBlock?: string } = {}): Promise { try { const currentState = this.stateManager.getCurrentState(); const stateHash = currentState?.hash || 'screenshot'; @@ -260,7 +270,7 @@ class Action { } } - async execute(code: string): Promise { + private async executeOnce(code: string): Promise { let error: Error | null = null; setActivity('🔎 Browsing...', 'action'); @@ -301,7 +311,7 @@ class Action { codeString = executedSteps.join('\n'); } - const pageState = await this.capturePageState({ codeBlock: codeString }); + const pageState = await this.captureOnce({ codeBlock: codeString }); this.actionResult = pageState; this.assertionSteps = assertionSteps; @@ -366,6 +376,8 @@ class Action { export default Action; +export type RecoveryRunner = (fn: () => Promise) => Promise; + function errorToString(error: any): string { if (error.cliMessage) { return error.cliMessage(); diff --git a/src/ai/agent.ts b/src/ai/agent.ts index 646238c..75fd319 100644 --- a/src/ai/agent.ts +++ b/src/ai/agent.ts @@ -1,3 +1,23 @@ +import type { RequestStore } from '../api/request-store.ts'; +import type { ExplorbotConfig } from '../config.ts'; +import type Explorer from '../explorer.ts'; +import type { KnowledgeTracker } from '../knowledge-tracker.ts'; +import type { PlaywrightRecorder } from '../playwright-recorder.ts'; +import type { StateManager } from '../state-manager.ts'; +import type { AIProvider } from './provider.ts'; + export interface Agent { emoji?: string; } + +export interface AgentDeps { + explorer: Explorer; + ai: AIProvider; + config: ExplorbotConfig; + stateManager: StateManager; + knowledgeTracker: KnowledgeTracker; + requestStore: RequestStore; + playwrightRecorder: PlaywrightRecorder; +} + +export type ToolDeps = Pick; diff --git a/src/ai/captain.ts b/src/ai/captain.ts index 1b96bb0..1c4cb74 100644 --- a/src/ai/captain.ts +++ b/src/ai/captain.ts @@ -49,7 +49,7 @@ export class Captain extends CaptainBase implements Agent { private getHooksRunner(): HooksRunner { if (!this.hooksRunner) { const explorer = this.explorBot.getExplorer(); - this.hooksRunner = new HooksRunner(explorer, explorer.getConfig()); + this.hooksRunner = new HooksRunner(explorer, this.explorBot.getConfig()); } return this.hooksRunner; } @@ -89,17 +89,16 @@ export class Captain extends CaptainBase implements Agent { getMode(): CaptainMode { const explorer = this.explorBot.getExplorer(); const activeTest = explorer.activeTest; - const page = explorer.playwrightHelper?.page; - if (activeTest && (!page || page.isClosed?.())) return 'heal'; + if (activeTest && !explorer.page) return 'heal'; if (activeTest) return 'test'; - if (explorer.getStateManager().getCurrentState()) return 'web'; + if (this.explorBot.stateManager().getCurrentState()) return 'web'; return 'idle'; } private systemPrompt(): string { const mode = this.getMode(); - const currentUrl = this.explorBot.getExplorer().getStateManager().getCurrentState()?.url; + const currentUrl = this.explorBot.stateManager().getCurrentState()?.url; const customPrompt = this.explorBot.getProvider().getSystemPromptForAgent('captain', currentUrl); return dedent` @@ -147,7 +146,7 @@ export class Captain extends CaptainBase implements Agent { } private async getPageContext(): Promise { - const state = this.explorBot.getExplorer().getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { return 'No page loaded'; } @@ -339,7 +338,7 @@ export class Captain extends CaptainBase implements Agent { } const pilotAnalysis = this.explorBot.agentPilot().getLastAnalysis() || ''; - const currentUrl = this.explorBot.getExplorer().getStateManager().getCurrentState()?.url || ''; + const currentUrl = this.explorBot.stateManager().getCurrentState()?.url || ''; const schema = z.object({ action: z.enum(['inject', 'stop', 'pass', 'skip']), @@ -390,7 +389,7 @@ export class Captain extends CaptainBase implements Agent { async processExecutionError(error: Error, activeTest: Test): Promise { const explorer = this.explorBot.getExplorer(); - const result = await explorer.handleExecutionError(error); + const result = await explorer.recover(error); return { ...result, message: result.recovered ? `${result.message}\nContinue the test "${activeTest.scenario}" from the restored page.` : result.message, @@ -402,7 +401,7 @@ export class Captain extends CaptainBase implements Agent { } async handle(input: string, options: { reset?: boolean } = {}): Promise { - const stateManager = this.explorBot.getExplorer().getStateManager(); + const stateManager = this.explorBot.stateManager(); const initialState = stateManager.getCurrentState(); const conversation = options.reset ? this.resetConversation() : this.ensureConversation(); diff --git a/src/ai/captain/idle-mode.ts b/src/ai/captain/idle-mode.ts index a49abb2..c99f425 100644 --- a/src/ai/captain/idle-mode.ts +++ b/src/ai/captain/idle-mode.ts @@ -66,7 +66,7 @@ export function WithIdleMode(Base: T) { if (!action || action === 'replace') { plan.tests.length = 0; } - const currentUrl = ctx.explorBot.getExplorer().getStateManager().getCurrentState()?.url || ''; + const currentUrl = ctx.explorBot.stateManager().getCurrentState()?.url || ''; for (const testInput of tests) { const priority = testInput.priority || 'normal'; const expected = testInput.expected?.length ? testInput.expected : []; diff --git a/src/ai/captain/test-mode.ts b/src/ai/captain/test-mode.ts index a29361e..105b04a 100644 --- a/src/ai/captain/test-mode.ts +++ b/src/ai/captain/test-mode.ts @@ -146,7 +146,7 @@ export function WithTestMode(Base: T) { if (!t) return { success: false, message: `Test "${session}" not found` }; states = t.states; } else { - const stateManager = ctx.explorBot.getExplorer().getStateManager(); + const stateManager = ctx.explorBot.stateManager(); const history = stateManager.getStateHistory(); const seen = new Set(); states = history diff --git a/src/ai/captain/web-mode.ts b/src/ai/captain/web-mode.ts index e31295f..185d4b5 100644 --- a/src/ai/captain/web-mode.ts +++ b/src/ai/captain/web-mode.ts @@ -9,9 +9,10 @@ export function WithWebMode(Base: T) { return class extends Base { webModeTools(ctx: ModeContext): Record { const explorer = ctx.explorBot.getExplorer(); - const codeceptTools = createCodeceptJSTools(explorer, ctx.task); + const toolDeps = { explorer, stateManager: ctx.explorBot.stateManager(), ai: ctx.explorBot.getProvider() }; + const codeceptTools = createCodeceptJSTools(toolDeps, ctx.task); const agentTools = createAgentTools({ - explorer, + ...toolDeps, researcher: ctx.explorBot.agentResearcher(), navigator: ctx.explorBot.agentNavigator(), }); @@ -27,7 +28,7 @@ export function WithWebMode(Base: T) { try { debugLog('navigate', destination); await ctx.explorBot.agentNavigator().visit(destination); - const stateManager = ctx.explorBot.getExplorer().getStateManager(); + const stateManager = ctx.explorBot.stateManager(); const state = stateManager.getCurrentState(); return { success: true, url: state?.url, title: state?.title }; } catch (error: any) { @@ -45,48 +46,42 @@ export function WithWebMode(Base: T) { - closeTabs: Close all browser tabs except the current one - reload: Reload the current page - screenshot: Take a screenshot of current page - - recover: Recover from a closed/crashed page using Explorer recovery - - restart: Restart the browser when page/context recovery is not enough + - recover: Recover from a closed/crashed page, escalating to a full browser restart when needed - openFreshTab: Open a fresh tab in the current browser context `, inputSchema: z.object({ - action: z.enum(['status', 'evaluate', 'closeTabs', 'reload', 'screenshot', 'recover', 'restart', 'openFreshTab']).describe('Browser action to perform'), + action: z.enum(['status', 'evaluate', 'closeTabs', 'reload', 'screenshot', 'recover', 'openFreshTab']).describe('Browser action to perform'), code: z.string().optional().describe('JavaScript code for evaluate action'), }), execute: async ({ action, code }) => { const explorer = ctx.explorBot.getExplorer(); if (action === 'status') { - const page = explorer.playwrightHelper?.page; + const page = explorer.page; const pages = page?.context?.().pages?.() || []; return { success: true, hasPage: !!page, - isClosed: page?.isClosed?.() || false, - url: page && !page.isClosed?.() ? await page.url() : null, - title: page && !page.isClosed?.() ? await page.title().catch(() => null) : null, + isClosed: !page, + url: page ? await page.url() : null, + title: page ? await page.title().catch(() => null) : null, tabs: pages.length, }; } if (action === 'recover') { - const recovered = await explorer.recoverFromBrowserError(); - return { success: recovered, message: recovered ? 'Browser page recovered' : 'Browser recovery failed' }; - } - - if (action === 'restart') { - const restarted = await explorer.restartBrowser(); - return { success: restarted, message: restarted ? 'Browser restarted' : 'Browser restart failed' }; + const { ok } = await explorer.recover(); + return { success: ok, message: ok ? 'Browser page recovered' : 'Browser recovery failed' }; } if (action === 'openFreshTab') { await ctx.explorBot.openFreshTab(); - const state = explorer.getStateManager().getCurrentState(); + const state = ctx.explorBot.stateManager().getCurrentState(); return { success: true, url: state?.url, title: state?.title }; } - const page = explorer.playwrightHelper?.page; - if (!page || page.isClosed?.()) return { success: false, message: 'No browser page available. Try browser({ action: "recover" }) first.' }; + const page = explorer.page; + if (!page) return { success: false, message: 'No browser page available. Try browser({ action: "recover" }) first.' }; if (action === 'evaluate') { if (!code) return { success: false, message: 'Code required for evaluate action' }; diff --git a/src/ai/driller.ts b/src/ai/driller.ts index 0a064e6..8199f63 100644 --- a/src/ai/driller.ts +++ b/src/ai/driller.ts @@ -3,13 +3,9 @@ import dedent from 'dedent'; import { z } from 'zod'; import { ActionResult } from '../action-result.ts'; import { setActivity } from '../activity.ts'; -import type { ExperienceTracker } from '../experience-tracker.ts'; -import type Explorer from '../explorer.ts'; -import type { KnowledgeTracker } from '../knowledge-tracker.ts'; import { Observability } from '../observability.ts'; import { Plan, Test, TestResult } from '../test-plan.ts'; import { collectInteractiveNodes } from '../utils/aria.ts'; -import { HooksRunner } from '../utils/hooks-runner.ts'; import { EXPLORBOT_ATTRS, HTML_COMPOSITE_AREA_HINTS, @@ -26,8 +22,10 @@ import { } from '../utils/html.ts'; import { createDebug, tag } from '../utils/logger.ts'; import { loop, pause } from '../utils/loop.ts'; +import { annotatePageElements } from '../utils/web-annotate.ts'; +import { eidxInContainer } from '../utils/web-eidx.ts'; import { WebElement } from '../utils/web-element.ts'; -import type { Agent } from './agent.ts'; +import type { Agent, AgentDeps } from './agent.ts'; import type { Navigator } from './navigator.ts'; import type { Provider } from './provider.ts'; import { drillLocatorRule } from './rules.ts'; @@ -79,10 +77,7 @@ interface DrillOptions { export class Driller extends TaskAgent implements Agent { protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form']; emoji = 'D'; - private explorer: Explorer; - private provider: Provider; private navigator: Navigator; - private hooksRunner: HooksRunner; private currentPlan?: Plan; private allResults: InteractionResult[] = []; private verifiedAction: { componentId: string; toolName: string; code?: string; canonicalCode?: string } | null = null; @@ -90,32 +85,17 @@ export class Driller extends TaskAgent implements Agent { MAX_COMPONENT_ITERATIONS = 12; - constructor(explorer: Explorer, provider: Provider, navigator: Navigator) { - super(); - this.explorer = explorer; - this.provider = provider; + constructor(deps: AgentDeps, navigator: Navigator) { + super(deps); this.navigator = navigator; - this.hooksRunner = new HooksRunner(explorer, explorer.getConfig()); } protected getNavigator(): Navigator { return this.navigator; } - protected getExperienceTracker(): ExperienceTracker { - return this.explorer.getStateManager().getExperienceTracker(); - } - - protected getKnowledgeTracker(): KnowledgeTracker { - return this.explorer.getKnowledgeTracker(); - } - - protected getProvider(): Provider { - return this.provider; - } - getSystemMessage(component?: ComponentInfo): string { - const currentUrl = this.explorer.getStateManager().getCurrentState()?.url; + const currentUrl = this.stateManager.getCurrentState()?.url; const customPrompt = this.provider.getSystemPromptForAgent('driller', currentUrl); return dedent` @@ -159,7 +139,7 @@ export class Driller extends TaskAgent implements Agent { async drill(opts: DrillOptions = {}): Promise { const { knowledgePath, maxComponents = 30, interactive = isInteractive() } = opts; - const currentState = this.explorer.getStateManager().getCurrentState(); + const currentState = this.stateManager.getCurrentState(); if (!currentState) throw new Error('No page state available'); const sessionName = `driller_${Date.now().toString(36)}`; @@ -207,15 +187,15 @@ export class Driller extends TaskAgent implements Agent { private async captureAnnotatedState(): Promise { setActivity(`${this.emoji} Capturing annotated page state...`, 'action'); - const action = this.explorer.createAction(); + const action = this.explorer.action(); try { const annotated = await Promise.race([ - this.explorer.annotateElements(), + this.explorer.withPage(annotatePageElements), new Promise((_, reject) => { setTimeout(() => reject(new Error('annotateElements timeout')), 15000); }), ]); - return action.capturePageState({ ariaSnapshot: annotated.ariaSnapshot }); + return action.capturePageState(); } catch (error) { tag('warning').log(`Annotated capture failed, falling back to plain page state: ${error instanceof Error ? error.message : error}`); return action.capturePageState(); @@ -226,9 +206,10 @@ export class Driller extends TaskAgent implements Agent { private async collectComponents(state: ActionResult, maxComponents: number): Promise { setActivity(`${this.emoji} Collecting components...`, 'action'); - const page = this.explorer.playwrightHelper.page; - const eidxList = await this.explorer.getEidxInContainer(null); - const webElements = await WebElement.fromEidxList(page, eidxList); + const webElements = await this.explorer.withPage(async (page) => { + const eidxList = await eidxInContainer(page, null); + return WebElement.fromEidxList(page, eidxList); + }); const ariaNodes = collectInteractiveNodes(state.ariaSnapshot); const scored = webElements .filter((element) => isDrillableElement(element)) @@ -324,7 +305,7 @@ export class Driller extends TaskAgent implements Agent { conversation.addUserText(await this.buildComponentPrompt(originalState, component)); let finished = false; - const actionTools = this.createVerifiedActionTools(createCodeceptJSTools(this.explorer, test), component); + const actionTools = this.createVerifiedActionTools(createCodeceptJSTools(this.toolDeps, test), component); const tools = { ...actionTools, ...this.createDrillFlowTools(originalState, test, interactive) }; await loop( @@ -333,7 +314,7 @@ export class Driller extends TaskAgent implements Agent { setActivity(`${this.emoji} Drilling ${component.name}...`, 'action'); if (iteration > 1) { - const currentState = ActionResult.fromState(this.explorer.getStateManager().getCurrentState() || originalState); + const currentState = ActionResult.fromState(this.stateManager.getCurrentState() || originalState); conversation.addUserText(await this.buildContextUpdate(currentState, component)); if (this.pendingNestedContext) { conversation.addUserText(this.pendingNestedContext); @@ -577,7 +558,7 @@ export class Driller extends TaskAgent implements Agent { execute: async ({ reason }) => { await this.restoreOriginalState(originalState, reason); await this.captureAnnotatedState(); - const currentState = this.explorer.getStateManager().getCurrentState(); + const currentState = this.stateManager.getCurrentState(); return { success: true, url: currentState?.url || originalState.url }; }, }), @@ -598,9 +579,9 @@ export class Driller extends TaskAgent implements Agent { } private async restoreOriginalState(originalState: ActionResult, reason: string): Promise { - const currentState = this.explorer.getStateManager().getCurrentState(); + const currentState = this.stateManager.getCurrentState(); const targetUrl = originalState.fullUrl || originalState.url; - const action = this.explorer.createAction(); + const action = this.explorer.action(); if (currentState?.url !== originalState.url) { await action.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, `${reason} (restore URL)`); @@ -665,7 +646,7 @@ export class Driller extends TaskAgent implements Agent { const overlayHtml = await this.getVisibleOverlayHtml(); if (!overlayHtml) return null; - const state = this.explorer.getStateManager().getCurrentState(); + const state = this.stateManager.getCurrentState(); if (!state) return null; const currentState = ActionResult.fromState(state); return dedent` @@ -686,40 +667,42 @@ export class Driller extends TaskAgent implements Agent { } private async getVisibleOverlayHtml(): Promise { - const page = this.explorer.playwrightHelper.page; - return page.evaluate( - ({ extractorSource, config }) => { - const extract = new Function(`return ${extractorSource}`)() as (config: any) => string; - return extract(config); - }, - { - extractorSource: getVisibleOverlayHtmlExtractorSource(), - config: { - interactiveContentSelector: HTML_SELECTORS.interactiveContent, - limits: HTML_EXTRACTION_LIMITS, - overlaySelectors: HTML_SELECTORS.semanticOverlays, - visibilityLimits: HTML_VISIBILITY_LIMITS, + return this.explorer.withPage((page) => + page.evaluate( + ({ extractorSource, config }) => { + const extract = new Function(`return ${extractorSource}`)() as (config: any) => string; + return extract(config); }, - } + { + extractorSource: getVisibleOverlayHtmlExtractorSource(), + config: { + interactiveContentSelector: HTML_SELECTORS.interactiveContent, + limits: HTML_EXTRACTION_LIMITS, + overlaySelectors: HTML_SELECTORS.semanticOverlays, + visibilityLimits: HTML_VISIBILITY_LIMITS, + }, + } + ) ); } private async getComponentScopeHtml(component: ComponentInfo, originalState: ActionResult): Promise { - const page = this.explorer.playwrightHelper.page; - const scopedHtml = await page.evaluate( - ({ eidx, extractorSource, config }) => { - const extract = new Function(`return ${extractorSource}`)() as (eidx: string, config: any) => string; - return extract(eidx, config); - }, - { - eidx: component.eidx, - extractorSource: getComponentScopeHtmlExtractorSource(), - config: { - eidxAttr: EXPLORBOT_ATTRS.eidx, - interactiveControlSelector: HTML_SELECTORS.interactiveControl, - limits: HTML_EXTRACTION_LIMITS, + const scopedHtml = await this.explorer.withPage((page) => + page.evaluate( + ({ eidx, extractorSource, config }) => { + const extract = new Function(`return ${extractorSource}`)() as (eidx: string, config: any) => string; + return extract(eidx, config); }, - } + { + eidx: component.eidx, + extractorSource: getComponentScopeHtmlExtractorSource(), + config: { + eidxAttr: EXPLORBOT_ATTRS.eidx, + interactiveControlSelector: HTML_SELECTORS.interactiveControl, + limits: HTML_EXTRACTION_LIMITS, + }, + } + ) ); if (scopedHtml) return scopedHtml; diff --git a/src/ai/historian.ts b/src/ai/historian.ts index 25ae7c3..dc34513 100644 --- a/src/ai/historian.ts +++ b/src/ai/historian.ts @@ -1,6 +1,7 @@ import { readFileSync, writeFileSync } from 'node:fs'; import type { ExplorbotConfig } from '../config.ts'; import type { ExperienceTracker } from '../experience-tracker.ts'; +import type Explorer from '../explorer.ts'; import type { PlaywrightRecorder } from '../playwright-recorder.ts'; import type { Reporter } from '../reporter.ts'; import type { StateManager } from '../state-manager.ts'; @@ -23,10 +24,10 @@ export class Historian extends HistorianBase { declare reporter: Reporter | undefined; declare stateManager: StateManager | undefined; declare config: ExplorbotConfig | undefined; - declare playwright: { recorder: PlaywrightRecorder; helper: any } | undefined; + declare playwright: { recorder: PlaywrightRecorder; explorer: Explorer } | undefined; declare savedFiles: Set; - constructor(provider: Provider, experienceTracker: ExperienceTracker, reporter?: Reporter, stateManager?: StateManager, config?: ExplorbotConfig, playwright?: { recorder: PlaywrightRecorder; helper: any }) { + constructor(provider: Provider, experienceTracker: ExperienceTracker, reporter?: Reporter, stateManager?: StateManager, config?: ExplorbotConfig, playwright?: { recorder: PlaywrightRecorder; explorer: Explorer }) { super(); this.provider = provider; this.experienceTracker = experienceTracker; diff --git a/src/ai/historian/codeceptjs.ts b/src/ai/historian/codeceptjs.ts index c313a15..b48e340 100644 --- a/src/ai/historian/codeceptjs.ts +++ b/src/ai/historian/codeceptjs.ts @@ -6,8 +6,8 @@ import type { StateManager } from '../../state-manager.ts'; import type { Plan } from '../../test-plan.ts'; import { tag } from '../../utils/logger.ts'; import { relativeToCwd } from '../../utils/next-steps.ts'; -import { safeFilename } from '../../utils/strings.ts'; import { CODECEPT_TOOLS, isNonReusableCode, stripComments } from '../../utils/step-analyzer.ts'; +import { safeFilename } from '../../utils/strings.ts'; import type { Conversation } from '../conversation.ts'; import { ASSERTION_TOOLS } from '../tools.ts'; import type { Constructor } from './mixin.ts'; diff --git a/src/ai/historian/playwright.ts b/src/ai/historian/playwright.ts index 7386ee8..df8b7e7 100644 --- a/src/ai/historian/playwright.ts +++ b/src/ai/historian/playwright.ts @@ -2,13 +2,14 @@ import { mkdirSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { ActionResult } from '../../action-result.ts'; import { ConfigParser } from '../../config.ts'; -import type { StateManager } from '../../state-manager.ts'; +import type Explorer from '../../explorer.ts'; import { type PlaywrightRecorder, type TraceCall, renderAssertion, renderCall } from '../../playwright-recorder.ts'; +import type { StateManager } from '../../state-manager.ts'; import type { Plan } from '../../test-plan.ts'; import { tag } from '../../utils/logger.ts'; import { relativeToCwd } from '../../utils/next-steps.ts'; -import { safeFilename } from '../../utils/strings.ts'; import { CODECEPT_TOOLS } from '../../utils/step-analyzer.ts'; +import { safeFilename } from '../../utils/strings.ts'; import type { Conversation } from '../conversation.ts'; import { ASSERTION_TOOLS } from '../tools.ts'; import type { Constructor } from './mixin.ts'; @@ -23,7 +24,7 @@ export interface PlaywrightMethods { export function WithPlaywright(Base: T) { return class extends Base { - declare playwright: { recorder: PlaywrightRecorder; helper: any } | undefined; + declare playwright: { recorder: PlaywrightRecorder; explorer: Explorer } | undefined; declare savedFiles: Set; declare stateManager?: StateManager; diff --git a/src/ai/historian/screencast.ts b/src/ai/historian/screencast.ts index 1384345..60e0085 100644 --- a/src/ai/historian/screencast.ts +++ b/src/ai/historian/screencast.ts @@ -4,6 +4,7 @@ import { join } from 'node:path'; import * as codeceptjs from 'codeceptjs'; import { outputPath } from '../../config.ts'; import type { ExplorbotConfig } from '../../config.ts'; +import type Explorer from '../../explorer.ts'; import type { PlaywrightRecorder } from '../../playwright-recorder.ts'; import { tag } from '../../utils/logger.ts'; import { relativeToCwd } from '../../utils/next-steps.ts'; @@ -22,7 +23,7 @@ export function WithScreencast(Base: T) { return class extends Base { declare config: ExplorbotConfig | undefined; declare savedFiles: Set; - declare playwright: { recorder: PlaywrightRecorder; helper: any } | undefined; + declare playwright: { recorder: PlaywrightRecorder; explorer: Explorer } | undefined; private screencastPage: any = null; private screencastActive = false; @@ -41,7 +42,7 @@ export function WithScreencast(Base: T) { attachScreencast(): void { if (this.screencastListenersInstalled) return; if (!this.config?.ai?.agents?.historian?.screencast) return; - if (!this.playwright?.helper) return; + if (!this.playwright?.explorer) return; this.onTestBefore = (test: any) => { void this.startScreencast(test); @@ -62,7 +63,7 @@ export function WithScreencast(Base: T) { private async startScreencast(test: any): Promise { if (this.screencastActive) return; - const page = this.playwright?.helper?.page; + const page = this.playwright?.explorer?.page; if (!page?.screencast?.start) return; const task = test?._explorbotTest; diff --git a/src/ai/navigator.ts b/src/ai/navigator.ts index c15b2d1..cae50e0 100644 --- a/src/ai/navigator.ts +++ b/src/ai/navigator.ts @@ -3,17 +3,18 @@ import dedent from 'dedent'; import { z } from 'zod'; import { ActionResult } from '../action-result.js'; import type Action from '../action.ts'; +import type { ExplorbotConfig } from '../config.ts'; import type { ExperienceTracker } from '../experience-tracker.js'; import Explorer from '../explorer.ts'; import type { KnowledgeTracker } from '../knowledge-tracker.js'; -import { normalizeUrl } from '../state-manager.js'; +import { type StateManager, normalizeUrl } from '../state-manager.js'; import { extractCodeBlocks } from '../utils/code-extractor.js'; import { HooksRunner } from '../utils/hooks-runner.ts'; import { createDebug, pluralize, tag } from '../utils/logger.js'; import { loop, pause } from '../utils/loop.js'; import { RulesLoader } from '../utils/rules-loader.ts'; import { extractStatePath } from '../utils/url-matcher.js'; -import type { Agent } from './agent.js'; +import type { Agent, AgentDeps } from './agent.js'; import type { Conversation } from './conversation.js'; import type { Provider } from './provider.js'; import { Researcher } from './researcher.ts'; @@ -70,25 +71,29 @@ class Navigator implements Agent { `; private explorer: Explorer; - - constructor(explorer: Explorer, provider: Provider) { - this.provider = provider; - this.explorer = explorer; - this.knowledgeTracker = explorer.getKnowledgeTracker(); - this.experienceTracker = explorer.getStateManager().getExperienceTracker(); - this.hooksRunner = new HooksRunner(explorer, explorer.getConfig()); + private config: ExplorbotConfig; + private stateManager: StateManager; + + constructor(deps: AgentDeps) { + this.provider = deps.ai; + this.explorer = deps.explorer; + this.config = deps.config; + this.stateManager = deps.stateManager; + this.knowledgeTracker = deps.knowledgeTracker; + this.experienceTracker = deps.stateManager.getExperienceTracker(); + this.hooksRunner = new HooksRunner(deps.explorer, deps.config); } private get verifyAttempts(): number { - return this.explorer.getConfig().ai?.agents?.navigator?.verifyAttempts ?? 3; + return this.config.ai?.agents?.navigator?.verifyAttempts ?? 3; } private get verifyTimeout(): number { - return this.explorer.getConfig().ai?.agents?.navigator?.verifyTimeout ?? 1500; + return this.config.ai?.agents?.navigator?.verifyTimeout ?? 1500; } private getBaseOrigin(): string | null { - const baseUrl = this.explorer.getConfig().playwright.url; + const baseUrl = this.config.playwright.url; try { return new URL(baseUrl).origin; } catch { @@ -133,12 +138,12 @@ class Navigator implements Agent { } async visit(url: string): Promise { - return this.explorer.runWithBrowserRecovery('navigator.visit', () => this.visitOnce(url)); + return this.visitOnce(url); } private async visitOnce(url: string): Promise { try { - const action = this.explorer.createAction(); + const action = this.explorer.action(); await action.execute(`I.amOnPage('${url}')`); @@ -171,7 +176,7 @@ class Navigator implements Agent { throw new Error(`Navigation to ${url} failed: ${action.lastError?.message}`); } } - await this.explorer.capturePageWithScreenshot(); + await this.explorer.capture({ screenshot: true }); await this.hooksRunner.runAfterHook('navigator', url); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); @@ -188,7 +193,7 @@ class Navigator implements Agent { tag('info').log('AI Navigator resolving state at', actionResult.url); debugLog('Resolution message:', message); - const action = opts?.action ?? this.explorer.createAction(); + const action = opts?.action ?? this.explorer.action(); const expectedUrl = opts?.expectedUrl; const knowledge = this.knowledgeTracker.renderRelevantKnowledge(actionResult); @@ -341,7 +346,7 @@ class Navigator implements Agent { codeBlockIndex++; totalAttempts++; - await this.explorer.switchToMainFrame(); + await this.explorer.exitIframe(); const prevActionResult = action.actionResult ?? actionResult; const prevHash = prevActionResult.getStateHash(); @@ -373,7 +378,7 @@ class Navigator implements Agent { // URL did not transition to expectedUrl within timeout } } - const freshState = await this.explorer.capturePageState(); + const freshState = await this.explorer.capture(); const currentUrl = /^https?:\/\//i.test(expectedUrl) ? freshState.fullUrl || freshState.url || '' : freshState.url || ''; const urlMatches = this.isSameExpectedOrigin(expectedUrl, action.stateManager) && normalizeUrl(currentUrl) === normalizeUrl(expectedUrl); const stateChanged = freshState.getStateHash() !== actionResult.getStateHash(); @@ -473,7 +478,7 @@ class Navigator implements Agent { private buildExperienceTools(): { learnExperience: unknown } | undefined { if (!this.experienceTracker) return undefined; - const stateManager = this.explorer.getStateManager(); + const stateManager = this.stateManager; const getState = () => { const s = stateManager.getCurrentState(); return s ? ActionResult.fromState(s) : null; @@ -482,7 +487,7 @@ class Navigator implements Agent { } async freeSail(opts?: { strategy?: 'deep' | 'shallow'; scope?: string; visitedUrls?: Set }, actionResult?: ActionResult): Promise<{ target: string; reason: string } | null> { - const stateManager = this.explorer.getStateManager(); + const stateManager = this.stateManager; const state = stateManager.getCurrentState(); if (!state) { return null; @@ -690,11 +695,11 @@ class Navigator implements Agent { const successfulCodes: string[] = []; const assertionSteps: Array<{ name: string; args: any[] }> = []; - const action = this.explorer.createAction(); + const action = this.explorer.action(); let failures = 0; - const page = this.explorer.playwrightHelper?.page; - const originalTimeout = this.explorer.playwrightHelper?.options?.timeout ?? 3000; + const page = this.explorer.page; + const originalTimeout = this.config.playwright.timeout ?? 3000; page?.setDefaultTimeout(this.verifyTimeout); try { @@ -726,7 +731,7 @@ class Navigator implements Agent { return; } - await this.explorer.switchToMainFrame(); + await this.explorer.exitIframe(); const verified = await action.attempt(codeBlock, message); @@ -764,7 +769,7 @@ class Navigator implements Agent { if (alreadyVerified) verified = true; actionResult.addVerification(message, verified); - this.explorer.getStateManager().updateState(actionResult); + this.stateManager.updateState(actionResult); return { verified, successfulCodes, assertionSteps, totalAttempted }; } diff --git a/src/ai/pilot.ts b/src/ai/pilot.ts index 9983217..1aa6e09 100644 --- a/src/ai/pilot.ts +++ b/src/ai/pilot.ts @@ -2,8 +2,11 @@ import { tool } from 'ai'; import dedent from 'dedent'; import { z } from 'zod'; import { ActionResult } from '../action-result.ts'; +import type { RequestStore } from '../api/request-store.ts'; import { ConfigParser } from '../config.ts'; import type Explorer from '../explorer.ts'; +import type { PlaywrightRecorder } from '../playwright-recorder.ts'; +import type { StateManager } from '../state-manager.ts'; import { type Test, TestResult } from '../test-plan.ts'; import { collectInteractiveNodes, detectFocusArea, extractFocusedElement } from '../utils/aria.ts'; import { ErrorPageError } from '../utils/error-page.ts'; @@ -11,7 +14,7 @@ import { createDebug, tag } from '../utils/logger.ts'; const debugLog = createDebug('explorbot:pilot'); import { truncateJson } from '../utils/strings.ts'; -import type { Agent } from './agent.ts'; +import type { Agent, AgentDeps } from './agent.ts'; import type { Conversation } from './conversation.ts'; import type { Fisherman } from './fisherman.ts'; import type { Navigator } from './navigator.ts'; @@ -30,13 +33,19 @@ export class Pilot implements Agent { private conversation: Conversation | null = null; private researcher: Researcher; private explorer: Explorer; + private stateManager: StateManager; + private requestStore: RequestStore; + private playwrightRecorder: PlaywrightRecorder; private fisherman: Fisherman | null = null; - constructor(provider: Provider, agentTools: any, researcher: Researcher, explorer: Explorer) { - this.provider = provider; + constructor(deps: AgentDeps, agentTools: any, researcher: Researcher) { + this.provider = deps.ai; this.agentTools = agentTools; this.researcher = researcher; - this.explorer = explorer; + this.explorer = deps.explorer; + this.stateManager = deps.stateManager; + this.requestStore = deps.requestStore; + this.playwrightRecorder = deps.playwrightRecorder; } setFisherman(fisherman: Fisherman): void { @@ -91,7 +100,7 @@ export class Pilot implements Agent { let screenshotState: ActionResult | null = null; if (type === 'finish' && this.provider.hasVision()) { try { - screenshotState = await this.explorer.capturePageWithScreenshot(); + screenshotState = await this.explorer.capture({ screenshot: true }); if (screenshotState.screenshot) { visualAnalysis = (await this.researcher.answerQuestionAboutScreenshot(screenshotState, `Describe current page state relevant to: ${task.scenario}`)) || ''; } @@ -175,7 +184,7 @@ export class Pilot implements Agent { tag('substep').log(`Pilot requesting verification: ${result.requestVerification}`); const verifyResult = await navigator.verifyState(result.requestVerification, currentState).catch(() => null); if (verifyResult?.verified && verifyResult.assertionSteps?.length) { - this.explorer.getPlaywrightRecorder().recordVerification(verifyResult.assertionSteps); + this.playwrightRecorder.recordVerification(verifyResult.assertionSteps); } } @@ -595,9 +604,9 @@ export class Pilot implements Agent { } private getExperienceToc(): string { - const state = this.explorer.getStateManager().getCurrentState(); + const state = this.stateManager.getCurrentState(); if (!state) return ''; - return this.explorer.getStateManager().getExperienceTracker().renderExperienceTocFor(ActionResult.fromState(state)); + return this.stateManager.getExperienceTracker().renderExperienceTocFor(ActionResult.fromState(state)); } private pickPlanningTools() { @@ -660,7 +669,7 @@ export class Pilot implements Agent { private async checkDataAvailability(task: Test, requestedData: string, fishermanReason: string | undefined): Promise { if (!this.provider.hasVision()) return null; - const screenshotState = await this.explorer.capturePageWithScreenshot().catch(() => null); + const screenshotState = await this.explorer.capture({ screenshot: true }).catch(() => null); if (!screenshotState?.screenshot) return null; const question = dedent` @@ -712,8 +721,8 @@ export class Pilot implements Agent { lines.push('modal: none'); } - if (this.explorer.hasOtherTabs()) { - const tabs = this.explorer.getOtherTabsInfo(); + const tabs = this.stateManager.otherTabs; + if (tabs.length > 0) { lines.push(`other tabs: ${tabs.length} (${tabs.map((t) => `${t.url} - ${t.title}`).join(', ')})`); } else { lines.push('other tabs: none'); @@ -736,7 +745,7 @@ export class Pilot implements Agent { lines.push('console errors: none'); } - const failedRequests = this.explorer.getRequestStore()?.getFailedRequests() ?? []; + const failedRequests = this.requestStore.getFailedRequests(); if (failedRequests.length > 0) { const sample = failedRequests .slice(-5) @@ -817,7 +826,7 @@ export class Pilot implements Agent { private formatSessionLog(testerConversation: Conversation): string { const executions = testerConversation.getToolExecutions().filter((t) => !META_TOOLS.includes(t.toolName)); - const stateHistory = this.explorer.getStateManager().getStateHistory(); + const stateHistory = this.stateManager.getStateHistory(); const initialUrl = stateHistory[0]?.toState?.url || ''; let currentUrl = initialUrl; diff --git a/src/ai/planner.ts b/src/ai/planner.ts index 07c838b..879dfec 100644 --- a/src/ai/planner.ts +++ b/src/ai/planner.ts @@ -14,7 +14,7 @@ import { createDebug, tag } from '../utils/logger.js'; import { jsonToTable } from '../utils/markdown-parser.ts'; import { mdq } from '../utils/markdown-query.js'; import { planToCompactAiContext } from '../utils/test-plan-markdown.ts'; -import type { Agent } from './agent.js'; +import type { Agent, AgentDeps } from './agent.js'; import { Conversation } from './conversation.ts'; import type { Fisherman } from './fisherman.ts'; import { WithSessionDedup } from './planner/session-dedup.ts'; @@ -63,13 +63,13 @@ export class Planner extends PlannerBase implements Agent { researcher: Researcher; private fisherman: Fisherman | null = null; - constructor(explorer: Explorer, provider: Provider, researcher: Researcher) { + constructor(deps: AgentDeps, researcher: Researcher) { super(); - this.explorer = explorer; - this.provider = provider; + this.explorer = deps.explorer; + this.provider = deps.ai; this.researcher = researcher; - this.stateManager = explorer.getStateManager(); - this.experienceTracker = explorer.getStateManager().getExperienceTracker(); + this.stateManager = deps.stateManager; + this.experienceTracker = deps.stateManager.getExperienceTracker(); } setFisherman(fisherman: Fisherman): void { diff --git a/src/ai/quartermaster.ts b/src/ai/quartermaster.ts index d5c0f50..a98dedf 100644 --- a/src/ai/quartermaster.ts +++ b/src/ai/quartermaster.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import { z } from 'zod'; import type { ActionResult } from '../action-result.ts'; import { outputPath } from '../config.ts'; +import type Explorer from '../explorer.ts'; import type { StateManager, StateTransition, WebPageState } from '../state-manager.ts'; import type { Task } from '../test-plan.ts'; import { createDebug, tag } from '../utils/logger.ts'; @@ -18,7 +19,7 @@ export class Quartermaster { private outputDir: string; private pageAnalyses: Map = new Map(); private unsubscribe: (() => void) | null = null; - private playwrightHelper: any = null; + private explorer: Explorer | null = null; private pendingAnalyses: Promise[] = []; constructor(provider: Provider, options?: { model?: any }) { @@ -28,8 +29,8 @@ export class Quartermaster { this.outputDir = outputPath('a11y'); } - start(playwrightHelper: any, stateManager: StateManager): void { - this.playwrightHelper = playwrightHelper; + start(explorer: Explorer, stateManager: StateManager): void { + this.explorer = explorer; mkdirSync(this.outputDir, { recursive: true }); this.unsubscribe = stateManager.onStateChange((event) => { @@ -60,7 +61,7 @@ export class Quartermaster { } private async runAxeAnalysis(state: WebPageState): Promise { - const page = this.playwrightHelper?.page; + const page = this.explorer?.page; if (!page || !state.hash) { debugLog('No page available for axe analysis'); return; diff --git a/src/ai/rerunner.ts b/src/ai/rerunner.ts index 323e151..2a1113c 100644 --- a/src/ai/rerunner.ts +++ b/src/ai/rerunner.ts @@ -12,16 +12,13 @@ import figureSet from 'figures'; import { z } from 'zod'; import { ActionResult } from '../action-result.ts'; import { setActivity } from '../activity.ts'; -import type { ExperienceTracker } from '../experience-tracker.ts'; -import type Explorer from '../explorer.ts'; -import type { KnowledgeTracker } from '../knowledge-tracker.ts'; import { Stats } from '../stats.ts'; import { Task, Test, TestResult } from '../test-plan.ts'; import { formatHeadings } from '../utils/context-formatter.ts'; import { createDebug, tag } from '../utils/logger.ts'; import { loop } from '../utils/loop.ts'; import { RulesLoader } from '../utils/rules-loader.ts'; -import type { Agent } from './agent.ts'; +import type { Agent, AgentDeps } from './agent.ts'; import { toolExecutionLabel } from './conversation.ts'; import type { Navigator } from './navigator.ts'; import { Provider } from './provider.ts'; @@ -35,17 +32,13 @@ export class Rerunner extends TaskAgent implements Agent { protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form']; emoji = '🔄'; - private explorer: Explorer; - private provider: Provider; private agentTools: any; private healedSteps: Array<{ original: string; healed: string }> = []; private traceDir = ''; private static pluginsWired = false; - constructor(explorer: Explorer, provider: Provider, agentTools?: any) { - super(); - this.explorer = explorer; - this.provider = provider; + constructor(deps: AgentDeps, agentTools?: any) { + super(deps); this.agentTools = agentTools; } @@ -53,20 +46,8 @@ export class Rerunner extends TaskAgent implements Agent { throw new Error('Rerunner does not use Navigator'); } - protected getExperienceTracker(): ExperienceTracker { - return this.explorer.getStateManager().getExperienceTracker(); - } - - protected getKnowledgeTracker(): KnowledgeTracker { - return this.explorer.getKnowledgeTracker(); - } - - protected getProvider(): Provider { - return this.provider; - } - private get rerunnerConfig(): Record { - return (this.explorer.getConfig().ai?.agents?.rerunner as any) || {}; + return (this.config.ai?.agents?.rerunner as any) || {}; } private get healLimit(): number { @@ -325,7 +306,7 @@ export class Rerunner extends TaskAgent implements Agent { }); const healTask = new Task(`Heal: ${failedCode}`); - const codeceptTools = createCodeceptJSTools(this.explorer, healTask); + const codeceptTools = createCodeceptJSTools(this.toolDeps, healTask); let healed = false; let healedCommand = ''; @@ -345,9 +326,9 @@ export class Rerunner extends TaskAgent implements Agent { healTask.addNote(note); tag('substep').log(note); } - const action = this.explorer.createAction(); + const action = this.explorer.action(); await action.execute(`I.wait(${seconds})`); - const state = this.explorer.getStateManager().getCurrentState(); + const state = this.stateManager.getCurrentState(); const ar = state ? ActionResult.fromState(state) : null; return { success: true, @@ -438,8 +419,8 @@ export class Rerunner extends TaskAgent implements Agent { } private getHealSystemPrompt(): string { - const customRules = this.provider.getSystemPromptForAgent('rerunner', this.explorer.getStateManager().getCurrentState()?.url) || ''; - const currentUrl = this.explorer.getStateManager().getCurrentState()?.url || ''; + const customRules = this.provider.getSystemPromptForAgent('rerunner', this.stateManager.getCurrentState()?.url) || ''; + const currentUrl = this.stateManager.getCurrentState()?.url || ''; const approach = RulesLoader.loadRules('rerunner', ['healing-approach'], currentUrl); return dedent` @@ -474,7 +455,7 @@ export class Rerunner extends TaskAgent implements Agent { } private getHealUserPrompt(failedCode: string, error: Error): string { - const state = this.explorer.getStateManager().getCurrentState(); + const state = this.stateManager.getCurrentState(); const actionResult = state ? ActionResult.fromState(state) : null; const headings = formatHeadings(state || {}); diff --git a/src/ai/researcher.ts b/src/ai/researcher.ts index 9236241..3fe5e44 100644 --- a/src/ai/researcher.ts +++ b/src/ai/researcher.ts @@ -1,23 +1,22 @@ import dedent from 'dedent'; import { ActionResult } from '../action-result.js'; import { setActivity } from '../activity.ts'; -import { ConfigParser, outputPath } from '../config.ts'; +import { ConfigParser, type ExplorbotConfig, outputPath } from '../config.ts'; import { executionController } from '../execution-controller.ts'; import type { ExperienceTracker } from '../experience-tracker.ts'; import type Explorer from '../explorer.ts'; -import type { KnowledgeTracker } from '../knowledge-tracker.ts'; import { Observability } from '../observability.ts'; import type { StateManager } from '../state-manager.js'; import { WebPageState } from '../state-manager.js'; import { Stats } from '../stats.ts'; import { diffAriaSnapshots } from '../utils/aria.ts'; import { ErrorPageError, detectPageCondition } from '../utils/error-page.ts'; -import { HooksRunner } from '../utils/hooks-runner.ts'; import { isBodyEmpty } from '../utils/html.ts'; import { createDebug, tag } from '../utils/logger.js'; import { mdq } from '../utils/markdown-query.ts'; import { RulesLoader } from '../utils/rules-loader.ts'; -import type { Agent } from './agent.js'; +import { annotatePageElements } from '../utils/web-annotate.ts'; +import type { Agent, AgentDeps } from './agent.js'; import type { Navigator } from './navigator.ts'; import { ContextLengthError, type Provider } from './provider.js'; import { findSimilarResearch, getCachedResearch, saveResearch } from './researcher/cache.ts'; @@ -55,20 +54,16 @@ export class Researcher extends ResearcherBase implements Agent { declare explorer: Explorer; declare provider: Provider; declare stateManager: StateManager; + declare config: ExplorbotConfig; private experienceTracker!: ExperienceTracker; private hasScreenshotToAnalyze = false; declare actionResult: ActionResult | undefined; - private hooksRunner!: HooksRunner; - constructor(explorer: Explorer, provider: Provider) { - super(); - this.explorer = explorer; - this.provider = provider; - this.stateManager = explorer.getStateManager(); - this.experienceTracker = this.stateManager.getExperienceTracker(); - this.hooksRunner = new HooksRunner(explorer, explorer.getConfig()); + constructor(deps: AgentDeps) { + super(deps); + this.experienceTracker = deps.stateManager.getExperienceTracker(); - const ai = explorer.getConfig().ai; + const ai = deps.config.ai; if (ai) { ai.agents ??= {}; ai.agents.researcher ??= {}; @@ -80,18 +75,6 @@ export class Researcher extends ResearcherBase implements Agent { throw new Error('not implemented'); } - protected getExperienceTracker(): ExperienceTracker { - return this.experienceTracker; - } - - protected getKnowledgeTracker(): KnowledgeTracker { - return this.explorer.getKnowledgeTracker(); - } - - protected getProvider(): Provider { - return this.provider; - } - static getCachedResearch(state: WebPageState): string { return getCachedResearch(state.hash || ''); } @@ -110,7 +93,7 @@ export class Researcher extends ResearcherBase implements Agent { async research(state: WebPageState, opts: { screenshot?: boolean; force?: boolean; deep?: boolean; data?: boolean; fix?: boolean; _retriesLeft?: number } = {}): Promise { const { screenshot = false, force = false, deep = false, data = false, fix = true } = opts; - const maxRetries = (this.explorer.getConfig().ai?.agents?.researcher as any)?.retries ?? 2; + const maxRetries = (this.config.ai?.agents?.researcher as any)?.retries ?? 2; let retriesLeft = opts._retriesLeft ?? maxRetries; this.actionResult = ActionResult.fromState(state); const stateHash = state.hash || this.actionResult.getStateHash(); @@ -134,9 +117,9 @@ export class Researcher extends ResearcherBase implements Agent { await this.ensureNavigated(displayUrl, screenshot && this.provider.hasVision()); await this.hooksRunner.runBeforeHook('researcher', state.url); - const annotatedElements = await this.explorer.annotateElements(); + const { elements: annotatedElements } = await this.explorer.withPage(annotatePageElements); debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`); - this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() }); + this.actionResult = await this.explorer.capture({ screenshot: screenshot && this.provider.hasVision() }); const condition = detectPageCondition(this.actionResult!); if (condition === 'error') { @@ -245,7 +228,7 @@ export class Researcher extends ResearcherBase implements Agent { // Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring. if (!interrupted() && this.hasScreenshotToAnalyze) { const sections = parseResearchSections(result.text); - const focused = await this.explorer.runWithBrowserRecovery('detectFocusedSection', () => detectFocusedSection(this.explorer.playwrightHelper.page, sections)); + const focused = await this.explorer.withPage((page) => detectFocusedSection(page, sections)); if (focused) markSectionAsFocused(result, focused); } @@ -258,7 +241,7 @@ export class Researcher extends ResearcherBase implements Agent { const freshBroken = freshContainerLocs.filter((l) => l.valid === false).map((l) => l.locator); const containers = validContainers.filter((c) => !freshBroken.includes(c.css)); await this.visuallyAnnotateElements({ containers }); - this.actionResult = await this.explorer.capturePageWithScreenshot(); + this.actionResult = await this.explorer.capture({ screenshot: true }); const visualResult = await this.analyzeScreenshotForVisualProps(); if (visualResult.elements.size > 0) { await this.mergeVisualData(result, visualResult.elements); @@ -334,8 +317,7 @@ export class Researcher extends ResearcherBase implements Agent { private async ensureNavigated(url: string, screenshot?: boolean): Promise { if (!this.actionResult) { debugLog('No action result, navigating to URL'); - await this.explorer.visit(url); - this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot }); + this.actionResult = await this.explorer.visit(url, { screenshot }); return; } @@ -345,7 +327,7 @@ export class Researcher extends ResearcherBase implements Agent { if (!isEmpty && isOnCurrentState) { if ((!this.actionResult.screenshot && screenshot) || !this.actionResult.ariaSnapshot) { - this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot }); + this.actionResult = await this.explorer.capture({ screenshot }); } return; } @@ -353,8 +335,7 @@ export class Researcher extends ResearcherBase implements Agent { if (isEmpty && isOnCurrentState) { debugLog('HTML body empty on current URL, waiting for content'); tag('step').log('Page body is empty, waiting for content...'); - await this.explorer.visit(url); - this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false }); + this.actionResult = await this.explorer.visit(url, { screenshot: screenshot ?? false }); await this.waitUntilSettled(screenshot ?? false); return; } @@ -362,22 +343,21 @@ export class Researcher extends ResearcherBase implements Agent { debugLog('Not on current state, navigating to URL'); tag('step').log('Navigating to URL...'); - await this.explorer.visit(url); - this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false }); + this.actionResult = await this.explorer.visit(url, { screenshot: screenshot ?? false }); } private async waitUntilSettled(screenshot: boolean): Promise { - const errorPageTimeout = (this.explorer.getConfig().ai?.agents?.researcher as any)?.errorPageTimeout ?? 10; + const errorPageTimeout = (this.config.ai?.agents?.researcher as any)?.errorPageTimeout ?? 10; if (errorPageTimeout <= 0) return false; const includeScreenshot = screenshot && this.provider.hasVision(); try { - await this.explorer.runWithBrowserRecovery('waitUntilSettled', () => this.explorer.playwrightHelper.page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 })); + await this.explorer.withPage((page) => page.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 })); } catch {} - await this.explorer.annotateElements(); - this.actionResult = await this.explorer.capturePageState({ includeScreenshot }); + await this.explorer.withPage(annotatePageElements); + this.actionResult = await this.explorer.capture({ screenshot: includeScreenshot }); let condition = detectPageCondition(this.actionResult!); if (condition === 'error') { @@ -387,8 +367,8 @@ export class Researcher extends ResearcherBase implements Agent { for (let i = 0; i < 3; i++) { await new Promise((r) => setTimeout(r, 1000)); - await this.explorer.annotateElements(); - this.actionResult = await this.explorer.capturePageState({ includeScreenshot }); + await this.explorer.withPage(annotatePageElements); + this.actionResult = await this.explorer.capture({ screenshot: includeScreenshot }); condition = detectPageCondition(this.actionResult!); if (condition === 'error') { throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title, this.actionResult!.httpStatus); @@ -400,7 +380,7 @@ export class Researcher extends ResearcherBase implements Agent { } private getConfiguredSections(): Record { - const configSections = (this.explorer.getConfig().ai?.agents?.researcher as any)?.sections as string[] | undefined; + const configSections = (this.config.ai?.agents?.researcher as any)?.sections as string[] | undefined; if (!configSections?.length) return POSSIBLE_SECTIONS; const filtered: Record = {}; for (const key of configSections) { @@ -467,7 +447,7 @@ export class Researcher extends ResearcherBase implements Agent { if (!this.actionResult) throw new Error('actionResult is not set'); const html = await this.actionResult.combinedHtml(); - const knowledge = this.explorer.getKnowledgeTracker().renderRelevantKnowledge(this.actionResult); + const knowledge = this.knowledgeTracker.renderRelevantKnowledge(this.actionResult); const ariaSnapshot = this.actionResult.getCompactARIA(); @@ -728,9 +708,9 @@ export class Researcher extends ResearcherBase implements Agent { async cancelInUi() { const beforeAria = this.stateManager.getCurrentState()?.ariaSnapshot || null; - await this.explorer.executeAction('I.clickXY(0, 0)'); + await this.explorer.action().execute('I.clickXY(0, 0)'); if (diffAriaSnapshots(beforeAria, this.stateManager.getCurrentState()?.ariaSnapshot || null).text) return; - await this.explorer.executeAction(`I.pressKey('Escape')`); + await this.explorer.action().execute(`I.pressKey('Escape')`); } } diff --git a/src/ai/researcher/coordinates.ts b/src/ai/researcher/coordinates.ts index 3893fca..6e0dd65 100644 --- a/src/ai/researcher/coordinates.ts +++ b/src/ai/researcher/coordinates.ts @@ -4,6 +4,7 @@ import type { ActionResult } from '../../action-result.js'; import type Explorer from '../../explorer.ts'; import { tag } from '../../utils/logger.js'; import { mdq } from '../../utils/markdown-query.ts'; +import { eidxByLocator } from '../../utils/web-eidx.ts'; import { WebElement } from '../../utils/web-element.ts'; import type { Provider } from '../provider.js'; import { type Constructor, debugLog } from './mixin.ts'; @@ -81,7 +82,7 @@ export function WithCoordinates(Base: T) { } async visuallyAnnotateElements(opts?: { containers?: Array<{ css: string; label: string }> }): Promise { - return this.explorer.visuallyAnnotateElements(opts); + return this.explorer.withPage((page) => visuallyAnnotateContainers(page, opts?.containers || [])); } private async _analyzeScreenshotForVisualProps(): Promise { @@ -177,7 +178,7 @@ export function WithCoordinates(Base: T) { let eidx = el.eidx || null; if (!eidx) { const locator = el.css || el.xpath || (el.aria ? `role=${el.aria.role}[name="${el.aria.text}"]` : null); - if (locator) eidx = await this.explorer.getEidxByLocator(locator, section.containerCss); + if (locator) eidx = await this.explorer.withPage((page) => eidxByLocator(page, locator, section.containerCss)); } if (!eidx) continue; @@ -202,7 +203,7 @@ export function WithCoordinates(Base: T) { } if (eidxWithoutCoords.length === 0) return; - const webElements = await this.explorer.runWithBrowserRecovery('backfillCoordinates', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, eidxWithoutCoords)); + const webElements = await this.explorer.withPage((page) => WebElement.fromEidxList(page, eidxWithoutCoords)); if (webElements.length === 0) return; const rectMap = new Map(webElements.map((w) => [w.eidx!, w])); diff --git a/src/ai/researcher/deep-analysis.ts b/src/ai/researcher/deep-analysis.ts index b4dbf85..f81b22b 100644 --- a/src/ai/researcher/deep-analysis.ts +++ b/src/ai/researcher/deep-analysis.ts @@ -1,5 +1,6 @@ import dedent from 'dedent'; import { ActionResult, type Diff } from '../../action-result.js'; +import type { ExplorbotConfig } from '../../config.ts'; import { executionController } from '../../execution-controller.ts'; import type Explorer from '../../explorer.ts'; import type { StateManager } from '../../state-manager.js'; @@ -21,13 +22,14 @@ export function WithDeepAnalysis(Base: T) { declare explorer: Explorer; declare provider: Provider; declare stateManager: StateManager; + declare config: ExplorbotConfig; declare actionResult: ActionResult | undefined; async performDeepAnalysis(state: WebPageState, result: ResearchResult): Promise { tag('info').log('Starting deep analysis of expandable elements'); await (this as any).navigateTo(state.fullUrl || state.url); - const maxClicks = (this.explorer.getConfig().ai?.agents?.researcher as any)?.maxExpandableClicks ?? DEFAULT_MAX_EXPANDABLE_CLICKS; + const maxClicks = (this.config.ai?.agents?.researcher as any)?.maxExpandableClicks ?? DEFAULT_MAX_EXPANDABLE_CLICKS; const expandedSections: string[] = []; const navigationLinks: Array<{ code: string; url: string }> = []; @@ -358,10 +360,10 @@ export function WithDeepAnalysis(Base: T) { const isCoordinateClick = el.commands[0].startsWith('I.clickXY('); if (!isCoordinateClick) { const hoverCmd = el.commands[0].replace('I.click(', 'I.moveCursorTo('); - await this.explorer.attemptAction(hoverCmd, undefined); + await this.explorer.action().attempt(hoverCmd); await new Promise((r) => setTimeout(r, 500)); - await this.explorer.capturePageState(); + await this.explorer.capture(); const hoverAR = ActionResult.fromState(this.stateManager.getCurrentState()!); const hoverDiff = await hoverAR.diff(previousState); const hoverHtmlSize = hoverDiff.htmlParts.reduce((sum, p) => sum + p.subtree.length, 0); @@ -401,7 +403,7 @@ export function WithDeepAnalysis(Base: T) { const previousState = ActionResult.fromState(this.stateManager.getCurrentState()!); let clickCode: string | null = null; - const action = this.explorer.createAction(); + const action = this.explorer.action(); for (const cmd of commands) { if (await action.attempt(cmd, undefined)) { clickCode = cmd; @@ -417,7 +419,7 @@ export function WithDeepAnalysis(Base: T) { let diff: Diff; try { - await this.explorer.createAction().capturePageState(); + await this.explorer.capture(); const currAR = ActionResult.fromState(this.stateManager.getCurrentState()!); diff = await currAR.diff(previousState); } catch (err) { @@ -448,7 +450,7 @@ export function WithDeepAnalysis(Base: T) { private async _restorePageState(url: string, originalAria: string): Promise { try { await (this as any).cancelInUi(); - await this.explorer.capturePageState(); + await this.explorer.capture(); const currentAria = this.stateManager.getCurrentState()?.ariaSnapshot || ''; if (!diffAriaSnapshots(originalAria, currentAria).text) return; } catch (err) { diff --git a/src/ai/researcher/locators.ts b/src/ai/researcher/locators.ts index 62fa733..f9950b6 100644 --- a/src/ai/researcher/locators.ts +++ b/src/ai/researcher/locators.ts @@ -51,18 +51,18 @@ export function WithLocators(Base: T) { continue; } try { - const count = await this.explorer.playwrightLocatorCount((page) => { + const count = await this.explorer.withPage((page) => { const base = loc.container ? page.locator(loc.container) : page; if (loc.type === 'aria') { const parsed = parseAriaLocator(loc.locator); - if (!parsed) return page.locator('__invalid__'); - return base.getByRole(parsed.role as any, { name: parsed.text }); + if (!parsed) return page.locator('__invalid__').count(); + return base.getByRole(parsed.role as any, { name: parsed.text }).count(); } const converted = loc.locator.replace(/:contains\(/g, ':has-text('); if (converted !== loc.locator) { loc.locator = converted; } - return base.locator(loc.locator); + return base.locator(loc.locator).count(); }); loc.valid = count === 1; loc.pwLocator = buildPwLocatorString(loc); @@ -194,7 +194,7 @@ export function WithLocators(Base: T) { } if (needsXpath.length > 0) { - const webElements = await this.explorer.runWithBrowserRecovery('backfillBrokenLocators', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, needsXpath)); + const webElements = await this.explorer.withPage((page) => WebElement.fromEidxList(page, needsXpath)); const changedSections = new Set<(typeof sections)[0]>(); for (const w of webElements) { const entry = needsXpathEls.get(w.eidx!); @@ -217,7 +217,7 @@ export function WithLocators(Base: T) { let count = 0; try { - count = await this.explorer.playwrightLocatorCount((page) => page.locator(section.containerCss!)); + count = await this.explorer.withPage((page) => page.locator(section.containerCss!).count()); } catch {} if (count >= 1) continue; @@ -226,7 +226,7 @@ export function WithLocators(Base: T) { if (simplified) { let simplifiedCount = 0; try { - simplifiedCount = await this.explorer.playwrightLocatorCount((page) => page.locator(simplified)); + simplifiedCount = await this.explorer.withPage((page) => page.locator(simplified).count()); } catch {} if (simplifiedCount >= 1) { diff --git a/src/ai/researcher/sections.ts b/src/ai/researcher/sections.ts index 4659da3..243f098 100644 --- a/src/ai/researcher/sections.ts +++ b/src/ai/researcher/sections.ts @@ -1,5 +1,6 @@ import dedent from 'dedent'; import type { ActionResult } from '../../action-result.js'; +import type { ExplorbotConfig } from '../../config.ts'; import { executionController } from '../../execution-controller.ts'; import type Explorer from '../../explorer.ts'; import type { StateManager } from '../../state-manager.js'; @@ -20,6 +21,7 @@ export function WithSections(Base: T) { declare explorer: Explorer; declare provider: Provider; declare stateManager: StateManager; + declare config: ExplorbotConfig; declare actionResult: ActionResult | undefined; async researchBySections(): Promise { @@ -64,11 +66,11 @@ export function WithSections(Base: T) { } private async _detectFocusCss(): Promise { - const focusSections = (this.explorer.getConfig().ai?.agents?.researcher as any)?.focusSections as string[] | undefined; + const focusSections = (this.config.ai?.agents?.researcher as any)?.focusSections as string[] | undefined; if (!focusSections?.length) return null; for (const css of focusSections) { - const count = await this.explorer.playwrightLocatorCount((page: any) => page.locator(css)).catch(() => 0); + const count = await this.explorer.withPage((page) => page.locator(css).count()).catch(() => 0); if (count > 0) return css; } return null; diff --git a/src/ai/task-agent.ts b/src/ai/task-agent.ts index 8165998..9c635bc 100644 --- a/src/ai/task-agent.ts +++ b/src/ai/task-agent.ts @@ -1,6 +1,11 @@ import type { ActionResult } from '../action-result.js'; +import type { ExplorbotConfig } from '../config.ts'; import type { ExperienceTracker } from '../experience-tracker.js'; +import type Explorer from '../explorer.ts'; import type { KnowledgeTracker } from '../knowledge-tracker.js'; +import type { StateManager } from '../state-manager.ts'; +import { HooksRunner } from '../utils/hooks-runner.ts'; +import type { AgentDeps, ToolDeps } from './agent.ts'; import { Historian } from './historian.js'; import type { Navigator } from './navigator.js'; import type { Provider } from './provider.js'; @@ -17,6 +22,12 @@ function createNullProxy(): T { } export abstract class TaskAgent { + explorer!: Explorer; + provider!: Provider; + config!: ExplorbotConfig; + stateManager!: StateManager; + knowledgeTracker!: KnowledgeTracker; + protected hooksRunner!: HooksRunner; protected consecutiveFailures = 0; protected consecutiveEmptyResults = 0; protected recentToolCalls: any[] = []; @@ -25,10 +36,41 @@ export abstract class TaskAgent { private _historian: Historian | null = null; private _quartermaster: Quartermaster | null = null; + constructor(deps?: AgentDeps) { + if (!deps) return; + this.explorer = deps.explorer; + this.provider = deps.ai; + this.config = deps.config; + this.stateManager = deps.stateManager; + this.knowledgeTracker = deps.knowledgeTracker; + this.hooksRunner = new HooksRunner(deps.explorer, deps.config); + } + + setHistorian(historian: Historian): void { + this._historian = historian; + } + + setQuartermaster(quartermaster: Quartermaster): void { + this._quartermaster = quartermaster; + } + protected abstract getNavigator(): Navigator; - protected abstract getExperienceTracker(): ExperienceTracker; - protected abstract getKnowledgeTracker(): KnowledgeTracker; - protected abstract getProvider(): Provider; + + protected get toolDeps(): ToolDeps { + return { explorer: this.explorer, stateManager: this.stateManager, ai: this.provider }; + } + + protected getExperienceTracker(): ExperienceTracker { + return this.stateManager.getExperienceTracker(); + } + + protected getKnowledgeTracker(): KnowledgeTracker { + return this.knowledgeTracker; + } + + protected getProvider(): Provider { + return this.provider; + } protected getKnowledge(actionResult: ActionResult): string { return this.getKnowledgeTracker().renderRelevantKnowledge(actionResult); @@ -38,19 +80,11 @@ export abstract class TaskAgent { return this.getExperienceTracker().renderExperienceTocFor(actionResult); } - setHistorian(historian: Historian): void { - this._historian = historian; - } - protected getHistorian(): Historian { if (this._historian) return this._historian; return createNullProxy(); } - setQuartermaster(quartermaster: Quartermaster): void { - this._quartermaster = quartermaster; - } - protected getQuartermaster(): Quartermaster { if (this._quartermaster) return this._quartermaster; return createNullProxy(); diff --git a/src/ai/tester.ts b/src/ai/tester.ts index 0a57380..2e40725 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -5,18 +5,17 @@ import dedent from 'dedent'; import { z } from 'zod'; import { ActionResult } from '../action-result.ts'; import { clearActivity, setActivity } from '../activity.ts'; -import type { ExperienceTracker } from '../experience-tracker.ts'; -import type Explorer from '../explorer.ts'; +import type { RequestStore } from '../api/request-store.ts'; +import type { TestRun } from '../explorer.ts'; import { Observability } from '../observability.ts'; import type { StateTransition } from '../state-manager.ts'; import { Stats } from '../stats.ts'; import { type Test, TestResult, type TestResultType } from '../test-plan.ts'; import { detectFocusArea, extractFocusedElement } from '../utils/aria.ts'; import { ErrorPageError, isErrorPage } from '../utils/error-page.ts'; -import { HooksRunner } from '../utils/hooks-runner.ts'; import { createDebug, tag } from '../utils/logger.ts'; import { loop } from '../utils/loop.ts'; -import type { Agent } from './agent.ts'; +import type { Agent, AgentDeps } from './agent.ts'; import type { Captain } from './captain.ts'; import type { Conversation } from './conversation.ts'; import { Navigator } from './navigator.ts'; @@ -44,8 +43,8 @@ export class Tester extends TaskAgent implements Agent { protected readonly ACTION_TOOLS = ['click', 'hover', 'pressKey', 'form']; protected readonly SPECIAL_CONTEXT_ACTION_TOOLS = ['exitIframe']; emoji = '🧪'; - private explorer: Explorer; - private provider: Provider; + private requestStore: RequestStore; + private testRun: TestRun | null = null; private currentConversation: Conversation | null = null; private pilot: Pilot | null = null; private captain: Captain | null = null; @@ -60,38 +59,23 @@ export class Tester extends TaskAgent implements Agent { private previousStateHash: string | null = null; private pageStateHash: string | null = null; private pageActionResult: ActionResult | null = null; - private hooksRunner: HooksRunner; private seenUiMapUrls = new Set(); private lastAnalyzedStateHash: string | null = null; private stalledIterations = 0; private readonly MAX_STALLED_ITERATIONS = 3; - constructor(explorer: Explorer, provider: Provider, researcher: Researcher, navigator: Navigator, agentTools?: any) { - super(); - this.explorer = explorer; - this.provider = provider; + constructor(deps: AgentDeps, researcher: Researcher, navigator: Navigator, agentTools?: any) { + super(deps); + this.requestStore = deps.requestStore; this.researcher = researcher; this.navigator = navigator; this.agentTools = agentTools; - this.hooksRunner = new HooksRunner(explorer, explorer.getConfig()); } protected getNavigator(): Navigator { return this.navigator; } - protected getExperienceTracker(): ExperienceTracker { - return this.explorer.getStateManager().getExperienceTracker(); - } - - protected getKnowledgeTracker() { - return this.explorer.getKnowledgeTracker(); - } - - protected getProvider(): Provider { - return this.provider; - } - setPilot(pilot: Pilot): void { this.pilot = pilot; } @@ -101,11 +85,11 @@ export class Tester extends TaskAgent implements Agent { } private getCurrentState(): ActionResult { - return ActionResult.fromState(this.explorer.getStateManager().getCurrentState()!); + return ActionResult.fromState(this.stateManager.getCurrentState()!); } private get progressCheckInterval(): number { - return (this.explorer.getConfig().ai?.agents?.tester as any)?.progressCheckInterval ?? 3; + return (this.config.ai?.agents?.tester as any)?.progressCheckInterval ?? 3; } getConversation(): Conversation | null { @@ -114,7 +98,7 @@ export class Tester extends TaskAgent implements Agent { async test(task: Test): Promise<{ success: boolean }> { Stats.tests++; - const state = this.explorer.getStateManager().getCurrentState(); + const state = this.stateManager.getCurrentState(); if (!state) throw new Error('No state found'); setActivity(`🧪 Testing: ${task.scenario}`, 'action'); @@ -126,20 +110,20 @@ export class Tester extends TaskAgent implements Agent { this.seenUiMapUrls.clear(); this.lastAnalyzedStateHash = null; this.stalledIterations = 0; - this.explorer.getStateManager().clearHistory(); + this.stateManager.clearHistory(); this.resetFailureCount(); this.pilot?.reset(); - const requestStore = this.explorer.getRequestStore(); - requestStore?.clear(); - const offFailedRequest = requestStore?.onFailedRequest((r) => { + const requestStore = this.requestStore; + requestStore.clear(); + const offFailedRequest = requestStore.onFailedRequest((r) => { task.addNote(`Network error: ${r.method} ${r.path} → ${r.status}`, TestResult.FAILED); }); const initialState = ActionResult.fromState(state); if (isErrorPage(initialState)) { task.start(); - await this.explorer.startTest(task); + this.testRun = await this.explorer.beginTest(task); offFailedRequest?.(); return await this.abortStartedTestOnErrorPage(task, initialState); } @@ -196,7 +180,8 @@ export class Tester extends TaskAgent implements Agent { debugLog('Starting test execution with tools'); - if (!(await this.explorer.startTest(task))) { + this.testRun = await this.explorer.beginTest(task); + if (!this.testRun.started) { offFailedRequest?.(); await this.cleanupStartedTest(task); return { success: task.isSuccessful }; @@ -214,7 +199,7 @@ export class Tester extends TaskAgent implements Agent { } } - const startState = this.explorer.getStateManager().getCurrentState(); + const startState = this.stateManager.getCurrentState(); if (startState) { task.addUrlNote(startState); const startActionResult = ActionResult.fromState(startState); @@ -226,14 +211,14 @@ export class Tester extends TaskAgent implements Agent { const currentUrl = startState?.url || task.startUrl || ''; await this.hooksRunner.runBeforeHook('tester', currentUrl); - const offStateChange = this.explorer.getStateManager().onStateChange((event: StateTransition) => { + const offStateChange = this.stateManager.onStateChange((event: StateTransition) => { if (task.hasFinished) return; if (event.toState?.url === event.fromState?.url) return; if (event.toState) task.addUrlNote(event.toState, event.fromState || undefined); task.states.push(event.toState); }); - const codeceptjsTools = createCodeceptJSTools(this.explorer, task); + const codeceptjsTools = createCodeceptJSTools(this.toolDeps, task); let assertionPerformed = false; let extensions = 0; let shouldContinue = true; @@ -244,7 +229,7 @@ export class Tester extends TaskAgent implements Agent { await loop( async ({ stop, pause, iteration, userInput }) => { debugLog('iteration', iteration); - if (!(await this.explorer.ensurePageAvailable())) { + if (!(await this.explorer.recover()).ok) { task.addNote('Browser page is unavailable'); task.finish(TestResult.FAILED); stop(); @@ -258,12 +243,12 @@ export class Tester extends TaskAgent implements Agent { ...this.agentTools, }; if (currentState.isInsideIframe) { - Object.assign(tools, createIframeTools(this.explorer)); + Object.assign(tools, createIframeTools(this.toolDeps)); } debugLog(`Test ${task.scenario} iteration ${iteration}`); - if (this.explorer.getStateManager().isInDeadLoop()) { + if (this.stateManager.isInDeadLoop()) { task.addNote('Dead loop detected. Stopped'); stop(); return; @@ -412,7 +397,7 @@ export class Tester extends TaskAgent implements Agent { if (task.hasFinished) break; - if (!(await this.explorer.ensurePageAvailable())) break; + if (!(await this.explorer.recover()).ok) break; const finalState = this.getCurrentState(); const wantsContinue = await this.pilot!.finalReview(task, finalState, conversation, this.navigator); @@ -431,7 +416,7 @@ export class Tester extends TaskAgent implements Agent { shouldContinue = true; } - const finalUrl = this.explorer.getStateManager().getCurrentState()?.url || currentUrl; + const finalUrl = this.stateManager.getCurrentState()?.url || currentUrl; await this.hooksRunner.runAfterHook('tester', finalUrl); await this.getHistorian().saveSession(task, initialState, conversation); @@ -443,7 +428,7 @@ export class Tester extends TaskAgent implements Agent { offStateChange(); offFailedRequest?.(); await this.finishTest(task); - await this.explorer.stopTest(task, this.buildStopTestMeta(task)); + await this.testRun?.stop(this.buildStopTestMeta(task)); return { success: task.isSuccessful, @@ -559,7 +544,7 @@ export class Tester extends TaskAgent implements Agent { } if (currentState.isInsideIframe) { - const iframeInfo = currentState.iframeURL || this.explorer.getCurrentIframeInfo() || 'iframe context active'; + const iframeInfo = currentState.iframeURL || 'iframe context active'; context += dedent` INSIDE IFRAME: ${iframeInfo} @@ -568,10 +553,10 @@ export class Tester extends TaskAgent implements Agent { `; } - if (this.explorer.hasOtherTabs()) { - const otherTabs = this.explorer.getOtherTabsInfo(); + const otherTabs = this.stateManager.otherTabs; + if (otherTabs.length > 0) { context += multipleTabsRule(otherTabs); - this.explorer.clearOtherTabsInfo(); + this.stateManager.otherTabs = []; } if (isNewUrl) { @@ -675,7 +660,7 @@ export class Tester extends TaskAgent implements Agent { task.addNote(error.message, TestResult.FAILED, actionResult.screenshotFile, actionResult.fullUrl || actionResult.url); task.finish(TestResult.FAILED); this.finishTest(task); - await this.explorer.stopTest(task, this.buildStopTestMeta(task)); + await this.testRun?.stop(this.buildStopTestMeta(task)); clearActivity(true); return { success: false }; } @@ -790,7 +775,7 @@ export class Tester extends TaskAgent implements Agent { ${dataProtectionRules} - ${this.provider.getSystemPromptForAgent('tester', this.explorer.getStateManager().getCurrentState()?.url) || ''} + ${this.provider.getSystemPromptForAgent('tester', this.stateManager.getCurrentState()?.url) || ''} `; } @@ -837,7 +822,7 @@ export class Tester extends TaskAgent implements Agent { } private buildAvailableFiles(): string { - const userFiles = this.explorer.getConfig().files || {}; + const userFiles = this.config.files || {}; const codeceptDir = (global as any).codecept_dir || process.cwd(); const lines: string[] = []; @@ -896,10 +881,10 @@ export class Tester extends TaskAgent implements Agent { }), execute: async ({ reason }) => { if (this.getCurrentState().isInsideIframe) { - await this.explorer.switchToMainFrame(); + await this.explorer.exitIframe(); } - const currentState = this.explorer.getStateManager().getCurrentState(); + const currentState = this.stateManager.getCurrentState(); const currentUrl = currentState?.fullUrl || currentState?.url; if (currentUrl === resetUrl!) { return { @@ -927,7 +912,7 @@ export class Tester extends TaskAgent implements Agent { const explanation = reason ? `${reason} (RESET)` : 'Resetting to initial page'; const targetUrl = resetUrl!; task.addNote(explanation); - const resetAction = this.explorer.createAction(); + const resetAction = this.explorer.action(); const success = await resetAction.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, explanation); if (success) { @@ -1076,7 +1061,7 @@ export class Tester extends TaskAgent implements Agent { mappedStatus = TestResult.FAILED; } - const screenshotFile = this.explorer.getStateManager().getCurrentState()?.screenshotFile; + const screenshotFile = this.stateManager.getCurrentState()?.screenshotFile; for (const noteText of input.notes) { task.addNote(noteText, mappedStatus, screenshotFile); @@ -1117,7 +1102,7 @@ export class Tester extends TaskAgent implements Agent { const message = error instanceof Error ? error.message : String(error); if (!task.hasFinished) task.addNote(`Execution error: ${message}`); - const result = await this.explorer.handleExecutionError(error); + const result = await this.explorer.recover(error); tag('info').log(`Browser supervisor: ${result.action} - ${result.message}`); task.addNote(result.message); @@ -1152,7 +1137,7 @@ export class Tester extends TaskAgent implements Agent { private async cleanupStartedTest(task: Test): Promise { await this.finishTest(task); - await this.explorer.stopTest(task, { + await this.testRun?.stop({ startUrl: task.startUrl, style: task.style, sessionName: task.sessionName, diff --git a/src/ai/tools.ts b/src/ai/tools.ts index a1ffd85..1c0d36c 100644 --- a/src/ai/tools.ts +++ b/src/ai/tools.ts @@ -3,13 +3,13 @@ import dedent from 'dedent'; import { z } from 'zod'; import { ActionResult, type PageDiff, type ToolResultMetadata } from '../action-result.ts'; import type { ExperienceTracker } from '../experience-tracker.ts'; -import type Explorer from '../explorer.ts'; import { type Task, TestResult } from '../test-plan.js'; import { LARGE_ARIA_CHANGE_THRESHOLD, extractFocusedElement } from '../utils/aria.ts'; import { isFatalBrowserError } from '../utils/browser-errors.ts'; import { createDebug, tag } from '../utils/logger.js'; import { pause } from '../utils/loop.js'; import { WebElement } from '../utils/web-element.ts'; +import type { ToolDeps } from './agent.ts'; import { Navigator } from './navigator.ts'; import type { AIProvider } from './provider.ts'; import { Researcher } from './researcher.ts'; @@ -18,11 +18,16 @@ import { isInteractive } from './task-agent.ts'; const debugLog = createDebug('explorbot:tools'); -export const ASSERTION_TOOLS = ['verify'] as const; +interface AgentToolDeps extends ToolDeps { + researcher: Researcher; + navigator: Navigator; + supervisor?: boolean; + withExperience?: boolean; +} -export function createCodeceptJSTools(explorer: Explorer, task: Task) { - const stateManager = explorer.getStateManager(); +export const ASSERTION_TOOLS = ['verify'] as const; +export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps, task: Task) { return { click: tool({ description: dedent` @@ -81,7 +86,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { }); const previousState = ActionResult.fromState(stateManager.getCurrentState()!); - const action = explorer.createAction(); + const action = explorer.action(); const attempts: Array<{ command: string; success: boolean; error?: string }> = []; for (let i = 0; i < commands.length; i++) { @@ -101,7 +106,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { let disambiguated = null; if (attempts.some((a) => a.error?.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN))) { - disambiguated = await disambiguateElements(action.lastError, explanation, explorer.getAIProvider()); + disambiguated = await disambiguateElements(action.lastError, explanation, ai); } if (disambiguated) { @@ -193,7 +198,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { }); const previousState = ActionResult.fromState(stateManager.getCurrentState()!); - const action = explorer.createAction(); + const action = explorer.action(); const attempts: Array<{ command: string; success: boolean; error?: string }> = []; for (const command of commands) { @@ -263,7 +268,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { if (!isSingleChar && !isStandardKey) { const previousState = ActionResult.fromState(stateManager.getCurrentState()!); - const action = explorer.createAction(); + const action = explorer.action(); const typeCommand = `I.type(${JSON.stringify(key)})`; await action.attempt(typeCommand, explanation); const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, key); @@ -306,7 +311,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { } const previousState = ActionResult.fromState(stateManager.getCurrentState()!); - const action = explorer.createAction(); + const action = explorer.action(); let pressKeyCommand: string; if (modifier) { @@ -406,12 +411,11 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { const previousState = ActionResult.fromState(stateManager.getCurrentState()!); const formLocator = codeLines[0] || 'form'; - const action = explorer.createAction(); - const wasInIframe = await explorer.isInsideIframe(); + const action = explorer.action(); await action.attempt(codeBlock, explanation); - if (action.lastError && !wasInIframe && (await explorer.isInsideIframe())) { - await explorer.switchToMainFrame(); + if (action.lastError && !previousState.isInsideIframe) { + await explorer.exitIframe(); } const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, formLocator); @@ -422,7 +426,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { let formSuggestion = 'Look into error message and identify which commands passed and which failed. Continue execution using step-by-step approach using click() and form() tools.'; if (message.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN)) { - const disambiguated = await disambiguateElements(action.lastError, explanation, explorer.getAIProvider()); + const disambiguated = await disambiguateElements(action.lastError, explanation, ai); if (disambiguated) { formSuggestion = `Multiple elements matched. Add step.opts({ elementIndex: ${disambiguated.position} }) to the failing command. Fallback locator: ${disambiguated.xpath}`; } @@ -471,9 +475,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { }; } -export function createIframeTools(explorer: Explorer) { - const stateManager = explorer.getStateManager(); - +export function createIframeTools({ explorer, stateManager }: ToolDeps) { return { exitIframe: tool({ description: dedent` @@ -495,9 +497,9 @@ export function createIframeTools(explorer: Explorer) { }); } - await explorer.switchToMainFrame(); + await explorer.exitIframe(); - const nextState = await explorer.capturePageState(); + const nextState = await explorer.capture(); const toolResult = await nextState.toToolResult(previousState, 'I.switchTo()'); return successToolResult('exitIframe', { @@ -540,19 +542,7 @@ export function createLearnExperienceTool({ getExperienceTracker, getState }: { }); } -export function createAgentTools({ - explorer, - researcher, - navigator, - supervisor, - withExperience, -}: { - explorer: Explorer; - researcher: Researcher; - navigator: Navigator; - supervisor?: boolean; - withExperience?: boolean; -}): any { +export function createAgentTools({ explorer, stateManager, ai, researcher, navigator, supervisor, withExperience }: AgentToolDeps): any { let visionDisabled = false; const tools: Record = { @@ -577,7 +567,7 @@ export function createAgentTools({ } try { - const actionResult = await explorer.capturePageWithScreenshot(); + const actionResult = await explorer.capture({ screenshot: true }); if (!actionResult.screenshot) { return failedToolResult('see', 'Failed to capture screenshot for analysis'); @@ -625,7 +615,6 @@ export function createAgentTools({ }), execute: async ({ reason }) => { try { - const stateManager = explorer.getStateManager(); const currentState = stateManager.getCurrentState(); if (!currentState) { @@ -666,7 +655,7 @@ export function createAgentTools({ }), execute: async ({ assertion }) => { try { - const currentState = explorer.getStateManager().getCurrentState(); + const currentState = stateManager.getCurrentState(); const verifications = currentState?.verifications; if (verifications?.[assertion] !== undefined) { @@ -677,7 +666,7 @@ export function createAgentTools({ }); } - const actionResult = await explorer.capturePageState(); + const actionResult = await explorer.capture(); const result = await navigator.verifyState(assertion, actionResult); if (result.verified) { @@ -727,7 +716,6 @@ export function createAgentTools({ }), execute: async ({ reason }) => { try { - const stateManager = explorer.getStateManager(); const currentState = stateManager.getCurrentState(); if (!currentState) { @@ -771,7 +759,6 @@ export function createAgentTools({ }), execute: async ({ instruction }) => { try { - const stateManager = explorer.getStateManager(); const currentState = stateManager.getCurrentState(); if (!currentState) { @@ -826,7 +813,6 @@ export function createAgentTools({ } try { - const stateManager = explorer.getStateManager(); const currentState = stateManager.getCurrentState(); if (!currentState) { @@ -834,8 +820,8 @@ export function createAgentTools({ } const previousState = ActionResult.fromState(currentState); - const action = explorer.createAction(); - const actionResult = await explorer.capturePageWithScreenshot(); + const action = explorer.action(); + const actionResult = await explorer.capture({ screenshot: true }); if (!actionResult.screenshot) { return failedToolResult('visualClick', 'Failed to capture screenshot for visual analysis'); @@ -897,7 +883,6 @@ export function createAgentTools({ reason: z.string().describe('Why you need to go back'), }), execute: async ({ reason }) => { - const stateManager = explorer.getStateManager(); const currentState = stateManager.getCurrentState(); const currentUrl = currentState?.fullUrl || currentState?.url; const history = stateManager.getStateHistory(); @@ -920,7 +905,7 @@ export function createAgentTools({ let previousState: ActionResult | null = null; if (currentState) previousState = ActionResult.fromState(currentState); - const action = explorer.createAction(); + const action = explorer.action(); const success = await action.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, `${reason} (BACK to ${targetUrl})`); if (success) { @@ -942,7 +927,7 @@ export function createAgentTools({ description: 'List all previously visited page states (deduped by URL). Use to find pages to navigate back to.', inputSchema: z.object({}), execute: async () => { - const history = explorer.getStateManager().getStateHistory(); + const history = stateManager.getStateHistory(); const seen = new Set(); const states = history .map((t) => t.toState) @@ -975,7 +960,6 @@ export function createAgentTools({ reason: z.string().describe('What element you are looking for and why'), }), execute: async ({ xpath, reason }) => { - const stateManager = explorer.getStateManager(); const currentState = stateManager.getCurrentState(); if (!currentState) { @@ -1001,9 +985,9 @@ export function createAgentTools({ }); } - const action = explorer.createAction(); + const action = explorer.action(); const visible = await action.attempt(`I.seeElement(${JSON.stringify(xpath)})`, 'xpathCheck visibility'); - const liveElement = await WebElement.fromPlaywrightLocator(explorer.playwrightHelper.page.locator(`xpath=${xpath}`)); + const liveElement = await explorer.withPage((page) => WebElement.fromPlaywrightLocator(page.locator(`xpath=${xpath}`))); const matchesSummary = result.elements.map((el, i) => `${i + 1}. <${el.tag} ${el.keyAttrs}> text="${el.text}" html: ${el.outerHTML}`).join('\n'); @@ -1033,9 +1017,8 @@ export function createAgentTools({ if (withExperience !== false) { tools.learnExperience = createLearnExperienceTool({ - getExperienceTracker: () => explorer.getStateManager().getExperienceTracker(), + getExperienceTracker: () => stateManager.getExperienceTracker(), getState: () => { - const stateManager = explorer.getStateManager(); const currentState = stateManager.getCurrentState(); return currentState ? ActionResult.fromState(currentState) : null; }, diff --git a/src/command-handler.ts b/src/command-handler.ts index e79b874..2ee3b77 100644 --- a/src/command-handler.ts +++ b/src/command-handler.ts @@ -178,7 +178,7 @@ export class CommandHandler implements InputManager { } private async executeBrowserCommand(input: string): Promise { - const action = this.explorBot.getExplorer().createAction(); + const action = this.explorBot.getExplorer().action(); await action.execute(input); } diff --git a/src/commands/context-aria-command.ts b/src/commands/context-aria-command.ts index c6ec255..185c404 100644 --- a/src/commands/context-aria-command.ts +++ b/src/commands/context-aria-command.ts @@ -6,7 +6,7 @@ export class ContextAriaCommand extends BaseCommand { description = 'Print full ARIA snapshot for current page'; async execute(_args: string): Promise { - const state = this.explorBot.getExplorer().getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No active page to snapshot'); diff --git a/src/commands/context-command.ts b/src/commands/context-command.ts index d76b51c..0bd950a 100644 --- a/src/commands/context-command.ts +++ b/src/commands/context-command.ts @@ -1,9 +1,11 @@ import { ActionResult } from '../action-result.js'; import { Researcher } from '../ai/researcher.js'; +import { visuallyAnnotateContainers } from '../ai/researcher/coordinates.js'; import { outputPath } from '../config.js'; import { type ContextData, type ContextMode, formatContextSummary } from '../utils/context-formatter.js'; import { tag } from '../utils/logger.js'; import { extractValidContainers } from '../utils/research-parser.js'; +import { annotatePageElements } from '../utils/web-annotate.js'; import { BaseCommand, type Suggestion } from './base-command.js'; export class ContextCommand extends BaseCommand { @@ -25,7 +27,7 @@ export class ContextCommand extends BaseCommand { async execute(args: string): Promise { const explorer = this.explorBot.getExplorer(); - const state = explorer.getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No active page to show context for'); @@ -34,16 +36,16 @@ export class ContextCommand extends BaseCommand { const { opts } = this.parseArgs(args); const isVisual = !!(opts.visual || opts.screenshot); - await explorer.annotateElements(); + await explorer.withPage(annotatePageElements); if (isVisual) { const cachedResearch = Researcher.getCachedResearch(state); const containers = cachedResearch ? extractValidContainers(cachedResearch) : []; - await explorer.visuallyAnnotateElements({ containers }); + await explorer.withPage((page) => visuallyAnnotateContainers(page, containers)); } - const actionResult = await explorer.createAction().capturePageState({ includeScreenshot: isVisual }); - const experienceTracker = explorer.getStateManager().getExperienceTracker(); + const actionResult = await explorer.capture({ screenshot: isVisual }); + const experienceTracker = this.explorBot.experienceTracker(); const knowledgeTracker = this.explorBot.knowledgeTracker(); let mode: ContextMode = 'compact'; diff --git a/src/commands/context-data-command.ts b/src/commands/context-data-command.ts index 798e69f..2d67467 100644 --- a/src/commands/context-data-command.ts +++ b/src/commands/context-data-command.ts @@ -8,7 +8,7 @@ export class ContextDataCommand extends BaseCommand { async execute(_args: string): Promise { const explorer = this.explorBot.getExplorer(); - const state = explorer.getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No active page to extract data from'); @@ -18,7 +18,7 @@ export class ContextDataCommand extends BaseCommand { if (!actionResult.html || actionResult.html.trim().length < 100) { tag('info').log('Capturing fresh page content...'); - const freshResult = await explorer.createAction().capturePageState(); + const freshResult = await explorer.capture(); const table = await this.explorBot.agentResearcher().extractData(freshResult); tag('multiline').log(table); return; diff --git a/src/commands/context-experience-command.ts b/src/commands/context-experience-command.ts index 8a0f7d1..11b0516 100644 --- a/src/commands/context-experience-command.ts +++ b/src/commands/context-experience-command.ts @@ -10,14 +10,14 @@ export class ContextExperienceCommand extends BaseCommand { async execute(_args: string): Promise { const explorer = this.explorBot.getExplorer(); - const state = explorer.getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No active page'); } const actionResult = ActionResult.fromState(state); - const experienceTracker = explorer.getStateManager().getExperienceTracker(); + const experienceTracker = this.explorBot.experienceTracker(); const experience = experienceTracker.getRelevantExperience(actionResult); if (experience.length === 0) { diff --git a/src/commands/context-html-command.ts b/src/commands/context-html-command.ts index e051af1..5cecabb 100644 --- a/src/commands/context-html-command.ts +++ b/src/commands/context-html-command.ts @@ -8,7 +8,7 @@ export class ContextHtmlCommand extends BaseCommand { async execute(_args: string): Promise { const explorer = this.explorBot.getExplorer(); - const manager = explorer.getStateManager(); + const manager = this.explorBot.stateManager(); const state = manager.getCurrentState(); if (!state) { @@ -19,7 +19,7 @@ export class ContextHtmlCommand extends BaseCommand { if (!actionResult.html || actionResult.html.trim().length < 100) { tag('info').log('Capturing fresh page content...'); - actionResult = await explorer.createAction().capturePageState(); + actionResult = await explorer.capture(); } const html = await actionResult.combinedHtml(); diff --git a/src/commands/context-knowledge-command.ts b/src/commands/context-knowledge-command.ts index 4054d62..e3c2b9e 100644 --- a/src/commands/context-knowledge-command.ts +++ b/src/commands/context-knowledge-command.ts @@ -10,7 +10,7 @@ export class ContextKnowledgeCommand extends BaseCommand { async execute(_args: string): Promise { const explorer = this.explorBot.getExplorer(); - const state = explorer.getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No active page'); diff --git a/src/commands/drill-command.ts b/src/commands/drill-command.ts index 0f3cd80..a2328de 100644 --- a/src/commands/drill-command.ts +++ b/src/commands/drill-command.ts @@ -13,7 +13,7 @@ export class DrillCommand extends BaseCommand { const knowledgePath = this.parseKnowledgeArg(args); const maxComponents = this.parseMaxArg(args); - const state = this.explorBot.getExplorer().getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No active page to drill'); } diff --git a/src/commands/explore-command.ts b/src/commands/explore-command.ts index aa7f299..eae14f4 100644 --- a/src/commands/explore-command.ts +++ b/src/commands/explore-command.ts @@ -4,6 +4,7 @@ import { outputPath } from '../config.js'; import { normalizeUrl } from '../state-manager.js'; import { Stats } from '../stats.js'; import { type Plan, type Test, TestResult } from '../test-plan.js'; +import { browserErrorMessage } from '../utils/browser-errors.ts'; import { getCliName } from '../utils/cli-name.ts'; import { ErrorPageError, getStateErrorPageError } from '../utils/error-page.ts'; import { tag } from '../utils/logger.js'; @@ -38,7 +39,7 @@ export class ExploreCommand extends BaseCommand { private priorityFilter?: Set; private getCurrentPageUrl(): string | undefined { - const state = this.explorBot.getExplorer().getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); return state?.fullUrl || state?.url; } @@ -56,7 +57,7 @@ export class ExploreCommand extends BaseCommand { Stats.mode ??= 'explore'; Stats.focus ??= feature; const mainUrl = this.getCurrentPageUrl(); - const error = getStateErrorPageError(this.explorBot.getExplorer().getStateManager().getCurrentState()); + const error = getStateErrorPageError(this.explorBot.stateManager().getCurrentState()); if (error) { tag('warning').log(error.message); return; @@ -74,7 +75,7 @@ export class ExploreCommand extends BaseCommand { this.printResults(); return; } - if (mainUrl) await this.explorBot.visit(mainUrl); + if (mainUrl) await this.explorBot.visit(mainUrl).catch((err) => tag('warning').log(`Could not return to ${mainUrl}: ${browserErrorMessage(err)}`)); const savedPath = this.explorBot.savePlans(this.completedPlans); this.printResults(); await this.explorBot.printSessionAnalysis(); diff --git a/src/commands/freesail-command.ts b/src/commands/freesail-command.ts index 8a64f44..a6afd13 100644 --- a/src/commands/freesail-command.ts +++ b/src/commands/freesail-command.ts @@ -35,7 +35,7 @@ export class FreesailCommand extends BaseCommand { async (ctx) => { if (maxTests != null && testsRun >= maxTests) ctx.stop(); - const stateManager = this.explorBot.getExplorer().getStateManager(); + const stateManager = this.explorBot.stateManager(); const state = stateManager.getCurrentState(); if (state && !Researcher.getCachedResearch(state)) { diff --git a/src/commands/learn-command.ts b/src/commands/learn-command.ts index b4dcbd2..5c8ea82 100644 --- a/src/commands/learn-command.ts +++ b/src/commands/learn-command.ts @@ -14,13 +14,13 @@ export class LearnCommand extends BaseCommand { if (!note) { const AddKnowledge = (await import('../components/AddKnowledge.js')).default; const explorer = this.explorBot.getExplorer(); - const state = explorer.getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); const initialUrl = state?.url || ''; const { unmount } = render( React.createElement(AddKnowledge, { initialUrl, - knowledgeTracker: explorer.getKnowledgeTracker(), + knowledgeTracker: this.explorBot.knowledgeTracker(), onComplete: () => unmount(), onCancel: () => unmount(), }), @@ -33,14 +33,14 @@ export class LearnCommand extends BaseCommand { } const explorer = this.explorBot.getExplorer(); - const state = explorer.getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No active page to attach knowledge'); } const targetUrl = state.url || state.fullUrl || '/'; - explorer.getKnowledgeTracker().addKnowledge(targetUrl, note); + this.explorBot.knowledgeTracker().addKnowledge(targetUrl, note); tag('success').log('Knowledge saved for current page'); } } diff --git a/src/commands/path-command.ts b/src/commands/path-command.ts index 7e288a9..51b281e 100644 --- a/src/commands/path-command.ts +++ b/src/commands/path-command.ts @@ -16,7 +16,7 @@ export class PathCommand extends BaseCommand { async execute(args: string): Promise { const { opts } = this.parseArgs(args); const showLinks = !!opts.links; - const stateManager = this.explorBot.getExplorer().getStateManager(); + const stateManager = this.explorBot.stateManager(); const history = stateManager.getStateHistory(); if (history.length === 0) { diff --git a/src/commands/research-command.ts b/src/commands/research-command.ts index ab2952c..218ce4b 100644 --- a/src/commands/research-command.ts +++ b/src/commands/research-command.ts @@ -27,7 +27,7 @@ export class ResearchCommand extends BaseCommand { await this.explorBot.agentNavigator().visit(target); } - const state = this.explorBot.getExplorer().getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No active page to research'); } diff --git a/src/commands/test-command.ts b/src/commands/test-command.ts index 4eeb866..bc4267d 100644 --- a/src/commands/test-command.ts +++ b/src/commands/test-command.ts @@ -48,7 +48,7 @@ export class TestCommand extends BaseCommand { if (matching.length > 0) { toExecute.push(...matching); } else { - const state = this.explorBot.getExplorer().getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No page loaded. Please navigate to a page first.'); } diff --git a/src/components/App.tsx b/src/components/App.tsx index 785b36d..80be05c 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -68,7 +68,7 @@ export function App({ explorBot, initialShowInput = false, exitOnEmptyInput = fa }); }); - const manager = explorBot.getExplorer().getStateManager(); + const manager = explorBot.stateManager(); unsubscribe = manager.onStateChange((transition: StateTransition) => { if (mounted) { diff --git a/src/explorbot.ts b/src/explorbot.ts index 4d740d9..190a932 100644 --- a/src/explorbot.ts +++ b/src/explorbot.ts @@ -1,5 +1,6 @@ import { existsSync, mkdirSync } from 'node:fs'; import path from 'node:path'; +import type { AgentDeps } from './ai/agent.ts'; import { Captain } from './ai/captain.ts'; import { Driller } from './ai/driller.ts'; import { ExperienceCompactor } from './ai/experience-compactor.ts'; @@ -23,7 +24,9 @@ import { ConfigParser } from './config.ts'; import { ExperienceTracker } from './experience-tracker.ts'; import Explorer from './explorer.ts'; import { KnowledgeTracker } from './knowledge-tracker.ts'; -import { WebPageState } from './state-manager.ts'; +import { PlaywrightRecorder } from './playwright-recorder.ts'; +import { Reporter } from './reporter.ts'; +import { StateManager, WebPageState } from './state-manager.ts'; import { Stats } from './stats.ts'; import type { Suite } from './suite.ts'; import { Plan, type Test } from './test-plan.ts'; @@ -63,6 +66,10 @@ export class ExplorBot { private lastReportedTestCount = 0; private _experienceTracker?: ExperienceTracker; private _knowledgeTracker?: KnowledgeTracker; + private _stateManager?: StateManager; + private _reporter?: Reporter; + private _requestStore?: RequestStore; + private _playwrightRecorder?: PlaywrightRecorder; constructor(options: ExplorBotOptions = {}) { this.options = options; @@ -74,7 +81,7 @@ export class ExplorBot { } get isExploring(): boolean { - return this.explorer?.isStarted; + return !!this.explorer; } setUserResolve(fn: UserResolveFunction): void { @@ -82,13 +89,19 @@ export class ExplorBot { } async start(): Promise { - if (this.explorer?.isStarted) { + if (this.explorer) { return; } try { await this.startProviderOnly(); - this.explorer = new Explorer(this.config, this.provider, this.options, this.experienceTracker(), this.knowledgeTracker()); + this.explorer = new Explorer(this.config, this.options, { + stateManager: this.stateManager(), + knowledgeTracker: this.knowledgeTracker(), + reporter: this.reporter(), + requestStore: this.requestStore(), + playwrightRecorder: this.playwrightRecorder(), + }); await this.explorer.start(); if (!this.options.incognito) { await this.agentExperienceCompactor().autocompact(); @@ -126,7 +139,7 @@ export class ExplorBot { } getCurrentState(): WebPageState | null { - return this.explorer?.getStateManager().getCurrentState() ?? null; + return this._stateManager?.getCurrentState() ?? null; } getExplorer(): Explorer { @@ -141,6 +154,22 @@ export class ExplorBot { return (this._experienceTracker ||= new ExperienceTracker(this.knowledgeTracker())); } + stateManager(): StateManager { + return (this._stateManager ||= new StateManager(this.experienceTracker(), this.knowledgeTracker())); + } + + reporter(): Reporter { + return (this._reporter ||= new Reporter(this.config.reporter, this.stateManager())); + } + + requestStore(): RequestStore { + return (this._requestStore ||= new RequestStore(this.configParser.getOutputDir())); + } + + playwrightRecorder(): PlaywrightRecorder { + return (this._playwrightRecorder ||= new PlaywrightRecorder()); + } + getConfig(): ExplorbotConfig { return this.config; } @@ -153,11 +182,15 @@ export class ExplorBot { return this.provider; } - createAgent(factory: (deps: { explorer: Explorer; ai: AIProvider; config: ExplorbotConfig }) => T): T { + createAgent(factory: (deps: AgentDeps) => T): T { const agent = factory({ explorer: this.explorer, ai: this.provider, config: this.config, + stateManager: this.stateManager(), + knowledgeTracker: this.knowledgeTracker(), + requestStore: this.requestStore(), + playwrightRecorder: this.playwrightRecorder(), }); const agentName = (agent as any).constructor.name.toLowerCase(); @@ -167,18 +200,16 @@ export class ExplorBot { } agentResearcher(): Researcher { - return (this.agents.researcher ||= this.createAgent(({ ai, explorer }) => new Researcher(explorer, ai))); + return (this.agents.researcher ||= this.createAgent((deps) => new Researcher(deps))); } agentNavigator(): Navigator { - return (this.agents.navigator ||= this.createAgent(({ ai, explorer }) => { - return new Navigator(explorer, ai); - })); + return (this.agents.navigator ||= this.createAgent((deps) => new Navigator(deps))); } agentPlanner(): Planner { if (!this.agents.planner) { - this.agents.planner = this.createAgent(({ ai, explorer }) => new Planner(explorer, ai, this.agentResearcher())); + this.agents.planner = this.createAgent((deps) => new Planner(deps, this.agentResearcher())); const fisherman = this.agentFisherman(); if (fisherman) this.agents.planner.setFisherman(fisherman); } @@ -186,21 +217,21 @@ export class ExplorBot { } agentPilot(): Pilot { - return (this.agents.pilot ||= this.createAgent(({ ai, explorer }) => { + return (this.agents.pilot ||= this.createAgent((deps) => { const researcher = this.agentResearcher(); const navigator = this.agentNavigator(); - const tools = createAgentTools({ explorer, researcher, navigator, supervisor: true }); - return new Pilot(ai, tools, researcher, explorer); + const tools = createAgentTools({ ...deps, researcher, navigator, supervisor: true }); + return new Pilot(deps, tools, researcher); })); } agentTester(): Tester { if (!this.agents.tester) { - this.agents.tester = this.createAgent(({ ai, explorer }) => { + this.agents.tester = this.createAgent((deps) => { const researcher = this.agentResearcher(); const navigator = this.agentNavigator(); - const tools = createAgentTools({ explorer, researcher, navigator }); - return new Tester(explorer, ai, researcher, navigator, tools); + const tools = createAgentTools({ ...deps, researcher, navigator }); + return new Tester(deps, researcher, navigator, tools); }); const qm = this.agentQuartermaster(); @@ -234,29 +265,27 @@ export class ExplorBot { this.agents.quartermaster = new Quartermaster(this.provider, { model: config?.model, }); - this.agents.quartermaster.start(this.explorer.playwrightHelper, this.explorer.getStateManager()); + this.agents.quartermaster.start(this.explorer, this.stateManager()); } return this.agents.quartermaster; } agentHistorian(): Historian { - return (this.agents.historian ||= this.createAgent(({ ai, explorer, config }) => { - const experienceTracker = explorer.getStateManager().getExperienceTracker(); - const reporter = explorer.getReporter(); - return new Historian(ai, experienceTracker, reporter, explorer.getStateManager(), config, { - recorder: explorer.getPlaywrightRecorder(), - helper: explorer.playwrightHelper, + return (this.agents.historian ||= this.createAgent(({ ai, explorer, config, stateManager, playwrightRecorder }) => { + return new Historian(ai, this.experienceTracker(), this.reporter(), stateManager, config, { + recorder: playwrightRecorder, + explorer, }); })); } agentRerunner(): Rerunner { if (!this.agents.rerunner) { - this.agents.rerunner = this.createAgent(({ ai, explorer }) => { + this.agents.rerunner = this.createAgent((deps) => { const researcher = this.agentResearcher(); const navigator = this.agentNavigator(); - const tools = createAgentTools({ explorer, researcher, navigator, withExperience: false }); - return new Rerunner(explorer, ai, tools); + const tools = createAgentTools({ ...deps, researcher, navigator, withExperience: false }); + return new Rerunner(deps, tools); }); this.agents.rerunner.setHistorian(this.agentHistorian()); } @@ -264,9 +293,9 @@ export class ExplorBot { } agentDriller(): Driller { - return (this.agents.driller ||= this.createAgent(({ ai, explorer }) => { + return (this.agents.driller ||= this.createAgent((deps) => { const navigator = this.agentNavigator(); - return new Driller(explorer, ai, navigator); + return new Driller(deps, navigator); })); } @@ -283,7 +312,7 @@ export class ExplorBot { if (!this.agents.fisherman) { const apiConfig = this.config.api; const outputDir = this.configParser.getOutputDir(); - const requestStore = this.explorer.getRequestStore() || new RequestStore(outputDir); + const requestStore = this.requestStore(); const baseEndpoint = apiConfig?.baseEndpoint || this.config.playwright.url; const configHeaders = apiConfig?.headers || {}; const apiClient = new ApiClient(baseEndpoint); @@ -298,7 +327,11 @@ export class ExplorBot { } }; - const cookieProvider = () => this.explorer.extractCookies(); + const cookieProvider = async (): Promise> => { + const cookies = await this.explorer.withPage((page) => page.context().cookies()).catch(() => []); + if (!cookies.length) return {}; + return { Cookie: cookies.map((c: any) => `${c.name}=${c.value}`).join('; ') }; + }; this.agents.fisherman = this.createAgent(({ ai }) => { return new Fisherman(ai, apiClient, requestStore, specLoader, baseEndpoint, cookieProvider, configHeaders, hasApiConfig); @@ -332,7 +365,7 @@ export class ExplorBot { } if (!opts.extend && this.currentPlan?.url) { - const currentUrl = this.explorer?.getStateManager().getCurrentState()?.url; + const currentUrl = this._stateManager?.getCurrentState()?.url; if (currentUrl && currentUrl !== this.currentPlan.url) { tag('info').log('Different page detected, clearing previous plan'); this.clearPlan(); @@ -385,7 +418,7 @@ export class ExplorBot { } generatePlanFilename(feature?: string): string { - const state = this.explorer?.getStateManager().getCurrentState(); + const state = this._stateManager?.getCurrentState(); const urlPath = state?.url || '/'; const urlPart = sanitizeFilename(urlPath) || 'root'; const suffix = '.md'; @@ -465,7 +498,7 @@ export class ExplorBot { const filePath = this.agentSessionAnalyst().writeReport(markdown); tag('info').log(`Session report saved: ${relativeToCwd(filePath)}`); - const reporter = this.explorer?.getReporter(); + const reporter = this._reporter; if (reporter?.isEnabled()) { let description = markdown; const modelsTable = Stats.modelsTable(this.provider.getConfiguredModels()); diff --git a/src/explorer.ts b/src/explorer.ts index 6fc2a5a..ca8cc5b 100644 --- a/src/explorer.ts +++ b/src/explorer.ts @@ -1,31 +1,26 @@ import { existsSync, mkdirSync } from 'node:fs'; -import path, { join } from 'node:path'; +import path from 'node:path'; // @ts-ignore import * as codeceptjs from 'codeceptjs'; import stepsListener from 'codeceptjs/lib/listener/steps'; import storeListener from 'codeceptjs/lib/listener/store'; import { createTest } from 'codeceptjs/lib/mocha/test'; import dedent from 'dedent'; -import type { BrowserContextOptions } from 'playwright'; +import type { BrowserContextOptions, Page } from 'playwright'; import { ActionResult } from './action-result.ts'; import Action from './action.js'; -import { AIProvider } from './ai/provider.js'; -import { visuallyAnnotateContainers } from './ai/researcher/coordinates.ts'; -import { RequestStore } from './api/request-store.ts'; +import type { RequestStore } from './api/request-store.ts'; import { XhrCapture } from './api/xhr-capture.ts'; import type { ExplorbotConfig } from './config.js'; import { ConfigParser } from './config.js'; -import type { ExperienceTracker } from './experience-tracker.js'; -import { KnowledgeTracker } from './knowledge-tracker.js'; -import { PlaywrightRecorder } from './playwright-recorder.ts'; -import { Reporter } from './reporter.ts'; -import { StateManager } from './state-manager.js'; +import type { KnowledgeTracker } from './knowledge-tracker.js'; +import type { PlaywrightRecorder } from './playwright-recorder.ts'; +import type { Reporter } from './reporter.ts'; +import type { StateManager } from './state-manager.js'; import { Test, TestResult } from './test-plan.ts'; import { BrowserRecoveryError, browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts'; -import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from './utils/html.ts'; import { createDebug, log, tag } from './utils/logger.js'; import { sleep, waitForPageReadiness } from './utils/page-readiness.ts'; -import { WebElement } from './utils/web-element.ts'; declare global { namespace NodeJS { @@ -43,66 +38,224 @@ declare namespace CodeceptJS { const debugLog = createDebug('explorbot:explorer'); const RECOVERABLE_NAVIGATION_ERRORS = /net::ERR_ABORTED|page\.screenshot.*Timeout|waiting for fonts to load/i; - -interface TabInfo { - url: string; - title: string; -} - -interface BrowserExecutionErrorResult { - action: 'continue' | 'stop'; - message: string; - recovered?: boolean; -} +const RECOVERY_NAVIGATION = { waitUntil: 'domcontentloaded', timeout: 10000 } as const; class Explorer { - private aiProvider: AIProvider; - playwrightHelper: any; - public isStarted = false; + private config: ExplorbotConfig; + private options?: ExplorerOptions; + private stateManager: StateManager; + private knowledgeTracker: KnowledgeTracker; + private reporter: Reporter; + private requestStore: RequestStore; + private playwrightRecorder: PlaywrightRecorder; + private playwrightHelper: any; + private _actor!: CodeceptJS.I; + private started = false; private isSharedBrowser = false; - actor!: CodeceptJS.I; - private stateManager!: StateManager; - private knowledgeTracker!: KnowledgeTracker; - config: ExplorbotConfig; - private options?: { show?: boolean; headless?: boolean; incognito?: boolean; session?: string }; - private reporter!: Reporter; - private otherTabs: TabInfo[] = []; - private _activeTest: Test | null = null; private xhrCapture: XhrCapture | null = null; - private requestStore: RequestStore | null = null; - private playwrightRecorder: PlaywrightRecorder = new PlaywrightRecorder(); + private _activeTest: Test | null = null; private observedTestPages = new Set(); private testPageErrorHandler: ((error: Error) => void) | null = null; private testConsoleHandler: ((message: any) => void) | null = null; private testDialogHandler: ((dialog: any) => void) | null = null; - constructor(config: ExplorbotConfig, aiProvider: AIProvider, options: { show?: boolean; headless?: boolean; incognito?: boolean; session?: string } | undefined, experienceTracker: ExperienceTracker, knowledgeTracker: KnowledgeTracker) { + constructor(config: ExplorbotConfig, options: ExplorerOptions | undefined, deps: ExplorerDeps) { this.config = config; - this.aiProvider = aiProvider; this.options = options; + this.stateManager = deps.stateManager; + this.knowledgeTracker = deps.knowledgeTracker; + this.reporter = deps.reporter; + this.requestStore = deps.requestStore; + this.playwrightRecorder = deps.playwrightRecorder; this.initializeContainer(); - this.knowledgeTracker = knowledgeTracker; - this.stateManager = new StateManager(experienceTracker, knowledgeTracker); - this.reporter = new Reporter(config.reporter, this.stateManager); } - private initializeContainer() { - try { - // Use project root for output directory, not current working directory - const configParser = ConfigParser.getInstance(); - const projectRoot = configParser.getProjectRoot(); - (global as any).output_dir = path.join(projectRoot, 'output', 'states'); - (global as any).codecept_dir = projectRoot; + get actor(): CodeceptJS.I { + return this._actor; + } + + get page(): Page | null { + const page = this.playwrightHelper?.page; + if (!page || page.isClosed?.()) return null; + return page; + } + + get activeTest(): Test | null { + return this._activeTest; + } - configParser.validateConfig(this.config); + async start(): Promise { + if (this.started) return; - const codeceptConfig = this.convertToCodeceptConfig(this.config); + await codeceptjs.recorder.start(); + await codeceptjs.container.started(null); + storeListener(); + stepsListener(); - codeceptjs.container.create(codeceptConfig, {}); - } catch (error) { - log(`❌ Failed to initialize container: ${error}`); - throw error; + codeceptjs.recorder.retry({ + retries: this.config.action?.retries || 3, + when: (err: any) => !!err?.message?.includes?.('context'), + }); + + this.playwrightHelper = codeceptjs.container.helpers('Playwright'); + if (!this.playwrightHelper) { + throw new Error('Playwright helper not available'); + } + await this.connectOrLaunchBrowser(); + const hasSession = this.options?.session && existsSync(this.options.session); + await this.playwrightHelper._createContextPage(this.createBrowserContextOptions()); + await this.playwrightRecorder.start(this.playwrightHelper.browserContext); + this.attachXhrCapture(); + if (hasSession) { + tag('info').log(`Session restored from ${path.relative(process.cwd(), this.options!.session!)}`); + } + + this._actor = codeceptjs.container.support('I'); + this.started = true; + + this.listenToStateChanged(); + + codeceptjs.event.dispatcher.emit('global.before'); + tag('success').log('Browser started, ready to explore'); + } + + async stop(): Promise { + if (!this.started) return; + this.started = false; + + await this.stopCaptures(); + + if (this.options?.session && this.playwrightHelper?.browserContext) { + const dir = path.dirname(this.options.session); + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + await this.playwrightHelper.browserContext.storageState({ path: this.options.session }); + debugLog(`Session saved to ${path.relative(process.cwd(), this.options.session)}`); + } + + codeceptjs.event.dispatcher.emit('global.after'); + codeceptjs.event.dispatcher.emit('global.result'); + + if (!this.isSharedBrowser) { + await Promise.all([this.reporter.finishRun(), this.playwrightHelper._stopBrowser(), codeceptjs.recorder.stop()]); + return; + } + + tag('info').log('Closing browser context (persistent browser stays running)'); + await this.closeBrowserContext(); + this.playwrightHelper.browser = null; + this.playwrightHelper.isRunning = false; + await Promise.all([this.reporter.finishRun(), codeceptjs.recorder.stop()]); + } + + action(): Action { + return new Action(this._actor, this.stateManager, this.playwrightRecorder, (fn) => this.runWithRecovery('action', fn)); + } + + async visit(url: string, opts: CaptureOpts = {}): Promise { + return this.runWithRecovery('visit', async () => { + const action = await this.visitOnce(url); + if (opts.screenshot) return action.capturePageState({ includeScreenshot: true }); + return action.getActionResult() ?? action.capturePageState(); + }); + } + + async capture(opts: CaptureOpts = {}): Promise { + return this.action().capturePageState({ includeScreenshot: opts.screenshot }); + } + + async withPage(fn: (page: Page) => Promise): Promise { + return this.runWithRecovery('page operation', () => fn(this.playwrightHelper.page)); + } + + async exitIframe(): Promise { + if (!this.playwrightHelper.frame) return; + debugLog('Switching to main frame'); + await this.playwrightHelper.switchTo(); + } + + async recover(error?: unknown): Promise { + if (error) return this.recoverFromExecutionError(error); + + if (this.page) { + this.watchActiveTestPage(); + return { ok: true, action: 'continue', message: 'Page is available' }; + } + + if (!(await this.recoverOrRestart())) { + return { ok: false, action: 'stop', message: 'Browser page could not be recovered' }; } + this.watchActiveTestPage(); + return { ok: true, action: 'continue', recovered: true, message: 'Browser page was recovered' }; + } + + async beginTest(test: Test): Promise { + this._activeTest = test; + test.start(); + await this.reporter.reportTestStart(test); + await this.closeOtherTabs(); + this.stateManager.otherTabs = []; + if (!this.page && !(await this.recoverOrRestart())) { + this._activeTest = null; + return { started: false, stop: async () => {} }; + } + + const codeceptjsTest = toCodeceptjsTest(test); + + const stepHandler = (step: any, status?: string, error?: string, log?: string) => { + if (!step.toCode) return; + if (step?.name?.startsWith('grab')) return; + if (step?.name?.startsWith('save')) return; + + test.addStep(step.toCode(), step.duration, status, error, log); + + if (!this.stateManager.getCurrentState()) return; + + const lastScreenshot = ActionResult.fromState(this.stateManager.getCurrentState()!).screenshotFile; + test.setActiveNoteScreenshot(lastScreenshot); + }; + + this.watchActiveTestPage(); + + const onStepPassed = (step: any) => stepHandler(step, 'passed'); + const onStepFailed = (step: any, error: any) => { + stepHandler(step, 'failed', error?.message || String(error), error?.stack); + }; + const onTestAfter = () => { + codeceptjs.event.dispatcher.off('step.passed', onStepPassed); + codeceptjs.event.dispatcher.off('step.failed', onStepFailed); + codeceptjs.event.dispatcher.off('test.after', onTestAfter); + this.unwatchActiveTestPages(); + }; + + codeceptjs.event.dispatcher.emit('test.before', codeceptjsTest); + codeceptjs.event.dispatcher.emit('test.start', codeceptjsTest); + codeceptjs.event.dispatcher.on('step.passed', onStepPassed); + codeceptjs.event.dispatcher.on('step.failed', onStepFailed); + codeceptjs.event.dispatcher.on('test.after', onTestAfter); + + return { started: true, stop: (meta) => this.finishTest(test, meta) }; + } + + async openFreshTab(): Promise { + const oldPage = this.playwrightHelper?.page; + if (!oldPage) return; + + await this.activateNewPage(oldPage.context()); + await oldPage.close(); + this.stateManager.otherTabs = []; + + debugLog('Opened fresh tab, closed previous tab'); + } + + private initializeContainer() { + const configParser = ConfigParser.getInstance(); + const projectRoot = configParser.getProjectRoot(); + (global as any).output_dir = path.join(projectRoot, 'output', 'states'); + (global as any).codecept_dir = projectRoot; + + configParser.validateConfig(this.config); + + codeceptjs.container.create(this.convertToCodeceptConfig(this.config), {}); } private convertToCodeceptConfig(config: ExplorbotConfig): any { @@ -160,99 +313,6 @@ class Explorer { return codeceptConfig; } - public getConfig(): ExplorbotConfig { - return this.config; - } - - public getAIProvider(): AIProvider { - return this.aiProvider; - } - - public getStateManager(): StateManager { - return this.stateManager; - } - - public getKnowledgeTracker(): KnowledgeTracker { - return this.knowledgeTracker; - } - - public getReporter(): Reporter { - return this.reporter; - } - - public getRequestStore(): RequestStore | null { - return this.requestStore; - } - - async extractCookies(): Promise> { - if (!this.playwrightHelper?.browserContext) return {}; - try { - const cookies = await this.playwrightHelper.browserContext.cookies(); - if (!cookies.length) return {}; - const cookieString = cookies.map((c: any) => `${c.name}=${c.value}`).join('; '); - return { Cookie: cookieString }; - } catch { - return {}; - } - } - - private setupXhrCapture(reuseRequestStore = false): void { - const configParser = ConfigParser.getInstance(); - const outputDir = configParser.getOutputDir(); - if (!reuseRequestStore || !this.requestStore) { - this.requestStore = new RequestStore(outputDir); - } - const baseUrl = this.config.playwright.url; - this.xhrCapture = new XhrCapture(this.requestStore!, baseUrl); - this.xhrCapture.attach(this.playwrightHelper.page); - } - - async start() { - if (this.isStarted) { - return; - } - - await codeceptjs.recorder.start(); - await codeceptjs.container.started(null); - storeListener(); - stepsListener(); - - codeceptjs.recorder.retry({ - retries: this.config.action?.retries || 3, - when: (err: any) => { - if (!err || typeof err.message !== 'string') { - return false; - } - // ignore context errors - return err.message.includes('context'); - }, - }); - - this.playwrightHelper = codeceptjs.container.helpers('Playwright'); - if (!this.playwrightHelper) { - throw new Error('Playwright helper not available'); - } - await this.connectOrLaunchBrowser(); - const hasSession = this.options?.session && existsSync(this.options.session); - await this.playwrightHelper._createContextPage(this.createBrowserContextOptions()); - await this.playwrightRecorder.start(this.playwrightHelper.browserContext); - this.setupXhrCapture(); - if (hasSession) { - tag('info').log(`Session restored from ${path.relative(process.cwd(), this.options!.session!)}`); - } - const I = codeceptjs.container.support('I'); - - this.actor = I; - this.isStarted = true; - - this.listenToStateChanged(); - - codeceptjs.event.dispatcher.emit('global.before'); - tag('success').log('Browser started, ready to explore'); - - return I; - } - private async connectOrLaunchBrowser(): Promise { const { getAliveEndpoint } = await import('./browser-server.js'); const endpoint = await getAliveEndpoint(); @@ -284,12 +344,13 @@ class Explorer { return contextOptions; } - createAction() { - return new Action(this.actor, this.stateManager, this.playwrightRecorder); + private attachXhrCapture(): void { + this.xhrCapture = new XhrCapture(this.requestStore, this.config.playwright.url); + this.xhrCapture.attach(this.playwrightHelper.page); } - async runWithBrowserRecovery(label: string, operation: () => Promise): Promise { - if (!(await this.ensurePageAvailable())) { + private async runWithRecovery(label: string, operation: () => Promise): Promise { + if (!this.page && !(await this.recoverOrRestart())) { throw new Error(`Browser page is unavailable before ${label}`); } @@ -304,23 +365,21 @@ class Explorer { try { return await operation(); } catch (retryError) { - if (!isNavigationTransitionError(retryError) && !this.isFatalBrowserError(retryError)) throw retryError; + if (!isNavigationTransitionError(retryError) && !isFatalBrowserError(retryError)) throw retryError; recoveryError = retryError; } } - if (!this.isFatalBrowserError(recoveryError)) throw recoveryError; + if (!isFatalBrowserError(recoveryError)) throw recoveryError; tag('warning').log(`${label}: browser page is unavailable, recovering...`); - let recovered = await this.recoverFromBrowserError(); - if (!recovered) recovered = await this.restartBrowser(); - if (!recovered) throw new BrowserRecoveryError(label, recoveryError, false); + if (!(await this.recoverOrRestart())) throw new BrowserRecoveryError(label, recoveryError, false); if (!(await this.waitForPageReadiness())) throw new BrowserRecoveryError(label, recoveryError, true); try { return await operation(); } catch (retryError) { - if (this.isFatalBrowserError(retryError)) { + if (isFatalBrowserError(retryError)) { throw new BrowserRecoveryError(label, retryError, true); } throw retryError; @@ -328,31 +387,7 @@ class Explorer { } } - async capturePageState(opts: { includeScreenshot?: boolean } = {}): Promise { - return this.runWithBrowserRecovery('capturePageState', () => this.createAction().capturePageState(opts)); - } - - async capturePageWithScreenshot(): Promise { - return this.capturePageState({ includeScreenshot: true }); - } - - async executeAction(code: string): Promise { - return this.runWithBrowserRecovery('executeAction', () => this.createAction().execute(code)); - } - - async attemptAction(code: string, originalMessage?: string): Promise { - return this.runWithBrowserRecovery('attemptAction', () => this.createAction().attempt(code, originalMessage)); - } - - getPlaywrightRecorder(): PlaywrightRecorder { - return this.playwrightRecorder; - } - - async visit(url: string) { - return this.runWithBrowserRecovery('visit', () => this.visitOnce(url)); - } - - private async visitOnce(url: string) { + private async visitOnce(url: string): Promise { await this.closeOtherTabs(); const serializedUrl = JSON.stringify(url); @@ -361,7 +396,7 @@ class Explorer { const { statePush = false, wait, waitForElement, code } = this.knowledgeTracker.getStateParameters(actionResult, ['statePush', 'wait', 'waitForElement', 'code']); - const action = this.createAction(); + const action = new Action(this._actor, this.stateManager, this.playwrightRecorder); if (statePush) { await action.execute(`I.executeScript(() => { window.history.pushState({}, '', ${serializedUrl}); window.dispatchEvent(new PopStateEvent('popstate')); })`); @@ -394,94 +429,65 @@ class Explorer { return action; } - async annotateElements(): Promise { - return this.runWithBrowserRecovery('annotateElements', async () => { - const { elements } = await annotatePageElements(this.playwrightHelper.page); - return elements; - }); - } - - async visuallyAnnotateElements(opts?: { containers?: Array<{ css: string; label: string }> }): Promise { - return this.runWithBrowserRecovery('visuallyAnnotateElements', () => visuallyAnnotateContainers(this.playwrightHelper.page, opts?.containers || [])); - } + private async recoverFromExecutionError(error: unknown): Promise { + const message = browserErrorMessage(error); + tag('error').log(`Browser execution error: ${message}`); - async getEidxInContainer(containerCss: string | null): Promise { - const page = this.playwrightHelper.page; - try { - const selector = containerCss ? `${containerCss} [${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]` : `[${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]`; - const elements = await page.locator(selector).all(); - const result: string[] = []; - for (const el of elements) { - const attr = await el.getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx); - if (attr) result.push(attr); - } - return result; - } catch (error) { - if (this.isFatalBrowserError(error)) { - tag('warning').log(`getEidxInContainer: ${browserErrorMessage(error)}`); - await this.recoverFromBrowserError(); - } - return []; + if (error instanceof Error && error.name === 'AbortError') { + return { ok: false, action: 'stop', message }; } - } - async getEidxByLocator(locator: string, container?: string | null): Promise { - try { - const page = this.playwrightHelper.page; - const base = container ? page.locator(container) : page; - const el = locator.startsWith('//') ? base.locator(`xpath=${locator}`) : base.locator(locator); - return await el.first().getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx); - } catch (error) { - if (this.isFatalBrowserError(error)) { - tag('warning').log(`getEidxByLocator: ${browserErrorMessage(error)}`); - await this.recoverFromBrowserError(); - } - return null; + if (error instanceof BrowserRecoveryError) { + return { ok: false, action: 'stop', recovered: error.recovered, message: error.message }; } - } - - private resolveBrowserUrl(url?: string): string | null { - if (!url) return null; - try { - return new URL(url).toString(); - } catch {} - const baseUrl = this.config.playwright?.url || this.config.web?.url; - if (!baseUrl) return null; + if (!isFatalBrowserError(error)) { + return { + ok: true, + action: 'continue', + message: `Previous execution error: ${message}. Investigate the current state and choose a different approach.`, + }; + } - try { - return new URL(url, baseUrl).toString(); - } catch { - return null; + if (!(await this.recoverOrRestart())) { + return { ok: false, action: 'stop', recovered: false, message: `Browser could not be recovered after fatal error: ${message}` }; } + + this.watchActiveTestPage(); + return { + ok: true, + action: 'continue', + recovered: true, + message: dedent` + Browser was recovered after a fatal page error. + Continue from the restored page. + The interrupted browser action is not product evidence. + Inspect the restored page and retry the current step when it is still required. + `, + }; } - isFatalBrowserError(error: unknown): boolean { - return isFatalBrowserError(error); + private async recoverOrRestart(): Promise { + if (await this.recoverPage()) return true; + return this.restartBrowser(); } - async recoverFromBrowserError(): Promise { + private async recoverPage(): Promise { try { - if (!this.playwrightHelper?.page || this.playwrightHelper.page.isClosed?.()) { + if (!this.page) { const context = this.playwrightHelper?.browserContext; - if (!context) return await this.restartBrowser(); - const page = await context.newPage(); - await page.bringToFront(); - await this.playwrightHelper._setPage(page); - this.bindFrameNavigated(page); - if (this.xhrCapture) { - this.xhrCapture.attach(this.playwrightHelper.page); - } + if (!context) return false; + await this.activateNewPage(context); } const url = this.resolveBrowserUrl(this.stateManager.getCurrentState()?.url); if (url) { tag('warning').log(`Browser error detected, recovering by navigating to ${url}`); - await this.playwrightHelper.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 10000 }); + await this.playwrightHelper.page.goto(url, RECOVERY_NAVIGATION); return this.waitForPageReadiness(); } tag('warning').log('Browser error detected, reloading page'); - await this.playwrightHelper.page.reload({ waitUntil: 'domcontentloaded', timeout: 10000 }); + await this.playwrightHelper.page.reload(RECOVERY_NAVIGATION); return this.waitForPageReadiness(); } catch (err) { tag('error').log(`Browser recovery failed: ${browserErrorMessage(err)}`); @@ -489,24 +495,37 @@ class Explorer { } } - async restartBrowser(): Promise { - if (!this.playwrightHelper) return false; + private async activateNewPage(context: any): Promise { + const page = await context.newPage(); + await page.bringToFront(); + await this.playwrightHelper._setPage(page); + this.bindFrameNavigated(page); + this.xhrCapture?.attach(page); + } - const url = this.resolveBrowserUrl(this.stateManager.getCurrentState()?.url); + private async stopCaptures(): Promise { + if (this.xhrCapture && this.playwrightHelper?.page) { + this.xhrCapture.detach(this.playwrightHelper.page); + } + await this.playwrightRecorder.stop(); + } - try { - if (this.xhrCapture && this.playwrightHelper.page) { - this.xhrCapture.detach(this.playwrightHelper.page); - } + private async closeBrowserContext(): Promise { + if (!this.playwrightHelper.browserContext) return; + await this.playwrightHelper.browserContext.close().catch((err: unknown) => { + debugLog('Failed to close browser context:', err); + }); + this.playwrightHelper.browserContext = null; + } - await this.playwrightRecorder.stop(); + private async restartBrowser(): Promise { + if (!this.playwrightHelper) return false; - if (this.playwrightHelper.browserContext) { - await this.playwrightHelper.browserContext.close().catch((err: unknown) => { - debugLog('Failed to close browser context before restart:', err); - }); - this.playwrightHelper.browserContext = null; - } + const url = this.resolveBrowserUrl(this.stateManager.getCurrentState()?.url); + + try { + await this.stopCaptures(); + await this.closeBrowserContext(); if (!this.isSharedBrowser) { await this.playwrightHelper._stopBrowser().catch((err: unknown) => { @@ -517,11 +536,11 @@ class Explorer { await this.connectOrLaunchBrowser(); await this.playwrightHelper._createContextPage(this.createBrowserContextOptions()); await this.playwrightRecorder.start(this.playwrightHelper.browserContext); - this.setupXhrCapture(true); + this.attachXhrCapture(); this.listenToStateChanged(); if (url) { - await this.playwrightHelper.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 10000 }); + await this.playwrightHelper.page.goto(url, RECOVERY_NAVIGATION); if (!(await this.waitForPageReadiness())) return false; } @@ -533,10 +552,19 @@ class Explorer { } } - async switchToMainFrame() { - if (this.playwrightHelper.frame) { - debugLog('Switching to main frame'); - await this.playwrightHelper.switchTo(); + private resolveBrowserUrl(url?: string): string | null { + if (!url) return null; + try { + return new URL(url).toString(); + } catch {} + + const baseUrl = this.config.playwright?.url || this.config.web?.url; + if (!baseUrl) return null; + + try { + return new URL(url, baseUrl).toString(); + } catch { + return null; } } @@ -551,238 +579,112 @@ class Explorer { return true; } - async isInsideIframe(): Promise { - if (this.playwrightHelper.frame) return true; - - try { - const page = this.playwrightHelper.page; - if (!page) return false; - return await page.evaluate(() => window.top !== window.self); - } catch (error) { - if (this.isFatalBrowserError(error)) { - tag('warning').log(`isInsideIframe: ${browserErrorMessage(error)}`); - await this.recoverFromBrowserError(); - } - return false; - } - } - - getCurrentIframeInfo(): string | null { - if (!this.playwrightHelper?.frame) return null; - return 'iframe context active'; - } - - hasOtherTabs(): boolean { - return this.otherTabs.length > 0; - } - - getOtherTabsInfo(): TabInfo[] { - return [...this.otherTabs]; - } - - clearOtherTabsInfo(): void { - this.otherTabs = []; - } - private listenToStateChanged(): void { - if (!this.playwrightHelper) { - debugLog('Playwright helper not available for state monitoring'); + const page = this.playwrightHelper?.page; + if (!page) { + debugLog('Playwright page not available for state monitoring'); return; } + const initialPage = page; + const context = page.context(); - try { - const page = this.playwrightHelper.page; - if (!page) { - debugLog('Playwright page not available for state monitoring'); - return; - } - const initialPage = page; - const context = this.playwrightHelper.page.context(); + context.on('page', async (newPage: any) => { + if (newPage === initialPage) return; - context.on('page', async (newPage: any) => { - if (newPage === initialPage) { - return; + try { + if (newPage.url() === 'about:blank') { + await newPage.waitForURL(/^(?!about:blank$)/, { timeout: 5000 }).catch(() => {}); } + await newPage.waitForLoadState('domcontentloaded', { timeout: 5000 }); + const url = await newPage.url(); + const title = await newPage.title().catch(() => 'Unknown'); - try { - if (newPage.url() === 'about:blank') { - await newPage.waitForURL(/^(?!about:blank$)/, { timeout: 5000 }).catch(() => {}); - } - await newPage.waitForLoadState('domcontentloaded', { timeout: 5000 }); - const url = await newPage.url(); - const title = await newPage.title().catch(() => 'Unknown'); - - this.otherTabs.push({ url, title }); - if (url !== 'about:blank') { - tag('info').log(`New browser tab opened: ${url}`); - } - debugLog(`New tab detected: ${url} - ${title}`); - } catch (error) { - debugLog('Failed to get new tab info:', error); - this.otherTabs.push({ url: 'unknown', title: 'unknown' }); + this.stateManager.otherTabs.push({ url, title }); + if (url !== 'about:blank') { + tag('info').log(`New browser tab opened: ${url}`); } - }); + debugLog(`New tab detected: ${url} - ${title}`); + } catch (error) { + debugLog('Failed to get new tab info:', error); + this.stateManager.otherTabs.push({ url: 'unknown', title: 'unknown' }); + } + }); - this.bindFrameNavigated(page); + this.bindFrameNavigated(page); - debugLog('Listening for automatic state changes'); - } catch (error) { - debugLog('Failed to set up state change monitoring:', error); - } + debugLog('Listening for automatic state changes'); } - async stop(): Promise { - if (!this.isStarted) { - return; - } - - if (this.xhrCapture && this.playwrightHelper?.page) { - this.xhrCapture.detach(this.playwrightHelper.page); - } - - await this.playwrightRecorder.stop(); - - if (this.options?.session && this.playwrightHelper?.browserContext) { - const dir = path.dirname(this.options.session); - if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); - await this.playwrightHelper.browserContext.storageState({ path: this.options.session }); - debugLog(`Session saved to ${path.relative(process.cwd(), this.options.session)}`); - } + private bindFrameNavigated(page: any): void { + page.on('framenavigated', async (frame: any) => { + if (frame !== page.mainFrame()) return; - codeceptjs.event.dispatcher.emit('global.after'); - codeceptjs.event.dispatcher.emit('global.result'); + const newUrl = await frame.url(); + let newTitle = ''; - if (this.isSharedBrowser) { - tag('info').log('Closing browser context (persistent browser stays running)'); try { - if (this.playwrightHelper.browserContext) { - await this.playwrightHelper.browserContext.close(); - this.playwrightHelper.browserContext = null; - } - } catch (err) { - debugLog('Failed to close browser context:', err); + newTitle = await frame.title(); + } catch (error) { + debugLog('Failed to get page title:', error); } - this.playwrightHelper.browser = null; - this.playwrightHelper.isRunning = false; - await Promise.all([this.reporter.finishRun(), codeceptjs.recorder.stop()]); - } else { - await Promise.all([this.reporter.finishRun(), this.playwrightHelper._stopBrowser(), codeceptjs.recorder.stop()]); - } - } - - get activeTest(): Test | null { - return this._activeTest; - } - - async startTest(test: Test): Promise { - this._activeTest = test; - test.start(); - await this.reporter.reportTestStart(test); - await this.closeOtherTabs(); - this.otherTabs = []; - if (!(await this.ensurePageAvailable())) return false; - - const codeceptjsTest = toCodeceptjsTest(test); - - const stepHandler = (step: any, status?: string, error?: string, log?: string) => { - if (!step.toCode) return; - if (step?.name?.startsWith('grab')) return; - if (step?.name?.startsWith('save')) return; - test.addStep(step.toCode(), step.duration, status, error, log); + this.stateManager.updateStateFromBasic(newUrl, newTitle, 'navigation'); - if (!this.stateManager.getCurrentState()) return; + await sleep(500); + }); + } - const lastScreenshot = ActionResult.fromState(this.stateManager.getCurrentState()!).screenshotFile; - test.setActiveNoteScreenshot(lastScreenshot); - }; + private async closeOtherTabs(): Promise { + if (!this.playwrightHelper) return; - this.watchActiveTestPage(); + const context = this.playwrightHelper.page.context(); + const pages = context.pages(); - const onStepPassed = (step: any) => stepHandler(step, 'passed'); - const onStepFailed = (step: any, error: any) => { - stepHandler(step, 'failed', error?.message || String(error), error?.stack); - }; - const onTestAfter = () => { - codeceptjs.event.dispatcher.off('step.passed', onStepPassed); - codeceptjs.event.dispatcher.off('step.failed', onStepFailed); - codeceptjs.event.dispatcher.off('test.after', onTestAfter); - this.unwatchActiveTestPages(); - }; + if (pages.length <= 1) return; - codeceptjs.event.dispatcher.emit('test.before', codeceptjsTest); - codeceptjs.event.dispatcher.emit('test.start', codeceptjsTest); - codeceptjs.event.dispatcher.on('step.passed', onStepPassed); - codeceptjs.event.dispatcher.on('step.failed', onStepFailed); - codeceptjs.event.dispatcher.on('test.after', onTestAfter); + debugLog(`Found ${pages.length} tabs, cleaning up to keep only the first one`); - return true; - } + const firstPage = pages[0]; + const tabsToClose = pages.slice(1); - async ensurePageAvailable(): Promise { - const page = this.playwrightHelper?.page; - if (page && !page.isClosed?.()) { - this.watchActiveTestPage(page); - return true; + for (const page of tabsToClose) { + await page.close(); + debugLog(`Closed extra tab: ${await page.url()}`); } - const recovered = await this.recoverFromBrowserError(); - if (!recovered) return false; - this.watchActiveTestPage(); - return true; - } - - async handleExecutionError(error: unknown): Promise { - const message = browserErrorMessage(error); - tag('error').log(`Browser execution error: ${message}`); - - if (error instanceof Error && error.name === 'AbortError') { - return { - action: 'stop', - message, - }; - } + await firstPage.bringToFront(); + await this.playwrightHelper._setPage(firstPage); - if (error instanceof BrowserRecoveryError) { - return { - action: 'stop', - recovered: error.recovered, - message: error.message, - }; - } + debugLog(`Cleaned up tabs, now focused on: ${await firstPage.url()}`); + } - if (!this.isFatalBrowserError(error)) { - return { - action: 'continue', - message: `Previous execution error: ${message}. Investigate the current state and choose a different approach.`, - }; + private async finishTest(test: Test, meta?: Record): Promise { + this.unwatchActiveTestPages(); + this._activeTest = null; + const lastScreenshot = this.stateManager.getCurrentState()?.screenshotFile; + if (lastScreenshot) { + meta ||= {}; + meta.screenshotFile = lastScreenshot; } + await this.reporter.reportTest(test, meta); + const codeceptjsTest = toCodeceptjsTest(test); - let recovered = await this.recoverFromBrowserError(); - if (!recovered) recovered = await this.restartBrowser(); - - if (!recovered) { - return { - action: 'stop', - recovered: false, - message: `Browser could not be recovered after fatal error: ${message}`, - }; + if (test.isSuccessful) { + codeceptjsTest.state = 'passed'; + codeceptjs.event.dispatcher.emit('test.passed', codeceptjsTest); + } else if (test.isSkipped) { + codeceptjsTest.state = 'skipped'; + codeceptjs.event.dispatcher.emit('test.skipped', codeceptjsTest); + } else { + codeceptjsTest.state = 'failed'; + codeceptjs.event.dispatcher.emit('test.failed', codeceptjsTest); } - this.watchActiveTestPage(); - return { - action: 'continue', - recovered: true, - message: dedent` - Browser was recovered after a fatal page error. - Continue from the restored page. - The interrupted browser action is not product evidence. - Inspect the restored page and retry the current step when it is still required. - `, - }; + codeceptjs.event.dispatcher.emit('test.finish', codeceptjsTest); + codeceptjs.event.dispatcher.emit('test.after', codeceptjsTest); } - watchActiveTestPage(page = this.playwrightHelper?.page): void { + private watchActiveTestPage(page = this.playwrightHelper?.page): void { if (!this._activeTest) return; if (!page) return; if (this.observedTestPages.has(page)) return; @@ -806,32 +708,6 @@ class Explorer { this.observedTestPages.add(page); } - async stopTest(test: Test, meta?: Record) { - this.unwatchActiveTestPages(); - this._activeTest = null; - const lastScreenshot = this.stateManager.getCurrentState()?.screenshotFile; - if (lastScreenshot) { - meta ||= {}; - meta.screenshotFile = lastScreenshot; - } - await this.reporter.reportTest(test, meta); - const codeceptjsTest = toCodeceptjsTest(test); - - if (test.isSuccessful) { - codeceptjsTest.state = 'passed'; - codeceptjs.event.dispatcher.emit('test.passed', codeceptjsTest); - } else if (test.isSkipped) { - codeceptjsTest.state = 'skipped'; - codeceptjs.event.dispatcher.emit('test.skipped', codeceptjsTest); - } else { - codeceptjsTest.state = 'failed'; - codeceptjs.event.dispatcher.emit('test.failed', codeceptjsTest); - } - - codeceptjs.event.dispatcher.emit('test.finish', codeceptjsTest); - codeceptjs.event.dispatcher.emit('test.after', codeceptjsTest); - } - private unwatchActiveTestPages(): void { for (const page of this.observedTestPages) { if (this.testPageErrorHandler) page.off('pageerror', this.testPageErrorHandler); @@ -840,87 +716,6 @@ class Explorer { } this.observedTestPages.clear(); } - - async playwrightLocatorCount(locatorFn: (page: any) => any): Promise { - try { - const pwLocator = locatorFn(this.playwrightHelper.page); - return await pwLocator.count(); - } catch (error) { - if (this.isFatalBrowserError(error)) { - tag('warning').log(`playwrightLocatorCount: ${browserErrorMessage(error)}`); - await this.recoverFromBrowserError(); - } - throw error; - } - } - - private bindFrameNavigated(page: any): void { - page.on('framenavigated', async (frame: any) => { - if (frame !== page.mainFrame()) return; - - const newUrl = await frame.url(); - let newTitle = ''; - - try { - newTitle = await frame.title(); - } catch (error) { - debugLog('Failed to get page title:', error); - } - - this.stateManager.updateStateFromBasic(newUrl, newTitle, 'navigation'); - - await sleep(500); - }); - } - - async openFreshTab(): Promise { - if (!this.playwrightHelper?.page) return; - - const oldPage = this.playwrightHelper.page; - const context = oldPage.context(); - const newPage = await context.newPage(); - - await oldPage.close(); - await newPage.bringToFront(); - - await this.playwrightHelper._setPage(newPage); - this.otherTabs = []; - - this.bindFrameNavigated(newPage); - if (this.xhrCapture) { - this.xhrCapture.attach(newPage); - } - - debugLog('Opened fresh tab, closed previous tab'); - } - - private async closeOtherTabs(): Promise { - if (!this.playwrightHelper) { - return; - } - - const context = this.playwrightHelper.page.context(); - const pages = context.pages(); - - if (pages.length <= 1) { - return; - } - - debugLog(`Found ${pages.length} tabs, cleaning up to keep only the first one`); - - const firstPage = pages[0]; - const tabsToClose = pages.slice(1); - - for (const page of tabsToClose) { - await page.close(); - debugLog(`Closed extra tab: ${await page.url()}`); - } - - await firstPage.bringToFront(); - await this.playwrightHelper._setPage(firstPage); - - debugLog(`Cleaned up tabs, now focused on: ${await firstPage.url()}`); - } } function toCodeceptjsTest(test: Test): any { @@ -938,63 +733,35 @@ function toCodeceptjsTest(test: Test): any { return codeceptjsTest; } -const REF_LINE_PATTERN = /^(\s*)-\s+(\w+)\s*(?:"([^"]*)")?.*?\[ref=(e\d+)\]/; - -const ANNOTATABLE_ROLES = new Set(['button', 'link', 'textbox', 'searchbox', 'checkbox', 'radio', 'switch', 'combobox', 'tab', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'slider', 'spinbutton', 'treeitem']); - -function parseAriaRefs(ariaSnapshot: string): Array<{ role: string; name: string; ref: string }> { - const entries: Array<{ role: string; name: string; ref: string }> = []; - for (const line of ariaSnapshot.split('\n')) { - const match = line.match(REF_LINE_PATTERN); - if (!match) continue; - if (!ANNOTATABLE_ROLES.has(match[2])) continue; - entries.push({ role: match[2], name: match[3] || '', ref: match[4] }); - } - return entries; +export interface ExplorerOptions { + show?: boolean; + headless?: boolean; + incognito?: boolean; + session?: string; } -export async function annotatePageElements(page: any): Promise<{ ariaSnapshot: string; elements: WebElement[] }> { - const ariaSnapshot: string = await page.locator('body').ariaSnapshot({ mode: 'ai' }); - const refEntries = parseAriaRefs(ariaSnapshot); +export interface ExplorerDeps { + stateManager: StateManager; + knowledgeTracker: KnowledgeTracker; + reporter: Reporter; + requestStore: RequestStore; + playwrightRecorder: PlaywrightRecorder; +} - const byRole = new Map>(); - for (const { role, name, ref } of refEntries) { - let list = byRole.get(role); - if (!list) { - list = []; - byRole.set(role, list); - } - list.push({ name, ref }); - } +export interface Recovery { + ok: boolean; + action: 'continue' | 'stop'; + message: string; + recovered?: boolean; +} - const elements: WebElement[] = []; - for (const [role, entries] of byRole) { - try { - const rawList = await page.getByRole(role).evaluateAll( - (domElements: Element[], [data, extractFnStr, config]: [Array<{ name: string; ref: string }>, string, typeof ELEMENT_EXTRACTION_CONFIG]) => { - const extract = new Function(`return ${extractFnStr}`)() as (el: Element) => any; - const results: any[] = []; - let ariaIdx = 0; - for (const el of domElements) { - if (ariaIdx >= data.length) break; - el.setAttribute(config.attrs.eidx, data[ariaIdx].ref); - const elData = extract(el, config); - if (elData) results.push(elData); - ariaIdx++; - } - return results; - }, - [entries, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG] - ); - for (const raw of rawList) { - elements.push(WebElement.fromRawData(raw, role)); - } - } catch { - debugLog(`Failed to annotate role=${role}`); - } - } +export interface TestRun { + started: boolean; + stop(meta?: Record): Promise; +} - return { ariaSnapshot, elements }; +export interface CaptureOpts { + screenshot?: boolean; } export default Explorer; diff --git a/src/state-manager.ts b/src/state-manager.ts index f853acd..8918fee 100644 --- a/src/state-manager.ts +++ b/src/state-manager.ts @@ -1,6 +1,6 @@ import { ActionResult } from './action-result.js'; import type { ExperienceTracker } from './experience-tracker.js'; -import type { KnowledgeTracker, Knowledge } from './knowledge-tracker.js'; +import type { Knowledge, KnowledgeTracker } from './knowledge-tracker.js'; import { detectFocusArea } from './utils/aria.js'; import { createDebug } from './utils/logger.js'; import { slugify } from './utils/strings.js'; @@ -67,9 +67,15 @@ export interface StateTransition { export type StateChangeListener = (event: StateTransition) => void; +export interface TabInfo { + url: string; + title: string; +} + export type { Knowledge }; export class StateManager { + otherTabs: TabInfo[] = []; private currentState: WebPageState | null = null; private stateHistory: StateTransition[] = []; private allVisitedUrls: Set = new Set(); diff --git a/src/utils/hooks-runner.ts b/src/utils/hooks-runner.ts index 31d84b0..7811886 100644 --- a/src/utils/hooks-runner.ts +++ b/src/utils/hooks-runner.ts @@ -54,11 +54,9 @@ export class HooksRunner { private async executeHook(hook: Hook, url: string): Promise { try { if (hook.type === 'playwright') { - const page = this.explorer.playwrightHelper.page; - await hook.hook({ page, url }); + await this.explorer.withPage(async (page) => hook.hook({ page, url })); } else { - const I = this.explorer.actor; - await hook.hook({ I, url }); + await hook.hook({ I: this.explorer.actor, url }); } } catch (error) { debugLog(`Hook error: ${error}`); diff --git a/src/utils/web-annotate.ts b/src/utils/web-annotate.ts new file mode 100644 index 0000000..b73105f --- /dev/null +++ b/src/utils/web-annotate.ts @@ -0,0 +1,64 @@ +import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from './html.ts'; +import { createDebug } from './logger.js'; +import { WebElement } from './web-element.ts'; + +const debugLog = createDebug('explorbot:web-annotate'); + +const REF_LINE_PATTERN = /^(\s*)-\s+(\w+)\s*(?:"([^"]*)")?.*?\[ref=(e\d+)\]/; + +const ANNOTATABLE_ROLES = new Set(['button', 'link', 'textbox', 'searchbox', 'checkbox', 'radio', 'switch', 'combobox', 'tab', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'slider', 'spinbutton', 'treeitem']); + +function parseAriaRefs(ariaSnapshot: string): Array<{ role: string; name: string; ref: string }> { + const entries: Array<{ role: string; name: string; ref: string }> = []; + for (const line of ariaSnapshot.split('\n')) { + const match = line.match(REF_LINE_PATTERN); + if (!match) continue; + if (!ANNOTATABLE_ROLES.has(match[2])) continue; + entries.push({ role: match[2], name: match[3] || '', ref: match[4] }); + } + return entries; +} + +export async function annotatePageElements(page: any): Promise<{ ariaSnapshot: string; elements: WebElement[] }> { + const ariaSnapshot: string = await page.locator('body').ariaSnapshot({ mode: 'ai' }); + const refEntries = parseAriaRefs(ariaSnapshot); + + const byRole = new Map>(); + for (const { role, name, ref } of refEntries) { + let list = byRole.get(role); + if (!list) { + list = []; + byRole.set(role, list); + } + list.push({ name, ref }); + } + + const elements: WebElement[] = []; + for (const [role, entries] of byRole) { + try { + const rawList = await page.getByRole(role).evaluateAll( + (domElements: Element[], [data, extractFnStr, config]: [Array<{ name: string; ref: string }>, string, typeof ELEMENT_EXTRACTION_CONFIG]) => { + const extract = new Function(`return ${extractFnStr}`)() as (el: Element) => any; + const results: any[] = []; + let ariaIdx = 0; + for (const el of domElements) { + if (ariaIdx >= data.length) break; + el.setAttribute(config.attrs.eidx, data[ariaIdx].ref); + const elData = extract(el, config); + if (elData) results.push(elData); + ariaIdx++; + } + return results; + }, + [entries, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG] + ); + for (const raw of rawList) { + elements.push(WebElement.fromRawData(raw, role)); + } + } catch { + debugLog(`Failed to annotate role=${role}`); + } + } + + return { ariaSnapshot, elements }; +} diff --git a/src/utils/web-eidx.ts b/src/utils/web-eidx.ts new file mode 100644 index 0000000..c86f55c --- /dev/null +++ b/src/utils/web-eidx.ts @@ -0,0 +1,21 @@ +import { ELEMENT_EXTRACTION_CONFIG } from './html.ts'; + +export async function eidxInContainer(page: any, containerCss: string | null): Promise { + const selector = containerCss ? `${containerCss} [${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]` : `[${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]`; + const elements = await page.locator(selector).all(); + const result: string[] = []; + for (const el of elements) { + const attr = await el.getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx).catch(() => null); + if (attr) result.push(attr); + } + return result; +} + +export async function eidxByLocator(page: any, locator: string, container?: string | null): Promise { + const base = container ? page.locator(container) : page; + const el = locator.startsWith('//') ? base.locator(`xpath=${locator}`) : base.locator(locator); + return el + .first() + .getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx) + .catch(() => null); +} diff --git a/tests/integration/annotate-elements.test.ts b/tests/integration/annotate-elements.test.ts index de526cd..246ef29 100644 --- a/tests/integration/annotate-elements.test.ts +++ b/tests/integration/annotate-elements.test.ts @@ -1,7 +1,7 @@ import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; import { join } from 'node:path'; import { type Browser, type Page, chromium } from 'playwright'; -import { annotatePageElements } from '../../src/explorer.ts'; +import { annotatePageElements } from '../../src/utils/web-annotate.ts'; import type { WebElement } from '../../src/utils/web-element.ts'; let browser: Browser; diff --git a/tests/integration/planner.test.ts b/tests/integration/planner.test.ts index 4eb4762..ece52b5 100644 --- a/tests/integration/planner.test.ts +++ b/tests/integration/planner.test.ts @@ -63,7 +63,7 @@ const fakeState = { html: 'stub', }; -function createMockExplorer(state = fakeState) { +function createMockDeps(state = fakeState) { const mockExperienceTracker = { getSuccessfulExperience: () => [] }; const mockKnowledgeTracker = { getRelevantKnowledge: () => [] }; const mockStateManager = { @@ -72,10 +72,13 @@ function createMockExplorer(state = fakeState) { getExperienceTracker: () => mockExperienceTracker, }; return { - getStateManager: () => mockStateManager, - getKnowledgeTracker: () => mockKnowledgeTracker, - getConfig: () => ConfigParser.getInstance().getConfig(), - } as any; + explorer: {} as any, + config: ConfigParser.getInstance().getConfig(), + stateManager: mockStateManager, + knowledgeTracker: mockKnowledgeTracker, + requestStore: {}, + playwrightRecorder: {}, + }; } function extractPromptText(entry: any): string { @@ -124,7 +127,7 @@ describe('Planner with aimock', () => { clearSessionDedup(); clearStyleCache(); - planner = new Planner(createMockExplorer(), provider, { research: async () => taskBoardUiMap } as any); + planner = new Planner({ ...createMockDeps(), ai: provider } as any, { research: async () => taskBoardUiMap } as any); mock.on({}, { content: JSON.stringify(defaultScenarios) }); }); diff --git a/tests/integration/researcher-browser.test.ts b/tests/integration/researcher-browser.test.ts index e85db9f..947ff72 100644 --- a/tests/integration/researcher-browser.test.ts +++ b/tests/integration/researcher-browser.test.ts @@ -8,7 +8,7 @@ import { Provider } from '../../src/ai/provider.ts'; import { Researcher } from '../../src/ai/researcher.ts'; import { clearResearchCache } from '../../src/ai/researcher/cache.ts'; import { ConfigParser } from '../../src/config.ts'; -import { annotatePageElements } from '../../src/explorer.ts'; +import { annotatePageElements } from '../../src/utils/web-annotate.ts'; const TASK_BOARD_URL = `file://${join(process.cwd(), 'test-data', 'task-board.html')}`; @@ -81,7 +81,7 @@ describe('Researcher with real browser + aimock', () => { }; } - function buildExplorer(state: any) { + function buildDeps(state: any) { const mockExperienceTracker = { getSuccessfulExperience: () => [], updateSummary: () => {}, @@ -92,24 +92,23 @@ describe('Researcher with real browser + aimock', () => { getExperienceTracker: () => mockExperienceTracker, getRelevantKnowledge: () => [], }; - return { - getStateManager: () => mockStateManager, - getKnowledgeTracker: () => ({ getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '' }), - getConfig: () => ConfigParser.getInstance().getConfig(), - visit: async () => {}, - annotateElements: async () => (await annotatePageElements(page)).elements, - capturePageState: async () => ActionResult.fromState(state), - capturePageWithScreenshot: async () => ActionResult.fromState(state), - runWithBrowserRecovery: async (_label: string, operation: () => Promise) => operation(), - createAction: () => ({ + const explorer = { + visit: async () => ActionResult.fromState(state), + capture: async () => ActionResult.fromState(state), + withPage: async (fn: any) => fn(page), + action: () => ({ capturePageState: async () => ActionResult.fromState(state), }), - playwrightLocatorCount: async (cb: (p: any) => any) => { - const locator = cb(page); - return locator.count(); - }, - playwrightHelper: { page }, + page, } as any; + return { + explorer, + config: ConfigParser.getInstance().getConfig(), + stateManager: mockStateManager, + knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '' }, + requestStore: {}, + playwrightRecorder: {}, + }; } beforeAll(async () => { @@ -140,7 +139,7 @@ describe('Researcher with real browser + aimock', () => { ConfigParser.setupTestConfig(); const state = await captureRealState(); - researcher = new Researcher(buildExplorer(state), provider); + researcher = new Researcher({ ...buildDeps(state), ai: provider } as any); }); afterAll(async () => { diff --git a/tests/integration/researcher-sections.test.ts b/tests/integration/researcher-sections.test.ts index c71021f..c693a55 100644 --- a/tests/integration/researcher-sections.test.ts +++ b/tests/integration/researcher-sections.test.ts @@ -15,7 +15,7 @@ const fakeState = { ariaSnapshot: '- navigation:\n - link "Home"\n- main:\n - button "Click me"', }; -function createMockExplorer(configOverrides: Record = {}, playwrightLocatorCount: () => Promise = async () => 0) { +function createMockDeps(configOverrides: Record = {}, locatorCount: () => Promise = async () => 0) { const baseConfig = ConfigParser.getInstance().getConfig(); const config = { ...baseConfig, @@ -36,21 +36,29 @@ function createMockExplorer(configOverrides: Record = {}, playw getExperienceTracker: () => ({ getSuccessfulExperience: () => [], updateSummary: () => {} }), getRelevantKnowledge: () => [], }; - return { - getStateManager: () => stateManager, - getKnowledgeTracker: () => ({ getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '' }), - getConfig: () => config, - visit: async () => {}, - annotateElements: async () => [], - capturePageState: async () => ActionResult.fromState(fakeState), - capturePageWithScreenshot: async () => ActionResult.fromState(fakeState), - runWithBrowserRecovery: async (_label: string, operation: () => Promise) => operation(), - createAction: () => ({ + const fakePage: any = {}; + fakePage.locator = () => fakePage; + fakePage.getByRole = () => fakePage; + fakePage.count = locatorCount; + fakePage.ariaSnapshot = async () => ''; + fakePage.evaluateAll = async () => []; + const explorer = { + visit: async () => ActionResult.fromState(fakeState), + capture: async () => ActionResult.fromState(fakeState), + withPage: async (fn: any) => fn(fakePage), + action: () => ({ capturePageState: async () => ActionResult.fromState(fakeState), }), - playwrightLocatorCount, - playwrightHelper: { page: {} }, + page: fakePage, } as any; + return { + explorer, + config, + stateManager, + knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '' }, + requestStore: {}, + playwrightRecorder: {}, + }; } function extractPromptText(entry: any): string { @@ -101,9 +109,9 @@ describe('Researcher researchBySections', () => { await mock.stop(); }); - function makeResearcher(configOverrides: Record = {}, playwrightLocatorCount: () => Promise = async () => 0): Researcher { - const explorer = createMockExplorer(configOverrides, playwrightLocatorCount); - const researcher = new Researcher(explorer, provider); + function makeResearcher(configOverrides: Record = {}, locatorCount: () => Promise = async () => 0): Researcher { + const deps = createMockDeps(configOverrides, locatorCount); + const researcher = new Researcher({ ...deps, ai: provider } as any); researcher.actionResult = ActionResult.fromState(fakeState); return researcher; } diff --git a/tests/integration/researcher.test.ts b/tests/integration/researcher.test.ts index 944f055..1f1b79a 100644 --- a/tests/integration/researcher.test.ts +++ b/tests/integration/researcher.test.ts @@ -48,7 +48,7 @@ const fakeState = { ariaSnapshot: '- region "main":\n - button "Create Task"\n - textbox "Search tasks"\n - combobox "Assignee"\n - combobox "Sort by"', }; -function createMockExplorer(state = fakeState) { +function createMockDeps(state = fakeState) { const mockExperienceTracker = { getSuccessfulExperience: () => [], updateSummary: () => {}, @@ -63,21 +63,29 @@ function createMockExplorer(state = fakeState) { getExperienceTracker: () => mockExperienceTracker, getRelevantKnowledge: () => [], }; - return { - getStateManager: () => mockStateManager, - getKnowledgeTracker: () => mockKnowledgeTracker, - getConfig: () => ConfigParser.getInstance().getConfig(), - visit: async () => {}, - annotateElements: async () => [], - capturePageState: async () => ActionResult.fromState(state), - capturePageWithScreenshot: async () => ActionResult.fromState(state), - runWithBrowserRecovery: async (_label: string, operation: () => Promise) => operation(), - createAction: () => ({ + const fakePage: any = {}; + fakePage.locator = () => fakePage; + fakePage.getByRole = () => fakePage; + fakePage.count = async () => 1; + fakePage.ariaSnapshot = async () => ''; + fakePage.evaluateAll = async () => []; + const explorer = { + visit: async () => ActionResult.fromState(state), + capture: async () => ActionResult.fromState(state), + withPage: async (fn: any) => fn(fakePage), + action: () => ({ capturePageState: async () => ActionResult.fromState(state), }), - playwrightLocatorCount: async () => 1, - playwrightHelper: { page: {} }, + page: fakePage, } as any; + return { + explorer, + config: ConfigParser.getInstance().getConfig(), + stateManager: mockStateManager, + knowledgeTracker: mockKnowledgeTracker, + requestStore: {}, + playwrightRecorder: {}, + }; } function extractPromptText(entry: any): string { @@ -124,7 +132,7 @@ describe('Researcher with aimock', () => { clearResearchCache(); ConfigParser.setupTestConfig(); - researcher = new Researcher(createMockExplorer(), provider); + researcher = new Researcher({ ...createMockDeps(), ai: provider } as any); mock.on({}, { content: taskBoardResearch }); }); diff --git a/tests/unit/agent-tools.test.ts b/tests/unit/agent-tools.test.ts index 64405a9..6336459 100644 --- a/tests/unit/agent-tools.test.ts +++ b/tests/unit/agent-tools.test.ts @@ -17,14 +17,12 @@ describe('createAgentTools experience', () => { content: '## FLOW: use prior success', }), }; - const explorer = { - getStateManager: () => ({ - getCurrentState: () => state, - getExperienceTracker: () => experienceTracker, - }), + const stateManager = { + getCurrentState: () => state, + getExperienceTracker: () => experienceTracker, } as any; - const tools = createAgentTools({ explorer, researcher: {} as any, navigator: {} as any }); + const tools = createAgentTools({ explorer: {} as any, stateManager, ai: {} as any, researcher: {} as any, navigator: {} as any }); expect(tools.learnExperience).toBeDefined(); @@ -38,7 +36,7 @@ describe('createAgentTools experience', () => { }); it('omits learnExperience when withExperience is false', () => { - const tools = createAgentTools({ explorer: {} as any, researcher: {} as any, navigator: {} as any, withExperience: false }); + const tools = createAgentTools({ explorer: {} as any, stateManager: {} as any, ai: {} as any, researcher: {} as any, navigator: {} as any, withExperience: false }); expect(tools.learnExperience).toBeUndefined(); }); }); diff --git a/tests/unit/annotate-elements.test.ts b/tests/unit/annotate-elements.test.ts index 1d40c61..5855a30 100644 --- a/tests/unit/annotate-elements.test.ts +++ b/tests/unit/annotate-elements.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'bun:test'; -import { annotatePageElements } from '../../src/explorer.ts'; +import { annotatePageElements } from '../../src/utils/web-annotate.ts'; function createMockPage(ariaSnapshot: string, roleElements: Record) { return { diff --git a/tests/unit/captain-mode.test.ts b/tests/unit/captain-mode.test.ts index d5e37cf..667ac2e 100644 --- a/tests/unit/captain-mode.test.ts +++ b/tests/unit/captain-mode.test.ts @@ -2,18 +2,18 @@ import { describe, expect, it } from 'bun:test'; import { Captain } from '../../src/ai/captain.ts'; function buildCaptain(opts: { state?: any; activeTest?: any; page?: any }) { + let page = opts.page || null; + if (page?.isClosed?.()) page = null; const explorer = { activeTest: opts.activeTest || null, - playwrightHelper: { - page: opts.page, - }, - getStateManager: () => ({ - getCurrentState: () => opts.state || null, - }), + page, }; const explorBot = { getExplorer: () => explorer, + stateManager: () => ({ + getCurrentState: () => opts.state || null, + }), }; return Object.assign(Object.create(Captain.prototype), { explorBot }) as Captain; @@ -64,7 +64,7 @@ describe('Captain modes', () => { describe('Captain execution recovery', () => { it('continues after a fatal browser error is recovered', async () => { const captain = buildCaptainWithExplorer({ - handleExecutionError: async () => ({ + recover: async () => ({ action: 'continue', recovered: true, message: 'Browser was recovered after a fatal page error.', @@ -80,7 +80,7 @@ describe('Captain execution recovery', () => { it('stops when a fatal browser error cannot be recovered', async () => { const captain = buildCaptainWithExplorer({ - handleExecutionError: async () => ({ + recover: async () => ({ action: 'stop', recovered: false, message: 'Browser could not be recovered', @@ -95,7 +95,7 @@ describe('Captain execution recovery', () => { it('continues when browser restart recovers after page recovery fails', async () => { const captain = buildCaptainWithExplorer({ - handleExecutionError: async () => ({ + recover: async () => ({ action: 'continue', recovered: true, message: 'Browser was recovered after a fatal page error.', @@ -110,7 +110,7 @@ describe('Captain execution recovery', () => { it('continues with guidance for non-fatal execution errors', async () => { const captain = buildCaptainWithExplorer({ - handleExecutionError: async () => ({ + recover: async () => ({ action: 'continue', message: 'Previous execution error: Locator not found. Investigate the current state and choose a different approach.', }), diff --git a/tests/unit/doc-collector.test.ts b/tests/unit/doc-collector.test.ts index 8cd49ce..caa56ef 100644 --- a/tests/unit/doc-collector.test.ts +++ b/tests/unit/doc-collector.test.ts @@ -4,10 +4,10 @@ import { Documentarian } from '../../boat/doc-collector/src/ai/documentarian.ts' import { pickDocActionCandidates } from '../../boat/doc-collector/src/ai/tools.ts'; import { DocBot } from '../../boat/doc-collector/src/docbot.ts'; import { normalizeAction, renderPageDocumentation, renderSpecIndex } from '../../boat/doc-collector/src/docs-renderer.ts'; -import { renderMermaidBody } from '../../boat/doc-collector/src/state-diagram.ts'; import { getDocPageKey, shouldCrawlDocPath } from '../../boat/doc-collector/src/path-filter.ts'; import { extractResearchNavigationTargets } from '../../boat/doc-collector/src/research-navigation.ts'; import { captureDocumentationScreenshots, getScreenshotSections } from '../../boat/doc-collector/src/screenshots.ts'; +import { renderMermaidBody } from '../../boat/doc-collector/src/state-diagram.ts'; describe('doc-collector path filter', () => { it('allows regular documentation pages', () => { @@ -403,7 +403,7 @@ describe('doc-collector screenshots', () => { }; const screenshots = await captureDocumentationScreenshots( - { playwrightHelper: { page } } as any, + { page } as any, { url: '/users/sign_in' }, ` ## Navigation @@ -738,11 +738,9 @@ describe('documentarian interactive mode', () => { }; }, } as any; + const stateManager = { getCurrentState: () => states[stateIndex] } as any; const explorer = { - getStateManager() { - return { getCurrentState: () => states[stateIndex] }; - }, - createAction() { + action() { return { async attempt(command: string) { stateIndex = command.startsWith('I.amOnPage') ? 0 : 1; @@ -751,7 +749,7 @@ describe('documentarian interactive mode', () => { }; }, } as any; - const documentarian = new Documentarian(provider, { docs: { interactive: true } }, explorer); + const documentarian = new Documentarian(provider, { docs: { interactive: true } }, explorer, stateManager); const result = await documentarian.document( states[0], `## Content Controls @@ -783,11 +781,9 @@ describe('documentarian interactive mode', () => { }; }, } as any; + const stateManager = { getCurrentState: () => states[stateIndex] } as any; const explorer = { - getStateManager() { - return { getCurrentState: () => states[stateIndex] }; - }, - createAction() { + action() { return { async attempt(command: string) { stateIndex = command.startsWith('I.amOnPage') ? 0 : 1; @@ -796,7 +792,7 @@ describe('documentarian interactive mode', () => { }; }, } as any; - const documentarian = new Documentarian(provider, { docs: { interactive: true } }, explorer); + const documentarian = new Documentarian(provider, { docs: { interactive: true } }, explorer, stateManager); const result = await documentarian.document( states[0], `## Content @@ -892,19 +888,17 @@ describe('documentarian interactive mode', () => { }, } as any; - const mockExplorer = { - getStateManager() { + const mockStateManager = { + getCurrentState() { return { - getCurrentState() { - return { - url: '/test', - title: 'Test', - ariaSnapshot: '[role: button]', - }; - }, + url: '/test', + title: 'Test', + ariaSnapshot: '[role: button]', }; }, - createAction() { + } as any; + const mockExplorer = { + action() { return { async execute(command: string) { return true; @@ -913,7 +907,7 @@ describe('documentarian interactive mode', () => { }, } as any; - const documentarian = new Documentarian(provider, { docs: { interactive: true } }, mockExplorer); + const documentarian = new Documentarian(provider, { docs: { interactive: true } }, mockExplorer, mockStateManager); const result = await documentarian.document( { @@ -946,19 +940,17 @@ describe('documentarian interactive mode', () => { }, } as any; - const mockExplorer = { - getStateManager() { + const mockStateManager = { + getCurrentState() { return { - getCurrentState() { - return { - url: '/test', - title: 'Test', - ariaSnapshot: '[role: button]', - }; - }, + url: '/test', + title: 'Test', + ariaSnapshot: '[role: button]', }; }, - createAction() { + } as any; + const mockExplorer = { + action() { return { async execute(command: string) { return true; @@ -967,7 +959,7 @@ describe('documentarian interactive mode', () => { }, } as any; - const documentarian = new Documentarian(provider, { docs: { interactive: true } }, mockExplorer); + const documentarian = new Documentarian(provider, { docs: { interactive: true } }, mockExplorer, mockStateManager); const result = await documentarian.document( { @@ -1015,15 +1007,13 @@ describe('documentarian interactive mode', () => { ]; let stateIndex = 0; - const mockExplorer = { - getStateManager() { - return { - getCurrentState() { - return states[stateIndex]; - }, - }; + const mockStateManager = { + getCurrentState() { + return states[stateIndex]; }, - createAction() { + } as any; + const mockExplorer = { + action() { return { async attempt(command: string) { if (command.startsWith('I.click')) { @@ -1040,7 +1030,7 @@ describe('documentarian interactive mode', () => { }, } as any; - const documentarian = new Documentarian(provider, { docs: { interactive: true } }, mockExplorer); + const documentarian = new Documentarian(provider, { docs: { interactive: true } }, mockExplorer, mockStateManager); const result = await documentarian.document( { url: '/items', @@ -1082,19 +1072,17 @@ describe('documentarian interactive mode', () => { }, } as any; - const mockExplorer = { - getStateManager() { + const mockStateManager = { + getCurrentState() { return { - getCurrentState() { - return { - url: '/test', - title: 'Test', - ariaSnapshot: '[role: tab]', - }; - }, + url: '/test', + title: 'Test', + ariaSnapshot: '[role: tab]', }; }, - createAction() { + } as any; + const mockExplorer = { + action() { return { async execute(command: string) { return true; @@ -1103,7 +1091,7 @@ describe('documentarian interactive mode', () => { }, } as any; - const documentarian = new Documentarian(provider, { docs: { interactive: true } }, mockExplorer); + const documentarian = new Documentarian(provider, { docs: { interactive: true } }, mockExplorer, mockStateManager); const result = await documentarian.document( { @@ -1134,19 +1122,17 @@ describe('documentarian interactive defaults', () => { }, } as any; - const mockExplorer = { - getStateManager() { + const mockStateManager = { + getCurrentState() { return { - getCurrentState() { - return { - url: '/test', - title: 'Test', - ariaSnapshot: '[role: button]', - }; - }, + url: '/test', + title: 'Test', + ariaSnapshot: '[role: button]', }; }, - createAction() { + } as any; + const mockExplorer = { + action() { return { async execute() { return true; @@ -1155,7 +1141,7 @@ describe('documentarian interactive defaults', () => { }, } as any; - const documentarian = new Documentarian(provider, {}, mockExplorer); + const documentarian = new Documentarian(provider, {}, mockExplorer, mockStateManager); const result = await documentarian.document( { url: '/test', diff --git a/tests/unit/explore-command.test.ts b/tests/unit/explore-command.test.ts index 5d8dcb2..3dde816 100644 --- a/tests/unit/explore-command.test.ts +++ b/tests/unit/explore-command.test.ts @@ -148,7 +148,7 @@ describe('ExploreCommand picking algorithm (via dry-run execute)', () => { function setupCommand(plan: Plan, _configure: string, maxTests: number) { const explorBot = { - getExplorer: () => ({ getStateManager: () => ({ getCurrentState: () => ({ url: '/demo' }) }) }), + stateManager: () => ({ getCurrentState: () => ({ url: '/demo' }) }), generatePlanFilename: () => 'demo.md', loadPlans: () => [plan], agentPlanner: () => ({ registerPlanInSession: () => {}, collectSubPageCandidates: () => [], pickNextSubPage: async () => null }), @@ -255,15 +255,13 @@ describe('ExploreCommand error page guard', () => { let testerCalled = 0; let saveCalled = 0; const explorBot = { - getExplorer: () => ({ - getStateManager: () => ({ - getCurrentState: () => ({ - url: '/missing', - fullUrl: 'https://example.com/missing', - title: 'Application', - httpStatus: 404, - html: 'Short response', - }), + stateManager: () => ({ + getCurrentState: () => ({ + url: '/missing', + fullUrl: 'https://example.com/missing', + title: 'Application', + httpStatus: 404, + html: 'Short response', }), }), plan: async () => { @@ -292,6 +290,46 @@ describe('ExploreCommand error page guard', () => { }); }); +describe('ExploreCommand reporting survives a failed return to the main page', () => { + test('final visit failure does not suppress savePlans or session analysis', async () => { + const fakePlan = new Plan('Live'); + fakePlan.url = '/live/plan'; + fakePlan.addTest(new Test('seed loaded', 'critical', 'ok', '/live/plan')); + + let savePlansCalled = false; + let analysisCalled = false; + const explorBot = { + stateManager: () => ({ getCurrentState: () => ({ url: '/live' }) }), + generatePlanFilename: () => 'live.md', + loadPlans: () => [fakePlan], + agentPlanner: () => ({ registerPlanInSession: () => {}, collectSubPageCandidates: () => [], pickNextSubPage: async () => null }), + setCurrentPlan: () => {}, + agentTester: () => ({ test: async () => {} }), + visit: async (url: string) => { + if (url === '/live') throw new Error('Navigation to /live failed: redirected to /live/new and could not resolve'); + }, + savePlans: () => { + savePlansCalled = true; + return null; + }, + printSessionAnalysis: async () => { + analysisCalled = true; + }, + agentHistorian: () => ({ getSavedFiles: () => [] }), + plan: async () => undefined, + getCurrentPlan: () => fakePlan, + lastPlanError: null, + } as unknown as ExplorBot; + + const cmd = new ExploreCommand(explorBot); + cmd.maxTests = 1; + await cmd.execute('--configure "new:0%"'); + + expect(savePlansCalled).toBe(true); + expect(analysisCalled).toBe(true); + }); +}); + describe('ExploreCommand priority filter applies to NEW tests too', () => { test('new tests with disallowed priorities get disabled and do not run', async () => { const fakePlan = new Plan('Live'); @@ -300,7 +338,7 @@ describe('ExploreCommand priority filter applies to NEW tests too', () => { let planCalled = 0; const explorBot = { - getExplorer: () => ({ getStateManager: () => ({ getCurrentState: () => ({ url: '/live' }) }) }), + stateManager: () => ({ getCurrentState: () => ({ url: '/live' }) }), generatePlanFilename: () => 'live.md', loadPlans: () => [fakePlan], agentPlanner: () => ({ registerPlanInSession: () => {}, collectSubPageCandidates: () => [], pickNextSubPage: async () => null }), diff --git a/tests/unit/explorer-recovery-url.test.ts b/tests/unit/explorer-recovery-url.test.ts index aa43e4d..3666d28 100644 --- a/tests/unit/explorer-recovery-url.test.ts +++ b/tests/unit/explorer-recovery-url.test.ts @@ -52,9 +52,9 @@ describe('Explorer recovery URL resolution', () => { updateStateFromBasic: () => {}, }; - const recovered = await explorer.recoverFromBrowserError(); + const { ok } = await explorer.recover(); - expect(recovered).toBe(true); + expect(ok).toBe(true); expect((explorer as any).playwrightHelper.page).toBe(newPage); expect(navigated).toEqual(['https://the-internet.herokuapp.com/']); expect(boundEvents).toContain('framenavigated'); @@ -67,12 +67,12 @@ describe('Explorer recovery URL resolution', () => { (explorer as any).playwrightHelper = { page: { isClosed: () => false }, }; - (explorer as any).recoverFromBrowserError = async () => { + (explorer as any).recoverPage = async () => { recoveries++; return true; }; - const result = await explorer.runWithBrowserRecovery('test operation', async () => { + const result = await (explorer as any).runWithRecovery('test operation', async () => { attempts++; if (attempts === 1) throw new Error('Target closed'); return 'recovered'; @@ -90,12 +90,12 @@ describe('Explorer recovery URL resolution', () => { (explorer as any).playwrightHelper = { page: { isClosed: () => false }, }; - (explorer as any).recoverFromBrowserError = async () => { + (explorer as any).recoverPage = async () => { recoveries++; return true; }; - const result = await explorer.runWithBrowserRecovery('test operation', async () => { + const result = await (explorer as any).runWithRecovery('test operation', async () => { attempts++; if (attempts === 1) throw new Error('page.evaluate: Execution context was destroyed, most likely because of a navigation'); if (attempts === 2) throw new Error('Target closed'); @@ -107,26 +107,23 @@ describe('Explorer recovery URL resolution', () => { expect(recoveries).toBe(1); }); - it('recovers and retries action attempts through Explorer', async () => { + it('recovers and retries page operations through withPage', async () => { const explorer = buildExplorer('https://the-internet.herokuapp.com'); let attempts = 0; let recoveries = 0; (explorer as any).playwrightHelper = { page: { isClosed: () => false }, }; - (explorer as any).recoverFromBrowserError = async () => { + (explorer as any).recoverPage = async () => { recoveries++; return true; }; - (explorer as any).createAction = () => ({ - attempt: async () => { - attempts++; - if (attempts === 1) throw new Error('Target page, context or browser has been closed'); - return true; - }, - }); - const result = await explorer.attemptAction('I.click("Menu")', undefined); + const result = await explorer.withPage(async () => { + attempts++; + if (attempts === 1) throw new Error('Target page, context or browser has been closed'); + return true; + }); expect(result).toBe(true); expect(attempts).toBe(2); @@ -138,18 +135,18 @@ describe('Explorer recovery URL resolution', () => { (explorer as any).playwrightHelper = { page: { isClosed: () => false }, }; - (explorer as any).recoverFromBrowserError = async () => true; + (explorer as any).recoverPage = async () => true; let error: unknown; try { - await explorer.runWithBrowserRecovery('capturePageState', async () => { + await (explorer as any).runWithRecovery('capturePageState', async () => { throw new Error('Target page, context or browser has been closed'); }); } catch (err) { error = err; } - const result = await explorer.handleExecutionError(error); + const result = await explorer.recover(error); expect(result.action).toBe('stop'); expect(result.message).toContain('failed after browser recovery'); @@ -157,10 +154,10 @@ describe('Explorer recovery URL resolution', () => { it('returns a stop decision when browser recovery fails', async () => { const explorer = buildExplorer('https://the-internet.herokuapp.com'); - (explorer as any).recoverFromBrowserError = async () => false; + (explorer as any).recoverPage = async () => false; (explorer as any).restartBrowser = async () => false; - const result = await explorer.handleExecutionError(new Error('Target closed')); + const result = await explorer.recover(new Error('Target closed')); expect(result.action).toBe('stop'); expect(result.recovered).toBe(false); @@ -169,7 +166,7 @@ describe('Explorer recovery URL resolution', () => { it('returns guidance for non-browser execution errors', async () => { const explorer = buildExplorer('https://the-internet.herokuapp.com'); - const result = await explorer.handleExecutionError(new Error('Locator not found')); + const result = await explorer.recover(new Error('Locator not found')); expect(result.action).toBe('continue'); expect(result.recovered).toBeUndefined(); diff --git a/tests/unit/explorer-step-listeners.test.ts b/tests/unit/explorer-step-listeners.test.ts index 69f827a..1c41bde 100644 --- a/tests/unit/explorer-step-listeners.test.ts +++ b/tests/unit/explorer-step-listeners.test.ts @@ -33,12 +33,10 @@ function trackRegistrations(): { registered: Map>; restore: () function buildExplorer() { const explorer = Object.assign(Object.create(Explorer.prototype), { reporter: { reportTestStart: async () => {} }, - stateManager: { getCurrentState: () => null }, - otherTabs: [], + stateManager: { getCurrentState: () => null, otherTabs: [] }, playwrightHelper: { page: { isClosed: () => false } }, }) as any; explorer.closeOtherTabs = async () => {}; - explorer.ensurePageAvailable = async () => true; explorer.watchActiveTestPage = () => {}; explorer.unwatchActiveTestPages = () => {}; return explorer as Explorer; @@ -58,7 +56,7 @@ describe('Explorer step listener cleanup', () => { it('removes every listener it registers after a test lifecycle', async () => { const { registered, restore } = trackRegistrations(); try { - await buildExplorer().startTest(buildTest()); + await buildExplorer().beginTest(buildTest()); expect(registered.get('step.passed')!.size).toBe(1); expect(registered.get('test.after')!.size).toBe(1); @@ -76,7 +74,7 @@ describe('Explorer step listener cleanup', () => { const { registered, restore } = trackRegistrations(); try { for (let i = 0; i < 5; i++) { - await buildExplorer().startTest(buildTest()); + await buildExplorer().beginTest(buildTest()); dispatcher.emit('test.after'); } expect(registered.get('step.passed')!.size).toBe(0); diff --git a/tests/unit/explorer.test.ts b/tests/unit/explorer.test.ts index f18b1ba..c8240ac 100644 --- a/tests/unit/explorer.test.ts +++ b/tests/unit/explorer.test.ts @@ -206,12 +206,14 @@ mock.module('codeceptjs/lib/listener/store.js', () => ({ })); import { isTemplateMatch } from '../../src/ai/planner/subpages.ts'; -import { AIProvider } from '../../src/ai/provider.ts'; +import { RequestStore } from '../../src/api/request-store.ts'; import { ConfigParser } from '../../src/config.ts'; import { ExperienceTracker } from '../../src/experience-tracker.ts'; import Explorer from '../../src/explorer.ts'; import { KnowledgeTracker } from '../../src/knowledge-tracker.ts'; +import { PlaywrightRecorder } from '../../src/playwright-recorder.ts'; import { Reporter } from '../../src/reporter.ts'; +import { StateManager } from '../../src/state-manager.ts'; import { isDynamicSegment } from '../../src/utils/url-matcher.ts'; describe('isDynamicSegment', () => { @@ -290,6 +292,7 @@ describe('isTemplateMatch', () => { describe('Explorer', () => { let explorer: Explorer; + let stateManager: StateManager; const baseUrl = 'mock://explorer.test/'; beforeAll(async () => { @@ -323,12 +326,22 @@ describe('Explorer', () => { vi.spyOn(Reporter.prototype, 'reportTest').mockResolvedValue(); const knowledgeTracker = new KnowledgeTracker(); - explorer = new Explorer(config, new AIProvider(config.ai), { headless: true }, new ExperienceTracker(knowledgeTracker), knowledgeTracker); + stateManager = new StateManager(new ExperienceTracker(knowledgeTracker), knowledgeTracker); + explorer = new Explorer( + config, + { headless: true }, + { + stateManager, + knowledgeTracker, + reporter: new Reporter(undefined, stateManager), + requestStore: new RequestStore(config.dirs?.output || 'output'), + playwrightRecorder: new PlaywrightRecorder(), + } + ); await explorer.start(); }); beforeEach(() => { - const stateManager = explorer.getStateManager(); stateManager.updateStateFromBasic(baseUrl, 'Initial', 'manual'); }); @@ -339,16 +352,16 @@ describe('Explorer', () => { }); it('visit loads target page and updates state', async () => { - const action = await explorer.visit(baseUrl); - const currentState = explorer.getStateManager().getCurrentState(); + const result = await explorer.visit(baseUrl); + const currentState = stateManager.getCurrentState(); expect(currentState?.fullUrl).toBe(baseUrl); expect(currentState?.h1).toBe('Text on Page'); - expect(action.getActionResult()?.html).toContain('Welcome to Explorbot'); + expect(result.html).toContain('Welcome to Explorbot'); }); - it('createAction executes assertions against current page', async () => { + it('action executes assertions against current page', async () => { await explorer.visit(baseUrl); - const action = explorer.createAction(); + const action = explorer.action(); await action.execute('I.see("Text on Page")'); expect(action.getActionResult()?.html).toContain('Text on Page'); }); diff --git a/tests/unit/navigator-origin-guard.test.ts b/tests/unit/navigator-origin-guard.test.ts index 0d1e9da..ac20bf7 100644 --- a/tests/unit/navigator-origin-guard.test.ts +++ b/tests/unit/navigator-origin-guard.test.ts @@ -3,13 +3,11 @@ import { Navigator } from '../../src/ai/navigator.ts'; describe('Navigator origin guard', () => { function createNavigator(baseUrl = 'http://192.168.1.162:3000') { - const navigator = Object.create(Navigator.prototype) as Navigator & { explorer: any }; - navigator.explorer = { - getConfig: () => ({ - playwright: { - url: baseUrl, - }, - }), + const navigator = Object.create(Navigator.prototype) as Navigator & { config: any }; + navigator.config = { + playwright: { + url: baseUrl, + }, }; return navigator; } diff --git a/tests/unit/pilot-state-context.test.ts b/tests/unit/pilot-state-context.test.ts index 64bedc5..b545467 100644 --- a/tests/unit/pilot-state-context.test.ts +++ b/tests/unit/pilot-state-context.test.ts @@ -15,14 +15,17 @@ function buildActionResult(browserLogs: any[] = [], ariaSnapshot = ''): ActionRe } function buildPilotWithStore(store: RequestStore | null, hasOtherTabs = false): Pilot { - const explorer: any = { - getRequestStore: () => store, - hasOtherTabs: () => hasOtherTabs, - getOtherTabsInfo: () => [], + const otherTabs = []; + if (hasOtherTabs) otherTabs.push({ url: 'about:blank', title: 'tab' }); + const deps: any = { + ai: {}, + explorer: {}, + stateManager: { otherTabs }, + requestStore: store || new RequestStore('output'), + playwrightRecorder: {}, }; - const provider: any = {}; const researcher: any = {}; - return new Pilot(provider, {}, researcher, explorer); + return new Pilot(deps, {}, researcher); } function makeFailure(method: string, path: string, status: number, counter: number): RequestResult { diff --git a/tests/unit/rerunner-healing-teardown.test.ts b/tests/unit/rerunner-healing-teardown.test.ts index 647d3f0..0ef4cfc 100644 --- a/tests/unit/rerunner-healing-teardown.test.ts +++ b/tests/unit/rerunner-healing-teardown.test.ts @@ -41,7 +41,7 @@ afterAll(() => { }); function buildRerunner(): any { - return new Rerunner({ getConfig: () => ({ ai: { agents: {} } }) } as any, {} as any); + return new Rerunner({ explorer: {}, ai: {}, config: { ai: { agents: {} } }, stateManager: {}, knowledgeTracker: {}, requestStore: {}, playwrightRecorder: {} } as any, {} as any); } const settle = () => new Promise((resolve) => setTimeout(resolve, 50)); diff --git a/tests/unit/tester-error-page.test.ts b/tests/unit/tester-error-page.test.ts index 6070b19..b758f00 100644 --- a/tests/unit/tester-error-page.test.ts +++ b/tests/unit/tester-error-page.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, mock, test } from 'bun:test'; -import { Tester } from '../../src/ai/tester.ts'; import { clearActivity, getCurrentActivity } from '../../src/activity.ts'; +import { Tester } from '../../src/ai/tester.ts'; import { ConfigParser } from '../../src/config.ts'; import { Test, TestResult } from '../../src/test-plan.ts'; @@ -35,42 +35,41 @@ describe('Tester error page handling', () => { const currentState = createState('500 Internal Server Error', '

500 Internal Server Error

', '/broken'); const startConversation = mock(() => createConversation()); const visit = mock(async () => {}); - const startTest = mock(async () => true); const stopTest = mock(async () => {}); + const startTest = mock(async () => ({ started: true, stop: stopTest })); const explorer: any = { - getConfig: () => ({}), - getStateManager: () => ({ + beginTest: startTest, + visit, + }; + const provider: any = { + getSystemPromptForAgent: () => '', + startConversation, + invokeConversation: mock(async () => null), + }; + const researcher: any = {}; + const navigator: any = {}; + const deps: any = { + explorer, + ai: provider, + config: {}, + stateManager: { getCurrentState: () => currentState, clearHistory: () => {}, + otherTabs: [], getExperienceTracker: () => ({ getExperienceTableOfContents: () => [], renderExperienceTocFor: () => '', }), - }), - getKnowledgeTracker: () => ({ + }, + knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '', - }), - getRequestStore: () => null, - playwrightHelper: { - page: { - on: () => {}, - off: () => {}, - }, }, - startTest, - stopTest, - visit, - }; - const provider: any = { - getSystemPromptForAgent: () => '', - startConversation, - invokeConversation: mock(async () => null), + requestStore: { clear: () => {}, onFailedRequest: () => () => {}, getFailedRequests: () => [] }, + playwrightRecorder: {}, }; - const researcher: any = {}; - const navigator: any = {}; - const tester = new Tester(explorer, provider, researcher, navigator); + const tester = new Tester(deps, researcher, navigator); const task = new Test('check broken page', 'normal', 'page works', '/broken'); const result = await tester.test(task); @@ -93,36 +92,11 @@ describe('Tester error page handling', () => { const visit = mock(async () => { currentState = createState('Application', 'Short response', '/missing', 404); }); - const startTest = mock(async () => true); const stopTest = mock(async () => {}); + const startTest = mock(async () => ({ started: true, stop: stopTest })); const explorer: any = { - getConfig: () => ({}), - getStateManager: () => ({ - getCurrentState: () => currentState, - clearHistory: () => {}, - getExperienceTracker: () => ({ - getExperienceTableOfContents: () => [], - renderExperienceTocFor: () => '', - }), - }), - getKnowledgeTracker: () => ({ - getRelevantKnowledge: () => [], - renderRelevantKnowledge: () => '', - }), - getRequestStore: () => null, - hasOtherTabs: () => false, - getOtherTabsInfo: () => [], - clearOtherTabsInfo: () => {}, - getCurrentIframeInfo: () => null, - playwrightHelper: { - page: { - on: () => {}, - off: () => {}, - }, - }, - startTest, - stopTest, + beginTest: startTest, visit, }; const provider: any = { @@ -135,7 +109,27 @@ describe('Tester error page handling', () => { researchOverlay: mock(async () => null), }; const navigator: any = {}; - const tester = new Tester(explorer, provider, researcher, navigator); + const deps: any = { + explorer, + ai: provider, + config: {}, + stateManager: { + getCurrentState: () => currentState, + clearHistory: () => {}, + otherTabs: [], + getExperienceTracker: () => ({ + getExperienceTableOfContents: () => [], + renderExperienceTocFor: () => '', + }), + }, + knowledgeTracker: { + getRelevantKnowledge: () => [], + renderRelevantKnowledge: () => '', + }, + requestStore: { clear: () => {}, onFailedRequest: () => () => {}, getFailedRequests: () => [] }, + playwrightRecorder: {}, + }; + const tester = new Tester(deps, researcher, navigator); const task = new Test('check missing page', 'normal', 'page works', '/missing'); const result = await tester.test(task); diff --git a/tests/unit/tester-focus-scope.test.ts b/tests/unit/tester-focus-scope.test.ts index 465d74c..02ec3d7 100644 --- a/tests/unit/tester-focus-scope.test.ts +++ b/tests/unit/tester-focus-scope.test.ts @@ -5,24 +5,6 @@ import { renderExperienceToc } from '../../src/experience-tracker.ts'; import { Test } from '../../src/test-plan.ts'; function buildTester(): Tester { - const explorer: any = { - getConfig: () => ({}), - getStateManager: () => ({ - getCurrentState: () => null, - getExperienceTracker: () => ({ - getExperienceTableOfContents: () => [], - renderExperienceTocFor: () => '', - }), - }), - getKnowledgeTracker: () => ({ - getRelevantKnowledge: () => [], - renderRelevantKnowledge: () => '', - }), - getCurrentIframeInfo: () => null, - hasOtherTabs: () => false, - getOtherTabsInfo: () => [], - clearOtherTabsInfo: () => {}, - }; const provider: any = { getSystemPromptForAgent: () => '', }; @@ -31,7 +13,26 @@ function buildTester(): Tester { researchOverlay: async () => null, }; const navigator: any = {}; - return new Tester(explorer, provider, researcher, navigator); + const deps: any = { + explorer: {}, + ai: provider, + config: {}, + stateManager: { + getCurrentState: () => null, + otherTabs: [], + getExperienceTracker: () => ({ + getExperienceTableOfContents: () => [], + renderExperienceTocFor: () => '', + }), + }, + knowledgeTracker: { + getRelevantKnowledge: () => [], + renderRelevantKnowledge: () => '', + }, + requestStore: { clear: () => {}, onFailedRequest: () => () => {}, getFailedRequests: () => [] }, + playwrightRecorder: {}, + }; + return new Tester(deps, researcher, navigator); } function buildTesterWithExperience(): Tester { @@ -43,23 +44,28 @@ function buildTesterWithExperience(): Tester { sections: [{ index: 1, level: 2, title: 'FLOW: create item' }], }, ]; - const explorer: any = { - getConfig: () => ({ files: {} }), - getStateManager: () => ({ + const provider: any = {}; + const researcher: any = {}; + const navigator: any = {}; + const deps: any = { + explorer: {}, + ai: provider, + config: { files: {} }, + stateManager: { + otherTabs: [], getExperienceTracker: () => ({ getExperienceTableOfContents: () => experienceToc, renderExperienceTocFor: () => renderExperienceToc(experienceToc), }), - }), - getKnowledgeTracker: () => ({ + }, + knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '', - }), + }, + requestStore: { clear: () => {}, onFailedRequest: () => () => {}, getFailedRequests: () => [] }, + playwrightRecorder: {}, }; - const provider: any = {}; - const researcher: any = {}; - const navigator: any = {}; - return new Tester(explorer, provider, researcher, navigator); + return new Tester(deps, researcher, navigator); } function buildState(ariaSnapshot: string, url = '/page'): ActionResult { diff --git a/tests/unit/tools-back.test.ts b/tests/unit/tools-back.test.ts index d94cb21..a8c3d63 100644 --- a/tests/unit/tools-back.test.ts +++ b/tests/unit/tools-back.test.ts @@ -29,8 +29,7 @@ describe('back tool', () => { getStateHistory: () => [{ toState: after }], }; const explorer = { - getStateManager: () => stateManager, - createAction: () => ({ + action: () => ({ attempt: async () => { navigated = true; return true; @@ -41,9 +40,10 @@ describe('back tool', () => { const tools = createAgentTools({ explorer: explorer as any, + stateManager: stateManager as any, + ai: {} as any, researcher: {} as any, navigator: {} as any, - getState: () => before as any, }); const result: any = await tools.back.execute({ reason: 'went to the wrong page' }); diff --git a/tests/unit/tools.test.ts b/tests/unit/tools.test.ts index 18cbf74..5277d14 100644 --- a/tests/unit/tools.test.ts +++ b/tests/unit/tools.test.ts @@ -1,9 +1,11 @@ import { describe, expect, it } from 'bun:test'; import { createCodeceptJSTools, createIframeTools, createLearnExperienceTool } from '../../src/ai/tools.ts'; -function fakeExplorer(): any { +function fakeDeps(): any { return { - getStateManager: () => ({}), + explorer: {}, + stateManager: {}, + ai: {}, }; } @@ -18,7 +20,7 @@ function fakeTask(): any { describe('createCodeceptJSTools click validation', () => { it('rejects empty commands without touching the browser', async () => { const task = fakeTask(); - const tools = createCodeceptJSTools(fakeExplorer(), task); + const tools = createCodeceptJSTools(fakeDeps(), task); const result = await tools.click.execute({ commands: [], explanation: 'nothing' }, {} as any); @@ -29,7 +31,7 @@ describe('createCodeceptJSTools click validation', () => { it('rejects non-click I. commands as invalid', async () => { const task = fakeTask(); - const tools = createCodeceptJSTools(fakeExplorer(), task); + const tools = createCodeceptJSTools(fakeDeps(), task); const result = await tools.click.execute({ commands: ['I.fillField("name", "value")'], explanation: 'type' }, {} as any); @@ -40,7 +42,7 @@ describe('createCodeceptJSTools click validation', () => { it('rejects non-moveCursorTo commands in hover', async () => { const task = fakeTask(); - const tools = createCodeceptJSTools(fakeExplorer(), task); + const tools = createCodeceptJSTools(fakeDeps(), task); const result = await tools.hover.execute({ commands: ['I.click("Save")'], explanation: 'hover' }, {} as any); @@ -51,7 +53,7 @@ describe('createCodeceptJSTools click validation', () => { describe('createIframeTools', () => { it('always returns the exitIframe tool', () => { - const tools = createIframeTools(fakeExplorer()); + const tools = createIframeTools(fakeDeps()); expect(Object.keys(tools)).toContain('exitIframe'); }); }); From 89f6bda36a7df01a0a56f6c03e134d916c766d82 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Fri, 17 Jul 2026 23:54:44 +0300 Subject: [PATCH 2/4] refactor: move iframe-exit from Explorer to Action Frame context is acting-level state, so exitIframe() lives on Action (which already holds the helper) instead of Explorer's interface. Failed attempts now restore the main-frame context ambiently for every action, generalizing the guard the form tool had; the exitIframe tool runs I.switchTo() through the action seam, so the step is recorded for generated tests. Explorer is down to 12 public members. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 1 + src/action.ts | 8 ++++++++ src/ai/navigator.ts | 4 ++-- src/ai/tester.ts | 4 +--- src/ai/tools.ts | 10 +++------- src/explorer.ts | 6 ------ 6 files changed, 15 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff14015..e04d4a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Changes - Browser crash recovery now applies to every browser operation. Previously some interactions and page lookups could fail permanently when the page or browser crashed mid-session; now every action, capture, and element query automatically reattaches the page or restarts the browser and retries once before giving up. - [Captain] The `browser` tool's `restart` action was merged into `recover` — recovering a page now escalates to a full browser restart automatically when needed, so there is no separate action to choose. +- Failed page actions that accidentally entered an iframe now return to the main page automatically. Previously only failed form actions did this; now it applies to every action, so a failed click or keypress can no longer leave the session stuck inside an iframe. - [Pilot] Pilot's instructions and its list of available tools now stay identical across every call in a session, with the scenario details moved to the end. Providers can reuse the prompt they already processed instead of re-reading it on each call, which lowers the cost of long test runs. - [Planner] Planning rules and output format instructions now come before the page research, so planning more tests for the same page reuses an already-processed prompt. - When a click matches several elements, the follow-up question that picks the right one now uses the default model instead of the more expensive agentic model. diff --git a/src/action.ts b/src/action.ts index 140d7ef..dda7cb9 100644 --- a/src/action.ts +++ b/src/action.ts @@ -339,6 +339,7 @@ class Action { } public async attempt(codeBlock: string, originalMessage?: string): Promise { + const wasInFrame = !!this.playwrightHelper.frame; try { debugLog('Resolution attempt...'); setActivity('🦾 Acting in browser...', 'action'); @@ -353,11 +354,18 @@ class Action { } catch (error) { this.lastError = error as Error; if (isFatalBrowserError(error)) throw error; + if (!wasInFrame) await this.exitIframe().catch(() => {}); debugLog(`Attempt failed: ${codeBlock}: ${errorToString(error) || this.lastError?.toString()}`); return false; } } + async exitIframe(): Promise { + if (!this.playwrightHelper.frame) return; + debugLog('Switching to main frame'); + await this.playwrightHelper.switchTo(); + } + getActor(): CodeceptJS.I { return this.actor; } diff --git a/src/ai/navigator.ts b/src/ai/navigator.ts index cae50e0..0599b4a 100644 --- a/src/ai/navigator.ts +++ b/src/ai/navigator.ts @@ -346,7 +346,7 @@ class Navigator implements Agent { codeBlockIndex++; totalAttempts++; - await this.explorer.exitIframe(); + await action.exitIframe(); const prevActionResult = action.actionResult ?? actionResult; const prevHash = prevActionResult.getStateHash(); @@ -731,7 +731,7 @@ class Navigator implements Agent { return; } - await this.explorer.exitIframe(); + await action.exitIframe(); const verified = await action.attempt(codeBlock, message); diff --git a/src/ai/tester.ts b/src/ai/tester.ts index 2e40725..8fdf4e0 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -880,9 +880,7 @@ export class Tester extends TaskAgent implements Agent { reason: z.string().optional().describe('Explanation why reset is the only option'), }), execute: async ({ reason }) => { - if (this.getCurrentState().isInsideIframe) { - await this.explorer.exitIframe(); - } + await this.explorer.action().exitIframe(); const currentState = this.stateManager.getCurrentState(); const currentUrl = currentState?.fullUrl || currentState?.url; diff --git a/src/ai/tools.ts b/src/ai/tools.ts index 1c0d36c..526cd2d 100644 --- a/src/ai/tools.ts +++ b/src/ai/tools.ts @@ -414,10 +414,6 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps, const action = explorer.action(); await action.attempt(codeBlock, explanation); - if (action.lastError && !previousState.isInsideIframe) { - await explorer.exitIframe(); - } - const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, formLocator); if (action.lastError) { @@ -497,9 +493,9 @@ export function createIframeTools({ explorer, stateManager }: ToolDeps) { }); } - await explorer.exitIframe(); - - const nextState = await explorer.capture(); + const action = explorer.action(); + await action.execute('I.switchTo()'); + const nextState = action.getActionResult()!; const toolResult = await nextState.toToolResult(previousState, 'I.switchTo()'); return successToolResult('exitIframe', { diff --git a/src/explorer.ts b/src/explorer.ts index ca8cc5b..b847e7d 100644 --- a/src/explorer.ts +++ b/src/explorer.ts @@ -167,12 +167,6 @@ class Explorer { return this.runWithRecovery('page operation', () => fn(this.playwrightHelper.page)); } - async exitIframe(): Promise { - if (!this.playwrightHelper.frame) return; - debugLog('Switching to main frame'); - await this.playwrightHelper.switchTo(); - } - async recover(error?: unknown): Promise { if (error) return this.recoverFromExecutionError(error); From 22a96f20622c76c0b68115a0e4a5f777cbc1bfea Mon Sep 17 00:00:00 2001 From: DavertMik Date: Fri, 17 Jul 2026 23:56:48 +0300 Subject: [PATCH 3/4] refactor: rename openFreshTab to openTab Co-Authored-By: Claude Fable 5 --- src/ai/captain/web-mode.ts | 8 ++++---- src/commands/freesail-command.ts | 2 +- src/explorbot.ts | 4 ++-- src/explorer.ts | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ai/captain/web-mode.ts b/src/ai/captain/web-mode.ts index 185d4b5..ebbaaad 100644 --- a/src/ai/captain/web-mode.ts +++ b/src/ai/captain/web-mode.ts @@ -47,10 +47,10 @@ export function WithWebMode(Base: T) { - reload: Reload the current page - screenshot: Take a screenshot of current page - recover: Recover from a closed/crashed page, escalating to a full browser restart when needed - - openFreshTab: Open a fresh tab in the current browser context + - openTab: Open a fresh tab in the current browser context `, inputSchema: z.object({ - action: z.enum(['status', 'evaluate', 'closeTabs', 'reload', 'screenshot', 'recover', 'openFreshTab']).describe('Browser action to perform'), + action: z.enum(['status', 'evaluate', 'closeTabs', 'reload', 'screenshot', 'recover', 'openTab']).describe('Browser action to perform'), code: z.string().optional().describe('JavaScript code for evaluate action'), }), execute: async ({ action, code }) => { @@ -74,8 +74,8 @@ export function WithWebMode(Base: T) { return { success: ok, message: ok ? 'Browser page recovered' : 'Browser recovery failed' }; } - if (action === 'openFreshTab') { - await ctx.explorBot.openFreshTab(); + if (action === 'openTab') { + await ctx.explorBot.openTab(); const state = ctx.explorBot.stateManager().getCurrentState(); return { success: true, url: state?.url, title: state?.title }; } diff --git a/src/commands/freesail-command.ts b/src/commands/freesail-command.ts index a6afd13..67643e2 100644 --- a/src/commands/freesail-command.ts +++ b/src/commands/freesail-command.ts @@ -70,7 +70,7 @@ export class FreesailCommand extends BaseCommand { } tag('info').log(`Navigating to: ${suggestion.target} - ${suggestion.reason}`); - await this.explorBot.openFreshTab(); + await this.explorBot.openTab(); await this.explorBot.visit(suggestion.target); this.explorBot.clearPlan(); }, diff --git a/src/explorbot.ts b/src/explorbot.ts index 190a932..f9f7cf2 100644 --- a/src/explorbot.ts +++ b/src/explorbot.ts @@ -134,8 +134,8 @@ export class ExplorBot { return this.agentNavigator().visit(url); } - async openFreshTab(): Promise { - await this.explorer.openFreshTab(); + async openTab(): Promise { + await this.explorer.openTab(); } getCurrentState(): WebPageState | null { diff --git a/src/explorer.ts b/src/explorer.ts index b847e7d..ee7e401 100644 --- a/src/explorer.ts +++ b/src/explorer.ts @@ -230,7 +230,7 @@ class Explorer { return { started: true, stop: (meta) => this.finishTest(test, meta) }; } - async openFreshTab(): Promise { + async openTab(): Promise { const oldPage = this.playwrightHelper?.page; if (!oldPage) return; From aae88af4c5291ebcbb85aad6262dce95f72e3e1f Mon Sep 17 00:00:00 2001 From: DavertMik Date: Wed, 22 Jul 2026 03:21:05 +0300 Subject: [PATCH 4/4] fix: do not start test lifecycle when browser recovery fails in beginTest Recovery check now runs before test.start()/reportTestStart(), so a failed recovery no longer leaves a started-but-unfinished test in the reporter. Also replace stale playwrightLocatorCount mention in integration-test docs with the current withPage mock. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 5 +++++ docs/contributing/ai-integration-tests.md | 2 +- src/explorer.ts | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e04d4a0..506c4f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 2026-07-22 + +### Changes +- Fixed a test being reported as started when the browser could not be recovered before it began. Browser recovery now runs before the test starts, so a failed recovery no longer leaves a started-but-never-finished test in the report. + ## 2026-07-17 ### Changes diff --git a/docs/contributing/ai-integration-tests.md b/docs/contributing/ai-integration-tests.md index a092d7c..6eeecdc 100644 --- a/docs/contributing/ai-integration-tests.md +++ b/docs/contributing/ai-integration-tests.md @@ -10,7 +10,7 @@ Reference implementation: `tests/integration/planner.test.ts`. - **The AI provider** — via the aimock HTTP server. Point the Vercel AI SDK at `mock.url/v1` with `createOpenAI({ compatibility: 'compatible' })` and `openai.chat('model-name')`. The `compatibility: 'compatible'` option keeps the mock on the Chat Completions API; without it, the SDK defaults to the Responses API, which aimock does not fully implement. - **Explorer, StateManager, Researcher, ExperienceTracker** — duck-typed mocks with only the methods the agent under test calls. Each agent runs in isolation; downstream agents (such as Researcher when testing Planner) return canned output. -- **`playwrightLocatorCount`** — for agents that validate locators, since no browser runs in these tests. +- **`withPage`** — resolves with a fake page object for agents that validate locators, since no browser runs in these tests. ### What we don't mock diff --git a/src/explorer.ts b/src/explorer.ts index ee7e401..f4dbe27 100644 --- a/src/explorer.ts +++ b/src/explorer.ts @@ -183,15 +183,15 @@ class Explorer { } async beginTest(test: Test): Promise { + if (!this.page && !(await this.recoverOrRestart())) { + return { started: false, stop: async () => {} }; + } + this._activeTest = test; test.start(); await this.reporter.reportTestStart(test); await this.closeOtherTabs(); this.stateManager.otherTabs = []; - if (!this.page && !(await this.recoverOrRestart())) { - this._activeTest = null; - return { started: false, stop: async () => {} }; - } const codeceptjsTest = toCodeceptjsTest(test);