diff --git a/CHANGELOG.md b/CHANGELOG.md index 34736d7..9d7e1ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 2026-07-22 + +### Changes +- Fixed a test being reported as started when the browser could not be recovered before it began. Browser recovery now runs before the test starts, so a failed recovery no longer leaves a started-but-never-finished test in the report. + ## 2026-07-19 ### Environment Variables @@ -39,6 +44,9 @@ Config-free runs do not write experience and run with the Historian off, so no g ## 2026-07-17 ### Changes +- Browser crash recovery now applies to every browser operation. Previously some interactions and page lookups could fail permanently when the page or browser crashed mid-session; now every action, capture, and element query automatically reattaches the page or restarts the browser and retries once before giving up. +- [Captain] The `browser` tool's `restart` action was merged into `recover` — recovering a page now escalates to a full browser restart automatically when needed, so there is no separate action to choose. +- Failed page actions that accidentally entered an iframe now return to the main page automatically. Previously only failed form actions did this; now it applies to every action, so a failed click or keypress can no longer leave the session stuck inside an iframe. - Research no longer discards whole sections of a page when a container matches more than one element. A container only narrows down where child elements are looked up, so a selector matching every navigation bar or every card on the page is now perfectly usable — each child element inside it is still required to be unique. Previously such a section was declared broken and every element in it was sent back to the AI for repair. - When a container matches nothing on the page, Explorbot now repairs it by looking at the elements it found inside it and working out their real wrapper, instead of asking the AI to guess a new selector. The section is fixed instantly and keeps working locators for its elements. When several sections share the same broken container and only some of them can be repaired this way, the remaining sections are still sent to the AI repair step instead of being wrongly treated as fixed. - [Pilot] Pilot's instructions and its list of available tools now stay identical across every call in a session, with the scenario details moved to the end. Providers can reuse the prompt they already processed instead of re-reading it on each call, which lowers the cost of long test runs. diff --git a/bin/explorbot-cli.ts b/bin/explorbot-cli.ts index d710e29..4bb4007 100755 --- a/bin/explorbot-cli.ts +++ b/bin/explorbot-cli.ts @@ -718,11 +718,11 @@ addCommonOptions(program.command('shell ').description('Execute a await explorBot.start(); await explorBot.agentNavigator().visit(url); - const action = explorBot.getExplorer().createAction(); + const action = explorBot.getExplorer().action(); await action.execute(command); log('Command executed successfully'); - const state = explorBot.getExplorer().getStateManager().getCurrentState(); + const state = explorBot.stateManager().getCurrentState(); if (state) log(`URL: ${state.url}`); await explorBot.stop(); diff --git a/boat/doc-collector/src/ai/documentarian.ts b/boat/doc-collector/src/ai/documentarian.ts index 4a6b779..72555f6 100644 --- a/boat/doc-collector/src/ai/documentarian.ts +++ b/boat/doc-collector/src/ai/documentarian.ts @@ -2,6 +2,7 @@ import dedent from 'dedent'; import { z } from 'zod'; import type { AIProvider } from '../../../../src/ai/provider.ts'; import type Explorer from '../../../../src/explorer.ts'; +import type { StateManager } from '../../../../src/state-manager.ts'; import type { WebPageState } from '../../../../src/state-manager.ts'; import { tag } from '../../../../src/utils/logger.ts'; import type { DocbotConfig } from '../config.ts'; @@ -11,11 +12,13 @@ class Documentarian { private provider: AIProvider; private config: DocbotConfig; private explorer?: Explorer; + private stateManager?: StateManager; - constructor(provider: AIProvider, config: DocbotConfig = {}, explorer?: Explorer) { + constructor(provider: AIProvider, config: DocbotConfig = {}, explorer?: Explorer, stateManager?: StateManager) { this.provider = provider; this.config = config; this.explorer = explorer; + this.stateManager = stateManager; } async document(state: WebPageState, research: string, captureState?: CaptureInteractionState): Promise { @@ -46,7 +49,7 @@ class Documentarian { try { tag('info').log('Starting interactive exploration...'); - const deterministicInteractions = await collectDocInteractions(this.explorer!, state, research, this.config, captureState); + const deterministicInteractions = await collectDocInteractions(this.explorer!, this.stateManager!, state, research, this.config, captureState); meaningfulInteractions = this.getMeaningfulInteractions(deterministicInteractions); if (meaningfulInteractions.length > 0) { tag('success').log(`Collected ${meaningfulInteractions.length} deterministic interactions`); diff --git a/boat/doc-collector/src/ai/tools.ts b/boat/doc-collector/src/ai/tools.ts index 2f72b05..9b1ccc8 100644 --- a/boat/doc-collector/src/ai/tools.ts +++ b/boat/doc-collector/src/ai/tools.ts @@ -1,6 +1,6 @@ import { type ResearchElement, parseResearchSections } from '../../../../src/ai/researcher/parser.ts'; import type Explorer from '../../../../src/explorer.ts'; -import type { WebPageState } from '../../../../src/state-manager.ts'; +import type { StateManager, WebPageState } from '../../../../src/state-manager.ts'; import { detectFocusArea } from '../../../../src/utils/aria.ts'; import type { DocbotConfig } from '../config.ts'; @@ -58,14 +58,14 @@ const DEFAULT_WAIT_MS = 700; const TAB_WAIT_MS = 500; const DEFAULT_DENIED_ACTION_LABELS = ['delete', 'remove', 'destroy', 'archive', 'discard', 'logout', 'sign out', 'signout', 'sign_out', 'erase', 'drop']; -export async function collectDocInteractions(explorer: Explorer, state: WebPageState, research: string, config: DocbotConfig = {}, captureState?: CaptureInteractionState): Promise { +export async function collectDocInteractions(explorer: Explorer, stateManager: StateManager, state: WebPageState, research: string, config: DocbotConfig = {}, captureState?: CaptureInteractionState): Promise { const sections = parseResearchSections(research); const transitions: DocStateTransition[] = []; const maxInteractions = getPositiveConfigNumber(config.docs?.maxInteractions, DEFAULT_MAX_INTERACTIONS); const tabGroup = findTabGroup(sections); if (tabGroup) { - transitions.push(...(await exploreTabGroup(explorer, tabGroup, state.url, maxInteractions, captureState))); + transitions.push(...(await exploreTabGroup(explorer, stateManager, tabGroup, state.url, maxInteractions, captureState))); } for (const candidate of findActionCandidates(sections, config)) { @@ -73,7 +73,7 @@ export async function collectDocInteractions(explorer: Explorer, state: WebPageS break; } - const transition = await executeInteraction(explorer, candidate, state.url, DEFAULT_WAIT_MS, captureState); + const transition = await executeInteraction(explorer, stateManager, candidate, state.url, DEFAULT_WAIT_MS, captureState); if (!transition) { continue; } @@ -92,7 +92,7 @@ export function pickDocActionCandidates(research: string, config: DocbotConfig = })); } -async function exploreTabGroup(explorer: Explorer, tabGroup: { elements: ResearchElement[]; container?: string; sectionName: string }, restoreUrl: string, maxInteractions: number, captureState?: CaptureInteractionState): Promise { +async function exploreTabGroup(explorer: Explorer, stateManager: StateManager, tabGroup: { elements: ResearchElement[]; container?: string; sectionName: string }, restoreUrl: string, maxInteractions: number, captureState?: CaptureInteractionState): Promise { const transitions: DocStateTransition[] = []; for (const element of tabGroup.elements) { @@ -102,6 +102,7 @@ async function exploreTabGroup(explorer: Explorer, tabGroup: { elements: Researc const transition = await executeInteraction( explorer, + stateManager, { element, container: tabGroup.container, @@ -123,8 +124,8 @@ async function exploreTabGroup(explorer: Explorer, tabGroup: { elements: Researc return transitions; } -async function executeInteraction(explorer: Explorer, candidate: InteractionCandidate, restoreUrl: string, waitMs: number, captureState?: CaptureInteractionState): Promise { - const beforeState = explorer.getStateManager().getCurrentState(); +async function executeInteraction(explorer: Explorer, stateManager: StateManager, candidate: InteractionCandidate, restoreUrl: string, waitMs: number, captureState?: CaptureInteractionState): Promise { + const beforeState = stateManager.getCurrentState(); if (!beforeState) { return null; } @@ -136,7 +137,7 @@ async function executeInteraction(explorer: Explorer, candidate: InteractionCand await wait(waitMs); - const afterState = explorer.getStateManager().getCurrentState(); + const afterState = stateManager.getCurrentState(); if (!afterState) { return null; } @@ -164,7 +165,7 @@ async function executeInteraction(explorer: Explorer, candidate: InteractionCand } async function attemptInteraction(explorer: Explorer, candidate: InteractionCandidate): Promise { - const action = explorer.createAction(); + const action = explorer.action(); for (const command of buildClickCommands(candidate.element, candidate.container)) { const success = await action.attempt(command, buildPurpose(candidate)); @@ -178,7 +179,7 @@ async function attemptInteraction(explorer: Explorer, candidate: InteractionCand async function restoreInteractionState(explorer: Explorer, restoreUrl: string, primaryCommand?: string | null): Promise { if (primaryCommand) { - const action = explorer.createAction(); + const action = explorer.action(); const restored = await action.attempt(primaryCommand, `Restore initial state on ${restoreUrl}`); if (restored) { await wait(TAB_WAIT_MS); @@ -186,7 +187,7 @@ async function restoreInteractionState(explorer: Explorer, restoreUrl: string, p } } - const action = explorer.createAction(); + const action = explorer.action(); await action.attempt(`I.amOnPage(${JSON.stringify(restoreUrl)})`, `Restore page ${restoreUrl}`); } diff --git a/boat/doc-collector/src/docbot.ts b/boat/doc-collector/src/docbot.ts index ed00560..956629a 100644 --- a/boat/doc-collector/src/docbot.ts +++ b/boat/doc-collector/src/docbot.ts @@ -8,10 +8,10 @@ import { sanitizeFilename } from '../../../src/utils/strings.ts'; import { Documentarian, type PageDocumentation } from './ai/documentarian.ts'; import { type DocbotConfig, DocbotConfigParser } from './config.ts'; import { type DocumentedPage, type SkippedPage, renderPageDocumentation, renderSpecIndex } from './docs-renderer.ts'; -import { renderMermaidBody } from './state-diagram.ts'; import { getDocPageKey, shouldCrawlDocPath } from './path-filter.ts'; import { extractResearchNavigationTargets } from './research-navigation.ts'; import { type DocumentationScreenshot, captureDocumentationScreenshots, captureInteractionScreenshot } from './screenshots.ts'; +import { renderMermaidBody } from './state-diagram.ts'; class DocBot { private explorBot: ExplorBot; @@ -43,7 +43,7 @@ class DocBot { config: this.options.docsConfig, path: this.options.path, }); - this.documentarian = new Documentarian(this.explorBot.getProvider(), this.config, this.explorBot.getExplorer()); + this.documentarian = new Documentarian(this.explorBot.getProvider(), this.config, this.explorBot.getExplorer(), this.explorBot.stateManager()); this.ensureDirectory(this.configParser.getOutputDir()); this.ensureDirectory(this.getPagesDir()); } @@ -76,7 +76,7 @@ class DocBot { continue; } - const stateManager = this.explorBot.getExplorer().getStateManager(); + const stateManager = this.explorBot.stateManager(); if (stateManager.hasVisitedState(target)) { continue; } diff --git a/boat/doc-collector/src/screenshots.ts b/boat/doc-collector/src/screenshots.ts index 4e70d54..81c5ec8 100644 --- a/boat/doc-collector/src/screenshots.ts +++ b/boat/doc-collector/src/screenshots.ts @@ -11,7 +11,7 @@ import type { DocbotConfig } from './config.ts'; const DEFAULT_MAX_SECTION_SCREENSHOTS = 8; export async function captureDocumentationScreenshots(explorer: Explorer, state: WebPageState, research: string, options: DocumentationScreenshotOptions): Promise { - const page = explorer.playwrightHelper?.page; + const page = explorer.page; if (!page) { return []; } @@ -62,7 +62,7 @@ export function getScreenshotSections(research: string): ScreenshotSection[] { } export async function captureInteractionScreenshot(explorer: Explorer, state: WebPageState, transition: DocStateTransition, options: DocumentationScreenshotOptions): Promise { - const page = explorer.playwrightHelper?.page; + const page = explorer.page; if (!page) { return null; } diff --git a/docs/contributing/ai-integration-tests.md b/docs/contributing/ai-integration-tests.md index a092d7c..6eeecdc 100644 --- a/docs/contributing/ai-integration-tests.md +++ b/docs/contributing/ai-integration-tests.md @@ -10,7 +10,7 @@ Reference implementation: `tests/integration/planner.test.ts`. - **The AI provider** — via the aimock HTTP server. Point the Vercel AI SDK at `mock.url/v1` with `createOpenAI({ compatibility: 'compatible' })` and `openai.chat('model-name')`. The `compatibility: 'compatible'` option keeps the mock on the Chat Completions API; without it, the SDK defaults to the Responses API, which aimock does not fully implement. - **Explorer, StateManager, Researcher, ExperienceTracker** — duck-typed mocks with only the methods the agent under test calls. Each agent runs in isolation; downstream agents (such as Researcher when testing Planner) return canned output. -- **`playwrightLocatorCount`** — for agents that validate locators, since no browser runs in these tests. +- **`withPage`** — resolves with a fake page object for agents that validate locators, since no browser runs in these tests. ### What we don't mock diff --git a/src/action.ts b/src/action.ts index df5fd23..dda7cb9 100644 --- a/src/action.ts +++ b/src/action.ts @@ -14,8 +14,8 @@ import { browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } import { captureHtmlForSnapshot, htmlCombinedSnapshot, minifyHtml } from './utils/html.js'; import { createDebug, setStepSpanParent, tag } from './utils/logger.js'; import { sleep, waitForPageReadiness } from './utils/page-readiness.ts'; -import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts'; import { safeFilename } from './utils/strings.ts'; +import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts'; const debugLog = createDebug('explorbot:action'); const CAPTURE_NAVIGATION_TRANSITION_ATTEMPTS = 3; @@ -33,14 +33,16 @@ class Action { public playwrightGroupId: string | null = null; public assertionSteps: Array<{ name: string; args: any[] }> = []; private recorder?: PlaywrightRecorder; + private recovery: RecoveryRunner; private mainDocumentStatus: number | undefined = undefined; - constructor(actor: CodeceptJS.I, stateManager: StateManager, recorder?: PlaywrightRecorder) { + constructor(actor: CodeceptJS.I, stateManager: StateManager, recorder?: PlaywrightRecorder, recovery?: RecoveryRunner) { this.actor = actor; this.stateManager = stateManager; this.config = ConfigParser.getInstance().getConfig(); this.playwrightHelper = container.helpers('Playwright'); this.recorder = recorder; + this.recovery = recovery || ((fn) => fn()); } async saveScreenshot(): Promise { @@ -59,7 +61,15 @@ class Action { } } - async capturePageState({ includeScreenshot = false, codeBlock }: { includeScreenshot?: boolean; codeBlock?: string } = {}): Promise { + async capturePageState(opts: { includeScreenshot?: boolean; codeBlock?: string } = {}): Promise { + return this.recovery(() => this.captureOnce(opts)); + } + + async execute(code: string): Promise { + return this.recovery(() => this.executeOnce(code)); + } + + private async captureOnce({ includeScreenshot = false, codeBlock }: { includeScreenshot?: boolean; codeBlock?: string } = {}): Promise { try { const currentState = this.stateManager.getCurrentState(); const stateHash = currentState?.hash || 'screenshot'; @@ -260,7 +270,7 @@ class Action { } } - async execute(code: string): Promise { + private async executeOnce(code: string): Promise { let error: Error | null = null; setActivity('🔎 Browsing...', 'action'); @@ -301,7 +311,7 @@ class Action { codeString = executedSteps.join('\n'); } - const pageState = await this.capturePageState({ codeBlock: codeString }); + const pageState = await this.captureOnce({ codeBlock: codeString }); this.actionResult = pageState; this.assertionSteps = assertionSteps; @@ -329,6 +339,7 @@ class Action { } public async attempt(codeBlock: string, originalMessage?: string): Promise { + const wasInFrame = !!this.playwrightHelper.frame; try { debugLog('Resolution attempt...'); setActivity('🦾 Acting in browser...', 'action'); @@ -343,11 +354,18 @@ class Action { } catch (error) { this.lastError = error as Error; if (isFatalBrowserError(error)) throw error; + if (!wasInFrame) await this.exitIframe().catch(() => {}); debugLog(`Attempt failed: ${codeBlock}: ${errorToString(error) || this.lastError?.toString()}`); return false; } } + async exitIframe(): Promise { + if (!this.playwrightHelper.frame) return; + debugLog('Switching to main frame'); + await this.playwrightHelper.switchTo(); + } + getActor(): CodeceptJS.I { return this.actor; } @@ -366,6 +384,8 @@ class Action { export default Action; +export type RecoveryRunner = (fn: () => Promise) => Promise; + function errorToString(error: any): string { if (error.cliMessage) { return error.cliMessage(); diff --git a/src/ai/agent.ts b/src/ai/agent.ts index 646238c..75fd319 100644 --- a/src/ai/agent.ts +++ b/src/ai/agent.ts @@ -1,3 +1,23 @@ +import type { RequestStore } from '../api/request-store.ts'; +import type { ExplorbotConfig } from '../config.ts'; +import type Explorer from '../explorer.ts'; +import type { KnowledgeTracker } from '../knowledge-tracker.ts'; +import type { PlaywrightRecorder } from '../playwright-recorder.ts'; +import type { StateManager } from '../state-manager.ts'; +import type { AIProvider } from './provider.ts'; + export interface Agent { emoji?: string; } + +export interface AgentDeps { + explorer: Explorer; + ai: AIProvider; + config: ExplorbotConfig; + stateManager: StateManager; + knowledgeTracker: KnowledgeTracker; + requestStore: RequestStore; + playwrightRecorder: PlaywrightRecorder; +} + +export type ToolDeps = Pick; diff --git a/src/ai/captain.ts b/src/ai/captain.ts index bd87e64..2be8fbf 100644 --- a/src/ai/captain.ts +++ b/src/ai/captain.ts @@ -49,7 +49,7 @@ export class Captain extends CaptainBase implements Agent { private getHooksRunner(): HooksRunner { if (!this.hooksRunner) { const explorer = this.explorBot.getExplorer(); - this.hooksRunner = new HooksRunner(explorer, explorer.getConfig()); + this.hooksRunner = new HooksRunner(explorer, this.explorBot.getConfig()); } return this.hooksRunner; } @@ -89,17 +89,16 @@ export class Captain extends CaptainBase implements Agent { getMode(): CaptainMode { const explorer = this.explorBot.getExplorer(); const activeTest = explorer.activeTest; - const page = explorer.playwrightHelper?.page; - if (activeTest && (!page || page.isClosed?.())) return 'heal'; + if (activeTest && !explorer.page) return 'heal'; if (activeTest) return 'test'; - if (explorer.getStateManager().getCurrentState()) return 'web'; + if (this.explorBot.stateManager().getCurrentState()) return 'web'; return 'idle'; } private systemPrompt(): string { const mode = this.getMode(); - const currentUrl = this.explorBot.getExplorer().getStateManager().getCurrentState()?.url; + const currentUrl = this.explorBot.stateManager().getCurrentState()?.url; const customPrompt = this.explorBot.getProvider().getSystemPromptForAgent('captain', currentUrl); return dedent` @@ -147,7 +146,7 @@ export class Captain extends CaptainBase implements Agent { } private async getPageContext(): Promise { - const state = this.explorBot.getExplorer().getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { return 'No page loaded'; } @@ -326,7 +325,7 @@ export class Captain extends CaptainBase implements Agent { } const pilotAnalysis = this.explorBot.agentPilot().getLastAnalysis() || ''; - const currentUrl = this.explorBot.getExplorer().getStateManager().getCurrentState()?.url || ''; + const currentUrl = this.explorBot.stateManager().getCurrentState()?.url || ''; const schema = z.object({ action: z.enum(['inject', 'stop', 'pass', 'skip']), @@ -377,7 +376,7 @@ export class Captain extends CaptainBase implements Agent { async processExecutionError(error: Error, activeTest: Test): Promise { const explorer = this.explorBot.getExplorer(); - const result = await explorer.handleExecutionError(error); + const result = await explorer.recover(error); return { ...result, message: result.recovered ? `${result.message}\nContinue the test "${activeTest.scenario}" from the restored page.` : result.message, @@ -389,7 +388,7 @@ export class Captain extends CaptainBase implements Agent { } async handle(input: string, options: { reset?: boolean } = {}): Promise { - const stateManager = this.explorBot.getExplorer().getStateManager(); + const stateManager = this.explorBot.stateManager(); const initialState = stateManager.getCurrentState(); const conversation = options.reset ? this.resetConversation() : this.ensureConversation(); diff --git a/src/ai/captain/idle-mode.ts b/src/ai/captain/idle-mode.ts index a49abb2..c99f425 100644 --- a/src/ai/captain/idle-mode.ts +++ b/src/ai/captain/idle-mode.ts @@ -66,7 +66,7 @@ export function WithIdleMode(Base: T) { if (!action || action === 'replace') { plan.tests.length = 0; } - const currentUrl = ctx.explorBot.getExplorer().getStateManager().getCurrentState()?.url || ''; + const currentUrl = ctx.explorBot.stateManager().getCurrentState()?.url || ''; for (const testInput of tests) { const priority = testInput.priority || 'normal'; const expected = testInput.expected?.length ? testInput.expected : []; diff --git a/src/ai/captain/test-mode.ts b/src/ai/captain/test-mode.ts index a29361e..105b04a 100644 --- a/src/ai/captain/test-mode.ts +++ b/src/ai/captain/test-mode.ts @@ -146,7 +146,7 @@ export function WithTestMode(Base: T) { if (!t) return { success: false, message: `Test "${session}" not found` }; states = t.states; } else { - const stateManager = ctx.explorBot.getExplorer().getStateManager(); + const stateManager = ctx.explorBot.stateManager(); const history = stateManager.getStateHistory(); const seen = new Set(); states = history diff --git a/src/ai/captain/web-mode.ts b/src/ai/captain/web-mode.ts index e31295f..ebbaaad 100644 --- a/src/ai/captain/web-mode.ts +++ b/src/ai/captain/web-mode.ts @@ -9,9 +9,10 @@ export function WithWebMode(Base: T) { return class extends Base { webModeTools(ctx: ModeContext): Record { const explorer = ctx.explorBot.getExplorer(); - const codeceptTools = createCodeceptJSTools(explorer, ctx.task); + const toolDeps = { explorer, stateManager: ctx.explorBot.stateManager(), ai: ctx.explorBot.getProvider() }; + const codeceptTools = createCodeceptJSTools(toolDeps, ctx.task); const agentTools = createAgentTools({ - explorer, + ...toolDeps, researcher: ctx.explorBot.agentResearcher(), navigator: ctx.explorBot.agentNavigator(), }); @@ -27,7 +28,7 @@ export function WithWebMode(Base: T) { try { debugLog('navigate', destination); await ctx.explorBot.agentNavigator().visit(destination); - const stateManager = ctx.explorBot.getExplorer().getStateManager(); + const stateManager = ctx.explorBot.stateManager(); const state = stateManager.getCurrentState(); return { success: true, url: state?.url, title: state?.title }; } catch (error: any) { @@ -45,48 +46,42 @@ export function WithWebMode(Base: T) { - closeTabs: Close all browser tabs except the current one - reload: Reload the current page - screenshot: Take a screenshot of current page - - recover: Recover from a closed/crashed page using Explorer recovery - - restart: Restart the browser when page/context recovery is not enough - - openFreshTab: Open a fresh tab in the current browser context + - recover: Recover from a closed/crashed page, escalating to a full browser restart when needed + - openTab: Open a fresh tab in the current browser context `, inputSchema: z.object({ - action: z.enum(['status', 'evaluate', 'closeTabs', 'reload', 'screenshot', 'recover', 'restart', 'openFreshTab']).describe('Browser action to perform'), + action: z.enum(['status', 'evaluate', 'closeTabs', 'reload', 'screenshot', 'recover', 'openTab']).describe('Browser action to perform'), code: z.string().optional().describe('JavaScript code for evaluate action'), }), execute: async ({ action, code }) => { const explorer = ctx.explorBot.getExplorer(); if (action === 'status') { - const page = explorer.playwrightHelper?.page; + const page = explorer.page; const pages = page?.context?.().pages?.() || []; return { success: true, hasPage: !!page, - isClosed: page?.isClosed?.() || false, - url: page && !page.isClosed?.() ? await page.url() : null, - title: page && !page.isClosed?.() ? await page.title().catch(() => null) : null, + isClosed: !page, + url: page ? await page.url() : null, + title: page ? await page.title().catch(() => null) : null, tabs: pages.length, }; } if (action === 'recover') { - const recovered = await explorer.recoverFromBrowserError(); - return { success: recovered, message: recovered ? 'Browser page recovered' : 'Browser recovery failed' }; + const { ok } = await explorer.recover(); + return { success: ok, message: ok ? 'Browser page recovered' : 'Browser recovery failed' }; } - if (action === 'restart') { - const restarted = await explorer.restartBrowser(); - return { success: restarted, message: restarted ? 'Browser restarted' : 'Browser restart failed' }; - } - - if (action === 'openFreshTab') { - await ctx.explorBot.openFreshTab(); - const state = explorer.getStateManager().getCurrentState(); + if (action === 'openTab') { + await ctx.explorBot.openTab(); + const state = ctx.explorBot.stateManager().getCurrentState(); return { success: true, url: state?.url, title: state?.title }; } - const page = explorer.playwrightHelper?.page; - if (!page || page.isClosed?.()) return { success: false, message: 'No browser page available. Try browser({ action: "recover" }) first.' }; + const page = explorer.page; + if (!page) return { success: false, message: 'No browser page available. Try browser({ action: "recover" }) first.' }; if (action === 'evaluate') { if (!code) return { success: false, message: 'Code required for evaluate action' }; diff --git a/src/ai/driller.ts b/src/ai/driller.ts index 0a064e6..8199f63 100644 --- a/src/ai/driller.ts +++ b/src/ai/driller.ts @@ -3,13 +3,9 @@ import dedent from 'dedent'; import { z } from 'zod'; import { ActionResult } from '../action-result.ts'; import { setActivity } from '../activity.ts'; -import type { ExperienceTracker } from '../experience-tracker.ts'; -import type Explorer from '../explorer.ts'; -import type { KnowledgeTracker } from '../knowledge-tracker.ts'; import { Observability } from '../observability.ts'; import { Plan, Test, TestResult } from '../test-plan.ts'; import { collectInteractiveNodes } from '../utils/aria.ts'; -import { HooksRunner } from '../utils/hooks-runner.ts'; import { EXPLORBOT_ATTRS, HTML_COMPOSITE_AREA_HINTS, @@ -26,8 +22,10 @@ import { } from '../utils/html.ts'; import { createDebug, tag } from '../utils/logger.ts'; import { loop, pause } from '../utils/loop.ts'; +import { annotatePageElements } from '../utils/web-annotate.ts'; +import { eidxInContainer } from '../utils/web-eidx.ts'; import { WebElement } from '../utils/web-element.ts'; -import type { Agent } from './agent.ts'; +import type { Agent, AgentDeps } from './agent.ts'; import type { Navigator } from './navigator.ts'; import type { Provider } from './provider.ts'; import { drillLocatorRule } from './rules.ts'; @@ -79,10 +77,7 @@ interface DrillOptions { export class Driller extends TaskAgent implements Agent { protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form']; emoji = 'D'; - private explorer: Explorer; - private provider: Provider; private navigator: Navigator; - private hooksRunner: HooksRunner; private currentPlan?: Plan; private allResults: InteractionResult[] = []; private verifiedAction: { componentId: string; toolName: string; code?: string; canonicalCode?: string } | null = null; @@ -90,32 +85,17 @@ export class Driller extends TaskAgent implements Agent { MAX_COMPONENT_ITERATIONS = 12; - constructor(explorer: Explorer, provider: Provider, navigator: Navigator) { - super(); - this.explorer = explorer; - this.provider = provider; + constructor(deps: AgentDeps, navigator: Navigator) { + super(deps); this.navigator = navigator; - this.hooksRunner = new HooksRunner(explorer, explorer.getConfig()); } protected getNavigator(): Navigator { return this.navigator; } - protected getExperienceTracker(): ExperienceTracker { - return this.explorer.getStateManager().getExperienceTracker(); - } - - protected getKnowledgeTracker(): KnowledgeTracker { - return this.explorer.getKnowledgeTracker(); - } - - protected getProvider(): Provider { - return this.provider; - } - getSystemMessage(component?: ComponentInfo): string { - const currentUrl = this.explorer.getStateManager().getCurrentState()?.url; + const currentUrl = this.stateManager.getCurrentState()?.url; const customPrompt = this.provider.getSystemPromptForAgent('driller', currentUrl); return dedent` @@ -159,7 +139,7 @@ export class Driller extends TaskAgent implements Agent { async drill(opts: DrillOptions = {}): Promise { const { knowledgePath, maxComponents = 30, interactive = isInteractive() } = opts; - const currentState = this.explorer.getStateManager().getCurrentState(); + const currentState = this.stateManager.getCurrentState(); if (!currentState) throw new Error('No page state available'); const sessionName = `driller_${Date.now().toString(36)}`; @@ -207,15 +187,15 @@ export class Driller extends TaskAgent implements Agent { private async captureAnnotatedState(): Promise { setActivity(`${this.emoji} Capturing annotated page state...`, 'action'); - const action = this.explorer.createAction(); + const action = this.explorer.action(); try { const annotated = await Promise.race([ - this.explorer.annotateElements(), + this.explorer.withPage(annotatePageElements), new Promise((_, reject) => { setTimeout(() => reject(new Error('annotateElements timeout')), 15000); }), ]); - return action.capturePageState({ ariaSnapshot: annotated.ariaSnapshot }); + return action.capturePageState(); } catch (error) { tag('warning').log(`Annotated capture failed, falling back to plain page state: ${error instanceof Error ? error.message : error}`); return action.capturePageState(); @@ -226,9 +206,10 @@ export class Driller extends TaskAgent implements Agent { private async collectComponents(state: ActionResult, maxComponents: number): Promise { setActivity(`${this.emoji} Collecting components...`, 'action'); - const page = this.explorer.playwrightHelper.page; - const eidxList = await this.explorer.getEidxInContainer(null); - const webElements = await WebElement.fromEidxList(page, eidxList); + const webElements = await this.explorer.withPage(async (page) => { + const eidxList = await eidxInContainer(page, null); + return WebElement.fromEidxList(page, eidxList); + }); const ariaNodes = collectInteractiveNodes(state.ariaSnapshot); const scored = webElements .filter((element) => isDrillableElement(element)) @@ -324,7 +305,7 @@ export class Driller extends TaskAgent implements Agent { conversation.addUserText(await this.buildComponentPrompt(originalState, component)); let finished = false; - const actionTools = this.createVerifiedActionTools(createCodeceptJSTools(this.explorer, test), component); + const actionTools = this.createVerifiedActionTools(createCodeceptJSTools(this.toolDeps, test), component); const tools = { ...actionTools, ...this.createDrillFlowTools(originalState, test, interactive) }; await loop( @@ -333,7 +314,7 @@ export class Driller extends TaskAgent implements Agent { setActivity(`${this.emoji} Drilling ${component.name}...`, 'action'); if (iteration > 1) { - const currentState = ActionResult.fromState(this.explorer.getStateManager().getCurrentState() || originalState); + const currentState = ActionResult.fromState(this.stateManager.getCurrentState() || originalState); conversation.addUserText(await this.buildContextUpdate(currentState, component)); if (this.pendingNestedContext) { conversation.addUserText(this.pendingNestedContext); @@ -577,7 +558,7 @@ export class Driller extends TaskAgent implements Agent { execute: async ({ reason }) => { await this.restoreOriginalState(originalState, reason); await this.captureAnnotatedState(); - const currentState = this.explorer.getStateManager().getCurrentState(); + const currentState = this.stateManager.getCurrentState(); return { success: true, url: currentState?.url || originalState.url }; }, }), @@ -598,9 +579,9 @@ export class Driller extends TaskAgent implements Agent { } private async restoreOriginalState(originalState: ActionResult, reason: string): Promise { - const currentState = this.explorer.getStateManager().getCurrentState(); + const currentState = this.stateManager.getCurrentState(); const targetUrl = originalState.fullUrl || originalState.url; - const action = this.explorer.createAction(); + const action = this.explorer.action(); if (currentState?.url !== originalState.url) { await action.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, `${reason} (restore URL)`); @@ -665,7 +646,7 @@ export class Driller extends TaskAgent implements Agent { const overlayHtml = await this.getVisibleOverlayHtml(); if (!overlayHtml) return null; - const state = this.explorer.getStateManager().getCurrentState(); + const state = this.stateManager.getCurrentState(); if (!state) return null; const currentState = ActionResult.fromState(state); return dedent` @@ -686,40 +667,42 @@ export class Driller extends TaskAgent implements Agent { } private async getVisibleOverlayHtml(): Promise { - const page = this.explorer.playwrightHelper.page; - return page.evaluate( - ({ extractorSource, config }) => { - const extract = new Function(`return ${extractorSource}`)() as (config: any) => string; - return extract(config); - }, - { - extractorSource: getVisibleOverlayHtmlExtractorSource(), - config: { - interactiveContentSelector: HTML_SELECTORS.interactiveContent, - limits: HTML_EXTRACTION_LIMITS, - overlaySelectors: HTML_SELECTORS.semanticOverlays, - visibilityLimits: HTML_VISIBILITY_LIMITS, + return this.explorer.withPage((page) => + page.evaluate( + ({ extractorSource, config }) => { + const extract = new Function(`return ${extractorSource}`)() as (config: any) => string; + return extract(config); }, - } + { + extractorSource: getVisibleOverlayHtmlExtractorSource(), + config: { + interactiveContentSelector: HTML_SELECTORS.interactiveContent, + limits: HTML_EXTRACTION_LIMITS, + overlaySelectors: HTML_SELECTORS.semanticOverlays, + visibilityLimits: HTML_VISIBILITY_LIMITS, + }, + } + ) ); } private async getComponentScopeHtml(component: ComponentInfo, originalState: ActionResult): Promise { - const page = this.explorer.playwrightHelper.page; - const scopedHtml = await page.evaluate( - ({ eidx, extractorSource, config }) => { - const extract = new Function(`return ${extractorSource}`)() as (eidx: string, config: any) => string; - return extract(eidx, config); - }, - { - eidx: component.eidx, - extractorSource: getComponentScopeHtmlExtractorSource(), - config: { - eidxAttr: EXPLORBOT_ATTRS.eidx, - interactiveControlSelector: HTML_SELECTORS.interactiveControl, - limits: HTML_EXTRACTION_LIMITS, + const scopedHtml = await this.explorer.withPage((page) => + page.evaluate( + ({ eidx, extractorSource, config }) => { + const extract = new Function(`return ${extractorSource}`)() as (eidx: string, config: any) => string; + return extract(eidx, config); }, - } + { + eidx: component.eidx, + extractorSource: getComponentScopeHtmlExtractorSource(), + config: { + eidxAttr: EXPLORBOT_ATTRS.eidx, + interactiveControlSelector: HTML_SELECTORS.interactiveControl, + limits: HTML_EXTRACTION_LIMITS, + }, + } + ) ); if (scopedHtml) return scopedHtml; diff --git a/src/ai/historian.ts b/src/ai/historian.ts index 25ae7c3..dc34513 100644 --- a/src/ai/historian.ts +++ b/src/ai/historian.ts @@ -1,6 +1,7 @@ import { readFileSync, writeFileSync } from 'node:fs'; import type { ExplorbotConfig } from '../config.ts'; import type { ExperienceTracker } from '../experience-tracker.ts'; +import type Explorer from '../explorer.ts'; import type { PlaywrightRecorder } from '../playwright-recorder.ts'; import type { Reporter } from '../reporter.ts'; import type { StateManager } from '../state-manager.ts'; @@ -23,10 +24,10 @@ export class Historian extends HistorianBase { declare reporter: Reporter | undefined; declare stateManager: StateManager | undefined; declare config: ExplorbotConfig | undefined; - declare playwright: { recorder: PlaywrightRecorder; helper: any } | undefined; + declare playwright: { recorder: PlaywrightRecorder; explorer: Explorer } | undefined; declare savedFiles: Set; - constructor(provider: Provider, experienceTracker: ExperienceTracker, reporter?: Reporter, stateManager?: StateManager, config?: ExplorbotConfig, playwright?: { recorder: PlaywrightRecorder; helper: any }) { + constructor(provider: Provider, experienceTracker: ExperienceTracker, reporter?: Reporter, stateManager?: StateManager, config?: ExplorbotConfig, playwright?: { recorder: PlaywrightRecorder; explorer: Explorer }) { super(); this.provider = provider; this.experienceTracker = experienceTracker; diff --git a/src/ai/historian/codeceptjs.ts b/src/ai/historian/codeceptjs.ts index c313a15..b48e340 100644 --- a/src/ai/historian/codeceptjs.ts +++ b/src/ai/historian/codeceptjs.ts @@ -6,8 +6,8 @@ import type { StateManager } from '../../state-manager.ts'; import type { Plan } from '../../test-plan.ts'; import { tag } from '../../utils/logger.ts'; import { relativeToCwd } from '../../utils/next-steps.ts'; -import { safeFilename } from '../../utils/strings.ts'; import { CODECEPT_TOOLS, isNonReusableCode, stripComments } from '../../utils/step-analyzer.ts'; +import { safeFilename } from '../../utils/strings.ts'; import type { Conversation } from '../conversation.ts'; import { ASSERTION_TOOLS } from '../tools.ts'; import type { Constructor } from './mixin.ts'; diff --git a/src/ai/historian/playwright.ts b/src/ai/historian/playwright.ts index 7386ee8..df8b7e7 100644 --- a/src/ai/historian/playwright.ts +++ b/src/ai/historian/playwright.ts @@ -2,13 +2,14 @@ import { mkdirSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { ActionResult } from '../../action-result.ts'; import { ConfigParser } from '../../config.ts'; -import type { StateManager } from '../../state-manager.ts'; +import type Explorer from '../../explorer.ts'; import { type PlaywrightRecorder, type TraceCall, renderAssertion, renderCall } from '../../playwright-recorder.ts'; +import type { StateManager } from '../../state-manager.ts'; import type { Plan } from '../../test-plan.ts'; import { tag } from '../../utils/logger.ts'; import { relativeToCwd } from '../../utils/next-steps.ts'; -import { safeFilename } from '../../utils/strings.ts'; import { CODECEPT_TOOLS } from '../../utils/step-analyzer.ts'; +import { safeFilename } from '../../utils/strings.ts'; import type { Conversation } from '../conversation.ts'; import { ASSERTION_TOOLS } from '../tools.ts'; import type { Constructor } from './mixin.ts'; @@ -23,7 +24,7 @@ export interface PlaywrightMethods { export function WithPlaywright(Base: T) { return class extends Base { - declare playwright: { recorder: PlaywrightRecorder; helper: any } | undefined; + declare playwright: { recorder: PlaywrightRecorder; explorer: Explorer } | undefined; declare savedFiles: Set; declare stateManager?: StateManager; diff --git a/src/ai/historian/screencast.ts b/src/ai/historian/screencast.ts index 1384345..60e0085 100644 --- a/src/ai/historian/screencast.ts +++ b/src/ai/historian/screencast.ts @@ -4,6 +4,7 @@ import { join } from 'node:path'; import * as codeceptjs from 'codeceptjs'; import { outputPath } from '../../config.ts'; import type { ExplorbotConfig } from '../../config.ts'; +import type Explorer from '../../explorer.ts'; import type { PlaywrightRecorder } from '../../playwright-recorder.ts'; import { tag } from '../../utils/logger.ts'; import { relativeToCwd } from '../../utils/next-steps.ts'; @@ -22,7 +23,7 @@ export function WithScreencast(Base: T) { return class extends Base { declare config: ExplorbotConfig | undefined; declare savedFiles: Set; - declare playwright: { recorder: PlaywrightRecorder; helper: any } | undefined; + declare playwright: { recorder: PlaywrightRecorder; explorer: Explorer } | undefined; private screencastPage: any = null; private screencastActive = false; @@ -41,7 +42,7 @@ export function WithScreencast(Base: T) { attachScreencast(): void { if (this.screencastListenersInstalled) return; if (!this.config?.ai?.agents?.historian?.screencast) return; - if (!this.playwright?.helper) return; + if (!this.playwright?.explorer) return; this.onTestBefore = (test: any) => { void this.startScreencast(test); @@ -62,7 +63,7 @@ export function WithScreencast(Base: T) { private async startScreencast(test: any): Promise { if (this.screencastActive) return; - const page = this.playwright?.helper?.page; + const page = this.playwright?.explorer?.page; if (!page?.screencast?.start) return; const task = test?._explorbotTest; diff --git a/src/ai/navigator.ts b/src/ai/navigator.ts index c15b2d1..0599b4a 100644 --- a/src/ai/navigator.ts +++ b/src/ai/navigator.ts @@ -3,17 +3,18 @@ import dedent from 'dedent'; import { z } from 'zod'; import { ActionResult } from '../action-result.js'; import type Action from '../action.ts'; +import type { ExplorbotConfig } from '../config.ts'; import type { ExperienceTracker } from '../experience-tracker.js'; import Explorer from '../explorer.ts'; import type { KnowledgeTracker } from '../knowledge-tracker.js'; -import { normalizeUrl } from '../state-manager.js'; +import { type StateManager, normalizeUrl } from '../state-manager.js'; import { extractCodeBlocks } from '../utils/code-extractor.js'; import { HooksRunner } from '../utils/hooks-runner.ts'; import { createDebug, pluralize, tag } from '../utils/logger.js'; import { loop, pause } from '../utils/loop.js'; import { RulesLoader } from '../utils/rules-loader.ts'; import { extractStatePath } from '../utils/url-matcher.js'; -import type { Agent } from './agent.js'; +import type { Agent, AgentDeps } from './agent.js'; import type { Conversation } from './conversation.js'; import type { Provider } from './provider.js'; import { Researcher } from './researcher.ts'; @@ -70,25 +71,29 @@ class Navigator implements Agent { `; private explorer: Explorer; - - constructor(explorer: Explorer, provider: Provider) { - this.provider = provider; - this.explorer = explorer; - this.knowledgeTracker = explorer.getKnowledgeTracker(); - this.experienceTracker = explorer.getStateManager().getExperienceTracker(); - this.hooksRunner = new HooksRunner(explorer, explorer.getConfig()); + private config: ExplorbotConfig; + private stateManager: StateManager; + + constructor(deps: AgentDeps) { + this.provider = deps.ai; + this.explorer = deps.explorer; + this.config = deps.config; + this.stateManager = deps.stateManager; + this.knowledgeTracker = deps.knowledgeTracker; + this.experienceTracker = deps.stateManager.getExperienceTracker(); + this.hooksRunner = new HooksRunner(deps.explorer, deps.config); } private get verifyAttempts(): number { - return this.explorer.getConfig().ai?.agents?.navigator?.verifyAttempts ?? 3; + return this.config.ai?.agents?.navigator?.verifyAttempts ?? 3; } private get verifyTimeout(): number { - return this.explorer.getConfig().ai?.agents?.navigator?.verifyTimeout ?? 1500; + return this.config.ai?.agents?.navigator?.verifyTimeout ?? 1500; } private getBaseOrigin(): string | null { - const baseUrl = this.explorer.getConfig().playwright.url; + const baseUrl = this.config.playwright.url; try { return new URL(baseUrl).origin; } catch { @@ -133,12 +138,12 @@ class Navigator implements Agent { } async visit(url: string): Promise { - return this.explorer.runWithBrowserRecovery('navigator.visit', () => this.visitOnce(url)); + return this.visitOnce(url); } private async visitOnce(url: string): Promise { try { - const action = this.explorer.createAction(); + const action = this.explorer.action(); await action.execute(`I.amOnPage('${url}')`); @@ -171,7 +176,7 @@ class Navigator implements Agent { throw new Error(`Navigation to ${url} failed: ${action.lastError?.message}`); } } - await this.explorer.capturePageWithScreenshot(); + await this.explorer.capture({ screenshot: true }); await this.hooksRunner.runAfterHook('navigator', url); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); @@ -188,7 +193,7 @@ class Navigator implements Agent { tag('info').log('AI Navigator resolving state at', actionResult.url); debugLog('Resolution message:', message); - const action = opts?.action ?? this.explorer.createAction(); + const action = opts?.action ?? this.explorer.action(); const expectedUrl = opts?.expectedUrl; const knowledge = this.knowledgeTracker.renderRelevantKnowledge(actionResult); @@ -341,7 +346,7 @@ class Navigator implements Agent { codeBlockIndex++; totalAttempts++; - await this.explorer.switchToMainFrame(); + await action.exitIframe(); const prevActionResult = action.actionResult ?? actionResult; const prevHash = prevActionResult.getStateHash(); @@ -373,7 +378,7 @@ class Navigator implements Agent { // URL did not transition to expectedUrl within timeout } } - const freshState = await this.explorer.capturePageState(); + const freshState = await this.explorer.capture(); const currentUrl = /^https?:\/\//i.test(expectedUrl) ? freshState.fullUrl || freshState.url || '' : freshState.url || ''; const urlMatches = this.isSameExpectedOrigin(expectedUrl, action.stateManager) && normalizeUrl(currentUrl) === normalizeUrl(expectedUrl); const stateChanged = freshState.getStateHash() !== actionResult.getStateHash(); @@ -473,7 +478,7 @@ class Navigator implements Agent { private buildExperienceTools(): { learnExperience: unknown } | undefined { if (!this.experienceTracker) return undefined; - const stateManager = this.explorer.getStateManager(); + const stateManager = this.stateManager; const getState = () => { const s = stateManager.getCurrentState(); return s ? ActionResult.fromState(s) : null; @@ -482,7 +487,7 @@ class Navigator implements Agent { } async freeSail(opts?: { strategy?: 'deep' | 'shallow'; scope?: string; visitedUrls?: Set }, actionResult?: ActionResult): Promise<{ target: string; reason: string } | null> { - const stateManager = this.explorer.getStateManager(); + const stateManager = this.stateManager; const state = stateManager.getCurrentState(); if (!state) { return null; @@ -690,11 +695,11 @@ class Navigator implements Agent { const successfulCodes: string[] = []; const assertionSteps: Array<{ name: string; args: any[] }> = []; - const action = this.explorer.createAction(); + const action = this.explorer.action(); let failures = 0; - const page = this.explorer.playwrightHelper?.page; - const originalTimeout = this.explorer.playwrightHelper?.options?.timeout ?? 3000; + const page = this.explorer.page; + const originalTimeout = this.config.playwright.timeout ?? 3000; page?.setDefaultTimeout(this.verifyTimeout); try { @@ -726,7 +731,7 @@ class Navigator implements Agent { return; } - await this.explorer.switchToMainFrame(); + await action.exitIframe(); const verified = await action.attempt(codeBlock, message); @@ -764,7 +769,7 @@ class Navigator implements Agent { if (alreadyVerified) verified = true; actionResult.addVerification(message, verified); - this.explorer.getStateManager().updateState(actionResult); + this.stateManager.updateState(actionResult); return { verified, successfulCodes, assertionSteps, totalAttempted }; } diff --git a/src/ai/pilot.ts b/src/ai/pilot.ts index 9983217..1aa6e09 100644 --- a/src/ai/pilot.ts +++ b/src/ai/pilot.ts @@ -2,8 +2,11 @@ import { tool } from 'ai'; import dedent from 'dedent'; import { z } from 'zod'; import { ActionResult } from '../action-result.ts'; +import type { RequestStore } from '../api/request-store.ts'; import { ConfigParser } from '../config.ts'; import type Explorer from '../explorer.ts'; +import type { PlaywrightRecorder } from '../playwright-recorder.ts'; +import type { StateManager } from '../state-manager.ts'; import { type Test, TestResult } from '../test-plan.ts'; import { collectInteractiveNodes, detectFocusArea, extractFocusedElement } from '../utils/aria.ts'; import { ErrorPageError } from '../utils/error-page.ts'; @@ -11,7 +14,7 @@ import { createDebug, tag } from '../utils/logger.ts'; const debugLog = createDebug('explorbot:pilot'); import { truncateJson } from '../utils/strings.ts'; -import type { Agent } from './agent.ts'; +import type { Agent, AgentDeps } from './agent.ts'; import type { Conversation } from './conversation.ts'; import type { Fisherman } from './fisherman.ts'; import type { Navigator } from './navigator.ts'; @@ -30,13 +33,19 @@ export class Pilot implements Agent { private conversation: Conversation | null = null; private researcher: Researcher; private explorer: Explorer; + private stateManager: StateManager; + private requestStore: RequestStore; + private playwrightRecorder: PlaywrightRecorder; private fisherman: Fisherman | null = null; - constructor(provider: Provider, agentTools: any, researcher: Researcher, explorer: Explorer) { - this.provider = provider; + constructor(deps: AgentDeps, agentTools: any, researcher: Researcher) { + this.provider = deps.ai; this.agentTools = agentTools; this.researcher = researcher; - this.explorer = explorer; + this.explorer = deps.explorer; + this.stateManager = deps.stateManager; + this.requestStore = deps.requestStore; + this.playwrightRecorder = deps.playwrightRecorder; } setFisherman(fisherman: Fisherman): void { @@ -91,7 +100,7 @@ export class Pilot implements Agent { let screenshotState: ActionResult | null = null; if (type === 'finish' && this.provider.hasVision()) { try { - screenshotState = await this.explorer.capturePageWithScreenshot(); + screenshotState = await this.explorer.capture({ screenshot: true }); if (screenshotState.screenshot) { visualAnalysis = (await this.researcher.answerQuestionAboutScreenshot(screenshotState, `Describe current page state relevant to: ${task.scenario}`)) || ''; } @@ -175,7 +184,7 @@ export class Pilot implements Agent { tag('substep').log(`Pilot requesting verification: ${result.requestVerification}`); const verifyResult = await navigator.verifyState(result.requestVerification, currentState).catch(() => null); if (verifyResult?.verified && verifyResult.assertionSteps?.length) { - this.explorer.getPlaywrightRecorder().recordVerification(verifyResult.assertionSteps); + this.playwrightRecorder.recordVerification(verifyResult.assertionSteps); } } @@ -595,9 +604,9 @@ export class Pilot implements Agent { } private getExperienceToc(): string { - const state = this.explorer.getStateManager().getCurrentState(); + const state = this.stateManager.getCurrentState(); if (!state) return ''; - return this.explorer.getStateManager().getExperienceTracker().renderExperienceTocFor(ActionResult.fromState(state)); + return this.stateManager.getExperienceTracker().renderExperienceTocFor(ActionResult.fromState(state)); } private pickPlanningTools() { @@ -660,7 +669,7 @@ export class Pilot implements Agent { private async checkDataAvailability(task: Test, requestedData: string, fishermanReason: string | undefined): Promise { if (!this.provider.hasVision()) return null; - const screenshotState = await this.explorer.capturePageWithScreenshot().catch(() => null); + const screenshotState = await this.explorer.capture({ screenshot: true }).catch(() => null); if (!screenshotState?.screenshot) return null; const question = dedent` @@ -712,8 +721,8 @@ export class Pilot implements Agent { lines.push('modal: none'); } - if (this.explorer.hasOtherTabs()) { - const tabs = this.explorer.getOtherTabsInfo(); + const tabs = this.stateManager.otherTabs; + if (tabs.length > 0) { lines.push(`other tabs: ${tabs.length} (${tabs.map((t) => `${t.url} - ${t.title}`).join(', ')})`); } else { lines.push('other tabs: none'); @@ -736,7 +745,7 @@ export class Pilot implements Agent { lines.push('console errors: none'); } - const failedRequests = this.explorer.getRequestStore()?.getFailedRequests() ?? []; + const failedRequests = this.requestStore.getFailedRequests(); if (failedRequests.length > 0) { const sample = failedRequests .slice(-5) @@ -817,7 +826,7 @@ export class Pilot implements Agent { private formatSessionLog(testerConversation: Conversation): string { const executions = testerConversation.getToolExecutions().filter((t) => !META_TOOLS.includes(t.toolName)); - const stateHistory = this.explorer.getStateManager().getStateHistory(); + const stateHistory = this.stateManager.getStateHistory(); const initialUrl = stateHistory[0]?.toState?.url || ''; let currentUrl = initialUrl; diff --git a/src/ai/planner.ts b/src/ai/planner.ts index 07c838b..879dfec 100644 --- a/src/ai/planner.ts +++ b/src/ai/planner.ts @@ -14,7 +14,7 @@ import { createDebug, tag } from '../utils/logger.js'; import { jsonToTable } from '../utils/markdown-parser.ts'; import { mdq } from '../utils/markdown-query.js'; import { planToCompactAiContext } from '../utils/test-plan-markdown.ts'; -import type { Agent } from './agent.js'; +import type { Agent, AgentDeps } from './agent.js'; import { Conversation } from './conversation.ts'; import type { Fisherman } from './fisherman.ts'; import { WithSessionDedup } from './planner/session-dedup.ts'; @@ -63,13 +63,13 @@ export class Planner extends PlannerBase implements Agent { researcher: Researcher; private fisherman: Fisherman | null = null; - constructor(explorer: Explorer, provider: Provider, researcher: Researcher) { + constructor(deps: AgentDeps, researcher: Researcher) { super(); - this.explorer = explorer; - this.provider = provider; + this.explorer = deps.explorer; + this.provider = deps.ai; this.researcher = researcher; - this.stateManager = explorer.getStateManager(); - this.experienceTracker = explorer.getStateManager().getExperienceTracker(); + this.stateManager = deps.stateManager; + this.experienceTracker = deps.stateManager.getExperienceTracker(); } setFisherman(fisherman: Fisherman): void { diff --git a/src/ai/quartermaster.ts b/src/ai/quartermaster.ts index d5c0f50..a98dedf 100644 --- a/src/ai/quartermaster.ts +++ b/src/ai/quartermaster.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import { z } from 'zod'; import type { ActionResult } from '../action-result.ts'; import { outputPath } from '../config.ts'; +import type Explorer from '../explorer.ts'; import type { StateManager, StateTransition, WebPageState } from '../state-manager.ts'; import type { Task } from '../test-plan.ts'; import { createDebug, tag } from '../utils/logger.ts'; @@ -18,7 +19,7 @@ export class Quartermaster { private outputDir: string; private pageAnalyses: Map = new Map(); private unsubscribe: (() => void) | null = null; - private playwrightHelper: any = null; + private explorer: Explorer | null = null; private pendingAnalyses: Promise[] = []; constructor(provider: Provider, options?: { model?: any }) { @@ -28,8 +29,8 @@ export class Quartermaster { this.outputDir = outputPath('a11y'); } - start(playwrightHelper: any, stateManager: StateManager): void { - this.playwrightHelper = playwrightHelper; + start(explorer: Explorer, stateManager: StateManager): void { + this.explorer = explorer; mkdirSync(this.outputDir, { recursive: true }); this.unsubscribe = stateManager.onStateChange((event) => { @@ -60,7 +61,7 @@ export class Quartermaster { } private async runAxeAnalysis(state: WebPageState): Promise { - const page = this.playwrightHelper?.page; + const page = this.explorer?.page; if (!page || !state.hash) { debugLog('No page available for axe analysis'); return; diff --git a/src/ai/rerunner.ts b/src/ai/rerunner.ts index 323e151..2a1113c 100644 --- a/src/ai/rerunner.ts +++ b/src/ai/rerunner.ts @@ -12,16 +12,13 @@ import figureSet from 'figures'; import { z } from 'zod'; import { ActionResult } from '../action-result.ts'; import { setActivity } from '../activity.ts'; -import type { ExperienceTracker } from '../experience-tracker.ts'; -import type Explorer from '../explorer.ts'; -import type { KnowledgeTracker } from '../knowledge-tracker.ts'; import { Stats } from '../stats.ts'; import { Task, Test, TestResult } from '../test-plan.ts'; import { formatHeadings } from '../utils/context-formatter.ts'; import { createDebug, tag } from '../utils/logger.ts'; import { loop } from '../utils/loop.ts'; import { RulesLoader } from '../utils/rules-loader.ts'; -import type { Agent } from './agent.ts'; +import type { Agent, AgentDeps } from './agent.ts'; import { toolExecutionLabel } from './conversation.ts'; import type { Navigator } from './navigator.ts'; import { Provider } from './provider.ts'; @@ -35,17 +32,13 @@ export class Rerunner extends TaskAgent implements Agent { protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form']; emoji = '🔄'; - private explorer: Explorer; - private provider: Provider; private agentTools: any; private healedSteps: Array<{ original: string; healed: string }> = []; private traceDir = ''; private static pluginsWired = false; - constructor(explorer: Explorer, provider: Provider, agentTools?: any) { - super(); - this.explorer = explorer; - this.provider = provider; + constructor(deps: AgentDeps, agentTools?: any) { + super(deps); this.agentTools = agentTools; } @@ -53,20 +46,8 @@ export class Rerunner extends TaskAgent implements Agent { throw new Error('Rerunner does not use Navigator'); } - protected getExperienceTracker(): ExperienceTracker { - return this.explorer.getStateManager().getExperienceTracker(); - } - - protected getKnowledgeTracker(): KnowledgeTracker { - return this.explorer.getKnowledgeTracker(); - } - - protected getProvider(): Provider { - return this.provider; - } - private get rerunnerConfig(): Record { - return (this.explorer.getConfig().ai?.agents?.rerunner as any) || {}; + return (this.config.ai?.agents?.rerunner as any) || {}; } private get healLimit(): number { @@ -325,7 +306,7 @@ export class Rerunner extends TaskAgent implements Agent { }); const healTask = new Task(`Heal: ${failedCode}`); - const codeceptTools = createCodeceptJSTools(this.explorer, healTask); + const codeceptTools = createCodeceptJSTools(this.toolDeps, healTask); let healed = false; let healedCommand = ''; @@ -345,9 +326,9 @@ export class Rerunner extends TaskAgent implements Agent { healTask.addNote(note); tag('substep').log(note); } - const action = this.explorer.createAction(); + const action = this.explorer.action(); await action.execute(`I.wait(${seconds})`); - const state = this.explorer.getStateManager().getCurrentState(); + const state = this.stateManager.getCurrentState(); const ar = state ? ActionResult.fromState(state) : null; return { success: true, @@ -438,8 +419,8 @@ export class Rerunner extends TaskAgent implements Agent { } private getHealSystemPrompt(): string { - const customRules = this.provider.getSystemPromptForAgent('rerunner', this.explorer.getStateManager().getCurrentState()?.url) || ''; - const currentUrl = this.explorer.getStateManager().getCurrentState()?.url || ''; + const customRules = this.provider.getSystemPromptForAgent('rerunner', this.stateManager.getCurrentState()?.url) || ''; + const currentUrl = this.stateManager.getCurrentState()?.url || ''; const approach = RulesLoader.loadRules('rerunner', ['healing-approach'], currentUrl); return dedent` @@ -474,7 +455,7 @@ export class Rerunner extends TaskAgent implements Agent { } private getHealUserPrompt(failedCode: string, error: Error): string { - const state = this.explorer.getStateManager().getCurrentState(); + const state = this.stateManager.getCurrentState(); const actionResult = state ? ActionResult.fromState(state) : null; const headings = formatHeadings(state || {}); diff --git a/src/ai/researcher.ts b/src/ai/researcher.ts index 288f283..a2a834a 100644 --- a/src/ai/researcher.ts +++ b/src/ai/researcher.ts @@ -1,23 +1,22 @@ import dedent from 'dedent'; import { ActionResult } from '../action-result.js'; import { setActivity } from '../activity.ts'; -import { ConfigParser, outputPath } from '../config.ts'; +import { ConfigParser, type ExplorbotConfig, outputPath } from '../config.ts'; import { executionController } from '../execution-controller.ts'; import type { ExperienceTracker } from '../experience-tracker.ts'; import type Explorer from '../explorer.ts'; -import type { KnowledgeTracker } from '../knowledge-tracker.ts'; import { Observability } from '../observability.ts'; import type { StateManager } from '../state-manager.js'; import { WebPageState } from '../state-manager.js'; import { Stats } from '../stats.ts'; import { diffAriaSnapshots } from '../utils/aria.ts'; import { ErrorPageError, detectPageCondition } from '../utils/error-page.ts'; -import { HooksRunner } from '../utils/hooks-runner.ts'; import { isBodyEmpty } from '../utils/html.ts'; import { createDebug, tag } from '../utils/logger.js'; import { mdq } from '../utils/markdown-query.ts'; import { RulesLoader } from '../utils/rules-loader.ts'; -import type { Agent } from './agent.js'; +import { annotatePageElements } from '../utils/web-annotate.ts'; +import type { Agent, AgentDeps } from './agent.js'; import type { Navigator } from './navigator.ts'; import { ContextLengthError, type Provider } from './provider.js'; import { findSimilarResearch, getCachedResearch, saveResearch } from './researcher/cache.ts'; @@ -55,20 +54,16 @@ export class Researcher extends ResearcherBase implements Agent { declare explorer: Explorer; declare provider: Provider; declare stateManager: StateManager; + declare config: ExplorbotConfig; private experienceTracker!: ExperienceTracker; private hasScreenshotToAnalyze = false; declare actionResult: ActionResult | undefined; - private hooksRunner!: HooksRunner; - constructor(explorer: Explorer, provider: Provider) { - super(); - this.explorer = explorer; - this.provider = provider; - this.stateManager = explorer.getStateManager(); - this.experienceTracker = this.stateManager.getExperienceTracker(); - this.hooksRunner = new HooksRunner(explorer, explorer.getConfig()); + constructor(deps: AgentDeps) { + super(deps); + this.experienceTracker = deps.stateManager.getExperienceTracker(); - const ai = explorer.getConfig().ai; + const ai = deps.config.ai; if (ai) { ai.agents ??= {}; ai.agents.researcher ??= {}; @@ -80,18 +75,6 @@ export class Researcher extends ResearcherBase implements Agent { throw new Error('not implemented'); } - protected getExperienceTracker(): ExperienceTracker { - return this.experienceTracker; - } - - protected getKnowledgeTracker(): KnowledgeTracker { - return this.explorer.getKnowledgeTracker(); - } - - protected getProvider(): Provider { - return this.provider; - } - static getCachedResearch(state: WebPageState): string { return getCachedResearch(state.hash || ''); } @@ -110,7 +93,7 @@ export class Researcher extends ResearcherBase implements Agent { async research(state: WebPageState, opts: { screenshot?: boolean; force?: boolean; deep?: boolean; data?: boolean; fix?: boolean; _retriesLeft?: number } = {}): Promise { const { screenshot = false, force = false, deep = false, data = false, fix = true } = opts; - const maxRetries = (this.explorer.getConfig().ai?.agents?.researcher as any)?.retries ?? 2; + const maxRetries = (this.config.ai?.agents?.researcher as any)?.retries ?? 2; let retriesLeft = opts._retriesLeft ?? maxRetries; this.actionResult = ActionResult.fromState(state); const stateHash = state.hash || this.actionResult.getStateHash(); @@ -134,9 +117,9 @@ export class Researcher extends ResearcherBase implements Agent { await this.ensureNavigated(displayUrl, screenshot && this.provider.hasVision()); await this.hooksRunner.runBeforeHook('researcher', state.url); - const annotatedElements = await this.explorer.annotateElements(); + const { elements: annotatedElements } = await this.explorer.withPage(annotatePageElements); debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`); - this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() }); + this.actionResult = await this.explorer.capture({ screenshot: screenshot && this.provider.hasVision() }); const condition = detectPageCondition(this.actionResult!); if (condition === 'error') { @@ -236,7 +219,7 @@ export class Researcher extends ResearcherBase implements Agent { // Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring. if (!interrupted() && this.hasScreenshotToAnalyze) { const sections = parseResearchSections(result.text); - const focused = await this.explorer.runWithBrowserRecovery('detectFocusedSection', () => detectFocusedSection(this.explorer.playwrightHelper.page, sections)); + const focused = await this.explorer.withPage((page) => detectFocusedSection(page, sections)); if (focused) markSectionAsFocused(result, focused); } @@ -249,7 +232,7 @@ export class Researcher extends ResearcherBase implements Agent { const freshBroken = freshContainerLocs.filter((l) => l.valid === false).map((l) => l.locator); const containers = validContainers.filter((c) => !freshBroken.includes(c.css)); await this.visuallyAnnotateElements({ containers }); - this.actionResult = await this.explorer.capturePageWithScreenshot(); + this.actionResult = await this.explorer.capture({ screenshot: true }); const visualResult = await this.analyzeScreenshotForVisualProps(); if (visualResult.elements.size > 0) { await this.mergeVisualData(result, visualResult.elements); @@ -325,8 +308,7 @@ export class Researcher extends ResearcherBase implements Agent { private async ensureNavigated(url: string, screenshot?: boolean): Promise { if (!this.actionResult) { debugLog('No action result, navigating to URL'); - await this.explorer.visit(url); - this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot }); + this.actionResult = await this.explorer.visit(url, { screenshot }); return; } @@ -336,7 +318,7 @@ export class Researcher extends ResearcherBase implements Agent { if (!isEmpty && isOnCurrentState) { if ((!this.actionResult.screenshot && screenshot) || !this.actionResult.ariaSnapshot) { - this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot }); + this.actionResult = await this.explorer.capture({ screenshot }); } return; } @@ -344,8 +326,7 @@ export class Researcher extends ResearcherBase implements Agent { if (isEmpty && isOnCurrentState) { debugLog('HTML body empty on current URL, waiting for content'); tag('step').log('Page body is empty, waiting for content...'); - await this.explorer.visit(url); - this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false }); + this.actionResult = await this.explorer.visit(url, { screenshot: screenshot ?? false }); await this.waitUntilSettled(screenshot ?? false); return; } @@ -353,22 +334,21 @@ export class Researcher extends ResearcherBase implements Agent { debugLog('Not on current state, navigating to URL'); tag('step').log('Navigating to URL...'); - await this.explorer.visit(url); - this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false }); + this.actionResult = await this.explorer.visit(url, { screenshot: screenshot ?? false }); } private async waitUntilSettled(screenshot: boolean): Promise { - const errorPageTimeout = (this.explorer.getConfig().ai?.agents?.researcher as any)?.errorPageTimeout ?? 10; + const errorPageTimeout = (this.config.ai?.agents?.researcher as any)?.errorPageTimeout ?? 10; if (errorPageTimeout <= 0) return false; const includeScreenshot = screenshot && this.provider.hasVision(); try { - await this.explorer.runWithBrowserRecovery('waitUntilSettled', () => this.explorer.playwrightHelper.page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 })); + await this.explorer.withPage((page) => page.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 })); } catch {} - await this.explorer.annotateElements(); - this.actionResult = await this.explorer.capturePageState({ includeScreenshot }); + await this.explorer.withPage(annotatePageElements); + this.actionResult = await this.explorer.capture({ screenshot: includeScreenshot }); let condition = detectPageCondition(this.actionResult!); if (condition === 'error') { @@ -378,8 +358,8 @@ export class Researcher extends ResearcherBase implements Agent { for (let i = 0; i < 3; i++) { await new Promise((r) => setTimeout(r, 1000)); - await this.explorer.annotateElements(); - this.actionResult = await this.explorer.capturePageState({ includeScreenshot }); + await this.explorer.withPage(annotatePageElements); + this.actionResult = await this.explorer.capture({ screenshot: includeScreenshot }); condition = detectPageCondition(this.actionResult!); if (condition === 'error') { throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title, this.actionResult!.httpStatus); @@ -391,7 +371,7 @@ export class Researcher extends ResearcherBase implements Agent { } private getConfiguredSections(): Record { - const configSections = (this.explorer.getConfig().ai?.agents?.researcher as any)?.sections as string[] | undefined; + const configSections = (this.config.ai?.agents?.researcher as any)?.sections as string[] | undefined; if (!configSections?.length) return POSSIBLE_SECTIONS; const filtered: Record = {}; for (const key of configSections) { @@ -458,7 +438,7 @@ export class Researcher extends ResearcherBase implements Agent { if (!this.actionResult) throw new Error('actionResult is not set'); const html = await this.actionResult.combinedHtml(); - const knowledge = this.explorer.getKnowledgeTracker().renderRelevantKnowledge(this.actionResult); + const knowledge = this.knowledgeTracker.renderRelevantKnowledge(this.actionResult); const ariaSnapshot = this.actionResult.getCompactARIA(); @@ -719,9 +699,9 @@ export class Researcher extends ResearcherBase implements Agent { async cancelInUi() { const beforeAria = this.stateManager.getCurrentState()?.ariaSnapshot || null; - await this.explorer.executeAction('I.clickXY(0, 0)'); + await this.explorer.action().execute('I.clickXY(0, 0)'); if (diffAriaSnapshots(beforeAria, this.stateManager.getCurrentState()?.ariaSnapshot || null).text) return; - await this.explorer.executeAction(`I.pressKey('Escape')`); + await this.explorer.action().execute(`I.pressKey('Escape')`); } } diff --git a/src/ai/researcher/coordinates.ts b/src/ai/researcher/coordinates.ts index 3893fca..6e0dd65 100644 --- a/src/ai/researcher/coordinates.ts +++ b/src/ai/researcher/coordinates.ts @@ -4,6 +4,7 @@ import type { ActionResult } from '../../action-result.js'; import type Explorer from '../../explorer.ts'; import { tag } from '../../utils/logger.js'; import { mdq } from '../../utils/markdown-query.ts'; +import { eidxByLocator } from '../../utils/web-eidx.ts'; import { WebElement } from '../../utils/web-element.ts'; import type { Provider } from '../provider.js'; import { type Constructor, debugLog } from './mixin.ts'; @@ -81,7 +82,7 @@ export function WithCoordinates(Base: T) { } async visuallyAnnotateElements(opts?: { containers?: Array<{ css: string; label: string }> }): Promise { - return this.explorer.visuallyAnnotateElements(opts); + return this.explorer.withPage((page) => visuallyAnnotateContainers(page, opts?.containers || [])); } private async _analyzeScreenshotForVisualProps(): Promise { @@ -177,7 +178,7 @@ export function WithCoordinates(Base: T) { let eidx = el.eidx || null; if (!eidx) { const locator = el.css || el.xpath || (el.aria ? `role=${el.aria.role}[name="${el.aria.text}"]` : null); - if (locator) eidx = await this.explorer.getEidxByLocator(locator, section.containerCss); + if (locator) eidx = await this.explorer.withPage((page) => eidxByLocator(page, locator, section.containerCss)); } if (!eidx) continue; @@ -202,7 +203,7 @@ export function WithCoordinates(Base: T) { } if (eidxWithoutCoords.length === 0) return; - const webElements = await this.explorer.runWithBrowserRecovery('backfillCoordinates', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, eidxWithoutCoords)); + const webElements = await this.explorer.withPage((page) => WebElement.fromEidxList(page, eidxWithoutCoords)); if (webElements.length === 0) return; const rectMap = new Map(webElements.map((w) => [w.eidx!, w])); diff --git a/src/ai/researcher/deep-analysis.ts b/src/ai/researcher/deep-analysis.ts index b4dbf85..f81b22b 100644 --- a/src/ai/researcher/deep-analysis.ts +++ b/src/ai/researcher/deep-analysis.ts @@ -1,5 +1,6 @@ import dedent from 'dedent'; import { ActionResult, type Diff } from '../../action-result.js'; +import type { ExplorbotConfig } from '../../config.ts'; import { executionController } from '../../execution-controller.ts'; import type Explorer from '../../explorer.ts'; import type { StateManager } from '../../state-manager.js'; @@ -21,13 +22,14 @@ export function WithDeepAnalysis(Base: T) { declare explorer: Explorer; declare provider: Provider; declare stateManager: StateManager; + declare config: ExplorbotConfig; declare actionResult: ActionResult | undefined; async performDeepAnalysis(state: WebPageState, result: ResearchResult): Promise { tag('info').log('Starting deep analysis of expandable elements'); await (this as any).navigateTo(state.fullUrl || state.url); - const maxClicks = (this.explorer.getConfig().ai?.agents?.researcher as any)?.maxExpandableClicks ?? DEFAULT_MAX_EXPANDABLE_CLICKS; + const maxClicks = (this.config.ai?.agents?.researcher as any)?.maxExpandableClicks ?? DEFAULT_MAX_EXPANDABLE_CLICKS; const expandedSections: string[] = []; const navigationLinks: Array<{ code: string; url: string }> = []; @@ -358,10 +360,10 @@ export function WithDeepAnalysis(Base: T) { const isCoordinateClick = el.commands[0].startsWith('I.clickXY('); if (!isCoordinateClick) { const hoverCmd = el.commands[0].replace('I.click(', 'I.moveCursorTo('); - await this.explorer.attemptAction(hoverCmd, undefined); + await this.explorer.action().attempt(hoverCmd); await new Promise((r) => setTimeout(r, 500)); - await this.explorer.capturePageState(); + await this.explorer.capture(); const hoverAR = ActionResult.fromState(this.stateManager.getCurrentState()!); const hoverDiff = await hoverAR.diff(previousState); const hoverHtmlSize = hoverDiff.htmlParts.reduce((sum, p) => sum + p.subtree.length, 0); @@ -401,7 +403,7 @@ export function WithDeepAnalysis(Base: T) { const previousState = ActionResult.fromState(this.stateManager.getCurrentState()!); let clickCode: string | null = null; - const action = this.explorer.createAction(); + const action = this.explorer.action(); for (const cmd of commands) { if (await action.attempt(cmd, undefined)) { clickCode = cmd; @@ -417,7 +419,7 @@ export function WithDeepAnalysis(Base: T) { let diff: Diff; try { - await this.explorer.createAction().capturePageState(); + await this.explorer.capture(); const currAR = ActionResult.fromState(this.stateManager.getCurrentState()!); diff = await currAR.diff(previousState); } catch (err) { @@ -448,7 +450,7 @@ export function WithDeepAnalysis(Base: T) { private async _restorePageState(url: string, originalAria: string): Promise { try { await (this as any).cancelInUi(); - await this.explorer.capturePageState(); + await this.explorer.capture(); const currentAria = this.stateManager.getCurrentState()?.ariaSnapshot || ''; if (!diffAriaSnapshots(originalAria, currentAria).text) return; } catch (err) { diff --git a/src/ai/researcher/locators.ts b/src/ai/researcher/locators.ts index c729ce0..520a86f 100644 --- a/src/ai/researcher/locators.ts +++ b/src/ai/researcher/locators.ts @@ -52,18 +52,18 @@ export function WithLocators(Base: T) { continue; } try { - const count = await this.explorer.playwrightLocatorCount((page) => { + const count = await this.explorer.withPage((page) => { const base = loc.container ? page.locator(loc.container) : page; if (loc.type === 'aria') { const parsed = parseAriaLocator(loc.locator); - if (!parsed) return page.locator('__invalid__'); - return base.getByRole(parsed.role as any, { name: parsed.text }); + if (!parsed) return page.locator('__invalid__').count(); + return base.getByRole(parsed.role as any, { name: parsed.text }).count(); } const converted = loc.locator.replace(/:contains\(/g, ':has-text('); if (converted !== loc.locator) { loc.locator = converted; } - return base.locator(loc.locator); + return base.locator(loc.locator).count(); }); loc.valid = count === 1; if (opts.scope && count > 1) loc.valid = true; @@ -196,7 +196,7 @@ export function WithLocators(Base: T) { } if (needsXpath.length > 0) { - const webElements = await this.explorer.runWithBrowserRecovery('backfillBrokenLocators', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, needsXpath)); + const webElements = await this.explorer.withPage((page) => WebElement.fromEidxList(page, needsXpath)); const changedSections = new Set<(typeof sections)[0]>(); for (const w of webElements) { const entry = needsXpathEls.get(w.eidx!); @@ -273,7 +273,7 @@ export function WithLocators(Base: T) { let count = 0; try { - count = await this.explorer.playwrightLocatorCount((page) => page.locator(section.containerCss!)); + count = await this.explorer.withPage((page) => page.locator(section.containerCss!).count()); } catch {} if (count >= 1) continue; @@ -282,7 +282,7 @@ export function WithLocators(Base: T) { if (simplified) { let simplifiedCount = 0; try { - simplifiedCount = await this.explorer.playwrightLocatorCount((page) => page.locator(simplified)); + simplifiedCount = await this.explorer.withPage((page) => page.locator(simplified).count()); } catch {} if (simplifiedCount >= 1) { diff --git a/src/ai/researcher/sections.ts b/src/ai/researcher/sections.ts index 4659da3..243f098 100644 --- a/src/ai/researcher/sections.ts +++ b/src/ai/researcher/sections.ts @@ -1,5 +1,6 @@ import dedent from 'dedent'; import type { ActionResult } from '../../action-result.js'; +import type { ExplorbotConfig } from '../../config.ts'; import { executionController } from '../../execution-controller.ts'; import type Explorer from '../../explorer.ts'; import type { StateManager } from '../../state-manager.js'; @@ -20,6 +21,7 @@ export function WithSections(Base: T) { declare explorer: Explorer; declare provider: Provider; declare stateManager: StateManager; + declare config: ExplorbotConfig; declare actionResult: ActionResult | undefined; async researchBySections(): Promise { @@ -64,11 +66,11 @@ export function WithSections(Base: T) { } private async _detectFocusCss(): Promise { - const focusSections = (this.explorer.getConfig().ai?.agents?.researcher as any)?.focusSections as string[] | undefined; + const focusSections = (this.config.ai?.agents?.researcher as any)?.focusSections as string[] | undefined; if (!focusSections?.length) return null; for (const css of focusSections) { - const count = await this.explorer.playwrightLocatorCount((page: any) => page.locator(css)).catch(() => 0); + const count = await this.explorer.withPage((page) => page.locator(css).count()).catch(() => 0); if (count > 0) return css; } return null; diff --git a/src/ai/task-agent.ts b/src/ai/task-agent.ts index 8165998..9c635bc 100644 --- a/src/ai/task-agent.ts +++ b/src/ai/task-agent.ts @@ -1,6 +1,11 @@ import type { ActionResult } from '../action-result.js'; +import type { ExplorbotConfig } from '../config.ts'; import type { ExperienceTracker } from '../experience-tracker.js'; +import type Explorer from '../explorer.ts'; import type { KnowledgeTracker } from '../knowledge-tracker.js'; +import type { StateManager } from '../state-manager.ts'; +import { HooksRunner } from '../utils/hooks-runner.ts'; +import type { AgentDeps, ToolDeps } from './agent.ts'; import { Historian } from './historian.js'; import type { Navigator } from './navigator.js'; import type { Provider } from './provider.js'; @@ -17,6 +22,12 @@ function createNullProxy(): T { } export abstract class TaskAgent { + explorer!: Explorer; + provider!: Provider; + config!: ExplorbotConfig; + stateManager!: StateManager; + knowledgeTracker!: KnowledgeTracker; + protected hooksRunner!: HooksRunner; protected consecutiveFailures = 0; protected consecutiveEmptyResults = 0; protected recentToolCalls: any[] = []; @@ -25,10 +36,41 @@ export abstract class TaskAgent { private _historian: Historian | null = null; private _quartermaster: Quartermaster | null = null; + constructor(deps?: AgentDeps) { + if (!deps) return; + this.explorer = deps.explorer; + this.provider = deps.ai; + this.config = deps.config; + this.stateManager = deps.stateManager; + this.knowledgeTracker = deps.knowledgeTracker; + this.hooksRunner = new HooksRunner(deps.explorer, deps.config); + } + + setHistorian(historian: Historian): void { + this._historian = historian; + } + + setQuartermaster(quartermaster: Quartermaster): void { + this._quartermaster = quartermaster; + } + protected abstract getNavigator(): Navigator; - protected abstract getExperienceTracker(): ExperienceTracker; - protected abstract getKnowledgeTracker(): KnowledgeTracker; - protected abstract getProvider(): Provider; + + protected get toolDeps(): ToolDeps { + return { explorer: this.explorer, stateManager: this.stateManager, ai: this.provider }; + } + + protected getExperienceTracker(): ExperienceTracker { + return this.stateManager.getExperienceTracker(); + } + + protected getKnowledgeTracker(): KnowledgeTracker { + return this.knowledgeTracker; + } + + protected getProvider(): Provider { + return this.provider; + } protected getKnowledge(actionResult: ActionResult): string { return this.getKnowledgeTracker().renderRelevantKnowledge(actionResult); @@ -38,19 +80,11 @@ export abstract class TaskAgent { return this.getExperienceTracker().renderExperienceTocFor(actionResult); } - setHistorian(historian: Historian): void { - this._historian = historian; - } - protected getHistorian(): Historian { if (this._historian) return this._historian; return createNullProxy(); } - setQuartermaster(quartermaster: Quartermaster): void { - this._quartermaster = quartermaster; - } - protected getQuartermaster(): Quartermaster { if (this._quartermaster) return this._quartermaster; return createNullProxy(); diff --git a/src/ai/tester.ts b/src/ai/tester.ts index 0a57380..8fdf4e0 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -5,18 +5,17 @@ import dedent from 'dedent'; import { z } from 'zod'; import { ActionResult } from '../action-result.ts'; import { clearActivity, setActivity } from '../activity.ts'; -import type { ExperienceTracker } from '../experience-tracker.ts'; -import type Explorer from '../explorer.ts'; +import type { RequestStore } from '../api/request-store.ts'; +import type { TestRun } from '../explorer.ts'; import { Observability } from '../observability.ts'; import type { StateTransition } from '../state-manager.ts'; import { Stats } from '../stats.ts'; import { type Test, TestResult, type TestResultType } from '../test-plan.ts'; import { detectFocusArea, extractFocusedElement } from '../utils/aria.ts'; import { ErrorPageError, isErrorPage } from '../utils/error-page.ts'; -import { HooksRunner } from '../utils/hooks-runner.ts'; import { createDebug, tag } from '../utils/logger.ts'; import { loop } from '../utils/loop.ts'; -import type { Agent } from './agent.ts'; +import type { Agent, AgentDeps } from './agent.ts'; import type { Captain } from './captain.ts'; import type { Conversation } from './conversation.ts'; import { Navigator } from './navigator.ts'; @@ -44,8 +43,8 @@ export class Tester extends TaskAgent implements Agent { protected readonly ACTION_TOOLS = ['click', 'hover', 'pressKey', 'form']; protected readonly SPECIAL_CONTEXT_ACTION_TOOLS = ['exitIframe']; emoji = '🧪'; - private explorer: Explorer; - private provider: Provider; + private requestStore: RequestStore; + private testRun: TestRun | null = null; private currentConversation: Conversation | null = null; private pilot: Pilot | null = null; private captain: Captain | null = null; @@ -60,38 +59,23 @@ export class Tester extends TaskAgent implements Agent { private previousStateHash: string | null = null; private pageStateHash: string | null = null; private pageActionResult: ActionResult | null = null; - private hooksRunner: HooksRunner; private seenUiMapUrls = new Set(); private lastAnalyzedStateHash: string | null = null; private stalledIterations = 0; private readonly MAX_STALLED_ITERATIONS = 3; - constructor(explorer: Explorer, provider: Provider, researcher: Researcher, navigator: Navigator, agentTools?: any) { - super(); - this.explorer = explorer; - this.provider = provider; + constructor(deps: AgentDeps, researcher: Researcher, navigator: Navigator, agentTools?: any) { + super(deps); + this.requestStore = deps.requestStore; this.researcher = researcher; this.navigator = navigator; this.agentTools = agentTools; - this.hooksRunner = new HooksRunner(explorer, explorer.getConfig()); } protected getNavigator(): Navigator { return this.navigator; } - protected getExperienceTracker(): ExperienceTracker { - return this.explorer.getStateManager().getExperienceTracker(); - } - - protected getKnowledgeTracker() { - return this.explorer.getKnowledgeTracker(); - } - - protected getProvider(): Provider { - return this.provider; - } - setPilot(pilot: Pilot): void { this.pilot = pilot; } @@ -101,11 +85,11 @@ export class Tester extends TaskAgent implements Agent { } private getCurrentState(): ActionResult { - return ActionResult.fromState(this.explorer.getStateManager().getCurrentState()!); + return ActionResult.fromState(this.stateManager.getCurrentState()!); } private get progressCheckInterval(): number { - return (this.explorer.getConfig().ai?.agents?.tester as any)?.progressCheckInterval ?? 3; + return (this.config.ai?.agents?.tester as any)?.progressCheckInterval ?? 3; } getConversation(): Conversation | null { @@ -114,7 +98,7 @@ export class Tester extends TaskAgent implements Agent { async test(task: Test): Promise<{ success: boolean }> { Stats.tests++; - const state = this.explorer.getStateManager().getCurrentState(); + const state = this.stateManager.getCurrentState(); if (!state) throw new Error('No state found'); setActivity(`🧪 Testing: ${task.scenario}`, 'action'); @@ -126,20 +110,20 @@ export class Tester extends TaskAgent implements Agent { this.seenUiMapUrls.clear(); this.lastAnalyzedStateHash = null; this.stalledIterations = 0; - this.explorer.getStateManager().clearHistory(); + this.stateManager.clearHistory(); this.resetFailureCount(); this.pilot?.reset(); - const requestStore = this.explorer.getRequestStore(); - requestStore?.clear(); - const offFailedRequest = requestStore?.onFailedRequest((r) => { + const requestStore = this.requestStore; + requestStore.clear(); + const offFailedRequest = requestStore.onFailedRequest((r) => { task.addNote(`Network error: ${r.method} ${r.path} → ${r.status}`, TestResult.FAILED); }); const initialState = ActionResult.fromState(state); if (isErrorPage(initialState)) { task.start(); - await this.explorer.startTest(task); + this.testRun = await this.explorer.beginTest(task); offFailedRequest?.(); return await this.abortStartedTestOnErrorPage(task, initialState); } @@ -196,7 +180,8 @@ export class Tester extends TaskAgent implements Agent { debugLog('Starting test execution with tools'); - if (!(await this.explorer.startTest(task))) { + this.testRun = await this.explorer.beginTest(task); + if (!this.testRun.started) { offFailedRequest?.(); await this.cleanupStartedTest(task); return { success: task.isSuccessful }; @@ -214,7 +199,7 @@ export class Tester extends TaskAgent implements Agent { } } - const startState = this.explorer.getStateManager().getCurrentState(); + const startState = this.stateManager.getCurrentState(); if (startState) { task.addUrlNote(startState); const startActionResult = ActionResult.fromState(startState); @@ -226,14 +211,14 @@ export class Tester extends TaskAgent implements Agent { const currentUrl = startState?.url || task.startUrl || ''; await this.hooksRunner.runBeforeHook('tester', currentUrl); - const offStateChange = this.explorer.getStateManager().onStateChange((event: StateTransition) => { + const offStateChange = this.stateManager.onStateChange((event: StateTransition) => { if (task.hasFinished) return; if (event.toState?.url === event.fromState?.url) return; if (event.toState) task.addUrlNote(event.toState, event.fromState || undefined); task.states.push(event.toState); }); - const codeceptjsTools = createCodeceptJSTools(this.explorer, task); + const codeceptjsTools = createCodeceptJSTools(this.toolDeps, task); let assertionPerformed = false; let extensions = 0; let shouldContinue = true; @@ -244,7 +229,7 @@ export class Tester extends TaskAgent implements Agent { await loop( async ({ stop, pause, iteration, userInput }) => { debugLog('iteration', iteration); - if (!(await this.explorer.ensurePageAvailable())) { + if (!(await this.explorer.recover()).ok) { task.addNote('Browser page is unavailable'); task.finish(TestResult.FAILED); stop(); @@ -258,12 +243,12 @@ export class Tester extends TaskAgent implements Agent { ...this.agentTools, }; if (currentState.isInsideIframe) { - Object.assign(tools, createIframeTools(this.explorer)); + Object.assign(tools, createIframeTools(this.toolDeps)); } debugLog(`Test ${task.scenario} iteration ${iteration}`); - if (this.explorer.getStateManager().isInDeadLoop()) { + if (this.stateManager.isInDeadLoop()) { task.addNote('Dead loop detected. Stopped'); stop(); return; @@ -412,7 +397,7 @@ export class Tester extends TaskAgent implements Agent { if (task.hasFinished) break; - if (!(await this.explorer.ensurePageAvailable())) break; + if (!(await this.explorer.recover()).ok) break; const finalState = this.getCurrentState(); const wantsContinue = await this.pilot!.finalReview(task, finalState, conversation, this.navigator); @@ -431,7 +416,7 @@ export class Tester extends TaskAgent implements Agent { shouldContinue = true; } - const finalUrl = this.explorer.getStateManager().getCurrentState()?.url || currentUrl; + const finalUrl = this.stateManager.getCurrentState()?.url || currentUrl; await this.hooksRunner.runAfterHook('tester', finalUrl); await this.getHistorian().saveSession(task, initialState, conversation); @@ -443,7 +428,7 @@ export class Tester extends TaskAgent implements Agent { offStateChange(); offFailedRequest?.(); await this.finishTest(task); - await this.explorer.stopTest(task, this.buildStopTestMeta(task)); + await this.testRun?.stop(this.buildStopTestMeta(task)); return { success: task.isSuccessful, @@ -559,7 +544,7 @@ export class Tester extends TaskAgent implements Agent { } if (currentState.isInsideIframe) { - const iframeInfo = currentState.iframeURL || this.explorer.getCurrentIframeInfo() || 'iframe context active'; + const iframeInfo = currentState.iframeURL || 'iframe context active'; context += dedent` INSIDE IFRAME: ${iframeInfo} @@ -568,10 +553,10 @@ export class Tester extends TaskAgent implements Agent { `; } - if (this.explorer.hasOtherTabs()) { - const otherTabs = this.explorer.getOtherTabsInfo(); + const otherTabs = this.stateManager.otherTabs; + if (otherTabs.length > 0) { context += multipleTabsRule(otherTabs); - this.explorer.clearOtherTabsInfo(); + this.stateManager.otherTabs = []; } if (isNewUrl) { @@ -675,7 +660,7 @@ export class Tester extends TaskAgent implements Agent { task.addNote(error.message, TestResult.FAILED, actionResult.screenshotFile, actionResult.fullUrl || actionResult.url); task.finish(TestResult.FAILED); this.finishTest(task); - await this.explorer.stopTest(task, this.buildStopTestMeta(task)); + await this.testRun?.stop(this.buildStopTestMeta(task)); clearActivity(true); return { success: false }; } @@ -790,7 +775,7 @@ export class Tester extends TaskAgent implements Agent { ${dataProtectionRules} - ${this.provider.getSystemPromptForAgent('tester', this.explorer.getStateManager().getCurrentState()?.url) || ''} + ${this.provider.getSystemPromptForAgent('tester', this.stateManager.getCurrentState()?.url) || ''} `; } @@ -837,7 +822,7 @@ export class Tester extends TaskAgent implements Agent { } private buildAvailableFiles(): string { - const userFiles = this.explorer.getConfig().files || {}; + const userFiles = this.config.files || {}; const codeceptDir = (global as any).codecept_dir || process.cwd(); const lines: string[] = []; @@ -895,11 +880,9 @@ export class Tester extends TaskAgent implements Agent { reason: z.string().optional().describe('Explanation why reset is the only option'), }), execute: async ({ reason }) => { - if (this.getCurrentState().isInsideIframe) { - await this.explorer.switchToMainFrame(); - } + await this.explorer.action().exitIframe(); - const currentState = this.explorer.getStateManager().getCurrentState(); + const currentState = this.stateManager.getCurrentState(); const currentUrl = currentState?.fullUrl || currentState?.url; if (currentUrl === resetUrl!) { return { @@ -927,7 +910,7 @@ export class Tester extends TaskAgent implements Agent { const explanation = reason ? `${reason} (RESET)` : 'Resetting to initial page'; const targetUrl = resetUrl!; task.addNote(explanation); - const resetAction = this.explorer.createAction(); + const resetAction = this.explorer.action(); const success = await resetAction.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, explanation); if (success) { @@ -1076,7 +1059,7 @@ export class Tester extends TaskAgent implements Agent { mappedStatus = TestResult.FAILED; } - const screenshotFile = this.explorer.getStateManager().getCurrentState()?.screenshotFile; + const screenshotFile = this.stateManager.getCurrentState()?.screenshotFile; for (const noteText of input.notes) { task.addNote(noteText, mappedStatus, screenshotFile); @@ -1117,7 +1100,7 @@ export class Tester extends TaskAgent implements Agent { const message = error instanceof Error ? error.message : String(error); if (!task.hasFinished) task.addNote(`Execution error: ${message}`); - const result = await this.explorer.handleExecutionError(error); + const result = await this.explorer.recover(error); tag('info').log(`Browser supervisor: ${result.action} - ${result.message}`); task.addNote(result.message); @@ -1152,7 +1135,7 @@ export class Tester extends TaskAgent implements Agent { private async cleanupStartedTest(task: Test): Promise { await this.finishTest(task); - await this.explorer.stopTest(task, { + await this.testRun?.stop({ startUrl: task.startUrl, style: task.style, sessionName: task.sessionName, diff --git a/src/ai/tools.ts b/src/ai/tools.ts index a1ffd85..526cd2d 100644 --- a/src/ai/tools.ts +++ b/src/ai/tools.ts @@ -3,13 +3,13 @@ import dedent from 'dedent'; import { z } from 'zod'; import { ActionResult, type PageDiff, type ToolResultMetadata } from '../action-result.ts'; import type { ExperienceTracker } from '../experience-tracker.ts'; -import type Explorer from '../explorer.ts'; import { type Task, TestResult } from '../test-plan.js'; import { LARGE_ARIA_CHANGE_THRESHOLD, extractFocusedElement } from '../utils/aria.ts'; import { isFatalBrowserError } from '../utils/browser-errors.ts'; import { createDebug, tag } from '../utils/logger.js'; import { pause } from '../utils/loop.js'; import { WebElement } from '../utils/web-element.ts'; +import type { ToolDeps } from './agent.ts'; import { Navigator } from './navigator.ts'; import type { AIProvider } from './provider.ts'; import { Researcher } from './researcher.ts'; @@ -18,11 +18,16 @@ import { isInteractive } from './task-agent.ts'; const debugLog = createDebug('explorbot:tools'); -export const ASSERTION_TOOLS = ['verify'] as const; +interface AgentToolDeps extends ToolDeps { + researcher: Researcher; + navigator: Navigator; + supervisor?: boolean; + withExperience?: boolean; +} -export function createCodeceptJSTools(explorer: Explorer, task: Task) { - const stateManager = explorer.getStateManager(); +export const ASSERTION_TOOLS = ['verify'] as const; +export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps, task: Task) { return { click: tool({ description: dedent` @@ -81,7 +86,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { }); const previousState = ActionResult.fromState(stateManager.getCurrentState()!); - const action = explorer.createAction(); + const action = explorer.action(); const attempts: Array<{ command: string; success: boolean; error?: string }> = []; for (let i = 0; i < commands.length; i++) { @@ -101,7 +106,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { let disambiguated = null; if (attempts.some((a) => a.error?.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN))) { - disambiguated = await disambiguateElements(action.lastError, explanation, explorer.getAIProvider()); + disambiguated = await disambiguateElements(action.lastError, explanation, ai); } if (disambiguated) { @@ -193,7 +198,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { }); const previousState = ActionResult.fromState(stateManager.getCurrentState()!); - const action = explorer.createAction(); + const action = explorer.action(); const attempts: Array<{ command: string; success: boolean; error?: string }> = []; for (const command of commands) { @@ -263,7 +268,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { if (!isSingleChar && !isStandardKey) { const previousState = ActionResult.fromState(stateManager.getCurrentState()!); - const action = explorer.createAction(); + const action = explorer.action(); const typeCommand = `I.type(${JSON.stringify(key)})`; await action.attempt(typeCommand, explanation); const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, key); @@ -306,7 +311,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { } const previousState = ActionResult.fromState(stateManager.getCurrentState()!); - const action = explorer.createAction(); + const action = explorer.action(); let pressKeyCommand: string; if (modifier) { @@ -406,14 +411,9 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { const previousState = ActionResult.fromState(stateManager.getCurrentState()!); const formLocator = codeLines[0] || 'form'; - const action = explorer.createAction(); - const wasInIframe = await explorer.isInsideIframe(); + const action = explorer.action(); await action.attempt(codeBlock, explanation); - if (action.lastError && !wasInIframe && (await explorer.isInsideIframe())) { - await explorer.switchToMainFrame(); - } - const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, formLocator); if (action.lastError) { @@ -422,7 +422,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { let formSuggestion = 'Look into error message and identify which commands passed and which failed. Continue execution using step-by-step approach using click() and form() tools.'; if (message.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN)) { - const disambiguated = await disambiguateElements(action.lastError, explanation, explorer.getAIProvider()); + const disambiguated = await disambiguateElements(action.lastError, explanation, ai); if (disambiguated) { formSuggestion = `Multiple elements matched. Add step.opts({ elementIndex: ${disambiguated.position} }) to the failing command. Fallback locator: ${disambiguated.xpath}`; } @@ -471,9 +471,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { }; } -export function createIframeTools(explorer: Explorer) { - const stateManager = explorer.getStateManager(); - +export function createIframeTools({ explorer, stateManager }: ToolDeps) { return { exitIframe: tool({ description: dedent` @@ -495,9 +493,9 @@ export function createIframeTools(explorer: Explorer) { }); } - await explorer.switchToMainFrame(); - - const nextState = await explorer.capturePageState(); + const action = explorer.action(); + await action.execute('I.switchTo()'); + const nextState = action.getActionResult()!; const toolResult = await nextState.toToolResult(previousState, 'I.switchTo()'); return successToolResult('exitIframe', { @@ -540,19 +538,7 @@ export function createLearnExperienceTool({ getExperienceTracker, getState }: { }); } -export function createAgentTools({ - explorer, - researcher, - navigator, - supervisor, - withExperience, -}: { - explorer: Explorer; - researcher: Researcher; - navigator: Navigator; - supervisor?: boolean; - withExperience?: boolean; -}): any { +export function createAgentTools({ explorer, stateManager, ai, researcher, navigator, supervisor, withExperience }: AgentToolDeps): any { let visionDisabled = false; const tools: Record = { @@ -577,7 +563,7 @@ export function createAgentTools({ } try { - const actionResult = await explorer.capturePageWithScreenshot(); + const actionResult = await explorer.capture({ screenshot: true }); if (!actionResult.screenshot) { return failedToolResult('see', 'Failed to capture screenshot for analysis'); @@ -625,7 +611,6 @@ export function createAgentTools({ }), execute: async ({ reason }) => { try { - const stateManager = explorer.getStateManager(); const currentState = stateManager.getCurrentState(); if (!currentState) { @@ -666,7 +651,7 @@ export function createAgentTools({ }), execute: async ({ assertion }) => { try { - const currentState = explorer.getStateManager().getCurrentState(); + const currentState = stateManager.getCurrentState(); const verifications = currentState?.verifications; if (verifications?.[assertion] !== undefined) { @@ -677,7 +662,7 @@ export function createAgentTools({ }); } - const actionResult = await explorer.capturePageState(); + const actionResult = await explorer.capture(); const result = await navigator.verifyState(assertion, actionResult); if (result.verified) { @@ -727,7 +712,6 @@ export function createAgentTools({ }), execute: async ({ reason }) => { try { - const stateManager = explorer.getStateManager(); const currentState = stateManager.getCurrentState(); if (!currentState) { @@ -771,7 +755,6 @@ export function createAgentTools({ }), execute: async ({ instruction }) => { try { - const stateManager = explorer.getStateManager(); const currentState = stateManager.getCurrentState(); if (!currentState) { @@ -826,7 +809,6 @@ export function createAgentTools({ } try { - const stateManager = explorer.getStateManager(); const currentState = stateManager.getCurrentState(); if (!currentState) { @@ -834,8 +816,8 @@ export function createAgentTools({ } const previousState = ActionResult.fromState(currentState); - const action = explorer.createAction(); - const actionResult = await explorer.capturePageWithScreenshot(); + const action = explorer.action(); + const actionResult = await explorer.capture({ screenshot: true }); if (!actionResult.screenshot) { return failedToolResult('visualClick', 'Failed to capture screenshot for visual analysis'); @@ -897,7 +879,6 @@ export function createAgentTools({ reason: z.string().describe('Why you need to go back'), }), execute: async ({ reason }) => { - const stateManager = explorer.getStateManager(); const currentState = stateManager.getCurrentState(); const currentUrl = currentState?.fullUrl || currentState?.url; const history = stateManager.getStateHistory(); @@ -920,7 +901,7 @@ export function createAgentTools({ let previousState: ActionResult | null = null; if (currentState) previousState = ActionResult.fromState(currentState); - const action = explorer.createAction(); + const action = explorer.action(); const success = await action.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, `${reason} (BACK to ${targetUrl})`); if (success) { @@ -942,7 +923,7 @@ export function createAgentTools({ description: 'List all previously visited page states (deduped by URL). Use to find pages to navigate back to.', inputSchema: z.object({}), execute: async () => { - const history = explorer.getStateManager().getStateHistory(); + const history = stateManager.getStateHistory(); const seen = new Set(); const states = history .map((t) => t.toState) @@ -975,7 +956,6 @@ export function createAgentTools({ reason: z.string().describe('What element you are looking for and why'), }), execute: async ({ xpath, reason }) => { - const stateManager = explorer.getStateManager(); const currentState = stateManager.getCurrentState(); if (!currentState) { @@ -1001,9 +981,9 @@ export function createAgentTools({ }); } - const action = explorer.createAction(); + const action = explorer.action(); const visible = await action.attempt(`I.seeElement(${JSON.stringify(xpath)})`, 'xpathCheck visibility'); - const liveElement = await WebElement.fromPlaywrightLocator(explorer.playwrightHelper.page.locator(`xpath=${xpath}`)); + const liveElement = await explorer.withPage((page) => WebElement.fromPlaywrightLocator(page.locator(`xpath=${xpath}`))); const matchesSummary = result.elements.map((el, i) => `${i + 1}. <${el.tag} ${el.keyAttrs}> text="${el.text}" html: ${el.outerHTML}`).join('\n'); @@ -1033,9 +1013,8 @@ export function createAgentTools({ if (withExperience !== false) { tools.learnExperience = createLearnExperienceTool({ - getExperienceTracker: () => explorer.getStateManager().getExperienceTracker(), + getExperienceTracker: () => stateManager.getExperienceTracker(), getState: () => { - const stateManager = explorer.getStateManager(); const currentState = stateManager.getCurrentState(); return currentState ? ActionResult.fromState(currentState) : null; }, diff --git a/src/command-handler.ts b/src/command-handler.ts index e79b874..2ee3b77 100644 --- a/src/command-handler.ts +++ b/src/command-handler.ts @@ -178,7 +178,7 @@ export class CommandHandler implements InputManager { } private async executeBrowserCommand(input: string): Promise { - const action = this.explorBot.getExplorer().createAction(); + const action = this.explorBot.getExplorer().action(); await action.execute(input); } diff --git a/src/commands/context-aria-command.ts b/src/commands/context-aria-command.ts index c6ec255..185c404 100644 --- a/src/commands/context-aria-command.ts +++ b/src/commands/context-aria-command.ts @@ -6,7 +6,7 @@ export class ContextAriaCommand extends BaseCommand { description = 'Print full ARIA snapshot for current page'; async execute(_args: string): Promise { - const state = this.explorBot.getExplorer().getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No active page to snapshot'); diff --git a/src/commands/context-command.ts b/src/commands/context-command.ts index d76b51c..0bd950a 100644 --- a/src/commands/context-command.ts +++ b/src/commands/context-command.ts @@ -1,9 +1,11 @@ import { ActionResult } from '../action-result.js'; import { Researcher } from '../ai/researcher.js'; +import { visuallyAnnotateContainers } from '../ai/researcher/coordinates.js'; import { outputPath } from '../config.js'; import { type ContextData, type ContextMode, formatContextSummary } from '../utils/context-formatter.js'; import { tag } from '../utils/logger.js'; import { extractValidContainers } from '../utils/research-parser.js'; +import { annotatePageElements } from '../utils/web-annotate.js'; import { BaseCommand, type Suggestion } from './base-command.js'; export class ContextCommand extends BaseCommand { @@ -25,7 +27,7 @@ export class ContextCommand extends BaseCommand { async execute(args: string): Promise { const explorer = this.explorBot.getExplorer(); - const state = explorer.getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No active page to show context for'); @@ -34,16 +36,16 @@ export class ContextCommand extends BaseCommand { const { opts } = this.parseArgs(args); const isVisual = !!(opts.visual || opts.screenshot); - await explorer.annotateElements(); + await explorer.withPage(annotatePageElements); if (isVisual) { const cachedResearch = Researcher.getCachedResearch(state); const containers = cachedResearch ? extractValidContainers(cachedResearch) : []; - await explorer.visuallyAnnotateElements({ containers }); + await explorer.withPage((page) => visuallyAnnotateContainers(page, containers)); } - const actionResult = await explorer.createAction().capturePageState({ includeScreenshot: isVisual }); - const experienceTracker = explorer.getStateManager().getExperienceTracker(); + const actionResult = await explorer.capture({ screenshot: isVisual }); + const experienceTracker = this.explorBot.experienceTracker(); const knowledgeTracker = this.explorBot.knowledgeTracker(); let mode: ContextMode = 'compact'; diff --git a/src/commands/context-data-command.ts b/src/commands/context-data-command.ts index 798e69f..2d67467 100644 --- a/src/commands/context-data-command.ts +++ b/src/commands/context-data-command.ts @@ -8,7 +8,7 @@ export class ContextDataCommand extends BaseCommand { async execute(_args: string): Promise { const explorer = this.explorBot.getExplorer(); - const state = explorer.getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No active page to extract data from'); @@ -18,7 +18,7 @@ export class ContextDataCommand extends BaseCommand { if (!actionResult.html || actionResult.html.trim().length < 100) { tag('info').log('Capturing fresh page content...'); - const freshResult = await explorer.createAction().capturePageState(); + const freshResult = await explorer.capture(); const table = await this.explorBot.agentResearcher().extractData(freshResult); tag('multiline').log(table); return; diff --git a/src/commands/context-experience-command.ts b/src/commands/context-experience-command.ts index 8a0f7d1..11b0516 100644 --- a/src/commands/context-experience-command.ts +++ b/src/commands/context-experience-command.ts @@ -10,14 +10,14 @@ export class ContextExperienceCommand extends BaseCommand { async execute(_args: string): Promise { const explorer = this.explorBot.getExplorer(); - const state = explorer.getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No active page'); } const actionResult = ActionResult.fromState(state); - const experienceTracker = explorer.getStateManager().getExperienceTracker(); + const experienceTracker = this.explorBot.experienceTracker(); const experience = experienceTracker.getRelevantExperience(actionResult); if (experience.length === 0) { diff --git a/src/commands/context-html-command.ts b/src/commands/context-html-command.ts index e051af1..5cecabb 100644 --- a/src/commands/context-html-command.ts +++ b/src/commands/context-html-command.ts @@ -8,7 +8,7 @@ export class ContextHtmlCommand extends BaseCommand { async execute(_args: string): Promise { const explorer = this.explorBot.getExplorer(); - const manager = explorer.getStateManager(); + const manager = this.explorBot.stateManager(); const state = manager.getCurrentState(); if (!state) { @@ -19,7 +19,7 @@ export class ContextHtmlCommand extends BaseCommand { if (!actionResult.html || actionResult.html.trim().length < 100) { tag('info').log('Capturing fresh page content...'); - actionResult = await explorer.createAction().capturePageState(); + actionResult = await explorer.capture(); } const html = await actionResult.combinedHtml(); diff --git a/src/commands/context-knowledge-command.ts b/src/commands/context-knowledge-command.ts index 4054d62..e3c2b9e 100644 --- a/src/commands/context-knowledge-command.ts +++ b/src/commands/context-knowledge-command.ts @@ -10,7 +10,7 @@ export class ContextKnowledgeCommand extends BaseCommand { async execute(_args: string): Promise { const explorer = this.explorBot.getExplorer(); - const state = explorer.getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No active page'); diff --git a/src/commands/drill-command.ts b/src/commands/drill-command.ts index 0f3cd80..a2328de 100644 --- a/src/commands/drill-command.ts +++ b/src/commands/drill-command.ts @@ -13,7 +13,7 @@ export class DrillCommand extends BaseCommand { const knowledgePath = this.parseKnowledgeArg(args); const maxComponents = this.parseMaxArg(args); - const state = this.explorBot.getExplorer().getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No active page to drill'); } diff --git a/src/commands/explore-command.ts b/src/commands/explore-command.ts index aa7f299..eae14f4 100644 --- a/src/commands/explore-command.ts +++ b/src/commands/explore-command.ts @@ -4,6 +4,7 @@ import { outputPath } from '../config.js'; import { normalizeUrl } from '../state-manager.js'; import { Stats } from '../stats.js'; import { type Plan, type Test, TestResult } from '../test-plan.js'; +import { browserErrorMessage } from '../utils/browser-errors.ts'; import { getCliName } from '../utils/cli-name.ts'; import { ErrorPageError, getStateErrorPageError } from '../utils/error-page.ts'; import { tag } from '../utils/logger.js'; @@ -38,7 +39,7 @@ export class ExploreCommand extends BaseCommand { private priorityFilter?: Set; private getCurrentPageUrl(): string | undefined { - const state = this.explorBot.getExplorer().getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); return state?.fullUrl || state?.url; } @@ -56,7 +57,7 @@ export class ExploreCommand extends BaseCommand { Stats.mode ??= 'explore'; Stats.focus ??= feature; const mainUrl = this.getCurrentPageUrl(); - const error = getStateErrorPageError(this.explorBot.getExplorer().getStateManager().getCurrentState()); + const error = getStateErrorPageError(this.explorBot.stateManager().getCurrentState()); if (error) { tag('warning').log(error.message); return; @@ -74,7 +75,7 @@ export class ExploreCommand extends BaseCommand { this.printResults(); return; } - if (mainUrl) await this.explorBot.visit(mainUrl); + if (mainUrl) await this.explorBot.visit(mainUrl).catch((err) => tag('warning').log(`Could not return to ${mainUrl}: ${browserErrorMessage(err)}`)); const savedPath = this.explorBot.savePlans(this.completedPlans); this.printResults(); await this.explorBot.printSessionAnalysis(); diff --git a/src/commands/freesail-command.ts b/src/commands/freesail-command.ts index 8a64f44..67643e2 100644 --- a/src/commands/freesail-command.ts +++ b/src/commands/freesail-command.ts @@ -35,7 +35,7 @@ export class FreesailCommand extends BaseCommand { async (ctx) => { if (maxTests != null && testsRun >= maxTests) ctx.stop(); - const stateManager = this.explorBot.getExplorer().getStateManager(); + const stateManager = this.explorBot.stateManager(); const state = stateManager.getCurrentState(); if (state && !Researcher.getCachedResearch(state)) { @@ -70,7 +70,7 @@ export class FreesailCommand extends BaseCommand { } tag('info').log(`Navigating to: ${suggestion.target} - ${suggestion.reason}`); - await this.explorBot.openFreshTab(); + await this.explorBot.openTab(); await this.explorBot.visit(suggestion.target); this.explorBot.clearPlan(); }, diff --git a/src/commands/learn-command.ts b/src/commands/learn-command.ts index b4dcbd2..5c8ea82 100644 --- a/src/commands/learn-command.ts +++ b/src/commands/learn-command.ts @@ -14,13 +14,13 @@ export class LearnCommand extends BaseCommand { if (!note) { const AddKnowledge = (await import('../components/AddKnowledge.js')).default; const explorer = this.explorBot.getExplorer(); - const state = explorer.getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); const initialUrl = state?.url || ''; const { unmount } = render( React.createElement(AddKnowledge, { initialUrl, - knowledgeTracker: explorer.getKnowledgeTracker(), + knowledgeTracker: this.explorBot.knowledgeTracker(), onComplete: () => unmount(), onCancel: () => unmount(), }), @@ -33,14 +33,14 @@ export class LearnCommand extends BaseCommand { } const explorer = this.explorBot.getExplorer(); - const state = explorer.getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No active page to attach knowledge'); } const targetUrl = state.url || state.fullUrl || '/'; - explorer.getKnowledgeTracker().addKnowledge(targetUrl, note); + this.explorBot.knowledgeTracker().addKnowledge(targetUrl, note); tag('success').log('Knowledge saved for current page'); } } diff --git a/src/commands/path-command.ts b/src/commands/path-command.ts index 7e288a9..51b281e 100644 --- a/src/commands/path-command.ts +++ b/src/commands/path-command.ts @@ -16,7 +16,7 @@ export class PathCommand extends BaseCommand { async execute(args: string): Promise { const { opts } = this.parseArgs(args); const showLinks = !!opts.links; - const stateManager = this.explorBot.getExplorer().getStateManager(); + const stateManager = this.explorBot.stateManager(); const history = stateManager.getStateHistory(); if (history.length === 0) { diff --git a/src/commands/research-command.ts b/src/commands/research-command.ts index ab2952c..218ce4b 100644 --- a/src/commands/research-command.ts +++ b/src/commands/research-command.ts @@ -27,7 +27,7 @@ export class ResearchCommand extends BaseCommand { await this.explorBot.agentNavigator().visit(target); } - const state = this.explorBot.getExplorer().getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No active page to research'); } diff --git a/src/commands/test-command.ts b/src/commands/test-command.ts index 4eeb866..bc4267d 100644 --- a/src/commands/test-command.ts +++ b/src/commands/test-command.ts @@ -48,7 +48,7 @@ export class TestCommand extends BaseCommand { if (matching.length > 0) { toExecute.push(...matching); } else { - const state = this.explorBot.getExplorer().getStateManager().getCurrentState(); + const state = this.explorBot.stateManager().getCurrentState(); if (!state) { throw new Error('No page loaded. Please navigate to a page first.'); } diff --git a/src/components/App.tsx b/src/components/App.tsx index 785b36d..80be05c 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -68,7 +68,7 @@ export function App({ explorBot, initialShowInput = false, exitOnEmptyInput = fa }); }); - const manager = explorBot.getExplorer().getStateManager(); + const manager = explorBot.stateManager(); unsubscribe = manager.onStateChange((transition: StateTransition) => { if (mounted) { diff --git a/src/explorbot.ts b/src/explorbot.ts index 5e31cf8..29991ab 100644 --- a/src/explorbot.ts +++ b/src/explorbot.ts @@ -1,5 +1,6 @@ import { existsSync, mkdirSync } from 'node:fs'; import path from 'node:path'; +import type { AgentDeps } from './ai/agent.ts'; import { Captain } from './ai/captain.ts'; import { Driller } from './ai/driller.ts'; import { ExperienceCompactor } from './ai/experience-compactor.ts'; @@ -23,7 +24,9 @@ import { ConfigParser } from './config.ts'; import { ExperienceTracker } from './experience-tracker.ts'; import Explorer from './explorer.ts'; import { KnowledgeTracker } from './knowledge-tracker.ts'; -import { WebPageState } from './state-manager.ts'; +import { PlaywrightRecorder } from './playwright-recorder.ts'; +import { Reporter } from './reporter.ts'; +import { StateManager, WebPageState } from './state-manager.ts'; import { Stats } from './stats.ts'; import type { Suite } from './suite.ts'; import { Plan, type Test } from './test-plan.ts'; @@ -63,6 +66,10 @@ export class ExplorBot { private lastReportedTestCount = 0; private _experienceTracker?: ExperienceTracker; private _knowledgeTracker?: KnowledgeTracker; + private _stateManager?: StateManager; + private _reporter?: Reporter; + private _requestStore?: RequestStore; + private _playwrightRecorder?: PlaywrightRecorder; constructor(options: ExplorBotOptions = {}) { this.options = options; @@ -74,7 +81,7 @@ export class ExplorBot { } get isExploring(): boolean { - return this.explorer?.isStarted; + return !!this.explorer; } setUserResolve(fn: UserResolveFunction): void { @@ -82,14 +89,20 @@ export class ExplorBot { } async start(): Promise { - if (this.explorer?.isStarted) { + if (this.explorer) { return; } try { await this.startProviderOnly(); const explorerOptions = { ...this.options, session: typeof this.options.session === 'string' ? this.options.session : undefined }; - this.explorer = new Explorer(this.config, this.provider, explorerOptions, this.experienceTracker(), this.knowledgeTracker()); + this.explorer = new Explorer(this.config, explorerOptions, { + stateManager: this.stateManager(), + knowledgeTracker: this.knowledgeTracker(), + reporter: this.reporter(), + requestStore: this.requestStore(), + playwrightRecorder: this.playwrightRecorder(), + }); await this.explorer.start(); if (!this.options.incognito) { await this.agentExperienceCompactor().autocompact(); @@ -123,12 +136,12 @@ export class ExplorBot { return this.agentNavigator().visit(url); } - async openFreshTab(): Promise { - await this.explorer.openFreshTab(); + async openTab(): Promise { + await this.explorer.openTab(); } getCurrentState(): WebPageState | null { - return this.explorer?.getStateManager().getCurrentState() ?? null; + return this._stateManager?.getCurrentState() ?? null; } getExplorer(): Explorer { @@ -143,6 +156,22 @@ export class ExplorBot { return (this._experienceTracker ||= new ExperienceTracker(this.knowledgeTracker())); } + stateManager(): StateManager { + return (this._stateManager ||= new StateManager(this.experienceTracker(), this.knowledgeTracker())); + } + + reporter(): Reporter { + return (this._reporter ||= new Reporter(this.config.reporter, this.stateManager())); + } + + requestStore(): RequestStore { + return (this._requestStore ||= new RequestStore(this.configParser.getOutputDir())); + } + + playwrightRecorder(): PlaywrightRecorder { + return (this._playwrightRecorder ||= new PlaywrightRecorder()); + } + getConfig(): ExplorbotConfig { return this.config; } @@ -155,11 +184,15 @@ export class ExplorBot { return this.provider; } - createAgent(factory: (deps: { explorer: Explorer; ai: AIProvider; config: ExplorbotConfig }) => T): T { + createAgent(factory: (deps: AgentDeps) => T): T { const agent = factory({ explorer: this.explorer, ai: this.provider, config: this.config, + stateManager: this.stateManager(), + knowledgeTracker: this.knowledgeTracker(), + requestStore: this.requestStore(), + playwrightRecorder: this.playwrightRecorder(), }); const agentName = (agent as any).constructor.name.toLowerCase(); @@ -169,18 +202,16 @@ export class ExplorBot { } agentResearcher(): Researcher { - return (this.agents.researcher ||= this.createAgent(({ ai, explorer }) => new Researcher(explorer, ai))); + return (this.agents.researcher ||= this.createAgent((deps) => new Researcher(deps))); } agentNavigator(): Navigator { - return (this.agents.navigator ||= this.createAgent(({ ai, explorer }) => { - return new Navigator(explorer, ai); - })); + return (this.agents.navigator ||= this.createAgent((deps) => new Navigator(deps))); } agentPlanner(): Planner { if (!this.agents.planner) { - this.agents.planner = this.createAgent(({ ai, explorer }) => new Planner(explorer, ai, this.agentResearcher())); + this.agents.planner = this.createAgent((deps) => new Planner(deps, this.agentResearcher())); const fisherman = this.agentFisherman(); if (fisherman) this.agents.planner.setFisherman(fisherman); } @@ -188,21 +219,21 @@ export class ExplorBot { } agentPilot(): Pilot { - return (this.agents.pilot ||= this.createAgent(({ ai, explorer }) => { + return (this.agents.pilot ||= this.createAgent((deps) => { const researcher = this.agentResearcher(); const navigator = this.agentNavigator(); - const tools = createAgentTools({ explorer, researcher, navigator, supervisor: true }); - return new Pilot(ai, tools, researcher, explorer); + const tools = createAgentTools({ ...deps, researcher, navigator, supervisor: true }); + return new Pilot(deps, tools, researcher); })); } agentTester(): Tester { if (!this.agents.tester) { - this.agents.tester = this.createAgent(({ ai, explorer }) => { + this.agents.tester = this.createAgent((deps) => { const researcher = this.agentResearcher(); const navigator = this.agentNavigator(); - const tools = createAgentTools({ explorer, researcher, navigator }); - return new Tester(explorer, ai, researcher, navigator, tools); + const tools = createAgentTools({ ...deps, researcher, navigator }); + return new Tester(deps, researcher, navigator, tools); }); const qm = this.agentQuartermaster(); @@ -236,29 +267,27 @@ export class ExplorBot { this.agents.quartermaster = new Quartermaster(this.provider, { model: config?.model, }); - this.agents.quartermaster.start(this.explorer.playwrightHelper, this.explorer.getStateManager()); + this.agents.quartermaster.start(this.explorer, this.stateManager()); } return this.agents.quartermaster; } agentHistorian(): Historian { - return (this.agents.historian ||= this.createAgent(({ ai, explorer, config }) => { - const experienceTracker = explorer.getStateManager().getExperienceTracker(); - const reporter = explorer.getReporter(); - return new Historian(ai, experienceTracker, reporter, explorer.getStateManager(), config, { - recorder: explorer.getPlaywrightRecorder(), - helper: explorer.playwrightHelper, + return (this.agents.historian ||= this.createAgent(({ ai, explorer, config, stateManager, playwrightRecorder }) => { + return new Historian(ai, this.experienceTracker(), this.reporter(), stateManager, config, { + recorder: playwrightRecorder, + explorer, }); })); } agentRerunner(): Rerunner { if (!this.agents.rerunner) { - this.agents.rerunner = this.createAgent(({ ai, explorer }) => { + this.agents.rerunner = this.createAgent((deps) => { const researcher = this.agentResearcher(); const navigator = this.agentNavigator(); - const tools = createAgentTools({ explorer, researcher, navigator, withExperience: false }); - return new Rerunner(explorer, ai, tools); + const tools = createAgentTools({ ...deps, researcher, navigator, withExperience: false }); + return new Rerunner(deps, tools); }); if (this.isHistorianEnabled()) this.agents.rerunner.setHistorian(this.agentHistorian()); } @@ -266,9 +295,9 @@ export class ExplorBot { } agentDriller(): Driller { - return (this.agents.driller ||= this.createAgent(({ ai, explorer }) => { + return (this.agents.driller ||= this.createAgent((deps) => { const navigator = this.agentNavigator(); - return new Driller(explorer, ai, navigator); + return new Driller(deps, navigator); })); } @@ -285,7 +314,7 @@ export class ExplorBot { if (!this.agents.fisherman) { const apiConfig = this.config.api; const outputDir = this.configParser.getOutputDir(); - const requestStore = this.explorer.getRequestStore() || new RequestStore(outputDir); + const requestStore = this.requestStore(); const baseEndpoint = apiConfig?.baseEndpoint || this.config.playwright.url; const configHeaders = apiConfig?.headers || {}; const apiClient = new ApiClient(baseEndpoint); @@ -300,7 +329,11 @@ export class ExplorBot { } }; - const cookieProvider = () => this.explorer.extractCookies(); + const cookieProvider = async (): Promise> => { + const cookies = await this.explorer.withPage((page) => page.context().cookies()).catch(() => []); + if (!cookies.length) return {}; + return { Cookie: cookies.map((c: any) => `${c.name}=${c.value}`).join('; ') }; + }; this.agents.fisherman = this.createAgent(({ ai }) => { return new Fisherman(ai, apiClient, requestStore, specLoader, baseEndpoint, cookieProvider, configHeaders, hasApiConfig); @@ -334,7 +367,7 @@ export class ExplorBot { } if (!opts.extend && this.currentPlan?.url) { - const currentUrl = this.explorer?.getStateManager().getCurrentState()?.url; + const currentUrl = this._stateManager?.getCurrentState()?.url; if (currentUrl && currentUrl !== this.currentPlan.url) { tag('info').log('Different page detected, clearing previous plan'); this.clearPlan(); @@ -387,7 +420,7 @@ export class ExplorBot { } generatePlanFilename(feature?: string): string { - const state = this.explorer?.getStateManager().getCurrentState(); + const state = this._stateManager?.getCurrentState(); const urlPath = state?.url || '/'; const urlPart = sanitizeFilename(urlPath) || 'root'; const suffix = '.md'; @@ -467,7 +500,7 @@ export class ExplorBot { const filePath = this.agentSessionAnalyst().writeReport(markdown); tag('info').log(`Session report saved: ${relativeToCwd(filePath)}`); - const reporter = this.explorer?.getReporter(); + const reporter = this._reporter; if (reporter?.isEnabled()) { let description = markdown; const modelsTable = Stats.modelsTable(this.provider.getConfiguredModels()); diff --git a/src/explorer.ts b/src/explorer.ts index 5a73c19..75acd67 100644 --- a/src/explorer.ts +++ b/src/explorer.ts @@ -1,31 +1,26 @@ import { existsSync, mkdirSync } from 'node:fs'; -import path, { join } from 'node:path'; +import path from 'node:path'; // @ts-ignore import * as codeceptjs from 'codeceptjs'; import stepsListener from 'codeceptjs/lib/listener/steps'; import storeListener from 'codeceptjs/lib/listener/store'; import { createTest } from 'codeceptjs/lib/mocha/test'; import dedent from 'dedent'; -import type { BrowserContextOptions } from 'playwright'; +import type { BrowserContextOptions, Page } from 'playwright'; import { ActionResult } from './action-result.ts'; import Action from './action.js'; -import { AIProvider } from './ai/provider.js'; -import { visuallyAnnotateContainers } from './ai/researcher/coordinates.ts'; -import { RequestStore } from './api/request-store.ts'; +import type { RequestStore } from './api/request-store.ts'; import { XhrCapture } from './api/xhr-capture.ts'; import type { ExplorbotConfig } from './config.js'; import { ConfigParser } from './config.js'; -import type { ExperienceTracker } from './experience-tracker.js'; -import { KnowledgeTracker } from './knowledge-tracker.js'; -import { PlaywrightRecorder } from './playwright-recorder.ts'; -import { Reporter } from './reporter.ts'; -import { StateManager } from './state-manager.js'; +import type { KnowledgeTracker } from './knowledge-tracker.js'; +import type { PlaywrightRecorder } from './playwright-recorder.ts'; +import type { Reporter } from './reporter.ts'; +import type { StateManager } from './state-manager.js'; import { Test, TestResult } from './test-plan.ts'; import { BrowserRecoveryError, browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts'; -import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from './utils/html.ts'; import { createDebug, log, tag } from './utils/logger.js'; import { sleep, waitForPageReadiness } from './utils/page-readiness.ts'; -import { WebElement } from './utils/web-element.ts'; declare global { namespace NodeJS { @@ -43,66 +38,218 @@ declare namespace CodeceptJS { const debugLog = createDebug('explorbot:explorer'); const RECOVERABLE_NAVIGATION_ERRORS = /net::ERR_ABORTED|page\.screenshot.*Timeout|waiting for fonts to load/i; - -interface TabInfo { - url: string; - title: string; -} - -interface BrowserExecutionErrorResult { - action: 'continue' | 'stop'; - message: string; - recovered?: boolean; -} +const RECOVERY_NAVIGATION = { waitUntil: 'domcontentloaded', timeout: 10000 } as const; class Explorer { - private aiProvider: AIProvider; - playwrightHelper: any; - public isStarted = false; + private config: ExplorbotConfig; + private options?: ExplorerOptions; + private stateManager: StateManager; + private knowledgeTracker: KnowledgeTracker; + private reporter: Reporter; + private requestStore: RequestStore; + private playwrightRecorder: PlaywrightRecorder; + private playwrightHelper: any; + private _actor!: CodeceptJS.I; + private started = false; private isSharedBrowser = false; - actor!: CodeceptJS.I; - private stateManager!: StateManager; - private knowledgeTracker!: KnowledgeTracker; - config: ExplorbotConfig; - private options?: { show?: boolean; headless?: boolean; incognito?: boolean; session?: string }; - private reporter!: Reporter; - private otherTabs: TabInfo[] = []; - private _activeTest: Test | null = null; private xhrCapture: XhrCapture | null = null; - private requestStore: RequestStore | null = null; - private playwrightRecorder: PlaywrightRecorder = new PlaywrightRecorder(); + private _activeTest: Test | null = null; private observedTestPages = new Set(); private testPageErrorHandler: ((error: Error) => void) | null = null; private testConsoleHandler: ((message: any) => void) | null = null; private testDialogHandler: ((dialog: any) => void) | null = null; - constructor(config: ExplorbotConfig, aiProvider: AIProvider, options: { show?: boolean; headless?: boolean; incognito?: boolean; session?: string } | undefined, experienceTracker: ExperienceTracker, knowledgeTracker: KnowledgeTracker) { + constructor(config: ExplorbotConfig, options: ExplorerOptions | undefined, deps: ExplorerDeps) { this.config = config; - this.aiProvider = aiProvider; this.options = options; + this.stateManager = deps.stateManager; + this.knowledgeTracker = deps.knowledgeTracker; + this.reporter = deps.reporter; + this.requestStore = deps.requestStore; + this.playwrightRecorder = deps.playwrightRecorder; this.initializeContainer(); - this.knowledgeTracker = knowledgeTracker; - this.stateManager = new StateManager(experienceTracker, knowledgeTracker); - this.reporter = new Reporter(config.reporter, this.stateManager); } - private initializeContainer() { - try { - // Use project root for output directory, not current working directory - const configParser = ConfigParser.getInstance(); - const projectRoot = configParser.getProjectRoot(); - (global as any).output_dir = configParser.getStatesDir(); - (global as any).codecept_dir = projectRoot; + get actor(): CodeceptJS.I { + return this._actor; + } + + get page(): Page | null { + const page = this.playwrightHelper?.page; + if (!page || page.isClosed?.()) return null; + return page; + } + + get activeTest(): Test | null { + return this._activeTest; + } - configParser.validateConfig(this.config); + async start(): Promise { + if (this.started) return; - const codeceptConfig = this.convertToCodeceptConfig(this.config); + await codeceptjs.recorder.start(); + await codeceptjs.container.started(null); + storeListener(); + stepsListener(); - codeceptjs.container.create(codeceptConfig, {}); - } catch (error) { - log(`❌ Failed to initialize container: ${error}`); - throw error; + codeceptjs.recorder.retry({ + retries: this.config.action?.retries || 3, + when: (err: any) => !!err?.message?.includes?.('context'), + }); + + this.playwrightHelper = codeceptjs.container.helpers('Playwright'); + if (!this.playwrightHelper) { + throw new Error('Playwright helper not available'); + } + await this.connectOrLaunchBrowser(); + const hasSession = this.options?.session && existsSync(this.options.session); + await this.playwrightHelper._createContextPage(this.createBrowserContextOptions()); + await this.playwrightRecorder.start(this.playwrightHelper.browserContext); + this.attachXhrCapture(); + if (hasSession) { + tag('info').log(`Session restored from ${path.relative(process.cwd(), this.options!.session!)}`); + } + + this._actor = codeceptjs.container.support('I'); + this.started = true; + + this.listenToStateChanged(); + + codeceptjs.event.dispatcher.emit('global.before'); + tag('success').log('Browser started, ready to explore'); + } + + async stop(): Promise { + if (!this.started) return; + this.started = false; + + await this.stopCaptures(); + + if (this.options?.session && this.playwrightHelper?.browserContext) { + const dir = path.dirname(this.options.session); + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + await this.playwrightHelper.browserContext.storageState({ path: this.options.session }); + debugLog(`Session saved to ${path.relative(process.cwd(), this.options.session)}`); + } + + codeceptjs.event.dispatcher.emit('global.after'); + codeceptjs.event.dispatcher.emit('global.result'); + + if (!this.isSharedBrowser) { + await Promise.all([this.reporter.finishRun(), this.playwrightHelper._stopBrowser(), codeceptjs.recorder.stop()]); + return; + } + + tag('info').log('Closing browser context (persistent browser stays running)'); + await this.closeBrowserContext(); + this.playwrightHelper.browser = null; + this.playwrightHelper.isRunning = false; + await Promise.all([this.reporter.finishRun(), codeceptjs.recorder.stop()]); + } + + action(): Action { + return new Action(this._actor, this.stateManager, this.playwrightRecorder, (fn) => this.runWithRecovery('action', fn)); + } + + async visit(url: string, opts: CaptureOpts = {}): Promise { + return this.runWithRecovery('visit', async () => { + const action = await this.visitOnce(url); + if (opts.screenshot) return action.capturePageState({ includeScreenshot: true }); + return action.getActionResult() ?? action.capturePageState(); + }); + } + + async capture(opts: CaptureOpts = {}): Promise { + return this.action().capturePageState({ includeScreenshot: opts.screenshot }); + } + + async withPage(fn: (page: Page) => Promise): Promise { + return this.runWithRecovery('page operation', () => fn(this.playwrightHelper.page)); + } + + async recover(error?: unknown): Promise { + if (error) return this.recoverFromExecutionError(error); + + if (this.page) { + this.watchActiveTestPage(); + return { ok: true, action: 'continue', message: 'Page is available' }; + } + + if (!(await this.recoverOrRestart())) { + return { ok: false, action: 'stop', message: 'Browser page could not be recovered' }; + } + this.watchActiveTestPage(); + return { ok: true, action: 'continue', recovered: true, message: 'Browser page was recovered' }; + } + + async beginTest(test: Test): Promise { + if (!this.page && !(await this.recoverOrRestart())) { + return { started: false, stop: async () => {} }; } + + this._activeTest = test; + test.start(); + await this.reporter.reportTestStart(test); + await this.closeOtherTabs(); + this.stateManager.otherTabs = []; + + const codeceptjsTest = toCodeceptjsTest(test); + + const stepHandler = (step: any, status?: string, error?: string, log?: string) => { + if (!step.toCode) return; + if (step?.name?.startsWith('grab')) return; + if (step?.name?.startsWith('save')) return; + + test.addStep(step.toCode(), step.duration, status, error, log); + + if (!this.stateManager.getCurrentState()) return; + + const lastScreenshot = ActionResult.fromState(this.stateManager.getCurrentState()!).screenshotFile; + test.setActiveNoteScreenshot(lastScreenshot); + }; + + this.watchActiveTestPage(); + + const onStepPassed = (step: any) => stepHandler(step, 'passed'); + const onStepFailed = (step: any, error: any) => { + stepHandler(step, 'failed', error?.message || String(error), error?.stack); + }; + const onTestAfter = () => { + codeceptjs.event.dispatcher.off('step.passed', onStepPassed); + codeceptjs.event.dispatcher.off('step.failed', onStepFailed); + codeceptjs.event.dispatcher.off('test.after', onTestAfter); + this.unwatchActiveTestPages(); + }; + + codeceptjs.event.dispatcher.emit('test.before', codeceptjsTest); + codeceptjs.event.dispatcher.emit('test.start', codeceptjsTest); + codeceptjs.event.dispatcher.on('step.passed', onStepPassed); + codeceptjs.event.dispatcher.on('step.failed', onStepFailed); + codeceptjs.event.dispatcher.on('test.after', onTestAfter); + + return { started: true, stop: (meta) => this.finishTest(test, meta) }; + } + + async openTab(): Promise { + const oldPage = this.playwrightHelper?.page; + if (!oldPage) return; + + await this.activateNewPage(oldPage.context()); + await oldPage.close(); + this.stateManager.otherTabs = []; + + debugLog('Opened fresh tab, closed previous tab'); + } + + private initializeContainer() { + const configParser = ConfigParser.getInstance(); + const projectRoot = configParser.getProjectRoot(); + (global as any).output_dir = configParser.getStatesDir(); + (global as any).codecept_dir = projectRoot; + + configParser.validateConfig(this.config); + + codeceptjs.container.create(this.convertToCodeceptConfig(this.config), {}); } private convertToCodeceptConfig(config: ExplorbotConfig): any { @@ -160,99 +307,6 @@ class Explorer { return codeceptConfig; } - public getConfig(): ExplorbotConfig { - return this.config; - } - - public getAIProvider(): AIProvider { - return this.aiProvider; - } - - public getStateManager(): StateManager { - return this.stateManager; - } - - public getKnowledgeTracker(): KnowledgeTracker { - return this.knowledgeTracker; - } - - public getReporter(): Reporter { - return this.reporter; - } - - public getRequestStore(): RequestStore | null { - return this.requestStore; - } - - async extractCookies(): Promise> { - if (!this.playwrightHelper?.browserContext) return {}; - try { - const cookies = await this.playwrightHelper.browserContext.cookies(); - if (!cookies.length) return {}; - const cookieString = cookies.map((c: any) => `${c.name}=${c.value}`).join('; '); - return { Cookie: cookieString }; - } catch { - return {}; - } - } - - private setupXhrCapture(reuseRequestStore = false): void { - const configParser = ConfigParser.getInstance(); - const outputDir = configParser.getOutputDir(); - if (!reuseRequestStore || !this.requestStore) { - this.requestStore = new RequestStore(outputDir); - } - const baseUrl = this.config.playwright.url; - this.xhrCapture = new XhrCapture(this.requestStore!, baseUrl); - this.xhrCapture.attach(this.playwrightHelper.page); - } - - async start() { - if (this.isStarted) { - return; - } - - await codeceptjs.recorder.start(); - await codeceptjs.container.started(null); - storeListener(); - stepsListener(); - - codeceptjs.recorder.retry({ - retries: this.config.action?.retries || 3, - when: (err: any) => { - if (!err || typeof err.message !== 'string') { - return false; - } - // ignore context errors - return err.message.includes('context'); - }, - }); - - this.playwrightHelper = codeceptjs.container.helpers('Playwright'); - if (!this.playwrightHelper) { - throw new Error('Playwright helper not available'); - } - await this.connectOrLaunchBrowser(); - const hasSession = this.options?.session && existsSync(this.options.session); - await this.playwrightHelper._createContextPage(this.createBrowserContextOptions()); - await this.playwrightRecorder.start(this.playwrightHelper.browserContext); - this.setupXhrCapture(); - if (hasSession) { - tag('info').log(`Session restored from ${path.relative(process.cwd(), this.options!.session!)}`); - } - const I = codeceptjs.container.support('I'); - - this.actor = I; - this.isStarted = true; - - this.listenToStateChanged(); - - codeceptjs.event.dispatcher.emit('global.before'); - tag('success').log('Browser started, ready to explore'); - - return I; - } - private async connectOrLaunchBrowser(): Promise { const { getAliveEndpoint } = await import('./browser-server.js'); const endpoint = await getAliveEndpoint(); @@ -284,12 +338,13 @@ class Explorer { return contextOptions; } - createAction() { - return new Action(this.actor, this.stateManager, this.playwrightRecorder); + private attachXhrCapture(): void { + this.xhrCapture = new XhrCapture(this.requestStore, this.config.playwright.url); + this.xhrCapture.attach(this.playwrightHelper.page); } - async runWithBrowserRecovery(label: string, operation: () => Promise): Promise { - if (!(await this.ensurePageAvailable())) { + private async runWithRecovery(label: string, operation: () => Promise): Promise { + if (!this.page && !(await this.recoverOrRestart())) { throw new Error(`Browser page is unavailable before ${label}`); } @@ -304,23 +359,21 @@ class Explorer { try { return await operation(); } catch (retryError) { - if (!isNavigationTransitionError(retryError) && !this.isFatalBrowserError(retryError)) throw retryError; + if (!isNavigationTransitionError(retryError) && !isFatalBrowserError(retryError)) throw retryError; recoveryError = retryError; } } - if (!this.isFatalBrowserError(recoveryError)) throw recoveryError; + if (!isFatalBrowserError(recoveryError)) throw recoveryError; tag('warning').log(`${label}: browser page is unavailable, recovering...`); - let recovered = await this.recoverFromBrowserError(); - if (!recovered) recovered = await this.restartBrowser(); - if (!recovered) throw new BrowserRecoveryError(label, recoveryError, false); + if (!(await this.recoverOrRestart())) throw new BrowserRecoveryError(label, recoveryError, false); if (!(await this.waitForPageReadiness())) throw new BrowserRecoveryError(label, recoveryError, true); try { return await operation(); } catch (retryError) { - if (this.isFatalBrowserError(retryError)) { + if (isFatalBrowserError(retryError)) { throw new BrowserRecoveryError(label, retryError, true); } throw retryError; @@ -328,31 +381,7 @@ class Explorer { } } - async capturePageState(opts: { includeScreenshot?: boolean } = {}): Promise { - return this.runWithBrowserRecovery('capturePageState', () => this.createAction().capturePageState(opts)); - } - - async capturePageWithScreenshot(): Promise { - return this.capturePageState({ includeScreenshot: true }); - } - - async executeAction(code: string): Promise { - return this.runWithBrowserRecovery('executeAction', () => this.createAction().execute(code)); - } - - async attemptAction(code: string, originalMessage?: string): Promise { - return this.runWithBrowserRecovery('attemptAction', () => this.createAction().attempt(code, originalMessage)); - } - - getPlaywrightRecorder(): PlaywrightRecorder { - return this.playwrightRecorder; - } - - async visit(url: string) { - return this.runWithBrowserRecovery('visit', () => this.visitOnce(url)); - } - - private async visitOnce(url: string) { + private async visitOnce(url: string): Promise { await this.closeOtherTabs(); const serializedUrl = JSON.stringify(url); @@ -361,7 +390,7 @@ class Explorer { const { statePush = false, wait, waitForElement, code } = this.knowledgeTracker.getStateParameters(actionResult, ['statePush', 'wait', 'waitForElement', 'code']); - const action = this.createAction(); + const action = new Action(this._actor, this.stateManager, this.playwrightRecorder); if (statePush) { await action.execute(`I.executeScript(() => { window.history.pushState({}, '', ${serializedUrl}); window.dispatchEvent(new PopStateEvent('popstate')); })`); @@ -394,94 +423,65 @@ class Explorer { return action; } - async annotateElements(): Promise { - return this.runWithBrowserRecovery('annotateElements', async () => { - const { elements } = await annotatePageElements(this.playwrightHelper.page); - return elements; - }); - } - - async visuallyAnnotateElements(opts?: { containers?: Array<{ css: string; label: string }> }): Promise { - return this.runWithBrowserRecovery('visuallyAnnotateElements', () => visuallyAnnotateContainers(this.playwrightHelper.page, opts?.containers || [])); - } + private async recoverFromExecutionError(error: unknown): Promise { + const message = browserErrorMessage(error); + tag('error').log(`Browser execution error: ${message}`); - async getEidxInContainer(containerCss: string | null): Promise { - const page = this.playwrightHelper.page; - try { - const selector = containerCss ? `${containerCss} [${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]` : `[${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]`; - const elements = await page.locator(selector).all(); - const result: string[] = []; - for (const el of elements) { - const attr = await el.getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx); - if (attr) result.push(attr); - } - return result; - } catch (error) { - if (this.isFatalBrowserError(error)) { - tag('warning').log(`getEidxInContainer: ${browserErrorMessage(error)}`); - await this.recoverFromBrowserError(); - } - return []; + if (error instanceof Error && error.name === 'AbortError') { + return { ok: false, action: 'stop', message }; } - } - async getEidxByLocator(locator: string, container?: string | null): Promise { - try { - const page = this.playwrightHelper.page; - const base = container ? page.locator(container) : page; - const el = locator.startsWith('//') ? base.locator(`xpath=${locator}`) : base.locator(locator); - return await el.first().getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx); - } catch (error) { - if (this.isFatalBrowserError(error)) { - tag('warning').log(`getEidxByLocator: ${browserErrorMessage(error)}`); - await this.recoverFromBrowserError(); - } - return null; + if (error instanceof BrowserRecoveryError) { + return { ok: false, action: 'stop', recovered: error.recovered, message: error.message }; } - } - - private resolveBrowserUrl(url?: string): string | null { - if (!url) return null; - try { - return new URL(url).toString(); - } catch {} - const baseUrl = this.config.playwright?.url || this.config.web?.url; - if (!baseUrl) return null; + if (!isFatalBrowserError(error)) { + return { + ok: true, + action: 'continue', + message: `Previous execution error: ${message}. Investigate the current state and choose a different approach.`, + }; + } - try { - return new URL(url, baseUrl).toString(); - } catch { - return null; + if (!(await this.recoverOrRestart())) { + return { ok: false, action: 'stop', recovered: false, message: `Browser could not be recovered after fatal error: ${message}` }; } + + this.watchActiveTestPage(); + return { + ok: true, + action: 'continue', + recovered: true, + message: dedent` + Browser was recovered after a fatal page error. + Continue from the restored page. + The interrupted browser action is not product evidence. + Inspect the restored page and retry the current step when it is still required. + `, + }; } - isFatalBrowserError(error: unknown): boolean { - return isFatalBrowserError(error); + private async recoverOrRestart(): Promise { + if (await this.recoverPage()) return true; + return this.restartBrowser(); } - async recoverFromBrowserError(): Promise { + private async recoverPage(): Promise { try { - if (!this.playwrightHelper?.page || this.playwrightHelper.page.isClosed?.()) { + if (!this.page) { const context = this.playwrightHelper?.browserContext; - if (!context) return await this.restartBrowser(); - const page = await context.newPage(); - await page.bringToFront(); - await this.playwrightHelper._setPage(page); - this.bindFrameNavigated(page); - if (this.xhrCapture) { - this.xhrCapture.attach(this.playwrightHelper.page); - } + if (!context) return false; + await this.activateNewPage(context); } const url = this.resolveBrowserUrl(this.stateManager.getCurrentState()?.url); if (url) { tag('warning').log(`Browser error detected, recovering by navigating to ${url}`); - await this.playwrightHelper.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 10000 }); + await this.playwrightHelper.page.goto(url, RECOVERY_NAVIGATION); return this.waitForPageReadiness(); } tag('warning').log('Browser error detected, reloading page'); - await this.playwrightHelper.page.reload({ waitUntil: 'domcontentloaded', timeout: 10000 }); + await this.playwrightHelper.page.reload(RECOVERY_NAVIGATION); return this.waitForPageReadiness(); } catch (err) { tag('error').log(`Browser recovery failed: ${browserErrorMessage(err)}`); @@ -489,24 +489,37 @@ class Explorer { } } - async restartBrowser(): Promise { - if (!this.playwrightHelper) return false; - - const url = this.resolveBrowserUrl(this.stateManager.getCurrentState()?.url); + private async activateNewPage(context: any): Promise { + const page = await context.newPage(); + await page.bringToFront(); + await this.playwrightHelper._setPage(page); + this.bindFrameNavigated(page); + this.xhrCapture?.attach(page); + } - try { - if (this.xhrCapture && this.playwrightHelper.page) { - this.xhrCapture.detach(this.playwrightHelper.page); - } + private async stopCaptures(): Promise { + if (this.xhrCapture && this.playwrightHelper?.page) { + this.xhrCapture.detach(this.playwrightHelper.page); + } + await this.playwrightRecorder.stop(); + } - await this.playwrightRecorder.stop(); + private async closeBrowserContext(): Promise { + if (!this.playwrightHelper.browserContext) return; + await this.playwrightHelper.browserContext.close().catch((err: unknown) => { + debugLog('Failed to close browser context:', err); + }); + this.playwrightHelper.browserContext = null; + } - if (this.playwrightHelper.browserContext) { - await this.playwrightHelper.browserContext.close().catch((err: unknown) => { - debugLog('Failed to close browser context before restart:', err); - }); - this.playwrightHelper.browserContext = null; - } + private async restartBrowser(): Promise { + if (!this.playwrightHelper) return false; + + const url = this.resolveBrowserUrl(this.stateManager.getCurrentState()?.url); + + try { + await this.stopCaptures(); + await this.closeBrowserContext(); if (!this.isSharedBrowser) { await this.playwrightHelper._stopBrowser().catch((err: unknown) => { @@ -517,11 +530,11 @@ class Explorer { await this.connectOrLaunchBrowser(); await this.playwrightHelper._createContextPage(this.createBrowserContextOptions()); await this.playwrightRecorder.start(this.playwrightHelper.browserContext); - this.setupXhrCapture(true); + this.attachXhrCapture(); this.listenToStateChanged(); if (url) { - await this.playwrightHelper.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 10000 }); + await this.playwrightHelper.page.goto(url, RECOVERY_NAVIGATION); if (!(await this.waitForPageReadiness())) return false; } @@ -533,10 +546,19 @@ class Explorer { } } - async switchToMainFrame() { - if (this.playwrightHelper.frame) { - debugLog('Switching to main frame'); - await this.playwrightHelper.switchTo(); + private resolveBrowserUrl(url?: string): string | null { + if (!url) return null; + try { + return new URL(url).toString(); + } catch {} + + const baseUrl = this.config.playwright?.url || this.config.web?.url; + if (!baseUrl) return null; + + try { + return new URL(url, baseUrl).toString(); + } catch { + return null; } } @@ -551,238 +573,112 @@ class Explorer { return true; } - async isInsideIframe(): Promise { - if (this.playwrightHelper.frame) return true; - - try { - const page = this.playwrightHelper.page; - if (!page) return false; - return await page.evaluate(() => window.top !== window.self); - } catch (error) { - if (this.isFatalBrowserError(error)) { - tag('warning').log(`isInsideIframe: ${browserErrorMessage(error)}`); - await this.recoverFromBrowserError(); - } - return false; - } - } - - getCurrentIframeInfo(): string | null { - if (!this.playwrightHelper?.frame) return null; - return 'iframe context active'; - } - - hasOtherTabs(): boolean { - return this.otherTabs.length > 0; - } - - getOtherTabsInfo(): TabInfo[] { - return [...this.otherTabs]; - } - - clearOtherTabsInfo(): void { - this.otherTabs = []; - } - private listenToStateChanged(): void { - if (!this.playwrightHelper) { - debugLog('Playwright helper not available for state monitoring'); + const page = this.playwrightHelper?.page; + if (!page) { + debugLog('Playwright page not available for state monitoring'); return; } + const initialPage = page; + const context = page.context(); - try { - const page = this.playwrightHelper.page; - if (!page) { - debugLog('Playwright page not available for state monitoring'); - return; - } - const initialPage = page; - const context = this.playwrightHelper.page.context(); + context.on('page', async (newPage: any) => { + if (newPage === initialPage) return; - context.on('page', async (newPage: any) => { - if (newPage === initialPage) { - return; + try { + if (newPage.url() === 'about:blank') { + await newPage.waitForURL(/^(?!about:blank$)/, { timeout: 5000 }).catch(() => {}); } + await newPage.waitForLoadState('domcontentloaded', { timeout: 5000 }); + const url = await newPage.url(); + const title = await newPage.title().catch(() => 'Unknown'); - try { - if (newPage.url() === 'about:blank') { - await newPage.waitForURL(/^(?!about:blank$)/, { timeout: 5000 }).catch(() => {}); - } - await newPage.waitForLoadState('domcontentloaded', { timeout: 5000 }); - const url = await newPage.url(); - const title = await newPage.title().catch(() => 'Unknown'); - - this.otherTabs.push({ url, title }); - if (url !== 'about:blank') { - tag('info').log(`New browser tab opened: ${url}`); - } - debugLog(`New tab detected: ${url} - ${title}`); - } catch (error) { - debugLog('Failed to get new tab info:', error); - this.otherTabs.push({ url: 'unknown', title: 'unknown' }); + this.stateManager.otherTabs.push({ url, title }); + if (url !== 'about:blank') { + tag('info').log(`New browser tab opened: ${url}`); } - }); + debugLog(`New tab detected: ${url} - ${title}`); + } catch (error) { + debugLog('Failed to get new tab info:', error); + this.stateManager.otherTabs.push({ url: 'unknown', title: 'unknown' }); + } + }); - this.bindFrameNavigated(page); + this.bindFrameNavigated(page); - debugLog('Listening for automatic state changes'); - } catch (error) { - debugLog('Failed to set up state change monitoring:', error); - } + debugLog('Listening for automatic state changes'); } - async stop(): Promise { - if (!this.isStarted) { - return; - } - - if (this.xhrCapture && this.playwrightHelper?.page) { - this.xhrCapture.detach(this.playwrightHelper.page); - } - - await this.playwrightRecorder.stop(); - - if (this.options?.session && this.playwrightHelper?.browserContext) { - const dir = path.dirname(this.options.session); - if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); - await this.playwrightHelper.browserContext.storageState({ path: this.options.session }); - debugLog(`Session saved to ${path.relative(process.cwd(), this.options.session)}`); - } + private bindFrameNavigated(page: any): void { + page.on('framenavigated', async (frame: any) => { + if (frame !== page.mainFrame()) return; - codeceptjs.event.dispatcher.emit('global.after'); - codeceptjs.event.dispatcher.emit('global.result'); + const newUrl = await frame.url(); + let newTitle = ''; - if (this.isSharedBrowser) { - tag('info').log('Closing browser context (persistent browser stays running)'); try { - if (this.playwrightHelper.browserContext) { - await this.playwrightHelper.browserContext.close(); - this.playwrightHelper.browserContext = null; - } - } catch (err) { - debugLog('Failed to close browser context:', err); + newTitle = await frame.title(); + } catch (error) { + debugLog('Failed to get page title:', error); } - this.playwrightHelper.browser = null; - this.playwrightHelper.isRunning = false; - await Promise.all([this.reporter.finishRun(), codeceptjs.recorder.stop()]); - } else { - await Promise.all([this.reporter.finishRun(), this.playwrightHelper._stopBrowser(), codeceptjs.recorder.stop()]); - } - } - - get activeTest(): Test | null { - return this._activeTest; - } - - async startTest(test: Test): Promise { - this._activeTest = test; - test.start(); - await this.reporter.reportTestStart(test); - await this.closeOtherTabs(); - this.otherTabs = []; - if (!(await this.ensurePageAvailable())) return false; - - const codeceptjsTest = toCodeceptjsTest(test); - - const stepHandler = (step: any, status?: string, error?: string, log?: string) => { - if (!step.toCode) return; - if (step?.name?.startsWith('grab')) return; - if (step?.name?.startsWith('save')) return; - test.addStep(step.toCode(), step.duration, status, error, log); + this.stateManager.updateStateFromBasic(newUrl, newTitle, 'navigation'); - if (!this.stateManager.getCurrentState()) return; + await sleep(500); + }); + } - const lastScreenshot = ActionResult.fromState(this.stateManager.getCurrentState()!).screenshotFile; - test.setActiveNoteScreenshot(lastScreenshot); - }; + private async closeOtherTabs(): Promise { + if (!this.playwrightHelper) return; - this.watchActiveTestPage(); + const context = this.playwrightHelper.page.context(); + const pages = context.pages(); - const onStepPassed = (step: any) => stepHandler(step, 'passed'); - const onStepFailed = (step: any, error: any) => { - stepHandler(step, 'failed', error?.message || String(error), error?.stack); - }; - const onTestAfter = () => { - codeceptjs.event.dispatcher.off('step.passed', onStepPassed); - codeceptjs.event.dispatcher.off('step.failed', onStepFailed); - codeceptjs.event.dispatcher.off('test.after', onTestAfter); - this.unwatchActiveTestPages(); - }; + if (pages.length <= 1) return; - codeceptjs.event.dispatcher.emit('test.before', codeceptjsTest); - codeceptjs.event.dispatcher.emit('test.start', codeceptjsTest); - codeceptjs.event.dispatcher.on('step.passed', onStepPassed); - codeceptjs.event.dispatcher.on('step.failed', onStepFailed); - codeceptjs.event.dispatcher.on('test.after', onTestAfter); + debugLog(`Found ${pages.length} tabs, cleaning up to keep only the first one`); - return true; - } + const firstPage = pages[0]; + const tabsToClose = pages.slice(1); - async ensurePageAvailable(): Promise { - const page = this.playwrightHelper?.page; - if (page && !page.isClosed?.()) { - this.watchActiveTestPage(page); - return true; + for (const page of tabsToClose) { + await page.close(); + debugLog(`Closed extra tab: ${await page.url()}`); } - const recovered = await this.recoverFromBrowserError(); - if (!recovered) return false; - this.watchActiveTestPage(); - return true; - } - - async handleExecutionError(error: unknown): Promise { - const message = browserErrorMessage(error); - tag('error').log(`Browser execution error: ${message}`); - - if (error instanceof Error && error.name === 'AbortError') { - return { - action: 'stop', - message, - }; - } + await firstPage.bringToFront(); + await this.playwrightHelper._setPage(firstPage); - if (error instanceof BrowserRecoveryError) { - return { - action: 'stop', - recovered: error.recovered, - message: error.message, - }; - } + debugLog(`Cleaned up tabs, now focused on: ${await firstPage.url()}`); + } - if (!this.isFatalBrowserError(error)) { - return { - action: 'continue', - message: `Previous execution error: ${message}. Investigate the current state and choose a different approach.`, - }; + private async finishTest(test: Test, meta?: Record): Promise { + this.unwatchActiveTestPages(); + this._activeTest = null; + const lastScreenshot = this.stateManager.getCurrentState()?.screenshotFile; + if (lastScreenshot) { + meta ||= {}; + meta.screenshotFile = lastScreenshot; } + await this.reporter.reportTest(test, meta); + const codeceptjsTest = toCodeceptjsTest(test); - let recovered = await this.recoverFromBrowserError(); - if (!recovered) recovered = await this.restartBrowser(); - - if (!recovered) { - return { - action: 'stop', - recovered: false, - message: `Browser could not be recovered after fatal error: ${message}`, - }; + if (test.isSuccessful) { + codeceptjsTest.state = 'passed'; + codeceptjs.event.dispatcher.emit('test.passed', codeceptjsTest); + } else if (test.isSkipped) { + codeceptjsTest.state = 'skipped'; + codeceptjs.event.dispatcher.emit('test.skipped', codeceptjsTest); + } else { + codeceptjsTest.state = 'failed'; + codeceptjs.event.dispatcher.emit('test.failed', codeceptjsTest); } - this.watchActiveTestPage(); - return { - action: 'continue', - recovered: true, - message: dedent` - Browser was recovered after a fatal page error. - Continue from the restored page. - The interrupted browser action is not product evidence. - Inspect the restored page and retry the current step when it is still required. - `, - }; + codeceptjs.event.dispatcher.emit('test.finish', codeceptjsTest); + codeceptjs.event.dispatcher.emit('test.after', codeceptjsTest); } - watchActiveTestPage(page = this.playwrightHelper?.page): void { + private watchActiveTestPage(page = this.playwrightHelper?.page): void { if (!this._activeTest) return; if (!page) return; if (this.observedTestPages.has(page)) return; @@ -806,32 +702,6 @@ class Explorer { this.observedTestPages.add(page); } - async stopTest(test: Test, meta?: Record) { - this.unwatchActiveTestPages(); - this._activeTest = null; - const lastScreenshot = this.stateManager.getCurrentState()?.screenshotFile; - if (lastScreenshot) { - meta ||= {}; - meta.screenshotFile = lastScreenshot; - } - await this.reporter.reportTest(test, meta); - const codeceptjsTest = toCodeceptjsTest(test); - - if (test.isSuccessful) { - codeceptjsTest.state = 'passed'; - codeceptjs.event.dispatcher.emit('test.passed', codeceptjsTest); - } else if (test.isSkipped) { - codeceptjsTest.state = 'skipped'; - codeceptjs.event.dispatcher.emit('test.skipped', codeceptjsTest); - } else { - codeceptjsTest.state = 'failed'; - codeceptjs.event.dispatcher.emit('test.failed', codeceptjsTest); - } - - codeceptjs.event.dispatcher.emit('test.finish', codeceptjsTest); - codeceptjs.event.dispatcher.emit('test.after', codeceptjsTest); - } - private unwatchActiveTestPages(): void { for (const page of this.observedTestPages) { if (this.testPageErrorHandler) page.off('pageerror', this.testPageErrorHandler); @@ -840,87 +710,6 @@ class Explorer { } this.observedTestPages.clear(); } - - async playwrightLocatorCount(locatorFn: (page: any) => any): Promise { - try { - const pwLocator = locatorFn(this.playwrightHelper.page); - return await pwLocator.count(); - } catch (error) { - if (this.isFatalBrowserError(error)) { - tag('warning').log(`playwrightLocatorCount: ${browserErrorMessage(error)}`); - await this.recoverFromBrowserError(); - } - throw error; - } - } - - private bindFrameNavigated(page: any): void { - page.on('framenavigated', async (frame: any) => { - if (frame !== page.mainFrame()) return; - - const newUrl = await frame.url(); - let newTitle = ''; - - try { - newTitle = await frame.title(); - } catch (error) { - debugLog('Failed to get page title:', error); - } - - this.stateManager.updateStateFromBasic(newUrl, newTitle, 'navigation'); - - await sleep(500); - }); - } - - async openFreshTab(): Promise { - if (!this.playwrightHelper?.page) return; - - const oldPage = this.playwrightHelper.page; - const context = oldPage.context(); - const newPage = await context.newPage(); - - await oldPage.close(); - await newPage.bringToFront(); - - await this.playwrightHelper._setPage(newPage); - this.otherTabs = []; - - this.bindFrameNavigated(newPage); - if (this.xhrCapture) { - this.xhrCapture.attach(newPage); - } - - debugLog('Opened fresh tab, closed previous tab'); - } - - private async closeOtherTabs(): Promise { - if (!this.playwrightHelper) { - return; - } - - const context = this.playwrightHelper.page.context(); - const pages = context.pages(); - - if (pages.length <= 1) { - return; - } - - debugLog(`Found ${pages.length} tabs, cleaning up to keep only the first one`); - - const firstPage = pages[0]; - const tabsToClose = pages.slice(1); - - for (const page of tabsToClose) { - await page.close(); - debugLog(`Closed extra tab: ${await page.url()}`); - } - - await firstPage.bringToFront(); - await this.playwrightHelper._setPage(firstPage); - - debugLog(`Cleaned up tabs, now focused on: ${await firstPage.url()}`); - } } function toCodeceptjsTest(test: Test): any { @@ -938,63 +727,35 @@ function toCodeceptjsTest(test: Test): any { return codeceptjsTest; } -const REF_LINE_PATTERN = /^(\s*)-\s+(\w+)\s*(?:"([^"]*)")?.*?\[ref=(e\d+)\]/; - -const ANNOTATABLE_ROLES = new Set(['button', 'link', 'textbox', 'searchbox', 'checkbox', 'radio', 'switch', 'combobox', 'tab', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'slider', 'spinbutton', 'treeitem']); - -function parseAriaRefs(ariaSnapshot: string): Array<{ role: string; name: string; ref: string }> { - const entries: Array<{ role: string; name: string; ref: string }> = []; - for (const line of ariaSnapshot.split('\n')) { - const match = line.match(REF_LINE_PATTERN); - if (!match) continue; - if (!ANNOTATABLE_ROLES.has(match[2])) continue; - entries.push({ role: match[2], name: match[3] || '', ref: match[4] }); - } - return entries; +export interface ExplorerOptions { + show?: boolean; + headless?: boolean; + incognito?: boolean; + session?: string; } -export async function annotatePageElements(page: any): Promise<{ ariaSnapshot: string; elements: WebElement[] }> { - const ariaSnapshot: string = await page.locator('body').ariaSnapshot({ mode: 'ai' }); - const refEntries = parseAriaRefs(ariaSnapshot); +export interface ExplorerDeps { + stateManager: StateManager; + knowledgeTracker: KnowledgeTracker; + reporter: Reporter; + requestStore: RequestStore; + playwrightRecorder: PlaywrightRecorder; +} - const byRole = new Map>(); - for (const { role, name, ref } of refEntries) { - let list = byRole.get(role); - if (!list) { - list = []; - byRole.set(role, list); - } - list.push({ name, ref }); - } +export interface Recovery { + ok: boolean; + action: 'continue' | 'stop'; + message: string; + recovered?: boolean; +} - const elements: WebElement[] = []; - for (const [role, entries] of byRole) { - try { - const rawList = await page.getByRole(role).evaluateAll( - (domElements: Element[], [data, extractFnStr, config]: [Array<{ name: string; ref: string }>, string, typeof ELEMENT_EXTRACTION_CONFIG]) => { - const extract = new Function(`return ${extractFnStr}`)() as (el: Element) => any; - const results: any[] = []; - let ariaIdx = 0; - for (const el of domElements) { - if (ariaIdx >= data.length) break; - el.setAttribute(config.attrs.eidx, data[ariaIdx].ref); - const elData = extract(el, config); - if (elData) results.push(elData); - ariaIdx++; - } - return results; - }, - [entries, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG] - ); - for (const raw of rawList) { - elements.push(WebElement.fromRawData(raw, role)); - } - } catch { - debugLog(`Failed to annotate role=${role}`); - } - } +export interface TestRun { + started: boolean; + stop(meta?: Record): Promise; +} - return { ariaSnapshot, elements }; +export interface CaptureOpts { + screenshot?: boolean; } export default Explorer; diff --git a/src/state-manager.ts b/src/state-manager.ts index f853acd..8918fee 100644 --- a/src/state-manager.ts +++ b/src/state-manager.ts @@ -1,6 +1,6 @@ import { ActionResult } from './action-result.js'; import type { ExperienceTracker } from './experience-tracker.js'; -import type { KnowledgeTracker, Knowledge } from './knowledge-tracker.js'; +import type { Knowledge, KnowledgeTracker } from './knowledge-tracker.js'; import { detectFocusArea } from './utils/aria.js'; import { createDebug } from './utils/logger.js'; import { slugify } from './utils/strings.js'; @@ -67,9 +67,15 @@ export interface StateTransition { export type StateChangeListener = (event: StateTransition) => void; +export interface TabInfo { + url: string; + title: string; +} + export type { Knowledge }; export class StateManager { + otherTabs: TabInfo[] = []; private currentState: WebPageState | null = null; private stateHistory: StateTransition[] = []; private allVisitedUrls: Set = new Set(); diff --git a/src/utils/hooks-runner.ts b/src/utils/hooks-runner.ts index 31d84b0..7811886 100644 --- a/src/utils/hooks-runner.ts +++ b/src/utils/hooks-runner.ts @@ -54,11 +54,9 @@ export class HooksRunner { private async executeHook(hook: Hook, url: string): Promise { try { if (hook.type === 'playwright') { - const page = this.explorer.playwrightHelper.page; - await hook.hook({ page, url }); + await this.explorer.withPage(async (page) => hook.hook({ page, url })); } else { - const I = this.explorer.actor; - await hook.hook({ I, url }); + await hook.hook({ I: this.explorer.actor, url }); } } catch (error) { debugLog(`Hook error: ${error}`); diff --git a/src/utils/web-annotate.ts b/src/utils/web-annotate.ts new file mode 100644 index 0000000..b73105f --- /dev/null +++ b/src/utils/web-annotate.ts @@ -0,0 +1,64 @@ +import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from './html.ts'; +import { createDebug } from './logger.js'; +import { WebElement } from './web-element.ts'; + +const debugLog = createDebug('explorbot:web-annotate'); + +const REF_LINE_PATTERN = /^(\s*)-\s+(\w+)\s*(?:"([^"]*)")?.*?\[ref=(e\d+)\]/; + +const ANNOTATABLE_ROLES = new Set(['button', 'link', 'textbox', 'searchbox', 'checkbox', 'radio', 'switch', 'combobox', 'tab', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'slider', 'spinbutton', 'treeitem']); + +function parseAriaRefs(ariaSnapshot: string): Array<{ role: string; name: string; ref: string }> { + const entries: Array<{ role: string; name: string; ref: string }> = []; + for (const line of ariaSnapshot.split('\n')) { + const match = line.match(REF_LINE_PATTERN); + if (!match) continue; + if (!ANNOTATABLE_ROLES.has(match[2])) continue; + entries.push({ role: match[2], name: match[3] || '', ref: match[4] }); + } + return entries; +} + +export async function annotatePageElements(page: any): Promise<{ ariaSnapshot: string; elements: WebElement[] }> { + const ariaSnapshot: string = await page.locator('body').ariaSnapshot({ mode: 'ai' }); + const refEntries = parseAriaRefs(ariaSnapshot); + + const byRole = new Map>(); + for (const { role, name, ref } of refEntries) { + let list = byRole.get(role); + if (!list) { + list = []; + byRole.set(role, list); + } + list.push({ name, ref }); + } + + const elements: WebElement[] = []; + for (const [role, entries] of byRole) { + try { + const rawList = await page.getByRole(role).evaluateAll( + (domElements: Element[], [data, extractFnStr, config]: [Array<{ name: string; ref: string }>, string, typeof ELEMENT_EXTRACTION_CONFIG]) => { + const extract = new Function(`return ${extractFnStr}`)() as (el: Element) => any; + const results: any[] = []; + let ariaIdx = 0; + for (const el of domElements) { + if (ariaIdx >= data.length) break; + el.setAttribute(config.attrs.eidx, data[ariaIdx].ref); + const elData = extract(el, config); + if (elData) results.push(elData); + ariaIdx++; + } + return results; + }, + [entries, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG] + ); + for (const raw of rawList) { + elements.push(WebElement.fromRawData(raw, role)); + } + } catch { + debugLog(`Failed to annotate role=${role}`); + } + } + + return { ariaSnapshot, elements }; +} diff --git a/src/utils/web-eidx.ts b/src/utils/web-eidx.ts new file mode 100644 index 0000000..c86f55c --- /dev/null +++ b/src/utils/web-eidx.ts @@ -0,0 +1,21 @@ +import { ELEMENT_EXTRACTION_CONFIG } from './html.ts'; + +export async function eidxInContainer(page: any, containerCss: string | null): Promise { + const selector = containerCss ? `${containerCss} [${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]` : `[${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]`; + const elements = await page.locator(selector).all(); + const result: string[] = []; + for (const el of elements) { + const attr = await el.getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx).catch(() => null); + if (attr) result.push(attr); + } + return result; +} + +export async function eidxByLocator(page: any, locator: string, container?: string | null): Promise { + const base = container ? page.locator(container) : page; + const el = locator.startsWith('//') ? base.locator(`xpath=${locator}`) : base.locator(locator); + return el + .first() + .getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx) + .catch(() => null); +} diff --git a/tests/integration/annotate-elements.test.ts b/tests/integration/annotate-elements.test.ts index de526cd..246ef29 100644 --- a/tests/integration/annotate-elements.test.ts +++ b/tests/integration/annotate-elements.test.ts @@ -1,7 +1,7 @@ import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; import { join } from 'node:path'; import { type Browser, type Page, chromium } from 'playwright'; -import { annotatePageElements } from '../../src/explorer.ts'; +import { annotatePageElements } from '../../src/utils/web-annotate.ts'; import type { WebElement } from '../../src/utils/web-element.ts'; let browser: Browser; diff --git a/tests/integration/planner.test.ts b/tests/integration/planner.test.ts index 4eb4762..ece52b5 100644 --- a/tests/integration/planner.test.ts +++ b/tests/integration/planner.test.ts @@ -63,7 +63,7 @@ const fakeState = { html: 'stub', }; -function createMockExplorer(state = fakeState) { +function createMockDeps(state = fakeState) { const mockExperienceTracker = { getSuccessfulExperience: () => [] }; const mockKnowledgeTracker = { getRelevantKnowledge: () => [] }; const mockStateManager = { @@ -72,10 +72,13 @@ function createMockExplorer(state = fakeState) { getExperienceTracker: () => mockExperienceTracker, }; return { - getStateManager: () => mockStateManager, - getKnowledgeTracker: () => mockKnowledgeTracker, - getConfig: () => ConfigParser.getInstance().getConfig(), - } as any; + explorer: {} as any, + config: ConfigParser.getInstance().getConfig(), + stateManager: mockStateManager, + knowledgeTracker: mockKnowledgeTracker, + requestStore: {}, + playwrightRecorder: {}, + }; } function extractPromptText(entry: any): string { @@ -124,7 +127,7 @@ describe('Planner with aimock', () => { clearSessionDedup(); clearStyleCache(); - planner = new Planner(createMockExplorer(), provider, { research: async () => taskBoardUiMap } as any); + planner = new Planner({ ...createMockDeps(), ai: provider } as any, { research: async () => taskBoardUiMap } as any); mock.on({}, { content: JSON.stringify(defaultScenarios) }); }); diff --git a/tests/integration/researcher-browser.test.ts b/tests/integration/researcher-browser.test.ts index e85db9f..947ff72 100644 --- a/tests/integration/researcher-browser.test.ts +++ b/tests/integration/researcher-browser.test.ts @@ -8,7 +8,7 @@ import { Provider } from '../../src/ai/provider.ts'; import { Researcher } from '../../src/ai/researcher.ts'; import { clearResearchCache } from '../../src/ai/researcher/cache.ts'; import { ConfigParser } from '../../src/config.ts'; -import { annotatePageElements } from '../../src/explorer.ts'; +import { annotatePageElements } from '../../src/utils/web-annotate.ts'; const TASK_BOARD_URL = `file://${join(process.cwd(), 'test-data', 'task-board.html')}`; @@ -81,7 +81,7 @@ describe('Researcher with real browser + aimock', () => { }; } - function buildExplorer(state: any) { + function buildDeps(state: any) { const mockExperienceTracker = { getSuccessfulExperience: () => [], updateSummary: () => {}, @@ -92,24 +92,23 @@ describe('Researcher with real browser + aimock', () => { getExperienceTracker: () => mockExperienceTracker, getRelevantKnowledge: () => [], }; - return { - getStateManager: () => mockStateManager, - getKnowledgeTracker: () => ({ getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '' }), - getConfig: () => ConfigParser.getInstance().getConfig(), - visit: async () => {}, - annotateElements: async () => (await annotatePageElements(page)).elements, - capturePageState: async () => ActionResult.fromState(state), - capturePageWithScreenshot: async () => ActionResult.fromState(state), - runWithBrowserRecovery: async (_label: string, operation: () => Promise) => operation(), - createAction: () => ({ + const explorer = { + visit: async () => ActionResult.fromState(state), + capture: async () => ActionResult.fromState(state), + withPage: async (fn: any) => fn(page), + action: () => ({ capturePageState: async () => ActionResult.fromState(state), }), - playwrightLocatorCount: async (cb: (p: any) => any) => { - const locator = cb(page); - return locator.count(); - }, - playwrightHelper: { page }, + page, } as any; + return { + explorer, + config: ConfigParser.getInstance().getConfig(), + stateManager: mockStateManager, + knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '' }, + requestStore: {}, + playwrightRecorder: {}, + }; } beforeAll(async () => { @@ -140,7 +139,7 @@ describe('Researcher with real browser + aimock', () => { ConfigParser.setupTestConfig(); const state = await captureRealState(); - researcher = new Researcher(buildExplorer(state), provider); + researcher = new Researcher({ ...buildDeps(state), ai: provider } as any); }); afterAll(async () => { diff --git a/tests/integration/researcher-sections.test.ts b/tests/integration/researcher-sections.test.ts index c71021f..c693a55 100644 --- a/tests/integration/researcher-sections.test.ts +++ b/tests/integration/researcher-sections.test.ts @@ -15,7 +15,7 @@ const fakeState = { ariaSnapshot: '- navigation:\n - link "Home"\n- main:\n - button "Click me"', }; -function createMockExplorer(configOverrides: Record = {}, playwrightLocatorCount: () => Promise = async () => 0) { +function createMockDeps(configOverrides: Record = {}, locatorCount: () => Promise = async () => 0) { const baseConfig = ConfigParser.getInstance().getConfig(); const config = { ...baseConfig, @@ -36,21 +36,29 @@ function createMockExplorer(configOverrides: Record = {}, playw getExperienceTracker: () => ({ getSuccessfulExperience: () => [], updateSummary: () => {} }), getRelevantKnowledge: () => [], }; - return { - getStateManager: () => stateManager, - getKnowledgeTracker: () => ({ getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '' }), - getConfig: () => config, - visit: async () => {}, - annotateElements: async () => [], - capturePageState: async () => ActionResult.fromState(fakeState), - capturePageWithScreenshot: async () => ActionResult.fromState(fakeState), - runWithBrowserRecovery: async (_label: string, operation: () => Promise) => operation(), - createAction: () => ({ + const fakePage: any = {}; + fakePage.locator = () => fakePage; + fakePage.getByRole = () => fakePage; + fakePage.count = locatorCount; + fakePage.ariaSnapshot = async () => ''; + fakePage.evaluateAll = async () => []; + const explorer = { + visit: async () => ActionResult.fromState(fakeState), + capture: async () => ActionResult.fromState(fakeState), + withPage: async (fn: any) => fn(fakePage), + action: () => ({ capturePageState: async () => ActionResult.fromState(fakeState), }), - playwrightLocatorCount, - playwrightHelper: { page: {} }, + page: fakePage, } as any; + return { + explorer, + config, + stateManager, + knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '' }, + requestStore: {}, + playwrightRecorder: {}, + }; } function extractPromptText(entry: any): string { @@ -101,9 +109,9 @@ describe('Researcher researchBySections', () => { await mock.stop(); }); - function makeResearcher(configOverrides: Record = {}, playwrightLocatorCount: () => Promise = async () => 0): Researcher { - const explorer = createMockExplorer(configOverrides, playwrightLocatorCount); - const researcher = new Researcher(explorer, provider); + function makeResearcher(configOverrides: Record = {}, locatorCount: () => Promise = async () => 0): Researcher { + const deps = createMockDeps(configOverrides, locatorCount); + const researcher = new Researcher({ ...deps, ai: provider } as any); researcher.actionResult = ActionResult.fromState(fakeState); return researcher; } diff --git a/tests/integration/researcher.test.ts b/tests/integration/researcher.test.ts index 944f055..1f1b79a 100644 --- a/tests/integration/researcher.test.ts +++ b/tests/integration/researcher.test.ts @@ -48,7 +48,7 @@ const fakeState = { ariaSnapshot: '- region "main":\n - button "Create Task"\n - textbox "Search tasks"\n - combobox "Assignee"\n - combobox "Sort by"', }; -function createMockExplorer(state = fakeState) { +function createMockDeps(state = fakeState) { const mockExperienceTracker = { getSuccessfulExperience: () => [], updateSummary: () => {}, @@ -63,21 +63,29 @@ function createMockExplorer(state = fakeState) { getExperienceTracker: () => mockExperienceTracker, getRelevantKnowledge: () => [], }; - return { - getStateManager: () => mockStateManager, - getKnowledgeTracker: () => mockKnowledgeTracker, - getConfig: () => ConfigParser.getInstance().getConfig(), - visit: async () => {}, - annotateElements: async () => [], - capturePageState: async () => ActionResult.fromState(state), - capturePageWithScreenshot: async () => ActionResult.fromState(state), - runWithBrowserRecovery: async (_label: string, operation: () => Promise) => operation(), - createAction: () => ({ + const fakePage: any = {}; + fakePage.locator = () => fakePage; + fakePage.getByRole = () => fakePage; + fakePage.count = async () => 1; + fakePage.ariaSnapshot = async () => ''; + fakePage.evaluateAll = async () => []; + const explorer = { + visit: async () => ActionResult.fromState(state), + capture: async () => ActionResult.fromState(state), + withPage: async (fn: any) => fn(fakePage), + action: () => ({ capturePageState: async () => ActionResult.fromState(state), }), - playwrightLocatorCount: async () => 1, - playwrightHelper: { page: {} }, + page: fakePage, } as any; + return { + explorer, + config: ConfigParser.getInstance().getConfig(), + stateManager: mockStateManager, + knowledgeTracker: mockKnowledgeTracker, + requestStore: {}, + playwrightRecorder: {}, + }; } function extractPromptText(entry: any): string { @@ -124,7 +132,7 @@ describe('Researcher with aimock', () => { clearResearchCache(); ConfigParser.setupTestConfig(); - researcher = new Researcher(createMockExplorer(), provider); + researcher = new Researcher({ ...createMockDeps(), ai: provider } as any); mock.on({}, { content: taskBoardResearch }); }); diff --git a/tests/unit/agent-tools.test.ts b/tests/unit/agent-tools.test.ts index 64405a9..6336459 100644 --- a/tests/unit/agent-tools.test.ts +++ b/tests/unit/agent-tools.test.ts @@ -17,14 +17,12 @@ describe('createAgentTools experience', () => { content: '## FLOW: use prior success', }), }; - const explorer = { - getStateManager: () => ({ - getCurrentState: () => state, - getExperienceTracker: () => experienceTracker, - }), + const stateManager = { + getCurrentState: () => state, + getExperienceTracker: () => experienceTracker, } as any; - const tools = createAgentTools({ explorer, researcher: {} as any, navigator: {} as any }); + const tools = createAgentTools({ explorer: {} as any, stateManager, ai: {} as any, researcher: {} as any, navigator: {} as any }); expect(tools.learnExperience).toBeDefined(); @@ -38,7 +36,7 @@ describe('createAgentTools experience', () => { }); it('omits learnExperience when withExperience is false', () => { - const tools = createAgentTools({ explorer: {} as any, researcher: {} as any, navigator: {} as any, withExperience: false }); + const tools = createAgentTools({ explorer: {} as any, stateManager: {} as any, ai: {} as any, researcher: {} as any, navigator: {} as any, withExperience: false }); expect(tools.learnExperience).toBeUndefined(); }); }); diff --git a/tests/unit/annotate-elements.test.ts b/tests/unit/annotate-elements.test.ts index 1d40c61..5855a30 100644 --- a/tests/unit/annotate-elements.test.ts +++ b/tests/unit/annotate-elements.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'bun:test'; -import { annotatePageElements } from '../../src/explorer.ts'; +import { annotatePageElements } from '../../src/utils/web-annotate.ts'; function createMockPage(ariaSnapshot: string, roleElements: Record) { return { diff --git a/tests/unit/captain-mode.test.ts b/tests/unit/captain-mode.test.ts index d5e37cf..667ac2e 100644 --- a/tests/unit/captain-mode.test.ts +++ b/tests/unit/captain-mode.test.ts @@ -2,18 +2,18 @@ import { describe, expect, it } from 'bun:test'; import { Captain } from '../../src/ai/captain.ts'; function buildCaptain(opts: { state?: any; activeTest?: any; page?: any }) { + let page = opts.page || null; + if (page?.isClosed?.()) page = null; const explorer = { activeTest: opts.activeTest || null, - playwrightHelper: { - page: opts.page, - }, - getStateManager: () => ({ - getCurrentState: () => opts.state || null, - }), + page, }; const explorBot = { getExplorer: () => explorer, + stateManager: () => ({ + getCurrentState: () => opts.state || null, + }), }; return Object.assign(Object.create(Captain.prototype), { explorBot }) as Captain; @@ -64,7 +64,7 @@ describe('Captain modes', () => { describe('Captain execution recovery', () => { it('continues after a fatal browser error is recovered', async () => { const captain = buildCaptainWithExplorer({ - handleExecutionError: async () => ({ + recover: async () => ({ action: 'continue', recovered: true, message: 'Browser was recovered after a fatal page error.', @@ -80,7 +80,7 @@ describe('Captain execution recovery', () => { it('stops when a fatal browser error cannot be recovered', async () => { const captain = buildCaptainWithExplorer({ - handleExecutionError: async () => ({ + recover: async () => ({ action: 'stop', recovered: false, message: 'Browser could not be recovered', @@ -95,7 +95,7 @@ describe('Captain execution recovery', () => { it('continues when browser restart recovers after page recovery fails', async () => { const captain = buildCaptainWithExplorer({ - handleExecutionError: async () => ({ + recover: async () => ({ action: 'continue', recovered: true, message: 'Browser was recovered after a fatal page error.', @@ -110,7 +110,7 @@ describe('Captain execution recovery', () => { it('continues with guidance for non-fatal execution errors', async () => { const captain = buildCaptainWithExplorer({ - handleExecutionError: async () => ({ + recover: async () => ({ action: 'continue', message: 'Previous execution error: Locator not found. Investigate the current state and choose a different approach.', }), diff --git a/tests/unit/doc-collector.test.ts b/tests/unit/doc-collector.test.ts index 8cd49ce..caa56ef 100644 --- a/tests/unit/doc-collector.test.ts +++ b/tests/unit/doc-collector.test.ts @@ -4,10 +4,10 @@ import { Documentarian } from '../../boat/doc-collector/src/ai/documentarian.ts' import { pickDocActionCandidates } from '../../boat/doc-collector/src/ai/tools.ts'; import { DocBot } from '../../boat/doc-collector/src/docbot.ts'; import { normalizeAction, renderPageDocumentation, renderSpecIndex } from '../../boat/doc-collector/src/docs-renderer.ts'; -import { renderMermaidBody } from '../../boat/doc-collector/src/state-diagram.ts'; import { getDocPageKey, shouldCrawlDocPath } from '../../boat/doc-collector/src/path-filter.ts'; import { extractResearchNavigationTargets } from '../../boat/doc-collector/src/research-navigation.ts'; import { captureDocumentationScreenshots, getScreenshotSections } from '../../boat/doc-collector/src/screenshots.ts'; +import { renderMermaidBody } from '../../boat/doc-collector/src/state-diagram.ts'; describe('doc-collector path filter', () => { it('allows regular documentation pages', () => { @@ -403,7 +403,7 @@ describe('doc-collector screenshots', () => { }; const screenshots = await captureDocumentationScreenshots( - { playwrightHelper: { page } } as any, + { page } as any, { url: '/users/sign_in' }, ` ## Navigation @@ -738,11 +738,9 @@ describe('documentarian interactive mode', () => { }; }, } as any; + const stateManager = { getCurrentState: () => states[stateIndex] } as any; const explorer = { - getStateManager() { - return { getCurrentState: () => states[stateIndex] }; - }, - createAction() { + action() { return { async attempt(command: string) { stateIndex = command.startsWith('I.amOnPage') ? 0 : 1; @@ -751,7 +749,7 @@ describe('documentarian interactive mode', () => { }; }, } as any; - const documentarian = new Documentarian(provider, { docs: { interactive: true } }, explorer); + const documentarian = new Documentarian(provider, { docs: { interactive: true } }, explorer, stateManager); const result = await documentarian.document( states[0], `## Content Controls @@ -783,11 +781,9 @@ describe('documentarian interactive mode', () => { }; }, } as any; + const stateManager = { getCurrentState: () => states[stateIndex] } as any; const explorer = { - getStateManager() { - return { getCurrentState: () => states[stateIndex] }; - }, - createAction() { + action() { return { async attempt(command: string) { stateIndex = command.startsWith('I.amOnPage') ? 0 : 1; @@ -796,7 +792,7 @@ describe('documentarian interactive mode', () => { }; }, } as any; - const documentarian = new Documentarian(provider, { docs: { interactive: true } }, explorer); + const documentarian = new Documentarian(provider, { docs: { interactive: true } }, explorer, stateManager); const result = await documentarian.document( states[0], `## Content @@ -892,19 +888,17 @@ describe('documentarian interactive mode', () => { }, } as any; - const mockExplorer = { - getStateManager() { + const mockStateManager = { + getCurrentState() { return { - getCurrentState() { - return { - url: '/test', - title: 'Test', - ariaSnapshot: '[role: button]', - }; - }, + url: '/test', + title: 'Test', + ariaSnapshot: '[role: button]', }; }, - createAction() { + } as any; + const mockExplorer = { + action() { return { async execute(command: string) { return true; @@ -913,7 +907,7 @@ describe('documentarian interactive mode', () => { }, } as any; - const documentarian = new Documentarian(provider, { docs: { interactive: true } }, mockExplorer); + const documentarian = new Documentarian(provider, { docs: { interactive: true } }, mockExplorer, mockStateManager); const result = await documentarian.document( { @@ -946,19 +940,17 @@ describe('documentarian interactive mode', () => { }, } as any; - const mockExplorer = { - getStateManager() { + const mockStateManager = { + getCurrentState() { return { - getCurrentState() { - return { - url: '/test', - title: 'Test', - ariaSnapshot: '[role: button]', - }; - }, + url: '/test', + title: 'Test', + ariaSnapshot: '[role: button]', }; }, - createAction() { + } as any; + const mockExplorer = { + action() { return { async execute(command: string) { return true; @@ -967,7 +959,7 @@ describe('documentarian interactive mode', () => { }, } as any; - const documentarian = new Documentarian(provider, { docs: { interactive: true } }, mockExplorer); + const documentarian = new Documentarian(provider, { docs: { interactive: true } }, mockExplorer, mockStateManager); const result = await documentarian.document( { @@ -1015,15 +1007,13 @@ describe('documentarian interactive mode', () => { ]; let stateIndex = 0; - const mockExplorer = { - getStateManager() { - return { - getCurrentState() { - return states[stateIndex]; - }, - }; + const mockStateManager = { + getCurrentState() { + return states[stateIndex]; }, - createAction() { + } as any; + const mockExplorer = { + action() { return { async attempt(command: string) { if (command.startsWith('I.click')) { @@ -1040,7 +1030,7 @@ describe('documentarian interactive mode', () => { }, } as any; - const documentarian = new Documentarian(provider, { docs: { interactive: true } }, mockExplorer); + const documentarian = new Documentarian(provider, { docs: { interactive: true } }, mockExplorer, mockStateManager); const result = await documentarian.document( { url: '/items', @@ -1082,19 +1072,17 @@ describe('documentarian interactive mode', () => { }, } as any; - const mockExplorer = { - getStateManager() { + const mockStateManager = { + getCurrentState() { return { - getCurrentState() { - return { - url: '/test', - title: 'Test', - ariaSnapshot: '[role: tab]', - }; - }, + url: '/test', + title: 'Test', + ariaSnapshot: '[role: tab]', }; }, - createAction() { + } as any; + const mockExplorer = { + action() { return { async execute(command: string) { return true; @@ -1103,7 +1091,7 @@ describe('documentarian interactive mode', () => { }, } as any; - const documentarian = new Documentarian(provider, { docs: { interactive: true } }, mockExplorer); + const documentarian = new Documentarian(provider, { docs: { interactive: true } }, mockExplorer, mockStateManager); const result = await documentarian.document( { @@ -1134,19 +1122,17 @@ describe('documentarian interactive defaults', () => { }, } as any; - const mockExplorer = { - getStateManager() { + const mockStateManager = { + getCurrentState() { return { - getCurrentState() { - return { - url: '/test', - title: 'Test', - ariaSnapshot: '[role: button]', - }; - }, + url: '/test', + title: 'Test', + ariaSnapshot: '[role: button]', }; }, - createAction() { + } as any; + const mockExplorer = { + action() { return { async execute() { return true; @@ -1155,7 +1141,7 @@ describe('documentarian interactive defaults', () => { }, } as any; - const documentarian = new Documentarian(provider, {}, mockExplorer); + const documentarian = new Documentarian(provider, {}, mockExplorer, mockStateManager); const result = await documentarian.document( { url: '/test', diff --git a/tests/unit/explore-command.test.ts b/tests/unit/explore-command.test.ts index 5d8dcb2..3dde816 100644 --- a/tests/unit/explore-command.test.ts +++ b/tests/unit/explore-command.test.ts @@ -148,7 +148,7 @@ describe('ExploreCommand picking algorithm (via dry-run execute)', () => { function setupCommand(plan: Plan, _configure: string, maxTests: number) { const explorBot = { - getExplorer: () => ({ getStateManager: () => ({ getCurrentState: () => ({ url: '/demo' }) }) }), + stateManager: () => ({ getCurrentState: () => ({ url: '/demo' }) }), generatePlanFilename: () => 'demo.md', loadPlans: () => [plan], agentPlanner: () => ({ registerPlanInSession: () => {}, collectSubPageCandidates: () => [], pickNextSubPage: async () => null }), @@ -255,15 +255,13 @@ describe('ExploreCommand error page guard', () => { let testerCalled = 0; let saveCalled = 0; const explorBot = { - getExplorer: () => ({ - getStateManager: () => ({ - getCurrentState: () => ({ - url: '/missing', - fullUrl: 'https://example.com/missing', - title: 'Application', - httpStatus: 404, - html: 'Short response', - }), + stateManager: () => ({ + getCurrentState: () => ({ + url: '/missing', + fullUrl: 'https://example.com/missing', + title: 'Application', + httpStatus: 404, + html: 'Short response', }), }), plan: async () => { @@ -292,6 +290,46 @@ describe('ExploreCommand error page guard', () => { }); }); +describe('ExploreCommand reporting survives a failed return to the main page', () => { + test('final visit failure does not suppress savePlans or session analysis', async () => { + const fakePlan = new Plan('Live'); + fakePlan.url = '/live/plan'; + fakePlan.addTest(new Test('seed loaded', 'critical', 'ok', '/live/plan')); + + let savePlansCalled = false; + let analysisCalled = false; + const explorBot = { + stateManager: () => ({ getCurrentState: () => ({ url: '/live' }) }), + generatePlanFilename: () => 'live.md', + loadPlans: () => [fakePlan], + agentPlanner: () => ({ registerPlanInSession: () => {}, collectSubPageCandidates: () => [], pickNextSubPage: async () => null }), + setCurrentPlan: () => {}, + agentTester: () => ({ test: async () => {} }), + visit: async (url: string) => { + if (url === '/live') throw new Error('Navigation to /live failed: redirected to /live/new and could not resolve'); + }, + savePlans: () => { + savePlansCalled = true; + return null; + }, + printSessionAnalysis: async () => { + analysisCalled = true; + }, + agentHistorian: () => ({ getSavedFiles: () => [] }), + plan: async () => undefined, + getCurrentPlan: () => fakePlan, + lastPlanError: null, + } as unknown as ExplorBot; + + const cmd = new ExploreCommand(explorBot); + cmd.maxTests = 1; + await cmd.execute('--configure "new:0%"'); + + expect(savePlansCalled).toBe(true); + expect(analysisCalled).toBe(true); + }); +}); + describe('ExploreCommand priority filter applies to NEW tests too', () => { test('new tests with disallowed priorities get disabled and do not run', async () => { const fakePlan = new Plan('Live'); @@ -300,7 +338,7 @@ describe('ExploreCommand priority filter applies to NEW tests too', () => { let planCalled = 0; const explorBot = { - getExplorer: () => ({ getStateManager: () => ({ getCurrentState: () => ({ url: '/live' }) }) }), + stateManager: () => ({ getCurrentState: () => ({ url: '/live' }) }), generatePlanFilename: () => 'live.md', loadPlans: () => [fakePlan], agentPlanner: () => ({ registerPlanInSession: () => {}, collectSubPageCandidates: () => [], pickNextSubPage: async () => null }), diff --git a/tests/unit/explorer-recovery-url.test.ts b/tests/unit/explorer-recovery-url.test.ts index aa43e4d..3666d28 100644 --- a/tests/unit/explorer-recovery-url.test.ts +++ b/tests/unit/explorer-recovery-url.test.ts @@ -52,9 +52,9 @@ describe('Explorer recovery URL resolution', () => { updateStateFromBasic: () => {}, }; - const recovered = await explorer.recoverFromBrowserError(); + const { ok } = await explorer.recover(); - expect(recovered).toBe(true); + expect(ok).toBe(true); expect((explorer as any).playwrightHelper.page).toBe(newPage); expect(navigated).toEqual(['https://the-internet.herokuapp.com/']); expect(boundEvents).toContain('framenavigated'); @@ -67,12 +67,12 @@ describe('Explorer recovery URL resolution', () => { (explorer as any).playwrightHelper = { page: { isClosed: () => false }, }; - (explorer as any).recoverFromBrowserError = async () => { + (explorer as any).recoverPage = async () => { recoveries++; return true; }; - const result = await explorer.runWithBrowserRecovery('test operation', async () => { + const result = await (explorer as any).runWithRecovery('test operation', async () => { attempts++; if (attempts === 1) throw new Error('Target closed'); return 'recovered'; @@ -90,12 +90,12 @@ describe('Explorer recovery URL resolution', () => { (explorer as any).playwrightHelper = { page: { isClosed: () => false }, }; - (explorer as any).recoverFromBrowserError = async () => { + (explorer as any).recoverPage = async () => { recoveries++; return true; }; - const result = await explorer.runWithBrowserRecovery('test operation', async () => { + const result = await (explorer as any).runWithRecovery('test operation', async () => { attempts++; if (attempts === 1) throw new Error('page.evaluate: Execution context was destroyed, most likely because of a navigation'); if (attempts === 2) throw new Error('Target closed'); @@ -107,26 +107,23 @@ describe('Explorer recovery URL resolution', () => { expect(recoveries).toBe(1); }); - it('recovers and retries action attempts through Explorer', async () => { + it('recovers and retries page operations through withPage', async () => { const explorer = buildExplorer('https://the-internet.herokuapp.com'); let attempts = 0; let recoveries = 0; (explorer as any).playwrightHelper = { page: { isClosed: () => false }, }; - (explorer as any).recoverFromBrowserError = async () => { + (explorer as any).recoverPage = async () => { recoveries++; return true; }; - (explorer as any).createAction = () => ({ - attempt: async () => { - attempts++; - if (attempts === 1) throw new Error('Target page, context or browser has been closed'); - return true; - }, - }); - const result = await explorer.attemptAction('I.click("Menu")', undefined); + const result = await explorer.withPage(async () => { + attempts++; + if (attempts === 1) throw new Error('Target page, context or browser has been closed'); + return true; + }); expect(result).toBe(true); expect(attempts).toBe(2); @@ -138,18 +135,18 @@ describe('Explorer recovery URL resolution', () => { (explorer as any).playwrightHelper = { page: { isClosed: () => false }, }; - (explorer as any).recoverFromBrowserError = async () => true; + (explorer as any).recoverPage = async () => true; let error: unknown; try { - await explorer.runWithBrowserRecovery('capturePageState', async () => { + await (explorer as any).runWithRecovery('capturePageState', async () => { throw new Error('Target page, context or browser has been closed'); }); } catch (err) { error = err; } - const result = await explorer.handleExecutionError(error); + const result = await explorer.recover(error); expect(result.action).toBe('stop'); expect(result.message).toContain('failed after browser recovery'); @@ -157,10 +154,10 @@ describe('Explorer recovery URL resolution', () => { it('returns a stop decision when browser recovery fails', async () => { const explorer = buildExplorer('https://the-internet.herokuapp.com'); - (explorer as any).recoverFromBrowserError = async () => false; + (explorer as any).recoverPage = async () => false; (explorer as any).restartBrowser = async () => false; - const result = await explorer.handleExecutionError(new Error('Target closed')); + const result = await explorer.recover(new Error('Target closed')); expect(result.action).toBe('stop'); expect(result.recovered).toBe(false); @@ -169,7 +166,7 @@ describe('Explorer recovery URL resolution', () => { it('returns guidance for non-browser execution errors', async () => { const explorer = buildExplorer('https://the-internet.herokuapp.com'); - const result = await explorer.handleExecutionError(new Error('Locator not found')); + const result = await explorer.recover(new Error('Locator not found')); expect(result.action).toBe('continue'); expect(result.recovered).toBeUndefined(); diff --git a/tests/unit/explorer-step-listeners.test.ts b/tests/unit/explorer-step-listeners.test.ts index 69f827a..1c41bde 100644 --- a/tests/unit/explorer-step-listeners.test.ts +++ b/tests/unit/explorer-step-listeners.test.ts @@ -33,12 +33,10 @@ function trackRegistrations(): { registered: Map>; restore: () function buildExplorer() { const explorer = Object.assign(Object.create(Explorer.prototype), { reporter: { reportTestStart: async () => {} }, - stateManager: { getCurrentState: () => null }, - otherTabs: [], + stateManager: { getCurrentState: () => null, otherTabs: [] }, playwrightHelper: { page: { isClosed: () => false } }, }) as any; explorer.closeOtherTabs = async () => {}; - explorer.ensurePageAvailable = async () => true; explorer.watchActiveTestPage = () => {}; explorer.unwatchActiveTestPages = () => {}; return explorer as Explorer; @@ -58,7 +56,7 @@ describe('Explorer step listener cleanup', () => { it('removes every listener it registers after a test lifecycle', async () => { const { registered, restore } = trackRegistrations(); try { - await buildExplorer().startTest(buildTest()); + await buildExplorer().beginTest(buildTest()); expect(registered.get('step.passed')!.size).toBe(1); expect(registered.get('test.after')!.size).toBe(1); @@ -76,7 +74,7 @@ describe('Explorer step listener cleanup', () => { const { registered, restore } = trackRegistrations(); try { for (let i = 0; i < 5; i++) { - await buildExplorer().startTest(buildTest()); + await buildExplorer().beginTest(buildTest()); dispatcher.emit('test.after'); } expect(registered.get('step.passed')!.size).toBe(0); diff --git a/tests/unit/explorer.test.ts b/tests/unit/explorer.test.ts index f18b1ba..c8240ac 100644 --- a/tests/unit/explorer.test.ts +++ b/tests/unit/explorer.test.ts @@ -206,12 +206,14 @@ mock.module('codeceptjs/lib/listener/store.js', () => ({ })); import { isTemplateMatch } from '../../src/ai/planner/subpages.ts'; -import { AIProvider } from '../../src/ai/provider.ts'; +import { RequestStore } from '../../src/api/request-store.ts'; import { ConfigParser } from '../../src/config.ts'; import { ExperienceTracker } from '../../src/experience-tracker.ts'; import Explorer from '../../src/explorer.ts'; import { KnowledgeTracker } from '../../src/knowledge-tracker.ts'; +import { PlaywrightRecorder } from '../../src/playwright-recorder.ts'; import { Reporter } from '../../src/reporter.ts'; +import { StateManager } from '../../src/state-manager.ts'; import { isDynamicSegment } from '../../src/utils/url-matcher.ts'; describe('isDynamicSegment', () => { @@ -290,6 +292,7 @@ describe('isTemplateMatch', () => { describe('Explorer', () => { let explorer: Explorer; + let stateManager: StateManager; const baseUrl = 'mock://explorer.test/'; beforeAll(async () => { @@ -323,12 +326,22 @@ describe('Explorer', () => { vi.spyOn(Reporter.prototype, 'reportTest').mockResolvedValue(); const knowledgeTracker = new KnowledgeTracker(); - explorer = new Explorer(config, new AIProvider(config.ai), { headless: true }, new ExperienceTracker(knowledgeTracker), knowledgeTracker); + stateManager = new StateManager(new ExperienceTracker(knowledgeTracker), knowledgeTracker); + explorer = new Explorer( + config, + { headless: true }, + { + stateManager, + knowledgeTracker, + reporter: new Reporter(undefined, stateManager), + requestStore: new RequestStore(config.dirs?.output || 'output'), + playwrightRecorder: new PlaywrightRecorder(), + } + ); await explorer.start(); }); beforeEach(() => { - const stateManager = explorer.getStateManager(); stateManager.updateStateFromBasic(baseUrl, 'Initial', 'manual'); }); @@ -339,16 +352,16 @@ describe('Explorer', () => { }); it('visit loads target page and updates state', async () => { - const action = await explorer.visit(baseUrl); - const currentState = explorer.getStateManager().getCurrentState(); + const result = await explorer.visit(baseUrl); + const currentState = stateManager.getCurrentState(); expect(currentState?.fullUrl).toBe(baseUrl); expect(currentState?.h1).toBe('Text on Page'); - expect(action.getActionResult()?.html).toContain('Welcome to Explorbot'); + expect(result.html).toContain('Welcome to Explorbot'); }); - it('createAction executes assertions against current page', async () => { + it('action executes assertions against current page', async () => { await explorer.visit(baseUrl); - const action = explorer.createAction(); + const action = explorer.action(); await action.execute('I.see("Text on Page")'); expect(action.getActionResult()?.html).toContain('Text on Page'); }); diff --git a/tests/unit/navigator-origin-guard.test.ts b/tests/unit/navigator-origin-guard.test.ts index 0d1e9da..ac20bf7 100644 --- a/tests/unit/navigator-origin-guard.test.ts +++ b/tests/unit/navigator-origin-guard.test.ts @@ -3,13 +3,11 @@ import { Navigator } from '../../src/ai/navigator.ts'; describe('Navigator origin guard', () => { function createNavigator(baseUrl = 'http://192.168.1.162:3000') { - const navigator = Object.create(Navigator.prototype) as Navigator & { explorer: any }; - navigator.explorer = { - getConfig: () => ({ - playwright: { - url: baseUrl, - }, - }), + const navigator = Object.create(Navigator.prototype) as Navigator & { config: any }; + navigator.config = { + playwright: { + url: baseUrl, + }, }; return navigator; } diff --git a/tests/unit/pilot-state-context.test.ts b/tests/unit/pilot-state-context.test.ts index 64bedc5..b545467 100644 --- a/tests/unit/pilot-state-context.test.ts +++ b/tests/unit/pilot-state-context.test.ts @@ -15,14 +15,17 @@ function buildActionResult(browserLogs: any[] = [], ariaSnapshot = ''): ActionRe } function buildPilotWithStore(store: RequestStore | null, hasOtherTabs = false): Pilot { - const explorer: any = { - getRequestStore: () => store, - hasOtherTabs: () => hasOtherTabs, - getOtherTabsInfo: () => [], + const otherTabs = []; + if (hasOtherTabs) otherTabs.push({ url: 'about:blank', title: 'tab' }); + const deps: any = { + ai: {}, + explorer: {}, + stateManager: { otherTabs }, + requestStore: store || new RequestStore('output'), + playwrightRecorder: {}, }; - const provider: any = {}; const researcher: any = {}; - return new Pilot(provider, {}, researcher, explorer); + return new Pilot(deps, {}, researcher); } function makeFailure(method: string, path: string, status: number, counter: number): RequestResult { diff --git a/tests/unit/rerunner-healing-teardown.test.ts b/tests/unit/rerunner-healing-teardown.test.ts index 647d3f0..0ef4cfc 100644 --- a/tests/unit/rerunner-healing-teardown.test.ts +++ b/tests/unit/rerunner-healing-teardown.test.ts @@ -41,7 +41,7 @@ afterAll(() => { }); function buildRerunner(): any { - return new Rerunner({ getConfig: () => ({ ai: { agents: {} } }) } as any, {} as any); + return new Rerunner({ explorer: {}, ai: {}, config: { ai: { agents: {} } }, stateManager: {}, knowledgeTracker: {}, requestStore: {}, playwrightRecorder: {} } as any, {} as any); } const settle = () => new Promise((resolve) => setTimeout(resolve, 50)); diff --git a/tests/unit/tester-error-page.test.ts b/tests/unit/tester-error-page.test.ts index 6070b19..b758f00 100644 --- a/tests/unit/tester-error-page.test.ts +++ b/tests/unit/tester-error-page.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, mock, test } from 'bun:test'; -import { Tester } from '../../src/ai/tester.ts'; import { clearActivity, getCurrentActivity } from '../../src/activity.ts'; +import { Tester } from '../../src/ai/tester.ts'; import { ConfigParser } from '../../src/config.ts'; import { Test, TestResult } from '../../src/test-plan.ts'; @@ -35,42 +35,41 @@ describe('Tester error page handling', () => { const currentState = createState('500 Internal Server Error', '

500 Internal Server Error

', '/broken'); const startConversation = mock(() => createConversation()); const visit = mock(async () => {}); - const startTest = mock(async () => true); const stopTest = mock(async () => {}); + const startTest = mock(async () => ({ started: true, stop: stopTest })); const explorer: any = { - getConfig: () => ({}), - getStateManager: () => ({ + beginTest: startTest, + visit, + }; + const provider: any = { + getSystemPromptForAgent: () => '', + startConversation, + invokeConversation: mock(async () => null), + }; + const researcher: any = {}; + const navigator: any = {}; + const deps: any = { + explorer, + ai: provider, + config: {}, + stateManager: { getCurrentState: () => currentState, clearHistory: () => {}, + otherTabs: [], getExperienceTracker: () => ({ getExperienceTableOfContents: () => [], renderExperienceTocFor: () => '', }), - }), - getKnowledgeTracker: () => ({ + }, + knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '', - }), - getRequestStore: () => null, - playwrightHelper: { - page: { - on: () => {}, - off: () => {}, - }, }, - startTest, - stopTest, - visit, - }; - const provider: any = { - getSystemPromptForAgent: () => '', - startConversation, - invokeConversation: mock(async () => null), + requestStore: { clear: () => {}, onFailedRequest: () => () => {}, getFailedRequests: () => [] }, + playwrightRecorder: {}, }; - const researcher: any = {}; - const navigator: any = {}; - const tester = new Tester(explorer, provider, researcher, navigator); + const tester = new Tester(deps, researcher, navigator); const task = new Test('check broken page', 'normal', 'page works', '/broken'); const result = await tester.test(task); @@ -93,36 +92,11 @@ describe('Tester error page handling', () => { const visit = mock(async () => { currentState = createState('Application', 'Short response', '/missing', 404); }); - const startTest = mock(async () => true); const stopTest = mock(async () => {}); + const startTest = mock(async () => ({ started: true, stop: stopTest })); const explorer: any = { - getConfig: () => ({}), - getStateManager: () => ({ - getCurrentState: () => currentState, - clearHistory: () => {}, - getExperienceTracker: () => ({ - getExperienceTableOfContents: () => [], - renderExperienceTocFor: () => '', - }), - }), - getKnowledgeTracker: () => ({ - getRelevantKnowledge: () => [], - renderRelevantKnowledge: () => '', - }), - getRequestStore: () => null, - hasOtherTabs: () => false, - getOtherTabsInfo: () => [], - clearOtherTabsInfo: () => {}, - getCurrentIframeInfo: () => null, - playwrightHelper: { - page: { - on: () => {}, - off: () => {}, - }, - }, - startTest, - stopTest, + beginTest: startTest, visit, }; const provider: any = { @@ -135,7 +109,27 @@ describe('Tester error page handling', () => { researchOverlay: mock(async () => null), }; const navigator: any = {}; - const tester = new Tester(explorer, provider, researcher, navigator); + const deps: any = { + explorer, + ai: provider, + config: {}, + stateManager: { + getCurrentState: () => currentState, + clearHistory: () => {}, + otherTabs: [], + getExperienceTracker: () => ({ + getExperienceTableOfContents: () => [], + renderExperienceTocFor: () => '', + }), + }, + knowledgeTracker: { + getRelevantKnowledge: () => [], + renderRelevantKnowledge: () => '', + }, + requestStore: { clear: () => {}, onFailedRequest: () => () => {}, getFailedRequests: () => [] }, + playwrightRecorder: {}, + }; + const tester = new Tester(deps, researcher, navigator); const task = new Test('check missing page', 'normal', 'page works', '/missing'); const result = await tester.test(task); diff --git a/tests/unit/tester-focus-scope.test.ts b/tests/unit/tester-focus-scope.test.ts index 465d74c..02ec3d7 100644 --- a/tests/unit/tester-focus-scope.test.ts +++ b/tests/unit/tester-focus-scope.test.ts @@ -5,24 +5,6 @@ import { renderExperienceToc } from '../../src/experience-tracker.ts'; import { Test } from '../../src/test-plan.ts'; function buildTester(): Tester { - const explorer: any = { - getConfig: () => ({}), - getStateManager: () => ({ - getCurrentState: () => null, - getExperienceTracker: () => ({ - getExperienceTableOfContents: () => [], - renderExperienceTocFor: () => '', - }), - }), - getKnowledgeTracker: () => ({ - getRelevantKnowledge: () => [], - renderRelevantKnowledge: () => '', - }), - getCurrentIframeInfo: () => null, - hasOtherTabs: () => false, - getOtherTabsInfo: () => [], - clearOtherTabsInfo: () => {}, - }; const provider: any = { getSystemPromptForAgent: () => '', }; @@ -31,7 +13,26 @@ function buildTester(): Tester { researchOverlay: async () => null, }; const navigator: any = {}; - return new Tester(explorer, provider, researcher, navigator); + const deps: any = { + explorer: {}, + ai: provider, + config: {}, + stateManager: { + getCurrentState: () => null, + otherTabs: [], + getExperienceTracker: () => ({ + getExperienceTableOfContents: () => [], + renderExperienceTocFor: () => '', + }), + }, + knowledgeTracker: { + getRelevantKnowledge: () => [], + renderRelevantKnowledge: () => '', + }, + requestStore: { clear: () => {}, onFailedRequest: () => () => {}, getFailedRequests: () => [] }, + playwrightRecorder: {}, + }; + return new Tester(deps, researcher, navigator); } function buildTesterWithExperience(): Tester { @@ -43,23 +44,28 @@ function buildTesterWithExperience(): Tester { sections: [{ index: 1, level: 2, title: 'FLOW: create item' }], }, ]; - const explorer: any = { - getConfig: () => ({ files: {} }), - getStateManager: () => ({ + const provider: any = {}; + const researcher: any = {}; + const navigator: any = {}; + const deps: any = { + explorer: {}, + ai: provider, + config: { files: {} }, + stateManager: { + otherTabs: [], getExperienceTracker: () => ({ getExperienceTableOfContents: () => experienceToc, renderExperienceTocFor: () => renderExperienceToc(experienceToc), }), - }), - getKnowledgeTracker: () => ({ + }, + knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '', - }), + }, + requestStore: { clear: () => {}, onFailedRequest: () => () => {}, getFailedRequests: () => [] }, + playwrightRecorder: {}, }; - const provider: any = {}; - const researcher: any = {}; - const navigator: any = {}; - return new Tester(explorer, provider, researcher, navigator); + return new Tester(deps, researcher, navigator); } function buildState(ariaSnapshot: string, url = '/page'): ActionResult { diff --git a/tests/unit/tools-back.test.ts b/tests/unit/tools-back.test.ts index d94cb21..a8c3d63 100644 --- a/tests/unit/tools-back.test.ts +++ b/tests/unit/tools-back.test.ts @@ -29,8 +29,7 @@ describe('back tool', () => { getStateHistory: () => [{ toState: after }], }; const explorer = { - getStateManager: () => stateManager, - createAction: () => ({ + action: () => ({ attempt: async () => { navigated = true; return true; @@ -41,9 +40,10 @@ describe('back tool', () => { const tools = createAgentTools({ explorer: explorer as any, + stateManager: stateManager as any, + ai: {} as any, researcher: {} as any, navigator: {} as any, - getState: () => before as any, }); const result: any = await tools.back.execute({ reason: 'went to the wrong page' }); diff --git a/tests/unit/tools.test.ts b/tests/unit/tools.test.ts index 18cbf74..5277d14 100644 --- a/tests/unit/tools.test.ts +++ b/tests/unit/tools.test.ts @@ -1,9 +1,11 @@ import { describe, expect, it } from 'bun:test'; import { createCodeceptJSTools, createIframeTools, createLearnExperienceTool } from '../../src/ai/tools.ts'; -function fakeExplorer(): any { +function fakeDeps(): any { return { - getStateManager: () => ({}), + explorer: {}, + stateManager: {}, + ai: {}, }; } @@ -18,7 +20,7 @@ function fakeTask(): any { describe('createCodeceptJSTools click validation', () => { it('rejects empty commands without touching the browser', async () => { const task = fakeTask(); - const tools = createCodeceptJSTools(fakeExplorer(), task); + const tools = createCodeceptJSTools(fakeDeps(), task); const result = await tools.click.execute({ commands: [], explanation: 'nothing' }, {} as any); @@ -29,7 +31,7 @@ describe('createCodeceptJSTools click validation', () => { it('rejects non-click I. commands as invalid', async () => { const task = fakeTask(); - const tools = createCodeceptJSTools(fakeExplorer(), task); + const tools = createCodeceptJSTools(fakeDeps(), task); const result = await tools.click.execute({ commands: ['I.fillField("name", "value")'], explanation: 'type' }, {} as any); @@ -40,7 +42,7 @@ describe('createCodeceptJSTools click validation', () => { it('rejects non-moveCursorTo commands in hover', async () => { const task = fakeTask(); - const tools = createCodeceptJSTools(fakeExplorer(), task); + const tools = createCodeceptJSTools(fakeDeps(), task); const result = await tools.hover.execute({ commands: ['I.click("Save")'], explanation: 'hover' }, {} as any); @@ -51,7 +53,7 @@ describe('createCodeceptJSTools click validation', () => { describe('createIframeTools', () => { it('always returns the exitIframe tool', () => { - const tools = createIframeTools(fakeExplorer()); + const tools = createIframeTools(fakeDeps()); expect(Object.keys(tools)).toContain('exitIframe'); }); });