diff --git a/__tests__/dockerfile-cache-cleanup.test.ts b/__tests__/dockerfile-cache-cleanup.test.ts index 6f93284dd..985e742f0 100644 --- a/__tests__/dockerfile-cache-cleanup.test.ts +++ b/__tests__/dockerfile-cache-cleanup.test.ts @@ -1,5 +1,6 @@ import { readFileSync } from "node:fs"; import { resolve } from "node:path"; +import { AGENT_BROWSER_IDLE_TIMEOUT_MS } from "@/lib/ai/tools/utils/agent-browser-runtime"; const dockerfilePath = resolve(process.cwd(), "docker/Dockerfile"); const dockerfile = readFileSync(dockerfilePath, "utf8"); @@ -81,6 +82,12 @@ describe("sandbox Dockerfile cache cleanup", () => { expect(doctorIndex).toBeGreaterThan(cleanupIndex); }); + test("keeps the browser daemon timeout aligned with the Agent runtime", () => { + expect(dockerfile).toContain( + `ENV AGENT_BROWSER_IDLE_TIMEOUT_MS=${AGENT_BROWSER_IDLE_TIMEOUT_MS}`, + ); + }); + test("removes Go module and build caches after installing binaries", () => { const goRun = findRun( "github.com/projectdiscovery/interactsh/cmd/interactsh-client", diff --git a/docker/Dockerfile b/docker/Dockerfile index 3dcc3848b..bb3dc82fc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -252,6 +252,9 @@ ENV AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium ENV AGENT_BROWSER_USER_AGENT="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" ENV AGENT_BROWSER_ARGS="--disable-blink-features=AutomationControlled,--no-first-run,--no-default-browser-check,--disable-dev-shm-usage,--no-sandbox,--lang=en-US" ENV AGENT_BROWSER_SCREENSHOT_DIR=/home/user/agent-browser-screenshots +# Keep the browser daemon available across one maximum-length terminal command +# plus a five-minute buffer, then reclaim abandoned Chromium processes. +ENV AGENT_BROWSER_IDLE_TIMEOUT_MS=900000 RUN npm install -g --cache /tmp/npm-cache agent-browser@0.26.0 && \ rm -rf /tmp/npm-cache && \ diff --git a/lib/__tests__/system-prompt.test.ts b/lib/__tests__/system-prompt.test.ts index d5891e56e..33006274b 100644 --- a/lib/__tests__/system-prompt.test.ts +++ b/lib/__tests__/system-prompt.test.ts @@ -442,10 +442,28 @@ Commands run directly on the host OS "workstation" without Docker isolation. Be expect(cloudPrompt).toContain( "Invoke `agent-browser` directly through the terminal command tool", ); + expect(cloudPrompt).toContain( + "shuts down after 15 minutes without an agent-browser command", + ); + expect(cloudPrompt).toContain( + "assume open tabs, in-memory browser state, and element refs are lost", + ); + expect(cloudPrompt).toContain( + "reopen the URL and take a fresh snapshot instead of reusing old tabs or refs", + ); + expect(cloudPrompt).toContain( + "authenticate again through the user-approved flow", + ); + expect(cloudPrompt).toContain( + "Do not save cookies, local storage, or other authentication state to sandbox files", + ); + expect(cloudPrompt).not.toContain("agent-browser state save"); + expect(cloudPrompt).not.toContain("agent-browser --state"); for (const prompt of [localPrompt, askPrompt]) { expect(prompt).not.toContain(""); expect(prompt).not.toContain("agent-browser doctor --fix"); + expect(prompt).not.toContain("authentication state to sandbox files"); expect(prompt).not.toContain( "Invoke `agent-browser` directly through the terminal command tool", ); diff --git a/lib/ai/tools/__tests__/run-terminal-cmd.test.ts b/lib/ai/tools/__tests__/run-terminal-cmd.test.ts index 5272c9b7c..7a67936b9 100644 --- a/lib/ai/tools/__tests__/run-terminal-cmd.test.ts +++ b/lib/ai/tools/__tests__/run-terminal-cmd.test.ts @@ -736,6 +736,86 @@ describe("run_terminal_cmd — PTY action dispatch", () => { expect(detectAgentBrowserUsage("agent-browser-next open")).toBeNull(); }); + test("injects the idle timeout only for cloud agent-browser commands", async () => { + const e2b = { + jupyterUrl: "http://fake", + sandboxId: "sandbox-browser-env", + setTimeout: jest.fn(async () => undefined), + isRunning: jest.fn(async () => true), + commands: { + run: jest.fn(async (command: string) => { + if (command === "echo ready") { + return { stdout: "ready\n", stderr: "", exitCode: 0 }; + } + + return { + pid: 4321, + stdout: "", + stderr: "", + wait: jest.fn(async () => ({ + stdout: "done\n", + stderr: "", + exitCode: 0, + })), + kill: jest.fn(async () => true), + }; + }), + }, + }; + const { context } = makeContext({ sandbox: e2b }); + const tool = createRunTerminalCmd(context); + + await runTool(tool, { + command: "agent-browser open https://example.com", + brief: "open a browser page", + is_background: false, + timeout: 5, + interactive: false, + }); + await runTool(tool, { + command: "echo ok", + brief: "print a status", + is_background: false, + timeout: 5, + interactive: false, + }); + + const browserCall = e2b.commands.run.mock.calls.find( + ([command]) => command === "agent-browser open https://example.com", + ); + const unrelatedCall = e2b.commands.run.mock.calls.find( + ([command]) => command === "echo ok", + ); + + expect(browserCall?.[1]).toMatchObject({ + envs: { AGENT_BROWSER_IDLE_TIMEOUT_MS: "900000" }, + }); + expect(unrelatedCall?.[1]).not.toHaveProperty("envs"); + }); + + test("injects the idle timeout into cloud interactive browser shells", async () => { + const fakeHandle = makeFakeHandle(); + const e2b = makeFakeE2BSandbox(); + mockCreateE2BPtyHandle.mockResolvedValue(fakeHandle); + const { context } = makeContext({ sandbox: e2b }); + + setTimeout(() => fakeHandle.resolveExit(0), 10); + await runTool(createRunTerminalCmd(context), { + command: "agent-browser snapshot -i", + brief: "inspect the browser page", + is_background: false, + timeout: 5, + interactive: true, + }); + + expect(mockCreateE2BPtyHandle).toHaveBeenCalledWith( + e2b, + expect.objectContaining({ + envs: { AGENT_BROWSER_IDLE_TIMEOUT_MS: "900000" }, + }), + ); + }); + test("regression: legacy schema {command, brief, is_background, timeout} still works", async () => { // Use a non-E2B sandbox (sandboxKind !== "centrifugo" is NOT enough after // the isE2BSandbox hardening — a sandbox with sandboxKind: "centrifugo" is @@ -1413,6 +1493,7 @@ describe("run_terminal_cmd — PTY action dispatch", () => { expect(JSON.stringify(mockPhEvent.mock.calls)).not.toContain( "secret.example", ); + expect(nonE2B.commands.run.mock.calls[0]?.[1]).not.toHaveProperty("envs"); }); test("schema defaults action=exec and interactive=false when omitted", async () => { diff --git a/lib/ai/tools/__tests__/todo-write.test.ts b/lib/ai/tools/__tests__/todo-write.test.ts index d17f52f01..b6fc81bc6 100644 --- a/lib/ai/tools/__tests__/todo-write.test.ts +++ b/lib/ai/tools/__tests__/todo-write.test.ts @@ -97,6 +97,60 @@ describe("todo_write", () => { }); }); + it("skips exact normalized duplicate assistant todos and reports their ids", async () => { + const context = makeContext(); + const result = await runTool(createTodoWrite(context), { + merge: false, + todos: [ + { id: "first", content: "Review auth flow", status: "in_progress" }, + { + id: "duplicate", + content: " review AUTH flow ", + status: "pending", + }, + { + id: "distinct", + content: "Review auth flow on mobile", + status: "pending", + }, + ], + }); + + expect(result).toMatchObject({ + result: expect.stringContaining( + "Skipped new to-do IDs with exact duplicate normalized content matching an earlier item in this write or a preserved manual to-do: duplicate.", + ), + skippedTodoIds: ["duplicate"], + counts: { completed: 0, total: 2 }, + currentTodos: [ + { id: "first", content: "Review auth flow" }, + { id: "distinct", content: "Review auth flow on mobile" }, + ], + }); + }); + + it("keeps manual todos and skips new assistant duplicates of them", async () => { + const context = makeContext([ + { id: "manual", content: "Review auth flow", status: "pending" }, + ]); + const result = await runTool(createTodoWrite(context), { + merge: true, + todos: [ + { id: "duplicate", content: "review auth flow", status: "pending" }, + { id: "new", content: "Verify remediation", status: "pending" }, + ], + }); + + expect(result).toMatchObject({ + skippedTodoIds: ["duplicate"], + counts: { completed: 0, total: 2 }, + currentTodos: [ + { id: "manual", content: "Review auth flow" }, + { id: "new", content: "Verify remediation" }, + ], + }); + }); + it("tracks unique todo changes without treating inherited todos as current-run work", async () => { const context = makeContext([ { id: "stale", content: "Old work", status: "pending" }, diff --git a/lib/ai/tools/run-terminal-cmd.ts b/lib/ai/tools/run-terminal-cmd.ts index 125cdd9b6..163435d07 100644 --- a/lib/ai/tools/run-terminal-cmd.ts +++ b/lib/ai/tools/run-terminal-cmd.ts @@ -49,7 +49,10 @@ import { stripAnsi, peekExited, } from "./utils/pty-wait-utils"; -import { captureAgentBrowserUsage } from "./utils/agent-browser-usage"; +import { + captureAgentBrowserUsage, + getAgentBrowserRuntimeEnv, +} from "./utils/agent-browser-usage"; import { RUN_TERMINAL_DEFAULT_STREAM_TIMEOUT_SECONDS, RUN_TERMINAL_MAX_TIMEOUT_SECONDS, @@ -392,6 +395,9 @@ export const createRunTerminalCmd = (context: ToolContext) => { interactive: true, isBackground: false, }); + const agentBrowserEnv = isE2B + ? getAgentBrowserRuntimeEnv(command) + : undefined; // Factory is invoked BY `ptySessionManager.create` — this ensures // that if the concurrency cap is hit, the factory is never called @@ -420,6 +426,7 @@ export const createRunTerminalCmd = (context: ToolContext) => { return createE2BPtyHandle(sandbox, { cols, rows, + envs: agentBrowserEnv, }); }, }); @@ -1041,6 +1048,9 @@ export const createRunTerminalCmd = (context: ToolContext) => { onStderr: forwardCommandOutput, }, ); + const agentBrowserEnv = isE2BSandbox(sandboxInstance) + ? getAgentBrowserRuntimeEnv(command) + : undefined; const runOptions = isCentrifugoSandbox(sandboxInstance) ? { ...commonOptions, @@ -1052,7 +1062,11 @@ export const createRunTerminalCmd = (context: ToolContext) => { }, } : isE2BSandbox(sandboxInstance) - ? { ...commonOptions, signal: abortSignal } + ? { + ...commonOptions, + ...(agentBrowserEnv && { envs: agentBrowserEnv }), + signal: abortSignal, + } : commonOptions; // Determine if an error is a permanent command failure (don't retry) diff --git a/lib/ai/tools/schemas.ts b/lib/ai/tools/schemas.ts index 0a0e7bbef..2f939e674 100644 --- a/lib/ai/tools/schemas.ts +++ b/lib/ai/tools/schemas.ts @@ -342,7 +342,7 @@ export const todoWriteToolInputSchema = z.object({ ) .min(1) .describe( - "Array of todo items to write to the workspace. For merge=false, new items should include content and status and replace the assistant-generated plan while preserving manually created todos. Partial items are treated as merge-style updates. For merge=true, existing items may be patched with partial updates, but new items should include content and status.", + "Array of todo items to write to the workspace. For merge=false, new items should include content and status and replace the assistant-generated plan while preserving manually created todos. Partial items are treated as merge-style updates. For merge=true, existing items may be patched with partial updates, but new items should include content and status. A new item whose exact normalized content matches an earlier new item in the same write or a preserved manual todo is skipped and reported by ID.", ), }); diff --git a/lib/ai/tools/todo-write.ts b/lib/ai/tools/todo-write.ts index 7626aef17..06eed440a 100644 --- a/lib/ai/tools/todo-write.ts +++ b/lib/ai/tools/todo-write.ts @@ -1,6 +1,10 @@ import { tool } from "ai"; import type { ToolContext, Todo } from "@/types"; import { todoWriteTool } from "./schemas"; +import { + dedupeNewAssistantTodosByContent, + dedupeTodosById, +} from "@/lib/utils/todo-utils"; export const createTodoWrite = (context: ToolContext) => { const { todoManager, assistantMessageId } = context; @@ -30,12 +34,17 @@ export const createTodoWrite = (context: ToolContext) => { t.status === null, ); - const existingTodoIds = new Set( - todoManager.getAllTodos().map((todo) => todo.id), - ); + const existingTodos = todoManager.getAllTodos(); + const existingTodoIds = new Set(existingTodos.map((todo) => todo.id)); + const uniqueTodos = dedupeTodosById(todos); + const { todos: contentDedupedTodos, skippedTodoIds } = + dedupeNewAssistantTodosByContent(uniqueTodos, { + existingTodoIds: shouldMerge ? existingTodoIds : new Set(), + manualTodos: existingTodos.filter((todo) => !todo.sourceMessageId), + }); const todosWithSourceMessageId: Array & { id: string }> = assistantMessageId - ? todos.map((todo) => { + ? contentDedupedTodos.map((todo) => { const isNewCompleteMergeTodo = shouldMerge && !existingTodoIds.has(todo.id) && @@ -48,7 +57,7 @@ export const createTodoWrite = (context: ToolContext) => { ? { ...todo, sourceMessageId: assistantMessageId } : todo; }) - : todos; + : contentDedupedTodos; // Update backend state first (TodoManager handles deduplication) const updatedTodos = todoManager.setTodos( @@ -74,13 +83,18 @@ export const createTodoWrite = (context: ToolContext) => { })); return { - result: `Successfully ${action} to-dos. Make sure to follow and update your to-do list as you make progress. Cancel and add new to-do tasks as needed when the user makes a correction or follow-up request.${ + result: `Successfully ${action} to-dos.${ + skippedTodoIds.length > 0 + ? ` Skipped new to-do IDs with exact duplicate normalized content matching an earlier item in this write or a preserved manual to-do: ${skippedTodoIds.join(", ")}.` + : "" + } Make sure to follow and update your to-do list as you make progress. Cancel and add new to-do tasks as needed when the user makes a correction or follow-up request.${ stats.inProgress === 0 ? " No to-dos are marked in-progress, make sure to mark them before starting the next." : "" }`, counts, currentTodos, + skippedTodoIds, }; } catch (error) { return { diff --git a/lib/ai/tools/utils/agent-browser-runtime.ts b/lib/ai/tools/utils/agent-browser-runtime.ts new file mode 100644 index 000000000..2b38d98f4 --- /dev/null +++ b/lib/ai/tools/utils/agent-browser-runtime.ts @@ -0,0 +1,8 @@ +import { MAX_COMMAND_EXECUTION_TIME } from "./sandbox-command-options"; + +const AGENT_BROWSER_IDLE_GRACE_MS = 5 * 60 * 1000; + +// Allow one maximum-length foreground command plus a short handoff window +// before reclaiming a browser daemon that receives no further commands. +export const AGENT_BROWSER_IDLE_TIMEOUT_MS = + MAX_COMMAND_EXECUTION_TIME + AGENT_BROWSER_IDLE_GRACE_MS; diff --git a/lib/ai/tools/utils/agent-browser-usage.ts b/lib/ai/tools/utils/agent-browser-usage.ts index 6c38d7cdc..f08af0ef2 100644 --- a/lib/ai/tools/utils/agent-browser-usage.ts +++ b/lib/ai/tools/utils/agent-browser-usage.ts @@ -1,6 +1,7 @@ import type { AnySandbox, SandboxType, ToolContext } from "@/types"; import { phLogger } from "@/lib/posthog/server"; import { isCentrifugoSandbox, isE2BSandbox } from "./sandbox-types"; +import { AGENT_BROWSER_IDLE_TIMEOUT_MS } from "./agent-browser-runtime"; const AGENT_BROWSER_INVOCATION_RE = /(?:^|[;&|()]\s*)(?:(?:env\s+)?(?:[A-Za-z_][A-Za-z0-9_]*=(?:"[^"]*"|'[^']*'|[^\s;&|()]+)\s+)*)?(?:npx\s+(?:--yes\s+|-y\s+)?)?agent-browser(?:@[^\s;&|()]+)?(?=$|\s|[;&|()])(?:\s+([^\s;&|()]+))?/g; @@ -99,6 +100,16 @@ export function detectAgentBrowserUsage( }; } +export function getAgentBrowserRuntimeEnv( + command: string, +): Record | undefined { + if (!detectAgentBrowserUsage(command)) return undefined; + + return { + AGENT_BROWSER_IDLE_TIMEOUT_MS: String(AGENT_BROWSER_IDLE_TIMEOUT_MS), + }; +} + function getAgentBrowserSandboxType( context: ToolContext, sandbox: AnySandbox, diff --git a/lib/system-prompt.ts b/lib/system-prompt.ts index 5989bf11e..b56647f94 100644 --- a/lib/system-prompt.ts +++ b/lib/system-prompt.ts @@ -109,6 +109,10 @@ Useful reading commands: - \`agent-browser get text @e1\`, \`agent-browser get attr @e1 href\`, \`agent-browser get url\`, and \`agent-browser get title\` for targeted extraction. - Use semantic locators such as \`agent-browser find role button click --name "Submit"\` when a snapshot ref is unavailable. +Session lifetime: +- The cloud browser shuts down after 15 minutes without an agent-browser command. The next command starts a new browser, so assume open tabs, in-memory browser state, and element refs are lost; reopen the URL and take a fresh snapshot instead of reusing old tabs or refs. +- If login state is lost after relaunch, authenticate again through the user-approved flow. Do not save cookies, local storage, or other authentication state to sandbox files for idle recovery because a user's cloud sandbox can be reused across Agent runs. + Recovery: - For daemon, socket, connection, or browser-not-running failures, run \`agent-browser doctor\`; use \`agent-browser doctor --fix\` only when the diagnosis identifies a repairable problem, then reopen the page and retry. - For malformed command syntax, correct the command. For stale or invalid element refs, run a fresh \`agent-browser snapshot -i\`; do not blindly retry the same failing action. diff --git a/lib/utils/__tests__/todo-utils.test.ts b/lib/utils/__tests__/todo-utils.test.ts index cb19da6f2..8a62ed7fa 100644 --- a/lib/utils/__tests__/todo-utils.test.ts +++ b/lib/utils/__tests__/todo-utils.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from "@jest/globals"; import { mergeTodos, applyTodoWriteUpdate, + dedupeNewAssistantTodosByContent, TodoUpdateError, hasPartialTodos, shouldTreatAsMerge, @@ -18,6 +19,57 @@ import { import type { Todo } from "@/types"; describe("todo-utils", () => { + describe("dedupeNewAssistantTodosByContent", () => { + it("skips only exact normalized new duplicates and preserves manual content", () => { + const result = dedupeNewAssistantTodosByContent( + [ + { id: "manual-copy", content: "Manual task", status: "pending" }, + { id: "first", content: "Review auth flow", status: "pending" }, + { + id: "duplicate", + content: " review AUTH flow ", + status: "in_progress", + }, + { + id: "distinct", + content: "Review auth flow on mobile", + status: "pending", + }, + ], + { + manualTodos: [ + { id: "manual", content: "manual task", status: "pending" }, + ], + }, + ); + + expect(result).toEqual({ + todos: [ + { id: "first", content: "Review auth flow", status: "pending" }, + { + id: "distinct", + content: "Review auth flow on mobile", + status: "pending", + }, + ], + skippedTodoIds: ["manual-copy", "duplicate"], + }); + }); + + it("does not suppress updates to existing todo ids", () => { + const result = dedupeNewAssistantTodosByContent( + [ + { id: "existing-a", content: "Same task", status: "completed" }, + { id: "existing-b", content: "same task", status: "cancelled" }, + ], + { existingTodoIds: new Set(["existing-a", "existing-b"]) }, + ); + + expect(result.todos).toHaveLength(2); + expect(result.skippedTodoIds).toEqual([]); + }); + }); + describe("mergeTodos", () => { it("should merge new todos with existing ones", () => { const currentTodos: Todo[] = [ diff --git a/lib/utils/todo-utils.ts b/lib/utils/todo-utils.ts index 0fca5cb70..f7b207869 100644 --- a/lib/utils/todo-utils.ts +++ b/lib/utils/todo-utils.ts @@ -91,6 +91,50 @@ export const dedupeTodosById = ( return deduped.reverse(); }; +const normalizeTodoContentForDeduplication = (content: string): string => + content.normalize("NFKC").trim().replace(/\s+/g, " ").toLowerCase(); + +export const dedupeNewAssistantTodosByContent = ( + todos: ReadonlyArray, + options: { + existingTodoIds?: ReadonlySet; + manualTodos?: ReadonlyArray; + } = {}, +): { todos: T[]; skippedTodoIds: string[] } => { + const existingTodoIds = options.existingTodoIds ?? new Set(); + const seenContent = new Set( + (options.manualTodos ?? []).map((todo) => + normalizeTodoContentForDeduplication(todo.content), + ), + ); + const deduped: T[] = []; + const skippedTodoIds: string[] = []; + + for (const todo of todos) { + if ( + existingTodoIds.has(todo.id) || + typeof todo.content !== "string" || + todo.content.trim() === "" + ) { + deduped.push(todo); + continue; + } + + const normalizedContent = normalizeTodoContentForDeduplication( + todo.content, + ); + if (seenContent.has(normalizedContent)) { + skippedTodoIds.push(todo.id); + continue; + } + + seenContent.add(normalizedContent); + deduped.push(todo); + } + + return { todos: deduped, skippedTodoIds }; +}; + export interface ApplyTodoWriteUpdateOptions { currentTodos: Todo[]; incomingTodos: ReadonlyArray;