From dcc806a57613479ea4e60a631a3addc919de15eb Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sun, 19 Jul 2026 23:49:39 +0300 Subject: [PATCH 1/5] feat: config-free env mode, model recommendations, and provider docs Run Explorbot without an explorbot.config.js via EXPLORBOT_* variables. EXPLORBOT_AI_PROVIDER selects a provider and fills every role from its recommended models; EXPLORBOT_AI_MODEL pins the main model (a model id for that provider, or a standalone provider/model-id). Recommended models live in models.json, read at runtime (a bare provider name resolves through it) and shipped with the npm package. Each provider's config block in docs/basics/providers.md is generated from models.json by `bunosh docs:models`, verified in CI on release; a NOTE is emitted when a provider lacks a role. Adds Mistral as a provider. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/publish.yml | 3 + Bunoshfile.js | 35 ++++ CHANGELOG.md | 36 ++++ README.md | 26 ++- bin/explorbot-cli.ts | 26 ++- boat/api-tester/src/config.ts | 48 ++++- bun.lock | 14 ++ docs/basics/providers.md | 225 ++++++++++++++++-------- docs/index.json | 3 +- docs/reference/commands.md | 28 ++- docs/reference/configuration.md | 10 ++ docs/workflow/agentic-usage.md | 202 +++++++++++++++++++++ models.json | 30 ++++ package.json | 4 +- scripts/build-npm.sh | 1 + src/config.ts | 165 ++++++++++++++++-- src/experience-tracker.ts | 2 +- src/explorbot.ts | 8 +- src/explorer.ts | 2 +- src/utils/test-files.ts | 2 +- tests/unit/apibot-config.test.ts | 76 ++++++++ tests/unit/config.test.ts | 242 +++++++++++++++++++++++++- tests/unit/experience-tracker.test.ts | 14 ++ 23 files changed, 1098 insertions(+), 104 deletions(-) create mode 100644 docs/workflow/agentic-usage.md create mode 100644 models.json create mode 100644 tests/unit/apibot-config.test.ts diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2ab6451..5009eff 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -35,6 +35,9 @@ jobs: - name: Install dependencies run: bun install + - name: Verify provider docs are up to date + run: bunx bunosh docs:models --check + - name: Run unit tests run: bun test tests/unit/ diff --git a/Bunoshfile.js b/Bunoshfile.js index b227107..9b4e761 100644 --- a/Bunoshfile.js +++ b/Bunoshfile.js @@ -44,6 +44,41 @@ export async function worktreeCreate(name = '') { say(`Created worktree for feature ${worktreeName} in ${newDir}`); } +/** + * Regenerate provider docs (docs/basics/providers.md) from models.json + * @param {object} options + * @param {boolean} [options.check=false] - Fail if the docs are stale instead of rewriting them (for CI/release) + */ +export async function docsModels(options = { check: false }) { + const doc = resolve('docs/basics/providers.md'); + const original = readFileSync(doc, 'utf8'); + let updated = original; + + const models = JSON.parse(readFileSync(resolve('models.json'), 'utf8')); + for (const [provider, roles] of Object.entries(models)) { + const config = Object.entries(roles) + .map(([role, id]) => ` ${role}: ${provider}('${id}'),`) + .join('\n'); + let block = `\`\`\`javascript\nexport default {\n ai: {\n${config}\n },\n};\n\`\`\``; + + const missing = ['model', 'visionModel', 'agenticModel'].filter((role) => !roles[role]); + if (missing.length) { + const list = missing.map((role) => `\`${role}\``).join(' and '); + block += `\n\n> [!NOTE]\n> This provider currently doesn't serve ${list}, which is required for Explorbot to run at optimal cost and speed.\n> It is recommended to pair it with another AI provider.`; + } + + updated = updated.replace(new RegExp(`()[\\s\\S]*?()`), (_m, start, end) => `${start}\n${block}\n${end}`); + } + + if (updated === original) return say('docs/basics/providers.md already current'); + if (options.check) { + yell('docs/basics/providers.md is out of date. Run `bunosh docs:models` and commit.'); + process.exit(1); + } + writeFileSync(doc, updated); + say('Updated docs/basics/providers.md from models.json'); +} + /** * List demo-worthy segments from an Explorbot session log * @param {string} log - Path to explorbot.log diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a8b3d7..55ac085 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,41 @@ # Changelog +## 2026-07-19 + +### Environment Variables + +Explorbot can now run without an `explorbot.config.js`. Set `EXPLORBOT_AI_PROVIDER` and the configuration is built from the environment instead — for CI one-liners, demos, and coding agents driving Explorbot as a terminal command. A config file always wins when one is present, so nothing changes for existing projects. + +```bash +EXPLORBOT_URL=https://app.example.com \ +EXPLORBOT_AI_PROVIDER=openrouter \ +EXPLORBOT_KNOWLEDGE="Log in as admin@example.com / secret123" \ + npx explorbot explore /login --max-tests 3 +``` + +- **`EXPLORBOT_AI_PROVIDER`** — A provider name that fills every model role from that provider's recommended models, so there are no model IDs to look up and the run follows the recommendations as they change. Setting this turns on config-free mode. Providers: `openai`, `anthropic`, `google`, `groq`, `mistral`, `openrouter`, `sambanova`, each using its conventional API-key variable. +- **`EXPLORBOT_AI_MODEL`** — Pins the main `model`. With `EXPLORBOT_AI_PROVIDER` set it is the model id for that provider, used verbatim; on its own it must be `provider/model-id`, split on the first slash so `openrouter/openai/gpt-oss-120b:nitro` selects OpenRouter with model `openai/gpt-oss-120b:nitro`. +- **`EXPLORBOT_URL`** — Base URL to test. The API boat reads it as the base endpoint. Optional when the command already carries an absolute URL, as `docs collect https://…` does. +- **`EXPLORBOT_VISION_MODEL`** — Model for screenshot analysis, overriding the provider recommendation. Takes a provider name or `provider/model-id`. +- **`EXPLORBOT_AGENTIC_MODEL`** — Model for Captain and Pilot decisions, overriding the provider recommendation. Takes a provider name or `provider/model-id`. Combine providers by role, for example `EXPLORBOT_AI_PROVIDER=groq` with `EXPLORBOT_AGENTIC_MODEL=anthropic`. +- **`EXPLORBOT_OUTPUT`** — Output root for states, plans, research, and reports. Defaults to a fresh temp directory, so nothing is written to your project. +- **`EXPLORBOT_KNOWLEDGE`** — Inline knowledge text applied to every page, the quickest way to hand over credentials. +- **`EXPLORBOT_KNOWLEDGE_FILE`** — Path to a knowledge markdown file; its frontmatter is preserved, so it can target specific URLs. +- **`EXPLORBOT_API_SPEC`** — OpenAPI spec path for the API boat. + +Config-free runs do not write experience and run with the Historian off, so no generated test files appear and a run leaves no state behind. Point `EXPLORBOT_OUTPUT` at a stable directory, or use a config file, when you want learning to accumulate across runs. + +### Configuration +- **`experience.disabled`** — Stop writing experience files while still reading any that exist. Default: `false`. + +### Changes +- Added `docs/workflow/agentic-usage.md` — driving Explorbot from a coding agent: the config-free one-liner API, and handing Explorbot a test plan the agent wrote itself (`npx explorbot test plan.md '*'`). Plan files are input only and are never rewritten, so they live in version control next to the code they cover. +- `npx explorbot --help` now lists the `EXPLORBOT_*` variables with an example, so an agent can discover the config-free mode without reading the docs. +- The recommended model IDs in `models.json` are now read at runtime, not just when regenerating the provider docs — they are what a bare provider name resolves to. `models.json` ships with the npm package. +- Documented every AI provider in `docs/basics/providers.md`, with the recommended-model block for each generated from `models.json` by `bunosh docs:models` and verified in CI on release. Added Mistral as a supported provider — `mistral-small-latest` for the `model` and `visionModel` roles (it reads screenshots), `mistral-large-latest` for `agenticModel`. +- CodeceptJS screenshots now follow `dirs.output` instead of always landing in `output/states`. +- Bare `--session` writes to `$EXPLORBOT_OUTPUT/session.json` when that variable is set. In a temp-directory run the path is not known when the flag is parsed, so pass an explicit `--session ` there. + ## 2026-07-17 ### Changes diff --git a/README.md b/README.md index 70be873..502e11a 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ npx explorbot start https://your-app.com It runs with no babysitting and reports back what it finds. This is vibe-testing. +Explorbot works with any AI provider through the [Vercel AI SDK](https://sdk.vercel.ai/providers). See [`models.json`](models.json) for the current recommended provider and model setup, and [Providers](docs/basics/providers.md) for how to configure each one. + New here? Read the [Getting Started guide](docs/basics/getting-started.md). ## Use Cases @@ -92,7 +94,7 @@ Explorbot won't replace your regression tests — it covers what they can't. You ## Requirements - Node.js 24+ or **Bun** -- An **AI provider key** — OpenRouter recommended; Groq, Cerebras, OpenAI, Anthropic, and others via the [Vercel AI SDK](https://sdk.vercel.ai/providers) +- An **AI provider key** — OpenRouter recommended; Groq, Cerebras, [OpenAI](docs/basics/providers.md#openai), Anthropic, and others via the [Vercel AI SDK](https://sdk.vercel.ai/providers) - A **modern terminal** — iTerm2, WARP, Kitty, Ghostty, or Windows Terminal with WSL - A **compatible web app** — CRUD-heavy apps fit best. See [Prerequisites](docs/basics/prerequisites.md) @@ -125,6 +127,19 @@ Type `/explore`, and Explorbot runs its loop on its own — research, plan, test That's the gist. The [**Getting Started guide**](docs/basics/getting-started.md) walks through the full setup — choosing models, teaching Explorbot to log in, and picking the right feature to start on. +### Or skip the config file + +For a CI job, a demo, or a coding agent, pass everything as environment variables. Name a provider and Explorbot picks its recommended models: + +```bash +EXPLORBOT_URL=https://app.example.com \ +EXPLORBOT_AI_PROVIDER=openrouter \ +EXPLORBOT_KNOWLEDGE="Log in as admin@example.com / secret123" \ + npx explorbot explore /admin/users --max-tests 3 +``` + +Output lands in a temp directory and nothing is written to your project. See [Agentic Usage](docs/workflow/agentic-usage.md). + ## Teaching Explorbot Explorbot gets better when you tell it about your app: @@ -146,12 +161,13 @@ When you're ready to go deeper, the [full documentation](docs/) covers everythin ## FAQ **Can I run it in Cursor or Claude Code?** -No, Explorbot is a separate application designed for constant testing. Cursor, Codex, and Claude Code are coding agents — not relevant here. - -> However, Explorbot can be used as a subagent or terminal command controlled by a coding agent. +Not as a replacement — Explorbot is a separate application designed for constant testing, while Cursor, Codex, and Claude Code are coding agents. But a coding agent can drive Explorbot as a terminal command or subagent: it writes the test plan, Explorbot executes it against the real app. See [Agentic Usage](docs/workflow/agentic-usage.md). **Can I bring a Cursor or OpenAI subscription?** -No. Their models are too slow for the way Explorbot works. Use pay-per-token providers like Groq and OpenRouter. +No. Explorbot needs an API key, not a chat subscription. Use pay-per-token access — Groq, OpenRouter, or OpenAI's own API. + +**Can I use OpenAI directly?** +Yes. Add your `OPENAI_API_KEY` and point the models at OpenAI — a nano-class model for `model` and `visionModel`, a stronger one for `agenticModel`. Expect it to run a bit slower than hosted OSS models on Groq or Cerebras. See [Providers](docs/basics/providers.md#openai) for the config. **I want to use Opus!!!** Opus is great for coding. Testing needs a simpler model that can safely consume lots of HTML tokens, fast. Save the expensive models for sophisticated decision-making. diff --git a/bin/explorbot-cli.ts b/bin/explorbot-cli.ts index 4a4d701..5c28fa5 100755 --- a/bin/explorbot-cli.ts +++ b/bin/explorbot-cli.ts @@ -43,7 +43,8 @@ interface CLIOptions { } function buildExplorBotOptions(from: string | undefined, options: CLIOptions): ExplorBotOptions { - const sessionFile = options.session === true ? path.join(path.resolve(options.path || process.cwd()), 'output', 'session.json') : options.session; + const sessionDir = process.env.EXPLORBOT_OUTPUT ? path.resolve(process.env.EXPLORBOT_OUTPUT) : path.join(path.resolve(options.path || process.cwd()), 'output'); + const sessionFile = options.session === true ? path.join(sessionDir, 'session.json') : options.session; return { from, @@ -876,4 +877,27 @@ import { createDocsCommands } from '../boat/doc-collector/src/cli.ts'; program.addCommand(createApiCommands('api')); program.addCommand(createDocsCommands('docs')); +program.addHelpText( + 'after', + ` +Environment variables (config-free one-liner mode): + Set EXPLORBOT_AI_PROVIDER to run without an explorbot.config.js. A config file always wins. + + EXPLORBOT_AI_PROVIDER provider name; fills every role from its recommended models + EXPLORBOT_AI_MODEL model id for that provider, or provider/model-id on its own + EXPLORBOT_URL base URL to test; API boat uses it as the base endpoint + EXPLORBOT_VISION_MODEL screenshot analysis; overrides the provider recommendation + EXPLORBOT_AGENTIC_MODEL Captain and Pilot decisions; overrides the provider recommendation + EXPLORBOT_OUTPUT output root (default: a fresh temp directory) + EXPLORBOT_KNOWLEDGE inline knowledge applied to every page + EXPLORBOT_KNOWLEDGE_FILE path to a knowledge markdown file + EXPLORBOT_API_SPEC OpenAPI spec path for the API boat + + Providers: openai, anthropic, google, groq, mistral, openrouter, sambanova + Example: + EXPLORBOT_URL=https://app.example.com EXPLORBOT_AI_PROVIDER=openrouter \\ + ${cli} explore /login --max-tests 3 +` +); + program.parse(); diff --git a/boat/api-tester/src/config.ts b/boat/api-tester/src/config.ts index a0c538d..947d0a9 100644 --- a/boat/api-tester/src/config.ts +++ b/boat/api-tester/src/config.ts @@ -1,7 +1,7 @@ import { existsSync, mkdirSync, readFileSync } from 'node:fs'; import path, { resolve } from 'node:path'; import { parseEnv } from 'node:util'; -import { type AIConfig, type ApiHookFn, type ApiConfig as BaseApiConfig, EXPLORBOT_CONFIG_PATHS } from '../../../src/config.ts'; +import { type AIConfig, type ApiHookFn, type ApiConfig as BaseApiConfig, EXPLORBOT_CONFIG_PATHS, createModel, materializeKnowledge, resolveModel, resolveOutputRoot } from '../../../src/config.ts'; export type { AIConfig }; @@ -54,8 +54,11 @@ export class ApibotConfigParser { const resolvedPath = options?.config || this.findConfigFile(); if (!resolvedPath) { - if (options?.path) process.chdir(originalCwd); - throw new Error('No configuration file found. Create apibot.config.js or apibot.config.ts'); + try { + return await this.loadEnvConfig(); + } finally { + if (options?.path && originalCwd !== process.cwd()) process.chdir(originalCwd); + } } try { @@ -123,6 +126,45 @@ export class ApibotConfigParser { } } + private async loadEnvConfig(): Promise { + const provider = process.env.EXPLORBOT_AI_PROVIDER; + const modelSpec = process.env.EXPLORBOT_AI_MODEL; + if (!provider && !modelSpec) { + throw new Error('No configuration file found. Create apibot.config.js or set EXPLORBOT_URL and EXPLORBOT_AI_PROVIDER environment variables'); + } + if (modelSpec && !provider && !modelSpec.includes('/')) { + throw new Error('EXPLORBOT_AI_MODEL needs a provider — set EXPLORBOT_AI_PROVIDER, or write it as "provider/model-id"'); + } + + const baseEndpoint = process.env.EXPLORBOT_URL; + if (!baseEndpoint) { + throw new Error('No API endpoint to test. Set EXPLORBOT_URL to the API base endpoint'); + } + + const outputRoot = resolveOutputRoot(); + materializeKnowledge(outputRoot); + + const api: ApiConfig = { baseEndpoint }; + if (process.env.EXPLORBOT_API_SPEC) { + api.spec = [process.env.EXPLORBOT_API_SPEC]; + } + + let model: any; + if (provider && modelSpec) model = await createModel(provider, modelSpec); + if (provider && !modelSpec) model = await resolveModel(provider, 'model'); + if (!provider) model = await resolveModel(modelSpec!, 'model'); + + this.config = { + ai: { model }, + api, + dirs: { output: '.', knowledge: 'knowledge' }, + }; + this.configPath = path.join(outputRoot, 'apibot.config.js'); + this.validateConfig(this.config); + + return this.config; + } + private findConfigFile(): string | null { const apibotPaths = ['apibot.config.js', 'apibot.config.mjs', 'apibot.config.ts']; for (const p of apibotPaths) { diff --git a/bun.lock b/bun.lock index ca3d07b..19615e2 100644 --- a/bun.lock +++ b/bun.lock @@ -6,7 +6,9 @@ "name": "explorbot", "dependencies": { "@ai-sdk/anthropic": "^4.0", + "@ai-sdk/google": "^4.0.18", "@ai-sdk/groq": "^4.0", + "@ai-sdk/mistral": "^4.0.13", "@ai-sdk/openai": "^4.0", "@ai-sdk/otel": "^1.0.2", "@axe-core/playwright": "^4.11.0", @@ -88,8 +90,12 @@ "@ai-sdk/gateway": ["@ai-sdk/gateway@4.0.2", "", { "dependencies": { "@ai-sdk/provider": "4.0.0", "@ai-sdk/provider-utils": "5.0.0", "@vercel/oidc": "3.2.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-Jz1BiiTSvhDsCBJrkFRSqLHDRMVjFtYk9GdbSi3UOqY+/epza+oIESMDzfN4m+YHT/1IYmNEmxaMfjXOvxKDjQ=="], + "@ai-sdk/google": ["@ai-sdk/google@4.0.18", "", { "dependencies": { "@ai-sdk/provider": "4.0.3", "@ai-sdk/provider-utils": "5.0.11" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-NRbXRAasXLgFLiZDTsDUiTUuQtEUDVZoqruB5Px3geltTEqOzOo2eHN5WnDp0/OgxwnrNH4olV/TVetza0PzmQ=="], + "@ai-sdk/groq": ["@ai-sdk/groq@4.0.0", "", { "dependencies": { "@ai-sdk/provider": "4.0.0", "@ai-sdk/provider-utils": "5.0.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-QjhpyudO8eGi+C/kjVzSKfVFsLcuXi4smOez8njQITrV+5wZlKJ1xvoHyQom1FlgC9PsR4Ee6icGIqogoSb1Vg=="], + "@ai-sdk/mistral": ["@ai-sdk/mistral@4.0.13", "", { "dependencies": { "@ai-sdk/provider": "4.0.3", "@ai-sdk/provider-utils": "5.0.11" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-VYEWPEY6MhhvUKE0Huu63NJXowu2/nbbO60kE6/sdV9mLTk36PahQ7lNZFkJ2XJQ3eUy19RRWQDqYmyxKMiPlA=="], + "@ai-sdk/openai": ["@ai-sdk/openai@4.0.0", "", { "dependencies": { "@ai-sdk/provider": "4.0.0", "@ai-sdk/provider-utils": "5.0.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-XcI/bJEG+Ymx8ZwQMY9GE9waHdEcSzT6wn2o+YW80hOQPf8IAGQvdNWKaD0mxLi6AmOM0L01y/p626w7rImblQ=="], "@ai-sdk/openai-compatible": ["@ai-sdk/openai-compatible@1.0.11", "", { "dependencies": { "@ai-sdk/provider": "2.0.0", "@ai-sdk/provider-utils": "3.0.5" }, "peerDependencies": { "zod": "^3.25.76 || ^4" } }, "sha512-eRD6dZviy31KYz4YvxAR/c6UEYx3p4pCiWZeDdYdAHj0rn8xZlGVxtQRs1qynhz6IYGOo4aLBf9zVW5w0tI/Uw=="], @@ -2638,6 +2644,14 @@ "zone.js": ["zone.js@0.15.1", "", {}, "sha512-XE96n56IQpJM7NAoXswY3XRLcWFW83xe0BiAOeMD7K5k5xecOeul3Qcpx6GqEeeHNkW5DWL5zOyTbEfB4eti8w=="], + "@ai-sdk/google/@ai-sdk/provider": ["@ai-sdk/provider@4.0.3", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-e0CpNWJUY7OxAFAnCZkw+ri9QOHWwTs1tXP42782KFGCU07qt8NiXCrCVowyCB5dP2r5/Uls+g2oPd8kOJn9dw=="], + + "@ai-sdk/google/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@5.0.11", "", { "dependencies": { "@ai-sdk/provider": "4.0.3", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-7/96wE+ZsKB35iS9ASyllrE4Ym/EolXEB7AkuJ5FI++fmS85BVTAs77890C+1Z2jwHfBKjBQSBmsliOsAh0iFQ=="], + + "@ai-sdk/mistral/@ai-sdk/provider": ["@ai-sdk/provider@4.0.3", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-e0CpNWJUY7OxAFAnCZkw+ri9QOHWwTs1tXP42782KFGCU07qt8NiXCrCVowyCB5dP2r5/Uls+g2oPd8kOJn9dw=="], + + "@ai-sdk/mistral/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@5.0.11", "", { "dependencies": { "@ai-sdk/provider": "4.0.3", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-7/96wE+ZsKB35iS9ASyllrE4Ym/EolXEB7AkuJ5FI++fmS85BVTAs77890C+1Z2jwHfBKjBQSBmsliOsAh0iFQ=="], + "@ai-sdk/openai-compatible/@ai-sdk/provider": ["@ai-sdk/provider@2.0.0", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA=="], "@ai-sdk/openai-compatible/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@3.0.5", "", { "dependencies": { "@ai-sdk/provider": "2.0.0", "@standard-schema/spec": "^1.0.0", "eventsource-parser": "^3.0.3", "zod-to-json-schema": "^3.24.1" }, "peerDependencies": { "zod": "^3.25.76 || ^4" } }, "sha512-HliwB/yzufw3iwczbFVE2Fiwf1XqROB/I6ng8EKUsPM5+2wnIa8f4VbljZcDx+grhFrPV+PnRZH7zBqi8WZM7Q=="], diff --git a/docs/basics/providers.md b/docs/basics/providers.md index 079f14a..59ec44a 100644 --- a/docs/basics/providers.md +++ b/docs/basics/providers.md @@ -2,6 +2,8 @@ Explorbot connects to AI providers through the [Vercel AI SDK](https://sdk.vercel.ai/). Use any supported provider, and mix providers across different models. +> The `export default` config block inside each `` marker is generated from [`models.json`](../../models.json). After editing that file, run `bunosh docs:models`. Everything else — including the import blocks — is hand-written. + ## Requirements Your model must support: @@ -10,130 +12,166 @@ Your model must support: To analyze screenshots, you also need a vision-capable model. -### OpenRouter (recommended) +Explorbot uses three roles: + +- `model` for token-heavy page reading of ARIA & HTMLs (cheap & fast). +- `visionModel` for screenshot analysis +- `agenticModel` as advisor and planner. + +Pick a fast, cheap model for the first two and a stronger one for the third. When a provider has no recommended model for one of these roles, combine it with another provider for that role. + +### OpenRouter Start with OpenRouter. One key reaches [many providers and models](https://openrouter.ai/models). +Openrouter is an optimal solution as you can balance the models and provider for best price and speed. +So if your goal is to optimize costs, choose Openrouter. + +> Openrouter is recommended to start as it serves best models + +Install the provider package: ```bash -bun add @openrouter/ai-sdk-provider +npm i @openrouter/ai-sdk-provider ``` +Import it inside `explorbot.config.ts` and create the client from your API key: + ```javascript import { createOpenRouter } from '@openrouter/ai-sdk-provider'; const openrouter = createOpenRouter({ apiKey: process.env.OPENROUTER_API_KEY, }); +``` + +Set the recommended models in the exported config: + +```javascript export default { ai: { model: openrouter('openai/gpt-oss-20b:nitro'), - visionModel: openrouter('google/gemma-4-31b-it'), - agenticModel: openrouter('minimax/minimax-m2.5:nitro'), + visionModel: openrouter('google/gemma-4-31b-it:nitro'), + agenticModel: openrouter('google/gemma-4-31b-it:nitro'), }, }; ``` + -Pick model IDs from [OpenRouter](https://openrouter.ai/) that support structured output and tools. +Pick model IDs from [OpenRouter](https://openrouter.ai/) that support structured output and tools. The `:nitro` variants route to the fastest available host. ### Groq +Install the provider package: + ```bash -bun add @ai-sdk/groq +npm i @ai-sdk/groq ``` +Import it inside `explorbot.config.ts` and create the client from your API key: + ```javascript import { createGroq } from '@ai-sdk/groq'; const groq = createGroq({ apiKey: process.env.GROQ_API_KEY, }); - -export default { - ai: { - model: groq('openai/gpt-oss-20b'), - visionModel: groq('meta-llama/llama-4-scout-17b-16e-instruct'), - agenticModel: groq('openai/gpt-oss-120b'), - }, -}; ``` -### Cerebras - -```bash -bun add @ai-sdk/cerebras -``` +Set the recommended models in the exported config: + ```javascript -import { createCerebras } from '@ai-sdk/cerebras'; - -const cerebras = createCerebras({ - apiKey: process.env.CEREBRAS_API_KEY, -}); - export default { ai: { - model: cerebras('gpt-oss-120b'), - visionModel: cerebras('llama-4-scout-17b-16e-instruct'), - agenticModel: cerebras('gpt-oss-120b'), + model: groq('openai/gpt-oss-20b'), + visionModel: groq('qwen/qwen3.6-27b'), + agenticModel: groq('qwen/qwen3.6-27b'), }, }; ``` + + +The gpt-oss models are fast and cheap; the larger 120B handles the agenticModel role. ### OpenAI +Install the provider package: + ```bash -bun add @ai-sdk/openai +npm i @ai-sdk/openai ``` +Import it inside `explorbot.config.ts` and create the client from your API key: + ```javascript import { createOpenAI } from '@ai-sdk/openai'; const openai = createOpenAI({ apiKey: process.env.OPENAI_API_KEY, }); +``` + +Set the recommended models in the exported config: + +```javascript export default { ai: { - model: openai('gpt-4o-mini'), - visionModel: openai('gpt-4o-mini'), - agenticModel: openai('gpt-4o-mini'), + model: openai('gpt-5.4-nano'), + visionModel: openai('gpt-5.4-nano'), + agenticModel: openai('gpt-5.6-luna'), }, }; ``` - -Note: OpenAI models are slower and more expensive than many hosted OSS options. + ### Anthropic +Claude Haiku is the only Anthropic model suited to Explorbot, and even it is too costly per token for the token-heavy roles, so we recommend it only for the low-volume agenticModel. Use a cheaper provider for `model` and `visionModel` (see [Multi-Provider Configuration](#multi-provider-configuration)). + +Install the provider package: + ```bash -bun add @ai-sdk/anthropic +npm i @ai-sdk/anthropic ``` +Import it inside `explorbot.config.ts` and create the client from your API key: + ```javascript import { createAnthropic } from '@ai-sdk/anthropic'; const anthropic = createAnthropic({ apiKey: process.env.ANTHROPIC_API_KEY, }); +``` +Set the recommended model in the exported config: + + +```javascript export default { ai: { - model: anthropic('claude-sonnet-4-20250514'), - visionModel: anthropic('claude-sonnet-4-20250514'), - agenticModel: anthropic('claude-sonnet-4-20250514'), + agenticModel: anthropic('claude-haiku-4-5-20251001'), }, }; ``` -Note: Anthropic models are slower but accurate. Use them when accuracy matters more than speed. +> [!NOTE] +> This provider currently doesn't serve `model` and `visionModel`, which is required for Explorbot to run at optimal cost and speed. +> It is recommended to pair it with another AI provider. + ### Azure OpenAI +Install the provider package: + ```bash -bun add @ai-sdk/azure +npm i @ai-sdk/azure ``` +Import it inside `explorbot.config.ts` and create the client from your resource name and API key: + ```javascript import { createAzure } from '@ai-sdk/azure'; @@ -141,7 +179,11 @@ const azure = createAzure({ resourceName: process.env.AZURE_RESOURCE_NAME, apiKey: process.env.AZURE_API_KEY, }); +``` + +Set the models in the exported config, using the deployment names you created in your resource: +```javascript export default { ai: { model: azure('your-deployment-name'), @@ -151,33 +193,85 @@ export default { }; ``` -Use separate deployment names if your Azure setup uses different endpoints for chat and vision. +Azure addresses models by the deployment names you create in your resource, not by public model IDs. Use separate deployments if chat and vision run on different endpoints. ### Google (Gemini) +An API key from [Google AI Studio](https://aistudio.google.com/apikey) already has the Gemini API enabled. A key from the Google Cloud console may need the API enabled first, and service-account (Vertex Express) keys use a different API surface than the one this provider targets. + +Install the provider package: + ```bash -bun add @ai-sdk/google +npm i @ai-sdk/google ``` +Import it inside `explorbot.config.ts` and create the client from your API key: + ```javascript import { createGoogleGenerativeAI } from '@ai-sdk/google'; const google = createGoogleGenerativeAI({ apiKey: process.env.GOOGLE_API_KEY, }); +``` +Set the recommended models in the exported config: + + +```javascript export default { ai: { - model: google('gemini-2.0-flash'), - visionModel: google('gemini-2.0-flash'), - agenticModel: google('gemini-2.0-flash'), + model: google('gemini-3.1-flash-lite'), + visionModel: google('gemini-3.1-flash-lite'), + agenticModel: google('gemini-3.5-flash'), }, }; ``` + + +The flash-lite tier is the cheapest current option for the token-heavy `model` and `visionModel` roles; the full flash is stronger for the low-volume `agenticModel`. + +On a free (no-billing) key the `agenticModel` is heavily rate-limited and will fail with quota errors mid-session — keep every role on the flash-lite model or enable billing. Google also retires older models for new accounts: `gemini-2.5-flash` and `gemini-2.5-flash-lite` return a 404 for keys created after their cutoff, and enabling billing does not bring them back. + +Note: Gemini models are slower than hosted OSS models on Groq or Cerebras. Everything works, sessions just take more wall-clock time. + +### Mistral + +Install the provider package: + +```bash +npm i @ai-sdk/mistral +``` + +Import it inside `explorbot.config.ts` and create the client from your API key: + +```javascript +import { createMistral } from '@ai-sdk/mistral'; + +const mistral = createMistral({ + apiKey: process.env.MISTRAL_API_KEY, +}); +``` + +Set the recommended models in the exported config: + + +```javascript +export default { + ai: { + model: mistral('mistral-small-latest'), + visionModel: mistral('mistral-small-latest'), + agenticModel: mistral('mistral-large-latest'), + }, +}; +``` + + +Mistral Small covers the token-heavy `model` and `visionModel` roles — it accepts image input, so it can read screenshots. Mistral Large, the larger multimodal flagship, handles the low-volume `agenticModel`. The `-latest` aliases track Mistral's newest release of each, so recommendations keep up as models ship. ## Multi-Provider Configuration -Mix clients the same way you assign `model`, `visionModel`, and `agenticModel`. Each field can use a different provider instance: +Mix clients the same way you assign `model`, `visionModel`, and `agenticModel`. Each field can use a different provider instance — a fast provider does the token-heavy reading while a stronger one makes the decisions: ```javascript import { createGroq } from '@ai-sdk/groq'; @@ -190,57 +284,40 @@ export default { ai: { model: groq('openai/gpt-oss-20b'), visionModel: groq('meta-llama/llama-4-scout-17b-16e-instruct'), - agenticModel: openrouter('moonshotai/kimi-k2-instruct-0905'), + agenticModel: openrouter('minimax/minimax-m2.5:nitro'), }, }; ``` ## Per-Agent Model Configuration -```javascript -import { createGroq } from '@ai-sdk/groq'; - -const groq = createGroq({ - apiKey: process.env.GROQ_API_KEY, -}); +Any agent can override the defaults. Add an `agents` block and set per-agent options — a different `model` (using any client shown above) or a `reasoning` level: +```javascript export default { ai: { - model: groq('openai/gpt-oss-20b'), - visionModel: groq('meta-llama/llama-4-scout-17b-16e-instruct'), - agenticModel: groq('openai/gpt-oss-20b'), + // ...your model, visionModel, agenticModel... agents: { - navigator: { model: groq('openai/gpt-oss-20b') }, - researcher: { model: groq('openai/gpt-oss-20b') }, - planner: { model: groq('openai/gpt-oss-20b') }, - tester: { model: groq('openai/gpt-oss-20b') }, + researcher: { reasoning: 'low' }, + planner: { reasoning: 'none' }, + tester: { reasoning: 'none' }, }, }, }; ``` +See [Configuration](../reference/configuration.md) for every per-agent option. + ## Environment Variables -Set your API key as an environment variable: +Set your API key as an environment variable, or use a `.env` file in your project root: ```bash -# OpenRouter export OPENROUTER_API_KEY=your-key-here - -# Groq export GROQ_API_KEY=your-key-here - -# Cerebras export CEREBRAS_API_KEY=your-key-here - -# OpenAI export OPENAI_API_KEY=your-key-here - -# Anthropic export ANTHROPIC_API_KEY=your-key-here - -# Google export GOOGLE_API_KEY=your-key-here +export MISTRAL_API_KEY=your-key-here ``` - -Or use a `.env` file in your project root. diff --git a/docs/index.json b/docs/index.json index fca8a56..c008165 100644 --- a/docs/index.json +++ b/docs/index.json @@ -55,7 +55,8 @@ { "title": "Test plans", "file": "workflow/test-plans.md", "description": "The plan file format and how plans are reused" }, { "title": "Planning styles", "file": "workflow/planning-styles.md", "description": "Normal, curious, psycho, and your own" }, { "title": "Reporting", "file": "workflow/reporting.md", "description": "Local reports and Testomat.io" }, - { "title": "Continuous integration", "file": "workflow/ci.md", "description": "Scheduled runs with cached experience on any CI" } + { "title": "Continuous integration", "file": "workflow/ci.md", "description": "Scheduled runs with cached experience on any CI" }, + { "title": "Agentic usage", "file": "workflow/agentic-usage.md", "description": "Drive Explorbot from a coding agent: config-free runs and agent-written plans" } ] }, { diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 1569517..73a9fb1 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -77,7 +77,33 @@ npx explorbot navigate /login --session # probe + capture auth in on npx explorbot research /dashboard --session auth.json # reuse captured auth ``` -Without a file path, the flag defaults to `output/session.json`. +Without a file path, the flag defaults to `output/session.json`, or to `$EXPLORBOT_OUTPUT/session.json` when that variable is set. In a temp-directory run the temp path is not known when the flag is parsed, so pass an explicit `--session `. + +## Environment Variables + +Every command accepts `EXPLORBOT_*` variables in place of a config file. Set `EXPLORBOT_AI_PROVIDER` and Explorbot builds its configuration from the environment when no `explorbot.config.*` is found: + +```bash +EXPLORBOT_URL=https://app.example.com \ +EXPLORBOT_AI_PROVIDER=openrouter \ + npx explorbot explore /login --max-tests 3 +``` + +| Variable | Meaning | +|---|---| +| `EXPLORBOT_AI_PROVIDER` | A provider name; fills every role from its recommended models. Setting it turns on config-free mode | +| `EXPLORBOT_AI_MODEL` | Pins the main model — a model id for the provider, or a standalone `provider/model-id` | +| `EXPLORBOT_URL` | Base URL to test; the API boat reads it as the base endpoint | +| `EXPLORBOT_VISION_MODEL` | Screenshot analysis; overrides the provider recommendation | +| `EXPLORBOT_AGENTIC_MODEL` | Captain and Pilot decisions; overrides the provider recommendation | +| `EXPLORBOT_OUTPUT` | Output root. Defaults to a fresh temp directory | +| `EXPLORBOT_KNOWLEDGE` | Inline knowledge text, applied to every page | +| `EXPLORBOT_KNOWLEDGE_FILE` | Path to a knowledge markdown file | +| `EXPLORBOT_API_SPEC` | OpenAPI spec path for the API boat | + +A config file always wins when present. A bare provider name fills every model role from the recommendations in [Providers](../basics/providers.md); a `provider/model-id` spec pins one model and splits on the first slash, so `openrouter/openai/gpt-oss-120b:nitro` selects OpenRouter with model `openai/gpt-oss-120b:nitro`. Supported providers: `openai`, `anthropic`, `google`, `groq`, `openrouter`, `sambanova`. + +In this mode experience is not written and the Historian is off, so no generated test files appear. See [Agentic Usage](../workflow/agentic-usage.md) for the full picture. ## Persistent Browser diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index b82abe9..d453a6b 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -389,6 +389,10 @@ Or pass a custom path: npx explorbot explore /dashboard --config ./custom/path/config.js ``` +### Running without a config file + +When no config file is found and `EXPLORBOT_AI_PROVIDER` is set, Explorbot synthesizes a configuration from `EXPLORBOT_*` environment variables. Output goes to a temp directory, experience is not written, and the Historian is off. This is meant for one-liner CI jobs, demos, and coding agents — see [Agentic Usage](../workflow/agentic-usage.md) for the variable list and the trade-offs. + ## Full configuration reference ```javascript @@ -504,6 +508,12 @@ export default { experience: 'experience', // Learned patterns output: 'output', // Test results and logs }, + + // Experience recording + experience: { + disabled: true, // Stop writing experience; reading still works + maxReadLines: 100, // Lines of experience injected into prompts + }, }; ``` diff --git a/docs/workflow/agentic-usage.md b/docs/workflow/agentic-usage.md new file mode 100644 index 0000000..bcb72ef --- /dev/null +++ b/docs/workflow/agentic-usage.md @@ -0,0 +1,202 @@ +# Agentic Usage + +Explorbot is a terminal command, so a coding agent — Claude Code, Cursor, Codex, or your own script — can drive it the same way it drives `git` or `npm`. This page covers the two things an agent needs: starting a run without a config file, and handing Explorbot a test plan it wrote itself. + +The division of labour that works best: the agent decides *what* to test and writes it down as a plan; Explorbot figures out *how* to click through the app and reports what actually happened. + +## One-liner API + +Explorbot normally reads `explorbot.config.js`. When that file is absent and `EXPLORBOT_AI_PROVIDER` is set, Explorbot builds a config from `EXPLORBOT_*` environment variables instead. Name a provider and you get its recommended models: + +```bash +EXPLORBOT_URL=https://app.example.com \ +EXPLORBOT_AI_PROVIDER=openrouter \ + npx explorbot explore /login --max-tests 3 +``` + +No `init`, no config file, no project directory, no model IDs to look up. A config file always wins when present, so adding these variables never changes the behavior of an existing project. + +### Variables + +| Variable | Required | Meaning | +|---|---|---| +| `EXPLORBOT_AI_PROVIDER` | yes | A provider name; fills every role from its recommended models. Setting it turns on this mode | +| `EXPLORBOT_AI_MODEL` | no | Pins the main `model` — a model id for the provider, or a standalone `provider/model-id` that turns on this mode by itself | +| `EXPLORBOT_URL` | yes | Base URL to test. The [API boat](../api-testing/basics.md) reads it as the base endpoint | +| `EXPLORBOT_VISION_MODEL` | no | Screenshot analysis. A provider name or `provider/model-id`; overrides the provider recommendation | +| `EXPLORBOT_AGENTIC_MODEL` | no | Captain and Pilot decisions. A provider name or `provider/model-id`; overrides the provider recommendation | +| `EXPLORBOT_OUTPUT` | no | Output root. Defaults to a fresh temp directory | +| `EXPLORBOT_KNOWLEDGE` | no | Inline knowledge text, applied to every page | +| `EXPLORBOT_KNOWLEDGE_FILE` | no | Path to a knowledge markdown file | +| `EXPLORBOT_API_SPEC` | no | OpenAPI spec path for the API boat | + +`EXPLORBOT_URL` is optional when the command itself carries an absolute URL, as `docs collect https://…` does. + +### Naming models + +Set `EXPLORBOT_AI_PROVIDER` to a provider name and Explorbot uses that provider's recommended model for every role — the same IDs listed in [Providers](../basics/providers.md), maintained in [`models.json`](../../models.json): + +```bash +EXPLORBOT_AI_PROVIDER=openrouter # model, visionModel, and agenticModel all filled in +``` + +This is the form to reach for when you do not care which model runs, only that the run works. Recommendations change as models are released, so a provider name keeps up while a pinned ID does not. + +To pin the main model, add `EXPLORBOT_AI_MODEL`. With a provider set, it is the model id for that provider, used verbatim — slashes and all: + +```bash +EXPLORBOT_AI_PROVIDER=openrouter \ +EXPLORBOT_AI_MODEL=openai/gpt-oss-120b:nitro \ + npx explorbot explore /checkout +``` + +On its own, without a provider, `EXPLORBOT_AI_MODEL` must carry the provider as `provider/model-id`, and it sets only the main `model` — `visionModel` and `agenticModel` stay unset unless you add `EXPLORBOT_AI_PROVIDER` or set them explicitly. It splits on the **first** slash, so provider-qualified IDs survive intact: + +``` +openrouter/openai/gpt-oss-120b:nitro → openrouter, model "openai/gpt-oss-120b:nitro" +groq/openai/gpt-oss-20b → groq, model "openai/gpt-oss-20b" +anthropic/claude-haiku-4-5-20251001 → anthropic, model "claude-haiku-4-5-20251001" +``` + +`EXPLORBOT_VISION_MODEL` and `EXPLORBOT_AGENTIC_MODEL` override those roles the same way — a provider name for its recommendation, or `provider/model-id` to pin one. Mix the forms to take a provider's recommendations and override one role: + +```bash +EXPLORBOT_AI_PROVIDER=groq \ +EXPLORBOT_AGENTIC_MODEL=anthropic \ + npx explorbot explore /checkout +``` + +Supported providers: `openai`, `anthropic`, `google`, `groq`, `mistral`, `openrouter`, `sambanova`. Each is created with its conventional API-key variable — `OPENROUTER_API_KEY`, `GROQ_API_KEY`, `MISTRAL_API_KEY`, and so on. + +Not every provider has a recommendation for every role — Anthropic is recommended only for `agenticModel`, since Claude models are accurate but costly for token-heavy page reading. Naming a provider that has no recommendation for a role you asked for is an error that names the role, so combine providers as in the example above. + +A `.env` file in the working directory is loaded before the config lookup, so `EXPLORBOT_*` variables and API keys can live there instead of on the command line. + +### Knowledge without a project + +Both knowledge variables write into the run's knowledge directory, and both can be set at once. + +`EXPLORBOT_KNOWLEDGE` is the fast path for credentials — it applies to every page: + +```bash +EXPLORBOT_KNOWLEDGE="Log in as admin@example.com / secret123. Dismiss the cookie banner first." \ +EXPLORBOT_URL=https://app.example.com \ +EXPLORBOT_AI_PROVIDER=openrouter \ + npx explorbot explore /admin/users +``` + +`EXPLORBOT_KNOWLEDGE_FILE` points at a markdown file the agent wrote. Its frontmatter is preserved, so it can target specific URLs — see [Knowledge](./knowledge.md) for the format: + +```bash +EXPLORBOT_KNOWLEDGE_FILE=./checkout-knowledge.md npx explorbot explore /checkout +``` + +### What this mode changes + +Config-free runs are built to leave no trace in the working directory: + +- **Output goes to a temp directory** unless `EXPLORBOT_OUTPUT` is set. Read the path from the `Configuration built from EXPLORBOT_* environment variables. Output: …` line. +- **Experience is not written.** Nothing accumulates between runs, so a run is reproducible. Reading existing experience still works if the directory has any. +- **The Historian is off.** No generated CodeceptJS or Playwright test files. Plans and reports are still written. + +For a long-lived agent that should learn across runs, point `EXPLORBOT_OUTPUT` at a stable directory or switch to a real config file. + +### Reading results + +Everything lands under the output root: + +| Path | Contents | +|---|---| +| `reports/-.md` | Session report: coverage, defects, execution issues | +| `plans/.md` | The plan that was generated or executed | +| `states/` | Per-state HTML, ARIA snapshots, and screenshots | +| `research/` | UI maps produced by the Researcher | + +The report is the artifact to parse. It clusters findings by root cause and is written for a reader, not a machine. + +`explore` and `test` exit `0` whenever the session completes, and non-zero only when the run itself fails to start — a failing scenario is a result, not a crash. Do not read pass/fail from their exit code; read the report. `navigate` is the exception and exits `1` when a URL is unreachable, which makes it a useful pre-flight check. + +## Running agent-prepared plans + +A [test plan](./test-plans.md) is plain markdown. An agent that has read the codebase usually knows what a feature is supposed to do better than an agent looking at rendered HTML, so writing the plan and executing it are worth separating. + +Write the plan: + +```markdown + +# Checkout + +### Prerequisite + +* URL: /cart + + +# Customer completes checkout with a saved card + +## Requirements +/cart + +## Steps +* Proceed to checkout from the cart +* Pick the saved card as the payment method +* Confirm the order + +## Expected +* The order confirmation page shows an order number +* The cart is empty afterwards +``` + +Then hand it to Explorbot: + +```bash +EXPLORBOT_URL=https://app.example.com \ +EXPLORBOT_AI_PROVIDER=openrouter \ + npx explorbot test checkout-plan.md '*' +``` + +The index argument selects tests: `1`, `1,3`, `1-5`, or `*` for all. The plan file is input only — Explorbot never rewrites it, so plans stay in version control next to the code they cover. + +Steps are guidance, not a script. The Tester adapts them to what the page actually shows, which is why steps should describe intent rather than selectors. Expected outcomes are the strict part: a test passes only when every one of them is verified. See the [Planner's outcome guidance](../web-testing/planner.md#built-in-styles) for what makes an outcome verifiable. + +To have Explorbot invent the scenarios instead, run `explorbot plan ` and read the generated file from `plans/`. + +## Inspecting a page without spending tokens + +Two commands help an agent orient itself before committing to a run: + +```bash +npx explorbot context /login # URL, headings, knowledge, interactive elements +npx explorbot shell /login 'I.click("Sign in")' # run one CodeceptJS command +``` + +`context` makes no AI calls. Use it to check that a page loads, that login knowledge applies, and that the elements a plan assumes are actually there. + +## The other boats + +The same variables drive API testing and doc collection. + +```bash +EXPLORBOT_URL=https://api.example.com \ +EXPLORBOT_API_SPEC=./openapi.yaml \ +EXPLORBOT_AI_PROVIDER=openrouter \ + npx explorbot api explore +``` + +```bash +EXPLORBOT_AI_PROVIDER=openrouter \ + npx explorbot docs collect https://app.example.com/dashboard --max-pages 20 +``` + +`docs collect` takes its base URL from the absolute path argument, so `EXPLORBOT_URL` is optional there. + +Knowledge written by `EXPLORBOT_KNOWLEDGE` carries `endpoint: '*'` frontmatter alongside `url: '*'`, matching the convention `api init` and `api know` use. The API boat does not read knowledge at runtime yet; the frontmatter is there for when it does, and the web side ignores it. + +## See Also + +- [Test Plans](./test-plans.md) — the plan format in full +- [Knowledge](./knowledge.md) — teaching Explorbot about your app +- [Commands](../reference/commands.md) — every CLI command +- [Continuous integration](./ci.md) — scheduled runs with cached experience +- [Scripting](../reference/scripting.md) — the programmatic API when a CLI call is not enough diff --git a/models.json b/models.json new file mode 100644 index 0000000..30ecf77 --- /dev/null +++ b/models.json @@ -0,0 +1,30 @@ +{ + "openrouter": { + "model": "openai/gpt-oss-20b:nitro", + "visionModel": "google/gemma-4-31b-it:nitro", + "agenticModel": "google/gemma-4-31b-it:nitro" + }, + "groq": { + "model": "openai/gpt-oss-20b", + "visionModel": "qwen/qwen3.6-27b", + "agenticModel": "qwen/qwen3.6-27b" + }, + "openai": { + "model": "gpt-5.4-nano", + "visionModel": "gpt-5.4-nano", + "agenticModel": "gpt-5.6-luna" + }, + "anthropic": { + "agenticModel": "claude-haiku-4-5-20251001" + }, + "mistral": { + "model": "mistral-small-latest", + "visionModel": "mistral-small-latest", + "agenticModel": "mistral-large-latest" + }, + "google": { + "model": "gemini-3.1-flash-lite", + "visionModel": "gemini-3.1-flash-lite", + "agenticModel": "gemini-3.5-flash" + } +} diff --git a/package.json b/package.json index ef6abbe..4b9b142 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "bin": { "explorbot": "./dist/bin/explorbot-cli.js" }, - "files": ["dist/", "src/**/*.ts", "src/**/*.tsx", "bin/**/*.ts", "boat/api-tester/src/**/*.ts", "boat/doc-collector/src/**/*.ts", "boat/doc-collector/bin/**/*.ts", "boat/doc-collector/package.json", "rules/", "assets/sample-files/"], + "files": ["dist/", "src/**/*.ts", "src/**/*.tsx", "bin/**/*.ts", "boat/api-tester/src/**/*.ts", "boat/doc-collector/src/**/*.ts", "boat/doc-collector/bin/**/*.ts", "boat/doc-collector/package.json", "rules/", "assets/sample-files/", "models.json"], "scripts": { "build": "bun run build:bin", "build:bin": "bun build bin/explorbot-cli.ts --outdir bin --target node --external commander --format esm", @@ -51,7 +51,9 @@ "author": "", "dependencies": { "@ai-sdk/anthropic": "^4.0", + "@ai-sdk/google": "^4.0.18", "@ai-sdk/groq": "^4.0", + "@ai-sdk/mistral": "^4.0.13", "@ai-sdk/openai": "^4.0", "@ai-sdk/otel": "^1.0.2", "@axe-core/playwright": "^4.11.0", diff --git a/scripts/build-npm.sh b/scripts/build-npm.sh index 6607c34..91c88af 100755 --- a/scripts/build-npm.sh +++ b/scripts/build-npm.sh @@ -17,6 +17,7 @@ for dir in rules assets/sample-files; do done cp package.json "$DIST_DIR/package.json" +cp models.json "$DIST_DIR/models.json" CLI="$DIST_DIR/bin/explorbot-cli.js" if [ -f "$CLI" ]; then diff --git a/src/config.ts b/src/config.ts index a859f92..ab7f800 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,8 +1,22 @@ -import { existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs'; -import path, { dirname, join, resolve } from 'node:path'; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path, { basename, dirname, join, resolve } from 'node:path'; import { parseEnv } from 'node:util'; +import matter from 'gray-matter'; import { log } from './utils/logger.js'; +const PROVIDERS: Record Promise<(modelId: string) => any>> = { + openai: async () => (await import('@ai-sdk/openai')).createOpenAI(), + anthropic: async () => (await import('@ai-sdk/anthropic')).createAnthropic(), + google: async () => (await import('@ai-sdk/google')).createGoogleGenerativeAI(), + groq: async () => (await import('@ai-sdk/groq')).createGroq(), + mistral: async () => (await import('@ai-sdk/mistral')).createMistral(), + openrouter: async () => (await import('@openrouter/ai-sdk-provider')).createOpenRouter(), + sambanova: async () => (await import('sambanova-ai-provider')).createSambaNova(), +}; + +let cachedOutputRoot: string | null = null; + interface PlaywrightConfig { browser: 'chromium' | 'firefox' | 'webkit'; url: string; @@ -216,6 +230,7 @@ interface ExplorbotConfig { }; experience?: { maxReadLines?: number; + disabled?: boolean; }; reporter?: ReporterConfig; api?: ApiConfig; @@ -268,6 +283,7 @@ export type { export class ConfigParser { private static instance: ConfigParser; + private static recommended: Record> | null = null; private config: ExplorbotConfig | null = null; private configPath: string | null = null; private runtimeBaseUrlOverride: string | null = null; @@ -280,6 +296,11 @@ export class ConfigParser { Object.assign(process.env, parseEnv(readFileSync(resolved, 'utf8'))); } + public static recommendedModels(): Record> { + ConfigParser.recommended ||= JSON.parse(readFileSync(new URL('../models.json', import.meta.url), 'utf8')); + return ConfigParser.recommended!; + } + public static getInstance(): ConfigParser { if (!ConfigParser.instance) { ConfigParser.instance = new ConfigParser(); @@ -312,22 +333,31 @@ export class ConfigParser { try { const resolvedPath = options?.config || this.findConfigFile(); - if (!resolvedPath) { - throw new Error('No configuration file found. Please create explorbot.config.js or explorbot.config.ts'); + let loadedConfig: ExplorbotConfig | null = null; + let sourcePath = resolvedPath; + + if (resolvedPath) { + const configModule = await this.loadConfigModule(resolvedPath); + loadedConfig = configModule.default || configModule; + + if (!loadedConfig) { + throw new Error('Configuration file is empty or invalid'); + } + + log(`Configuration loaded from: ${resolvedPath}`); } - const configModule = await this.loadConfigModule(resolvedPath); - const loadedConfig = configModule.default || configModule; + if (!resolvedPath) { + const outputRoot = resolveOutputRoot(); + loadedConfig = await this.buildEnvConfig(options?.baseUrl, outputRoot); + sourcePath = join(outputRoot, 'explorbot.config.js'); - if (!loadedConfig) { - throw new Error('Configuration file is empty or invalid'); + log(`Configuration built from EXPLORBOT_* environment variables. Output: ${outputRoot}`); } this.config = this.resolveConfig(loadedConfig as ExplorbotConfig, options); this.runtimeBaseUrlOverride = options?.baseUrl || null; - this.configPath = resolvedPath; - - log(`Configuration loaded from: ${resolvedPath}`); + this.configPath = sourcePath; // Restore original directory after successful config load if (options?.path && originalCwd !== process.cwd()) { @@ -388,6 +418,7 @@ export class ConfigParser { // For testing purposes only public static resetForTesting(): void { + cachedOutputRoot = null; if (ConfigParser.instance) { ConfigParser.instance.config = null; ConfigParser.instance.configPath = null; @@ -442,6 +473,52 @@ export class ConfigParser { } } + private async buildEnvConfig(baseUrl: string | undefined, outputRoot: string): Promise { + const provider = process.env.EXPLORBOT_AI_PROVIDER; + const modelSpec = process.env.EXPLORBOT_AI_MODEL; + if (!provider && !modelSpec) { + throw new Error('No configuration file found. Please create explorbot.config.js or set EXPLORBOT_URL and EXPLORBOT_AI_PROVIDER environment variables'); + } + if (modelSpec && !provider && !modelSpec.includes('/')) { + throw new Error('EXPLORBOT_AI_MODEL needs a provider — set EXPLORBOT_AI_PROVIDER, or write it as "provider/model-id"'); + } + + const url = process.env.EXPLORBOT_URL || baseUrl; + if (!url) { + throw new Error('No URL to explore. Set EXPLORBOT_URL or pass a URL to the command'); + } + + materializeKnowledge(outputRoot); + + let model: any; + if (provider && modelSpec) model = await createModel(provider, modelSpec); + if (provider && !modelSpec) model = await resolveModel(provider, 'model'); + if (!provider) model = await resolveModel(modelSpec!, 'model'); + + const ai: AIConfig = { + model, + agents: { historian: { enabled: false } }, + }; + + let recommended: Record = {}; + if (provider) recommended = ConfigParser.recommendedModels()[provider] || {}; + + const visionSpec = process.env.EXPLORBOT_VISION_MODEL; + if (visionSpec) ai.visionModel = await resolveModel(visionSpec, 'visionModel'); + if (!visionSpec && recommended.visionModel) ai.visionModel = await resolveModel(provider!, 'visionModel'); + + const agenticSpec = process.env.EXPLORBOT_AGENTIC_MODEL; + if (agenticSpec) ai.agenticModel = await resolveModel(agenticSpec, 'agenticModel'); + if (!agenticSpec && recommended.agenticModel) ai.agenticModel = await resolveModel(provider!, 'agenticModel'); + + return { + playwright: { browser: 'chromium', url, show: false }, + ai, + dirs: { knowledge: 'knowledge', experience: 'experience', output: '.' }, + experience: { disabled: true }, + }; + } + private findConfigFile(): string | null { const possiblePaths = [...EXPLORBOT_CONFIG_PATHS, 'config/explorbot.config.js', 'config/explorbot.config.mjs', 'config/explorbot.config.ts', 'src/config/explorbot.config.js', 'src/config/explorbot.config.mjs', 'src/config/explorbot.config.ts']; @@ -554,3 +631,69 @@ export class ConfigParser { export function outputPath(...segments: string[]): string { return path.join(ConfigParser.getInstance().getOutputDir(), ...segments); } + +export async function resolveModel(spec: string, role: ModelRole = 'model'): Promise { + const separator = spec.indexOf('/'); + if (separator > 0) { + return createModel(spec.slice(0, separator), spec.slice(separator + 1)); + } + + const recommended = ConfigParser.recommendedModels()[spec]; + if (!recommended) { + throw new Error(`No recommended models for "${spec}". Write it as "provider/model-id", or use a provider with recommendations: ${Object.keys(ConfigParser.recommendedModels()).join(', ')}`); + } + + const modelId = recommended[role]; + if (!modelId) { + throw new Error(`Provider "${spec}" has no recommended ${role}. Set it explicitly as "provider/model-id".`); + } + + return createModel(spec, modelId); +} + +export function resolveOutputRoot(): string { + if (cachedOutputRoot) return cachedOutputRoot; + + const configured = process.env.EXPLORBOT_OUTPUT; + if (!configured) { + cachedOutputRoot = mkdtempSync(join(tmpdir(), 'explorbot-')); + return cachedOutputRoot; + } + + cachedOutputRoot = resolve(configured); + mkdirSync(cachedOutputRoot, { recursive: true }); + return cachedOutputRoot; +} + +export function materializeKnowledge(outputRoot: string): void { + const inline = process.env.EXPLORBOT_KNOWLEDGE; + const knowledgeFile = process.env.EXPLORBOT_KNOWLEDGE_FILE; + if (!inline && !knowledgeFile) return; + + const knowledgeDir = join(outputRoot, 'knowledge'); + mkdirSync(knowledgeDir, { recursive: true }); + + if (inline) { + writeFileSync(join(knowledgeDir, 'global.md'), matter.stringify(inline, { url: '*', endpoint: '*' })); + } + + if (!knowledgeFile) return; + + const source = resolve(knowledgeFile); + if (!existsSync(source)) { + throw new Error(`Knowledge file from EXPLORBOT_KNOWLEDGE_FILE not found: ${source}`); + } + copyFileSync(source, join(knowledgeDir, basename(source))); +} + +export async function createModel(provider: string, modelId: string): Promise { + const factory = PROVIDERS[provider]; + if (!factory) { + throw new Error(`Unknown AI provider "${provider}". Supported providers: ${Object.keys(PROVIDERS).join(', ')}`); + } + return (await factory())(modelId); +} + +type ModelRole = 'model' | 'visionModel' | 'agenticModel'; + +export type { ModelRole }; diff --git a/src/experience-tracker.ts b/src/experience-tracker.ts index 2d56ba4..968ee67 100644 --- a/src/experience-tracker.ts +++ b/src/experience-tracker.ts @@ -42,7 +42,7 @@ export class ExperienceTracker { constructor(knowledgeTracker: KnowledgeTracker, options: { disabled?: boolean } = {}) { const configParser = ConfigParser.getInstance(); const config = configParser.getConfig(); - this.disabled = options.disabled ?? false; + this.disabled = options.disabled ?? config.experience?.disabled ?? false; this.knowledgeTracker = knowledgeTracker; // Resolve experience directory relative to the config file location (project root) diff --git a/src/explorbot.ts b/src/explorbot.ts index 4d740d9..8e395f3 100644 --- a/src/explorbot.ts +++ b/src/explorbot.ts @@ -205,7 +205,7 @@ export class ExplorBot { const qm = this.agentQuartermaster(); if (qm) this.agents.tester.setQuartermaster(qm); - this.agents.tester.setHistorian(this.agentHistorian()); + if (this.isHistorianEnabled()) this.agents.tester.setHistorian(this.agentHistorian()); this.agents.tester.setPilot(this.agentPilot()); this.agents.tester.setCaptain(this.agentCaptain()); @@ -258,7 +258,7 @@ export class ExplorBot { const tools = createAgentTools({ explorer, researcher, navigator, withExperience: false }); return new Rerunner(explorer, ai, tools); }); - this.agents.rerunner.setHistorian(this.agentHistorian()); + if (this.isHistorianEnabled()) this.agents.rerunner.setHistorian(this.agentHistorian()); } return this.agents.rerunner; } @@ -479,4 +479,8 @@ export class ExplorBot { tag('warning').log(`Session analysis failed: ${message}`); } } + + private isHistorianEnabled(): boolean { + return this.config.ai?.agents?.historian?.enabled !== false; + } } diff --git a/src/explorer.ts b/src/explorer.ts index 6fc2a5a..5a73c19 100644 --- a/src/explorer.ts +++ b/src/explorer.ts @@ -91,7 +91,7 @@ class Explorer { // Use project root for output directory, not current working directory const configParser = ConfigParser.getInstance(); const projectRoot = configParser.getProjectRoot(); - (global as any).output_dir = path.join(projectRoot, 'output', 'states'); + (global as any).output_dir = configParser.getStatesDir(); (global as any).codecept_dir = projectRoot; configParser.validateConfig(this.config); diff --git a/src/utils/test-files.ts b/src/utils/test-files.ts index 887152c..8459000 100644 --- a/src/utils/test-files.ts +++ b/src/utils/test-files.ts @@ -76,7 +76,7 @@ export async function dryRunTestFile(filePath: string): Promise { }, }; - (global as any).output_dir = path.join(projectRoot, 'output', 'states'); + (global as any).output_dir = ConfigParser.getInstance().getStatesDir(); (global as any).codecept_dir = projectRoot; codeceptjs.container.create(codeceptConfig, {}); diff --git a/tests/unit/apibot-config.test.ts b/tests/unit/apibot-config.test.ts new file mode 100644 index 0000000..9dbe443 --- /dev/null +++ b/tests/unit/apibot-config.test.ts @@ -0,0 +1,76 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { ApibotConfigParser } from '../../boat/api-tester/src/config.ts'; +import { ConfigParser } from '../../src/config.ts'; + +const ENV_KEYS = ['EXPLORBOT_AI_PROVIDER', 'EXPLORBOT_AI_MODEL', 'EXPLORBOT_URL', 'EXPLORBOT_OUTPUT', 'EXPLORBOT_API_SPEC']; + +describe('ApibotConfigParser environment fallback', () => { + let savedEnv: Record = {}; + let outputRoot: string; + let parser: ApibotConfigParser; + let originalFindConfigFile: any; + + beforeEach(() => { + savedEnv = {}; + for (const key of ENV_KEYS) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } + outputRoot = mkdtempSync(join(tmpdir(), 'apibot-env-test-')); + ConfigParser.resetForTesting(); + + parser = ApibotConfigParser.getInstance(); + (parser as any).config = null; + (parser as any).configPath = null; + originalFindConfigFile = (parser as any).findConfigFile; + (parser as any).findConfigFile = () => null; + }); + + afterEach(() => { + for (const key of ENV_KEYS) { + if (savedEnv[key] === undefined) delete process.env[key]; + else process.env[key] = savedEnv[key]; + } + (parser as any).findConfigFile = originalFindConfigFile; + (parser as any).config = null; + (parser as any).configPath = null; + rmSync(outputRoot, { recursive: true, force: true }); + ConfigParser.resetForTesting(); + }); + + it('maps EXPLORBOT_URL to the API base endpoint', async () => { + process.env.EXPLORBOT_AI_MODEL = 'openrouter/openai/gpt-oss-120b'; + process.env.EXPLORBOT_URL = 'https://api.example.com'; + process.env.EXPLORBOT_OUTPUT = outputRoot; + + const config = await parser.loadConfig(); + + expect(config.api.baseEndpoint).toBe('https://api.example.com'); + expect(parser.getConfigPath()).toBe(join(outputRoot, 'apibot.config.js')); + expect(parser.getOutputDir()).toBe(outputRoot); + expect(parser.getKnowledgeDir()).toBe(join(outputRoot, 'knowledge')); + }); + + it('maps EXPLORBOT_API_SPEC to api.spec', async () => { + process.env.EXPLORBOT_AI_MODEL = 'openrouter/openai/gpt-oss-120b'; + process.env.EXPLORBOT_URL = 'https://api.example.com'; + process.env.EXPLORBOT_OUTPUT = outputRoot; + process.env.EXPLORBOT_API_SPEC = './openapi.yaml'; + + const config = await parser.loadConfig(); + + expect(config.api.spec).toEqual(['./openapi.yaml']); + }); + + it('throws when EXPLORBOT_URL is unset', async () => { + process.env.EXPLORBOT_AI_MODEL = 'openrouter/openai/gpt-oss-120b'; + await expect(parser.loadConfig()).rejects.toThrow(/EXPLORBOT_URL/); + }); + + it('throws the config-file error when no env vars are set', async () => { + await expect(parser.loadConfig()).rejects.toThrow(/apibot.config.js/); + }); +}); diff --git a/tests/unit/config.test.ts b/tests/unit/config.test.ts index 1bfb587..a1a48bb 100644 --- a/tests/unit/config.test.ts +++ b/tests/unit/config.test.ts @@ -1,5 +1,8 @@ -import { beforeEach, describe, expect, it } from 'bun:test'; -import { ConfigParser } from '../../src/config.ts'; +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { ConfigParser, materializeKnowledge, resolveModel, resolveOutputRoot } from '../../src/config.ts'; describe('ConfigParser runtime baseUrl overrides', () => { beforeEach(() => { @@ -40,3 +43,238 @@ describe('ConfigParser runtime baseUrl overrides', () => { } }); }); + +const ENV_KEYS = ['EXPLORBOT_AI_PROVIDER', 'EXPLORBOT_AI_MODEL', 'EXPLORBOT_VISION_MODEL', 'EXPLORBOT_AGENTIC_MODEL', 'EXPLORBOT_URL', 'EXPLORBOT_OUTPUT', 'EXPLORBOT_KNOWLEDGE', 'EXPLORBOT_KNOWLEDGE_FILE']; + +let savedEnv: Record = {}; +let scratchDir: string; + +describe('ConfigParser environment mode', () => { + let parser: ConfigParser; + let originalFindConfigFile: any; + + beforeEach(() => { + savedEnv = {}; + for (const key of ENV_KEYS) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } + scratchDir = mkdtempSync(join(tmpdir(), 'env-config-test-')); + ConfigParser.resetForTesting(); + + parser = ConfigParser.getInstance(); + originalFindConfigFile = (parser as any).findConfigFile; + (parser as any).findConfigFile = () => null; + }); + + afterEach(() => { + for (const key of ENV_KEYS) { + if (savedEnv[key] === undefined) delete process.env[key]; + else process.env[key] = savedEnv[key]; + } + (parser as any).findConfigFile = originalFindConfigFile; + rmSync(scratchDir, { recursive: true, force: true }); + ConfigParser.resetForTesting(); + }); + + it('reports both options when no provider or model is set', async () => { + process.env.EXPLORBOT_URL = 'https://example.com'; + await expect(parser.loadConfig()).rejects.toThrow(/EXPLORBOT_AI_PROVIDER/); + }); + + it('prefers a config file over env vars', async () => { + process.env.EXPLORBOT_AI_PROVIDER = 'openrouter'; + process.env.EXPLORBOT_URL = 'https://env.example.com'; + + const originalLoadConfigModule = (parser as any).loadConfigModule; + (parser as any).findConfigFile = () => '/virtual/explorbot.config.ts'; + (parser as any).loadConfigModule = async () => ({ + default: { + playwright: { url: 'https://file.example.com', browser: 'chromium' }, + ai: { model: { modelId: 'test-model', provider: 'test' } }, + }, + }); + + try { + const config = await parser.loadConfig(); + expect(config.playwright.url).toBe('https://file.example.com'); + } finally { + (parser as any).loadConfigModule = originalLoadConfigModule; + } + }); + + it('throws when no URL comes from env or argument', async () => { + process.env.EXPLORBOT_AI_MODEL = 'openrouter/openai/gpt-oss-120b'; + await expect(parser.loadConfig()).rejects.toThrow(/EXPLORBOT_URL/); + }); + + it('uses the baseUrl argument when EXPLORBOT_URL is unset', async () => { + process.env.EXPLORBOT_AI_MODEL = 'openrouter/openai/gpt-oss-120b'; + process.env.EXPLORBOT_OUTPUT = scratchDir; + + const config = await parser.loadConfig({ baseUrl: 'https://from-argument.example.com' }); + + expect(config.playwright.url).toBe('https://from-argument.example.com'); + }); + + it('splits the model spec on the first slash only', async () => { + const model = await resolveModel('openrouter/openai/gpt-oss-120b:nitro'); + expect(model.modelId).toBe('openai/gpt-oss-120b:nitro'); + }); + + it('rejects an unknown provider and lists supported ones', async () => { + await expect(resolveModel('nosuchprovider/some-model')).rejects.toThrow(/openrouter/); + }); + + it('resolves a bare provider name to its recommended model', async () => { + const recommended = ConfigParser.recommendedModels().openrouter; + expect((await resolveModel('openrouter')).modelId).toBe(recommended.model); + expect((await resolveModel('openrouter', 'visionModel')).modelId).toBe(recommended.visionModel); + expect((await resolveModel('openrouter', 'agenticModel')).modelId).toBe(recommended.agenticModel); + }); + + it('rejects a bare name with no recommendations and lists the providers that have them', async () => { + await expect(resolveModel('gpt-oss-120b')).rejects.toThrow(/openrouter/); + }); + + it('rejects a role the provider has no recommendation for', async () => { + await expect(resolveModel('anthropic', 'model')).rejects.toThrow(/no recommended model/); + }); + + it('fills every role from one provider name', async () => { + process.env.EXPLORBOT_AI_PROVIDER = 'openrouter'; + process.env.EXPLORBOT_URL = 'https://example.com'; + process.env.EXPLORBOT_OUTPUT = scratchDir; + + const recommended = ConfigParser.recommendedModels().openrouter; + const config = await parser.loadConfig(); + + expect(config.ai.model.modelId).toBe(recommended.model); + expect(config.ai.visionModel.modelId).toBe(recommended.visionModel); + expect(config.ai.agenticModel.modelId).toBe(recommended.agenticModel); + }); + + it('lets an explicit role override the provider recommendation', async () => { + process.env.EXPLORBOT_AI_PROVIDER = 'openrouter'; + process.env.EXPLORBOT_AGENTIC_MODEL = 'groq/openai/gpt-oss-120b'; + process.env.EXPLORBOT_URL = 'https://example.com'; + process.env.EXPLORBOT_OUTPUT = scratchDir; + + const config = await parser.loadConfig(); + + expect(config.ai.model.modelId).toBe(ConfigParser.recommendedModels().openrouter.model); + expect(config.ai.agenticModel.modelId).toBe('openai/gpt-oss-120b'); + }); + + it('leaves other roles unset when an explicit model id is given', async () => { + process.env.EXPLORBOT_AI_MODEL = 'openrouter/openai/gpt-oss-120b'; + process.env.EXPLORBOT_URL = 'https://example.com'; + process.env.EXPLORBOT_OUTPUT = scratchDir; + + const config = await parser.loadConfig(); + + expect(config.ai.model.modelId).toBe('openai/gpt-oss-120b'); + expect(config.ai.visionModel).toBeUndefined(); + expect(config.ai.agenticModel).toBeUndefined(); + }); + + it('uses EXPLORBOT_AI_MODEL as the model id under EXPLORBOT_AI_PROVIDER', async () => { + process.env.EXPLORBOT_AI_PROVIDER = 'openrouter'; + process.env.EXPLORBOT_AI_MODEL = 'openai/gpt-oss-120b:nitro'; + process.env.EXPLORBOT_URL = 'https://example.com'; + process.env.EXPLORBOT_OUTPUT = scratchDir; + + const config = await parser.loadConfig(); + + expect(config.ai.model.modelId).toBe('openai/gpt-oss-120b:nitro'); + expect(config.ai.visionModel.modelId).toBe(ConfigParser.recommendedModels().openrouter.visionModel); + expect(config.ai.agenticModel.modelId).toBe(ConfigParser.recommendedModels().openrouter.agenticModel); + }); + + it('rejects a bare model id when no provider is set', async () => { + process.env.EXPLORBOT_AI_MODEL = 'some-model'; + process.env.EXPLORBOT_URL = 'https://example.com'; + await expect(parser.loadConfig()).rejects.toThrow(/EXPLORBOT_AI_PROVIDER/); + }); + + it('respects EXPLORBOT_OUTPUT as the output root', () => { + process.env.EXPLORBOT_OUTPUT = scratchDir; + expect(resolveOutputRoot()).toBe(scratchDir); + }); + + it('creates a temp output root when EXPLORBOT_OUTPUT is unset', () => { + const root = resolveOutputRoot(); + expect(root.startsWith(tmpdir())).toBe(true); + expect(existsSync(root)).toBe(true); + rmSync(root, { recursive: true, force: true }); + }); + + it('disables experience and historian and keeps output at the config root', async () => { + process.env.EXPLORBOT_AI_MODEL = 'openrouter/openai/gpt-oss-120b'; + process.env.EXPLORBOT_URL = 'https://example.com'; + process.env.EXPLORBOT_OUTPUT = scratchDir; + + const config = await parser.loadConfig(); + + expect(config.dirs?.output).toBe('.'); + expect(config.experience?.disabled).toBe(true); + expect(config.ai.agents?.historian?.enabled).toBe(false); + expect(parser.getOutputDir()).toBe(scratchDir); + }); + + it('resolves optional vision and agentic models', async () => { + process.env.EXPLORBOT_AI_MODEL = 'groq/openai/gpt-oss-20b'; + process.env.EXPLORBOT_VISION_MODEL = 'groq/meta-llama/llama-4-scout-17b-16e-instruct'; + process.env.EXPLORBOT_AGENTIC_MODEL = 'groq/openai/gpt-oss-120b'; + process.env.EXPLORBOT_URL = 'https://example.com'; + process.env.EXPLORBOT_OUTPUT = scratchDir; + + const config = await parser.loadConfig(); + + expect(config.ai.visionModel.modelId).toBe('meta-llama/llama-4-scout-17b-16e-instruct'); + expect(config.ai.agenticModel.modelId).toBe('openai/gpt-oss-120b'); + }); + + it('writes inline knowledge as a global file matching every url and endpoint', () => { + process.env.EXPLORBOT_KNOWLEDGE = 'Use admin/admin123 to log in'; + + materializeKnowledge(scratchDir); + + const content = readFileSync(join(scratchDir, 'knowledge', 'global.md'), 'utf8'); + expect(content).toContain("url: '*'"); + expect(content).toContain("endpoint: '*'"); + expect(content).toContain('Use admin/admin123 to log in'); + }); + + it('copies EXPLORBOT_KNOWLEDGE_FILE by basename', () => { + const source = join(scratchDir, 'login.md'); + writeFileSync(source, '---\nurl: /login\n---\n\nUse admin/admin123\n'); + process.env.EXPLORBOT_KNOWLEDGE_FILE = source; + + materializeKnowledge(scratchDir); + + expect(readFileSync(join(scratchDir, 'knowledge', 'login.md'), 'utf8')).toContain('url: /login'); + }); + + it('throws when EXPLORBOT_KNOWLEDGE_FILE does not exist', () => { + process.env.EXPLORBOT_KNOWLEDGE_FILE = join(scratchDir, 'missing.md'); + expect(() => materializeKnowledge(scratchDir)).toThrow(/not found/); + }); + + it('accepts inline knowledge and a knowledge file at once', () => { + const source = join(scratchDir, 'checkout.md'); + writeFileSync(source, 'Checkout uses a test card\n'); + process.env.EXPLORBOT_KNOWLEDGE_FILE = source; + process.env.EXPLORBOT_KNOWLEDGE = 'Global note'; + + materializeKnowledge(scratchDir); + + expect(existsSync(join(scratchDir, 'knowledge', 'global.md'))).toBe(true); + expect(existsSync(join(scratchDir, 'knowledge', 'checkout.md'))).toBe(true); + }); + + it('writes no knowledge dir when neither knowledge var is set', () => { + materializeKnowledge(scratchDir); + expect(existsSync(join(scratchDir, 'knowledge'))).toBe(false); + }); +}); diff --git a/tests/unit/experience-tracker.test.ts b/tests/unit/experience-tracker.test.ts index 37d0b30..da94a7d 100644 --- a/tests/unit/experience-tracker.test.ts +++ b/tests/unit/experience-tracker.test.ts @@ -291,6 +291,20 @@ describe('ExperienceTracker', () => { const filePath = `/tmp/experience/${stateHash}.md`; expect(existsSync(filePath)).toBe(false); }); + + it('respects experience.disabled from config', () => { + const configParser = ConfigParser.getInstance(); + (configParser as any).config.experience = { disabled: true }; + + try { + const disabledTracker = new ExperienceTracker(new KnowledgeTracker()); + const state = makeState(); + disabledTracker.writeFlow(state, sampleBody); + expect(existsSync(`/tmp/experience/${state.getStateHash()}.md`)).toBe(false); + } finally { + (configParser as any).config.experience = undefined; + } + }); }); describe('file path extraction', () => { From 55cb549dcd28b97121b1bc9177fdac3d41847623 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Mon, 20 Jul 2026 00:27:39 +0300 Subject: [PATCH 2/5] feat: single registry for EXPLORBOT_* env vars, injected into help and docs EXPLORBOT_ENV_VARS in src/config.ts is the one place every environment variable is declared. The CLI renders it into help text on the root command and every subcommand (both boats included), so an agent running any `--help` discovers the full set. `bunosh docs:env` generates the tables in agentic-usage.md and commands.md from the same list, verified in CI on release. Adds EXPLORBOT_NO_BANNER, which was undocumented. Two tests guard the registry against drift in either direction. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/publish.yml | 3 +++ Bunoshfile.js | 38 ++++++++++++++++++++++++++++++++++ bin/explorbot-cli.ts | 35 ++++++++++++++++--------------- docs/reference/commands.md | 9 +++++--- docs/workflow/agentic-usage.md | 19 ++++++++++------- src/config.ts | 23 ++++++++++++++++++-- tests/unit/config.test.ts | 28 ++++++++++++++++++++++++- 7 files changed, 125 insertions(+), 30 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5009eff..460af93 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -38,6 +38,9 @@ jobs: - name: Verify provider docs are up to date run: bunx bunosh docs:models --check + - name: Verify env var docs are up to date + run: bunx bunosh docs:env --check + - name: Run unit tests run: bun test tests/unit/ diff --git a/Bunoshfile.js b/Bunoshfile.js index 9b4e761..0b3944c 100644 --- a/Bunoshfile.js +++ b/Bunoshfile.js @@ -15,6 +15,7 @@ import { startFixture } from './tests/regression/fixture/server.ts'; import { parsePlansFromMarkdown } from './src/utils/test-plan-markdown.ts'; import { htmlCombinedSnapshot, htmlTextSnapshot, minifyHtml } from './src/utils/html.js'; import { analyzeDemoCandidates, createDemoVideo } from './.claude/skills/demo-video/demo-video.ts'; +import { EXPLORBOT_ENV_VARS } from './src/config.ts'; const { exec, shell, writeToFile, task, ai } = global.bunosh; @@ -79,6 +80,43 @@ export async function docsModels(options = { check: false }) { say('Updated docs/basics/providers.md from models.json'); } +/** + * Regenerate the EXPLORBOT_* env var tables in docs from src/config.ts + * @param {object} options + * @param {boolean} [options.check=false] - Fail if the docs are stale instead of rewriting them (for CI/release) + */ +export async function docsEnv(options = { check: false }) { + const docs = [ + { path: 'docs/workflow/agentic-usage.md', withRequired: true }, + { path: 'docs/reference/commands.md', withRequired: false }, + ]; + + const stale = []; + for (const { path, withRequired } of docs) { + const doc = resolve(path); + const original = readFileSync(doc, 'utf8'); + + const header = withRequired ? '| Variable | Required | Meaning |\n|---|---|---|' : '| Variable | Meaning |\n|---|---|'; + const rows = EXPLORBOT_ENV_VARS.map((v) => { + if (withRequired) return `| \`${v.name}\` | ${v.required ? 'yes' : 'no'} | ${v.description} |`; + return `| \`${v.name}\` | ${v.description} |`; + }).join('\n'); + + const updated = original.replace(/()[\s\S]*?()/, (_m, start, end) => `${start}\n${header}\n${rows}\n${end}`); + if (updated === original) continue; + + stale.push(path); + if (!options.check) writeFileSync(doc, updated); + } + + if (!stale.length) return say('env var docs already current'); + if (options.check) { + yell(`Out of date: ${stale.join(', ')}. Run \`bunosh docs:env\` and commit.`); + process.exit(1); + } + say(`Updated ${stale.join(', ')} from src/config.ts`); +} + /** * List demo-worthy segments from an Explorbot session log * @param {string} log - Path to explorbot.log diff --git a/bin/explorbot-cli.ts b/bin/explorbot-cli.ts index 5c28fa5..16108a4 100755 --- a/bin/explorbot-cli.ts +++ b/bin/explorbot-cli.ts @@ -9,7 +9,7 @@ import { render } from 'ink'; import React from 'react'; import { App } from '../src/components/App.js'; import { StatusPane } from '../src/components/StatusPane.js'; -import { ConfigParser } from '../src/config.js'; +import { ConfigParser, EXPLORBOT_ENV_VARS, PROVIDERS } from '../src/config.js'; import { ExplorBot, type ExplorBotOptions } from '../src/explorbot.js'; import { Stats } from '../src/stats.js'; import { Plan } from '../src/test-plan.js'; @@ -877,27 +877,28 @@ import { createDocsCommands } from '../boat/doc-collector/src/cli.ts'; program.addCommand(createApiCommands('api')); program.addCommand(createDocsCommands('docs')); -program.addHelpText( - 'after', - ` +const envHelp = () => { + const width = Math.max(...EXPLORBOT_ENV_VARS.map((v) => v.name.length)); + const rows = EXPLORBOT_ENV_VARS.map((v) => ` ${v.name.padEnd(width)} ${v.description}`).join('\n'); + + return ` Environment variables (config-free one-liner mode): Set EXPLORBOT_AI_PROVIDER to run without an explorbot.config.js. A config file always wins. - EXPLORBOT_AI_PROVIDER provider name; fills every role from its recommended models - EXPLORBOT_AI_MODEL model id for that provider, or provider/model-id on its own - EXPLORBOT_URL base URL to test; API boat uses it as the base endpoint - EXPLORBOT_VISION_MODEL screenshot analysis; overrides the provider recommendation - EXPLORBOT_AGENTIC_MODEL Captain and Pilot decisions; overrides the provider recommendation - EXPLORBOT_OUTPUT output root (default: a fresh temp directory) - EXPLORBOT_KNOWLEDGE inline knowledge applied to every page - EXPLORBOT_KNOWLEDGE_FILE path to a knowledge markdown file - EXPLORBOT_API_SPEC OpenAPI spec path for the API boat - - Providers: openai, anthropic, google, groq, mistral, openrouter, sambanova +${rows} + + Providers: ${Object.keys(PROVIDERS).join(', ')} Example: EXPLORBOT_URL=https://app.example.com EXPLORBOT_AI_PROVIDER=openrouter \\ ${cli} explore /login --max-tests 3 -` -); +`; +}; + +const addEnvHelp = (cmd: Command) => { + cmd.addHelpText('after', envHelp); + for (const sub of cmd.commands) addEnvHelp(sub); +}; + +addEnvHelp(program); program.parse(); diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 73a9fb1..72ae697 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -89,17 +89,20 @@ EXPLORBOT_AI_PROVIDER=openrouter \ npx explorbot explore /login --max-tests 3 ``` + | Variable | Meaning | |---|---| -| `EXPLORBOT_AI_PROVIDER` | A provider name; fills every role from its recommended models. Setting it turns on config-free mode | -| `EXPLORBOT_AI_MODEL` | Pins the main model — a model id for the provider, or a standalone `provider/model-id` | +| `EXPLORBOT_AI_PROVIDER` | Provider name; fills every model role from its recommended models. Turns on config-free mode | +| `EXPLORBOT_AI_MODEL` | Pins the main model — a model id for the provider, or a standalone provider/model-id | | `EXPLORBOT_URL` | Base URL to test; the API boat reads it as the base endpoint | | `EXPLORBOT_VISION_MODEL` | Screenshot analysis; overrides the provider recommendation | | `EXPLORBOT_AGENTIC_MODEL` | Captain and Pilot decisions; overrides the provider recommendation | -| `EXPLORBOT_OUTPUT` | Output root. Defaults to a fresh temp directory | +| `EXPLORBOT_OUTPUT` | Output root for states, plans, research, and reports. Defaults to a fresh temp directory | | `EXPLORBOT_KNOWLEDGE` | Inline knowledge text, applied to every page | | `EXPLORBOT_KNOWLEDGE_FILE` | Path to a knowledge markdown file | | `EXPLORBOT_API_SPEC` | OpenAPI spec path for the API boat | +| `EXPLORBOT_NO_BANNER` | Suppress the startup banner, for machine-readable output | + A config file always wins when present. A bare provider name fills every model role from the recommendations in [Providers](../basics/providers.md); a `provider/model-id` spec pins one model and splits on the first slash, so `openrouter/openai/gpt-oss-120b:nitro` selects OpenRouter with model `openai/gpt-oss-120b:nitro`. Supported providers: `openai`, `anthropic`, `google`, `groq`, `openrouter`, `sambanova`. diff --git a/docs/workflow/agentic-usage.md b/docs/workflow/agentic-usage.md index bcb72ef..c5df442 100644 --- a/docs/workflow/agentic-usage.md +++ b/docs/workflow/agentic-usage.md @@ -18,19 +18,24 @@ No `init`, no config file, no project directory, no model IDs to look up. A conf ### Variables + | Variable | Required | Meaning | |---|---|---| -| `EXPLORBOT_AI_PROVIDER` | yes | A provider name; fills every role from its recommended models. Setting it turns on this mode | -| `EXPLORBOT_AI_MODEL` | no | Pins the main `model` — a model id for the provider, or a standalone `provider/model-id` that turns on this mode by itself | -| `EXPLORBOT_URL` | yes | Base URL to test. The [API boat](../api-testing/basics.md) reads it as the base endpoint | -| `EXPLORBOT_VISION_MODEL` | no | Screenshot analysis. A provider name or `provider/model-id`; overrides the provider recommendation | -| `EXPLORBOT_AGENTIC_MODEL` | no | Captain and Pilot decisions. A provider name or `provider/model-id`; overrides the provider recommendation | -| `EXPLORBOT_OUTPUT` | no | Output root. Defaults to a fresh temp directory | +| `EXPLORBOT_AI_PROVIDER` | yes | Provider name; fills every model role from its recommended models. Turns on config-free mode | +| `EXPLORBOT_AI_MODEL` | no | Pins the main model — a model id for the provider, or a standalone provider/model-id | +| `EXPLORBOT_URL` | yes | Base URL to test; the API boat reads it as the base endpoint | +| `EXPLORBOT_VISION_MODEL` | no | Screenshot analysis; overrides the provider recommendation | +| `EXPLORBOT_AGENTIC_MODEL` | no | Captain and Pilot decisions; overrides the provider recommendation | +| `EXPLORBOT_OUTPUT` | no | Output root for states, plans, research, and reports. Defaults to a fresh temp directory | | `EXPLORBOT_KNOWLEDGE` | no | Inline knowledge text, applied to every page | | `EXPLORBOT_KNOWLEDGE_FILE` | no | Path to a knowledge markdown file | | `EXPLORBOT_API_SPEC` | no | OpenAPI spec path for the API boat | +| `EXPLORBOT_NO_BANNER` | no | Suppress the startup banner, for machine-readable output | + -`EXPLORBOT_URL` is optional when the command itself carries an absolute URL, as `docs collect https://…` does. +`EXPLORBOT_URL` is optional when the command itself carries an absolute URL, as `docs collect https://…` does. The [API boat](../api-testing/basics.md) reads it as the base endpoint. + +This table is generated from the registry in `src/config.ts`, which also feeds `explorbot --help` — so `npx explorbot --help` lists the same variables on any command, and an agent can discover them without reading these docs. ### Naming models diff --git a/src/config.ts b/src/config.ts index ab7f800..30dc4d1 100644 --- a/src/config.ts +++ b/src/config.ts @@ -5,7 +5,7 @@ import { parseEnv } from 'node:util'; import matter from 'gray-matter'; import { log } from './utils/logger.js'; -const PROVIDERS: Record Promise<(modelId: string) => any>> = { +export const PROVIDERS: Record Promise<(modelId: string) => any>> = { openai: async () => (await import('@ai-sdk/openai')).createOpenAI(), anthropic: async () => (await import('@ai-sdk/anthropic')).createAnthropic(), google: async () => (await import('@ai-sdk/google')).createGoogleGenerativeAI(), @@ -254,6 +254,19 @@ type RuleEntry = string | Record; export const EXPLORBOT_CONFIG_PATHS = ['explorbot.config.js', 'explorbot.config.mjs', 'explorbot.config.ts']; +export const EXPLORBOT_ENV_VARS: EnvVar[] = [ + { name: 'EXPLORBOT_AI_PROVIDER', required: true, description: 'Provider name; fills every model role from its recommended models. Turns on config-free mode' }, + { name: 'EXPLORBOT_AI_MODEL', description: 'Pins the main model — a model id for the provider, or a standalone provider/model-id' }, + { name: 'EXPLORBOT_URL', required: true, description: 'Base URL to test; the API boat reads it as the base endpoint' }, + { name: 'EXPLORBOT_VISION_MODEL', description: 'Screenshot analysis; overrides the provider recommendation' }, + { name: 'EXPLORBOT_AGENTIC_MODEL', description: 'Captain and Pilot decisions; overrides the provider recommendation' }, + { name: 'EXPLORBOT_OUTPUT', description: 'Output root for states, plans, research, and reports. Defaults to a fresh temp directory' }, + { name: 'EXPLORBOT_KNOWLEDGE', description: 'Inline knowledge text, applied to every page' }, + { name: 'EXPLORBOT_KNOWLEDGE_FILE', description: 'Path to a knowledge markdown file' }, + { name: 'EXPLORBOT_API_SPEC', description: 'OpenAPI spec path for the API boat' }, + { name: 'EXPLORBOT_NO_BANNER', description: 'Suppress the startup banner, for machine-readable output' }, +]; + export type { ExplorbotConfig, PlaywrightConfig, @@ -696,4 +709,10 @@ export async function createModel(provider: string, modelId: string): Promise { beforeEach(() => { @@ -278,3 +278,29 @@ describe('ConfigParser environment mode', () => { expect(existsSync(join(scratchDir, 'knowledge'))).toBe(false); }); }); + +describe('EXPLORBOT_ENV_VARS registry', () => { + it('documents every EXPLORBOT_ variable the code reads', () => { + const sources = ['src/config.ts', 'bin/explorbot-cli.ts', 'boat/api-tester/src/config.ts', 'boat/doc-collector/src/cli.ts']; + const used = new Set(); + + for (const source of sources) { + const code = readFileSync(source, 'utf8'); + for (const match of code.matchAll(/process\.env\.(EXPLORBOT_[A-Z_]+)/g)) { + used.add(match[1]); + } + } + + const documented = new Set(EXPLORBOT_ENV_VARS.map((v) => v.name)); + const undocumented = [...used].filter((name) => !documented.has(name)); + + expect(undocumented).toEqual([]); + }); + + it('lists no variable the code never reads', () => { + const code = ['src/config.ts', 'bin/explorbot-cli.ts', 'boat/api-tester/src/config.ts'].map((f) => readFileSync(f, 'utf8')).join('\n'); + const unused = EXPLORBOT_ENV_VARS.filter((v) => !code.includes(`process.env.${v.name}`)); + + expect(unused.map((v) => v.name)).toEqual([]); + }); +}); From a16587664b638ed8c61c4a83aec3d8dfdfc0f9e8 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Mon, 20 Jul 2026 00:44:46 +0300 Subject: [PATCH 3/5] refactor: unify docs generation under docs:sync, drop process.exit docs:models and docs:env become one docs:sync command with two non-exported helpers, so CI runs a single verification step. Signals staleness by throwing instead of calling process.exit, matching how regression:all already reports failure. A failed task() cannot be used for this: bunosh's test-environment guard checks `typeof Bun?.jest !== 'undefined'`, which is always true under Bun, so task failures never set exit code 1 here. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/publish.yml | 7 ++---- Bunoshfile.js | 46 +++++++++++++++-------------------- CHANGELOG.md | 2 +- docs/basics/providers.md | 2 +- 4 files changed, 24 insertions(+), 33 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 460af93..ee9cc8b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -35,11 +35,8 @@ jobs: - name: Install dependencies run: bun install - - name: Verify provider docs are up to date - run: bunx bunosh docs:models --check - - - name: Verify env var docs are up to date - run: bunx bunosh docs:env --check + - name: Verify generated docs are up to date + run: bunx bunosh docs:sync --check - name: Run unit tests run: bun test tests/unit/ diff --git a/Bunoshfile.js b/Bunoshfile.js index 0b3944c..9bb4c85 100644 --- a/Bunoshfile.js +++ b/Bunoshfile.js @@ -46,13 +46,22 @@ export async function worktreeCreate(name = '') { } /** - * Regenerate provider docs (docs/basics/providers.md) from models.json + * Regenerate generated docs: provider blocks from models.json, EXPLORBOT_* tables from src/config.ts * @param {object} options * @param {boolean} [options.check=false] - Fail if the docs are stale instead of rewriting them (for CI/release) */ -export async function docsModels(options = { check: false }) { - const doc = resolve('docs/basics/providers.md'); - const original = readFileSync(doc, 'utf8'); +export async function docsSync(options = { check: false }) { + const stale = [...syncProviderDocs(options.check), ...syncEnvDocs(options.check)]; + + if (!stale.length) return say('Generated docs already current'); + if (!options.check) return say(`Updated ${stale.join(', ')}`); + + throw new Error(`Stale: ${stale.join(', ')}. Run \`bunosh docs:sync\` and commit.`); +} + +function syncProviderDocs(check) { + const path = 'docs/basics/providers.md'; + const original = readFileSync(resolve(path), 'utf8'); let updated = original; const models = JSON.parse(readFileSync(resolve('models.json'), 'utf8')); @@ -71,21 +80,12 @@ export async function docsModels(options = { check: false }) { updated = updated.replace(new RegExp(`()[\\s\\S]*?()`), (_m, start, end) => `${start}\n${block}\n${end}`); } - if (updated === original) return say('docs/basics/providers.md already current'); - if (options.check) { - yell('docs/basics/providers.md is out of date. Run `bunosh docs:models` and commit.'); - process.exit(1); - } - writeFileSync(doc, updated); - say('Updated docs/basics/providers.md from models.json'); + if (updated === original) return []; + if (!check) writeFileSync(resolve(path), updated); + return [path]; } -/** - * Regenerate the EXPLORBOT_* env var tables in docs from src/config.ts - * @param {object} options - * @param {boolean} [options.check=false] - Fail if the docs are stale instead of rewriting them (for CI/release) - */ -export async function docsEnv(options = { check: false }) { +function syncEnvDocs(check) { const docs = [ { path: 'docs/workflow/agentic-usage.md', withRequired: true }, { path: 'docs/reference/commands.md', withRequired: false }, @@ -93,8 +93,7 @@ export async function docsEnv(options = { check: false }) { const stale = []; for (const { path, withRequired } of docs) { - const doc = resolve(path); - const original = readFileSync(doc, 'utf8'); + const original = readFileSync(resolve(path), 'utf8'); const header = withRequired ? '| Variable | Required | Meaning |\n|---|---|---|' : '| Variable | Meaning |\n|---|---|'; const rows = EXPLORBOT_ENV_VARS.map((v) => { @@ -106,15 +105,10 @@ export async function docsEnv(options = { check: false }) { if (updated === original) continue; stale.push(path); - if (!options.check) writeFileSync(doc, updated); + if (!check) writeFileSync(resolve(path), updated); } - if (!stale.length) return say('env var docs already current'); - if (options.check) { - yell(`Out of date: ${stale.join(', ')}. Run \`bunosh docs:env\` and commit.`); - process.exit(1); - } - say(`Updated ${stale.join(', ')} from src/config.ts`); + return stale; } /** diff --git a/CHANGELOG.md b/CHANGELOG.md index 55ac085..5ec2c51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,7 @@ Config-free runs do not write experience and run with the Historian off, so no g - Added `docs/workflow/agentic-usage.md` — driving Explorbot from a coding agent: the config-free one-liner API, and handing Explorbot a test plan the agent wrote itself (`npx explorbot test plan.md '*'`). Plan files are input only and are never rewritten, so they live in version control next to the code they cover. - `npx explorbot --help` now lists the `EXPLORBOT_*` variables with an example, so an agent can discover the config-free mode without reading the docs. - The recommended model IDs in `models.json` are now read at runtime, not just when regenerating the provider docs — they are what a bare provider name resolves to. `models.json` ships with the npm package. -- Documented every AI provider in `docs/basics/providers.md`, with the recommended-model block for each generated from `models.json` by `bunosh docs:models` and verified in CI on release. Added Mistral as a supported provider — `mistral-small-latest` for the `model` and `visionModel` roles (it reads screenshots), `mistral-large-latest` for `agenticModel`. +- Documented every AI provider in `docs/basics/providers.md`, with the recommended-model block for each generated from `models.json` by `bunosh docs:sync` and verified in CI on release. Added Mistral as a supported provider — `mistral-small-latest` for the `model` and `visionModel` roles (it reads screenshots), `mistral-large-latest` for `agenticModel`. - CodeceptJS screenshots now follow `dirs.output` instead of always landing in `output/states`. - Bare `--session` writes to `$EXPLORBOT_OUTPUT/session.json` when that variable is set. In a temp-directory run the path is not known when the flag is parsed, so pass an explicit `--session ` there. diff --git a/docs/basics/providers.md b/docs/basics/providers.md index 59ec44a..ab055b5 100644 --- a/docs/basics/providers.md +++ b/docs/basics/providers.md @@ -2,7 +2,7 @@ Explorbot connects to AI providers through the [Vercel AI SDK](https://sdk.vercel.ai/). Use any supported provider, and mix providers across different models. -> The `export default` config block inside each `` marker is generated from [`models.json`](../../models.json). After editing that file, run `bunosh docs:models`. Everything else — including the import blocks — is hand-written. +> The `export default` config block inside each `` marker is generated from [`models.json`](../../models.json). After editing that file, run `bunosh docs:sync`. Everything else — including the import blocks — is hand-written. ## Requirements From 0dbb85310f61155985dfc066b0a178dc6be66976 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Mon, 20 Jul 2026 01:25:08 +0300 Subject: [PATCH 4/5] refactor: drive docs:sync with tasks and stopOnFail One task per generated file, so a stale doc is reported by name. stopOnFail aborts the run at the first failure instead of accumulating a stale list and branching on it afterwards. Co-Authored-By: Claude Opus 4.8 --- Bunoshfile.js | 57 +++++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/Bunoshfile.js b/Bunoshfile.js index 9bb4c85..1ed16d8 100644 --- a/Bunoshfile.js +++ b/Bunoshfile.js @@ -17,7 +17,7 @@ import { htmlCombinedSnapshot, htmlTextSnapshot, minifyHtml } from './src/utils/ import { analyzeDemoCandidates, createDemoVideo } from './.claude/skills/demo-video/demo-video.ts'; import { EXPLORBOT_ENV_VARS } from './src/config.ts'; -const { exec, shell, writeToFile, task, ai } = global.bunosh; +const { exec, shell, writeToFile, task, stopOnFail, ai } = global.bunosh; const CLI = resolve('bin/explorbot-cli.ts'); const REG_ROOT = resolve('tests/regression'); @@ -45,18 +45,24 @@ export async function worktreeCreate(name = '') { say(`Created worktree for feature ${worktreeName} in ${newDir}`); } +const ENV_DOCS = [ + { path: 'docs/workflow/agentic-usage.md', withRequired: true }, + { path: 'docs/reference/commands.md', withRequired: false }, +]; + /** * Regenerate generated docs: provider blocks from models.json, EXPLORBOT_* tables from src/config.ts * @param {object} options * @param {boolean} [options.check=false] - Fail if the docs are stale instead of rewriting them (for CI/release) */ export async function docsSync(options = { check: false }) { - const stale = [...syncProviderDocs(options.check), ...syncEnvDocs(options.check)]; + stopOnFail(); - if (!stale.length) return say('Generated docs already current'); - if (!options.check) return say(`Updated ${stale.join(', ')}`); + await task('docs/basics/providers.md', () => syncProviderDocs(options.check)); - throw new Error(`Stale: ${stale.join(', ')}. Run \`bunosh docs:sync\` and commit.`); + for (const doc of ENV_DOCS) { + await task(doc.path, () => syncEnvDoc(doc, options.check)); + } } function syncProviderDocs(check) { @@ -80,35 +86,32 @@ function syncProviderDocs(check) { updated = updated.replace(new RegExp(`()[\\s\\S]*?()`), (_m, start, end) => `${start}\n${block}\n${end}`); } - if (updated === original) return []; - if (!check) writeFileSync(resolve(path), updated); - return [path]; + return applyGenerated(path, original, updated, check); } -function syncEnvDocs(check) { - const docs = [ - { path: 'docs/workflow/agentic-usage.md', withRequired: true }, - { path: 'docs/reference/commands.md', withRequired: false }, - ]; +function syncEnvDoc({ path, withRequired }, check) { + const original = readFileSync(resolve(path), 'utf8'); + + let header = '| Variable | Meaning |\n|---|---|'; + if (withRequired) header = '| Variable | Required | Meaning |\n|---|---|---|'; - const stale = []; - for (const { path, withRequired } of docs) { - const original = readFileSync(resolve(path), 'utf8'); + const rows = EXPLORBOT_ENV_VARS.map((v) => { + if (!withRequired) return `| \`${v.name}\` | ${v.description} |`; + if (v.required) return `| \`${v.name}\` | yes | ${v.description} |`; + return `| \`${v.name}\` | no | ${v.description} |`; + }).join('\n'); - const header = withRequired ? '| Variable | Required | Meaning |\n|---|---|---|' : '| Variable | Meaning |\n|---|---|'; - const rows = EXPLORBOT_ENV_VARS.map((v) => { - if (withRequired) return `| \`${v.name}\` | ${v.required ? 'yes' : 'no'} | ${v.description} |`; - return `| \`${v.name}\` | ${v.description} |`; - }).join('\n'); + const updated = original.replace(/()[\s\S]*?()/, (_m, start, end) => `${start}\n${header}\n${rows}\n${end}`); - const updated = original.replace(/()[\s\S]*?()/, (_m, start, end) => `${start}\n${header}\n${rows}\n${end}`); - if (updated === original) continue; + return applyGenerated(path, original, updated, check); +} - stale.push(path); - if (!check) writeFileSync(resolve(path), updated); - } +function applyGenerated(path, original, updated, check) { + if (updated === original) return 'already current'; + if (check) throw new Error('stale — run `bunosh docs:sync` and commit'); - return stale; + writeFileSync(resolve(path), updated); + return 'regenerated'; } /** From cec876ea54c3d357ba420ad72d5dfa69600533f3 Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Wed, 22 Jul 2026 11:31:03 +0200 Subject: [PATCH 5/5] Fix --- bin/explorbot-cli.ts | 7 ++----- boat/doc-collector/src/cli.ts | 7 +------ docs/reference/commands.md | 2 +- docs/workflow/agentic-usage.md | 2 +- src/explorbot.ts | 6 ++++-- 5 files changed, 9 insertions(+), 15 deletions(-) diff --git a/bin/explorbot-cli.ts b/bin/explorbot-cli.ts index 16108a4..d710e29 100755 --- a/bin/explorbot-cli.ts +++ b/bin/explorbot-cli.ts @@ -43,9 +43,6 @@ interface CLIOptions { } function buildExplorBotOptions(from: string | undefined, options: CLIOptions): ExplorBotOptions { - const sessionDir = process.env.EXPLORBOT_OUTPUT ? path.resolve(process.env.EXPLORBOT_OUTPUT) : path.join(path.resolve(options.path || process.cwd()), 'output'); - const sessionFile = options.session === true ? path.join(sessionDir, 'session.json') : options.session; - return { from, verbose: options.verbose || options.debug, @@ -54,7 +51,7 @@ function buildExplorBotOptions(from: string | undefined, options: CLIOptions): E show: options.show, headless: options.headless, incognito: options.incognito, - session: sessionFile, + session: options.session, } as ExplorBotOptions; } @@ -691,7 +688,7 @@ program path: options.path, config: options.config, headless: true, - session: options.session === true ? 'output/session.json' : options.session, + session: options.session, }; const explorBot = new ExplorBot(mainOptions); diff --git a/boat/doc-collector/src/cli.ts b/boat/doc-collector/src/cli.ts index 3f953f7..2fba9f7 100644 --- a/boat/doc-collector/src/cli.ts +++ b/boat/doc-collector/src/cli.ts @@ -5,11 +5,6 @@ import { setPreserveConsoleLogs } from '../../../src/utils/logger.ts'; import { DocBot, type DocbotOptions } from './docbot.ts'; function buildOptions(options: any): DocbotOptions { - let session = options.session; - if (options.session === true) { - session = 'output/session.json'; - } - return { verbose: options.verbose || options.debug, config: options.config, @@ -17,7 +12,7 @@ function buildOptions(options: any): DocbotOptions { show: options.show, headless: options.headless, incognito: options.incognito, - session, + session: options.session, docsConfig: options.docsConfig, }; } diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 72ae697..ab1571a 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -77,7 +77,7 @@ npx explorbot navigate /login --session # probe + capture auth in on npx explorbot research /dashboard --session auth.json # reuse captured auth ``` -Without a file path, the flag defaults to `output/session.json`, or to `$EXPLORBOT_OUTPUT/session.json` when that variable is set. In a temp-directory run the temp path is not known when the flag is parsed, so pass an explicit `--session `. +Without a file path, the flag defaults to `session.json` inside the resolved configuration's output directory. In config-free mode that is the temporary or explicitly configured `EXPLORBOT_OUTPUT` directory. ## Environment Variables diff --git a/docs/workflow/agentic-usage.md b/docs/workflow/agentic-usage.md index c5df442..544f768 100644 --- a/docs/workflow/agentic-usage.md +++ b/docs/workflow/agentic-usage.md @@ -104,7 +104,7 @@ Config-free runs are built to leave no trace in the working directory: - **Experience is not written.** Nothing accumulates between runs, so a run is reproducible. Reading existing experience still works if the directory has any. - **The Historian is off.** No generated CodeceptJS or Playwright test files. Plans and reports are still written. -For a long-lived agent that should learn across runs, point `EXPLORBOT_OUTPUT` at a stable directory or switch to a real config file. +For a long-lived agent that should learn across runs, use a real config file with experience writing enabled. A stable `EXPLORBOT_OUTPUT` preserves run artifacts, but config-free mode still does not write experience. ### Reading results diff --git a/src/explorbot.ts b/src/explorbot.ts index 8e395f3..5e31cf8 100644 --- a/src/explorbot.ts +++ b/src/explorbot.ts @@ -42,7 +42,7 @@ export interface ExplorBotOptions { show?: boolean; headless?: boolean; incognito?: boolean; - session?: string; + session?: string | boolean; } export type UserResolveFunction = (error?: Error, showWelcome?: boolean) => Promise; @@ -88,7 +88,8 @@ export class ExplorBot { try { await this.startProviderOnly(); - this.explorer = new Explorer(this.config, this.provider, this.options, this.experienceTracker(), this.knowledgeTracker()); + const explorerOptions = { ...this.options, session: typeof this.options.session === 'string' ? this.options.session : undefined }; + this.explorer = new Explorer(this.config, this.provider, explorerOptions, this.experienceTracker(), this.knowledgeTracker()); await this.explorer.start(); if (!this.options.incognito) { await this.agentExperienceCompactor().autocompact(); @@ -103,6 +104,7 @@ export class ExplorBot { async startProviderOnly(): Promise { if (this.provider) return; this.config = await this.configParser.loadConfig(this.options); + if (this.options.session === true) this.options.session = path.join(this.configParser.getOutputDir(), 'session.json'); this.provider = new AIProvider(this.config.ai); await this.provider.validateConnection(); }