diff --git a/README.md b/README.md index 6c68c27b..4ed97176 100644 --- a/README.md +++ b/README.md @@ -69,22 +69,32 @@ const result = await supervise( ### Improve an agent `improve` optimizes one part of an agent and **only ships a change if it beats the current agent on tasks it never practiced on**. -It accepts prompt, skill document, tool, MCP, hook, subagent, workflow, rollout-policy, whole-profile, and code surfaces through one call. -Prompt and skill-document optimization have built-in generators; structured surfaces take an explicit generator, and code runs from isolated incumbent and candidate checkouts. +It accepts prompt, skill document, curated memory, tool, MCP, hook, subagent, workflow, rollout-policy, whole-profile, and code surfaces through one call. +Prompt, skill-document, memory, and rollout-policy optimization have built-in generators; structured surfaces take an explicit generator, and code runs from isolated incumbent and candidate checkouts. ```ts import { improve } from '@tangle-network/agent-runtime' const { profile, shipped, lift } = await improve(baseProfile, findings, { - surface: 'prompt', // or skills/tools/mcp/hooks/subagents/workflow/agent-profile/code + surface: 'prompt', // or skills/memory/tools/mcp/hooks/subagents/workflow/agent-profile/rollout-policy/code gate: 'holdout', // certified on a held-back exam, never the practice set scenarios, judge, agent, // how to measure a candidate }) ``` +Curated memory is an external lesson document, not a knowledge store. Supply its current text and persist only the promoted winner: + +```ts +await improve(baseProfile, findings, { + surface: 'memory', + memory: { document: currentLessons, writeBack: saveLessons }, + scenarios, judge, agent, +}) +``` + ### Improve a knowledge base -`runKnowledgeImprovementJob` is the runtime-owned front door for KB, wiki, memory-backed, and RAG improvement jobs. It creates a candidate copy, runs supervised agents against it, checks readiness through `@tangle-network/agent-knowledge`, measures spend and timing, and promotes only when the candidate passes. +`runKnowledgeImprovementJob` is the runtime-owned front door for KB, wiki, memory-backed, and RAG improvement jobs. It creates a candidate copy, runs supervised agents against it, checks readiness through `@tangle-network/agent-knowledge`, measures spend and timing, and promotes only when the candidate passes. Use `improve(..., { surface: 'memory' })` for the agent's curated lesson document; use this job for source, retrieval, and knowledge-store changes. ```ts import { runKnowledgeImprovementJob } from '@tangle-network/agent-runtime/knowledge' diff --git a/docs/agent-optimization-map.md b/docs/agent-optimization-map.md deleted file mode 100644 index 09c1b29c..00000000 --- a/docs/agent-optimization-map.md +++ /dev/null @@ -1,91 +0,0 @@ -# The Agent Optimization Map - -**What this document is.** -The one-page truth about optimizing agents on this substrate: every lever an `AgentProfile` exposes, what machinery exists to optimize or create each one, what is actually wired to the paved path, and what has been proven on live infrastructure. -Written to be read by humans and by agents configuring an improvement run. -Audited 2026-07-06 against `agent-interface`, `agent-eval`, `agent-runtime`, `agent-knowledge`, and the supervisor-lab live campaigns (runs 1–6). - -## The mental model in three sentences - -An **AgentProfile is the complete specification of an agent**: its brain-instructions, capabilities, and starting world. -**`improve()` is the one verb** that optimizes any single lever of that profile against an executable judge, with a statistical held-out gate deciding promotion. -Everything else in the stack is either a **proposer** (something that generates candidate lever-values), a **judge** (something that scores results), or a **loop** (something that runs proposers against judges until the gate ships or holds). - -## Lever inventory — what an AgentProfile actually carries - -| Lever (your words) | Profile field | In the type? | -|---|---|---| -| system prompt | `prompt.systemPrompt` (+ per-mode `modes`) | yes | -| skills | `resources.skills` (SKILL.md packages) | yes | -| tools | `tools` (enable/disable map) + `resources.tools` (file-based tool defs) | yes | -| MCP | `mcp` (server map) + `connections` | yes | -| knowledge base | no first-class field — lives as `resources.files` content or app-side (agent-knowledge KB store) | partial | -| resources | `resources.{instructions,agents,commands}` | yes | -| starting files | `resources.files` (materialized into the workspace before execution) | yes | -| full sandbox/VM state | backend/image choice + `extensions` (non-portable) + `resources.files` | partial — files yes, image/toolchain is the tcloud backend's, not the profile's | - -Also in the type and often forgotten: `subagents` (native sub-agents), `permissions`, `model` hints, `hooks`. - -## Optimization matrix — lever × machinery × wiring × proof - -| Surface | Baseline/apply plumbing | Proposer that exists | Reachable from `improve()`? | Proven live? | -|---|---|---|---|---| -| `prompt` | yes | `gepaProposer` (Pareto frontier + crossover-merge + reflective mutation) | **yes (default)** | **yes — supervisor-lab runs 3–6, ~1,700 sandbox cells** | -| `skills` | yes | `skillOptProposer` (skill-document patching) | **yes (default)** | no live run anywhere yet | -| `tools` | yes (JSON string surface) | none wired; `parameterSweepProposer` is shape-compatible for toggle/config sweeps | no — fails loud, needs `generator` | no | -| `mcp` | yes (JSON string surface) | none | no — fails loud | no | -| `hooks` | yes (JSON string surface) | none | no — fails loud | no | -| `code` (tools/harness/anything in a repo) | yes | `improvementDriver` + `agenticGenerator` (real coding harness per candidate worktree, verify-gated) | **yes — `code: { repoRoot }` facade (#480)** | offline test only; live milestone pending | -| `resources.files` (starting world) | **no surface** | none | no | no | -| knowledge base | **no surface** | agent-knowledge loops exist (below) but nothing writes back into a profile | no | no | - -**The one-sentence verdict: every lever you believe in is representable; two are optimizable-and-proven (prompt today, code as of #480); skills is one command away; tools/mcp/hooks are string surfaces awaiting a config proposer; files/knowledge aren't surfaces at all yet.** - -## The proposer zoo — what each one is for (plain words) - -| Proposer | What it does | Wired to paved path? | -|---|---|---| -| `gepaProposer` | evolves prompt text: keeps a frontier of variants that each win somewhere, merges them, mutates with evidence from best/worst trials | yes (prompt default) | -| `skillOptProposer` | same idea for skill documents (structured patches) | yes (skills default) | -| `fapoProposer` | **the meta-policy**: attribute each failure to a level, propose ONE scoped change, escalate prompt → parameters → structure only when the cheaper level is exhausted — with pluggable per-level generators | **no** — exported, zero consumers | -| `traceAnalystProposer` | turns trace-analyst findings (an Ax agent that reads execution traces with tools) into candidates | **no** — exported, zero consumers | -| `aceProposer` | append-only "playbook" curator: accumulates provenance-tagged lessons without ever summarizing old ones away (anti context-collapse) | no | -| `memoryCurationProposer` | the dedup-and-replace contrast to ACE | no | -| `parameterSweepProposer` | sweeps numeric/config parameters | no | -| `haloProposer` | hierarchical analyst-driven optimization | no | - -Since #480/#310/#311 the paved path also has: per-generation **failure distiller** (the proposer automatically sees each generation's worst cells + judge reasons), **power preflight** (minimum detectable lift computed from baseline cells; structurally-hopeless budgets warn), durable `runDir`, and the statistical holdout gate everywhere (the point-estimate gate was folded away). - -## Creation loops — making new things, not just mutating strings - -What you asked for → what exists: - -- **"Create new skills by mining traces"** — components all exist, composition does not: `trace-analyst` (reads traces, emits findings) → `skillOptProposer`/`aceProposer` (turn findings into skill-document content) → `improve(surface: 'skills')` (prove them). Nobody has connected the three. One composition, high value. -- **"Auto-research loops that write code"** — exists as of #480: `improve(surface: 'code', code: { repoRoot, verify })` runs a real coding harness per candidate in a worktree, gated by any executable judge. "Create a new tool" = point it at the repo where tools live. -- **"Knowledge discovery / find relevant OSS libraries / find MCPs"** — the research machinery exists in `agent-knowledge` (`runVerifiedResearchLoop`: two-agent verified research with claim-grounding; `discovery`; collection drivers) and is consumed by physim/legal/tax products — but **no bridge writes research results into an AgentProfile lever**. The missing primitive is small: a `knowledge → resources.files/skills` materializer, after which "research the ecosystem, propose an MCP server for the profile, prove it on the benchmark" is a normal `improve()` run over the `mcp` surface. -- **"Full sandbox initial state as a lever"** — `resources.files` covers workspace seeding; making it an `ImproveSurface` (candidate = a file-set, judge = task performance) is unbuilt but fits the existing surface contract exactly. - -## The three gaps that matter (everything else is garnish) - -1. **Reachability**: the zoo (FAPO, trace-analyst, ACE, sweep) needs one dial — `improve(..., { proposer: 'fapo' })` — instead of hand-assembly. FAPO especially: it *is* the "optimize everything, escalate sensibly" policy this document describes, already grounded in the paper, already pluggable. -2. **The knowledge bridge**: research loops produce verified knowledge; profiles can carry files/skills; nothing connects them. -3. **This document didn't exist.** Now it does; keep it honest — every row's "proven live" column only flips with a linked run. - -## Recipes (copy-paste truth, today) - -```ts -// Optimize a prompt against any executable judge (proven, runs 3–6): -await improve(profile, findings, { surface: 'prompt', scenarios, judge, agent, runDir }) - -// Optimize the skill documents: -await improve(profile, findings, { surface: 'skills', scenarios, judge, agent, runDir }) - -// Evolve CODE (tools, harnesses, algorithms) with a real coding agent per candidate: -await improve(profile, findings, { - surface: 'code', - code: { repoRoot, verify: commandVerifier({ command: 'pnpm', args: ['test'] }) }, - scenarios, judge, agent, runDir, -}) - -// Every result now carries result.power — read it BEFORE buying a bigger search. -``` diff --git a/docs/api/index.md b/docs/api/index.md index 249a21b2..1d0c1b39 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -5688,7 +5688,7 @@ Test seam — inject the worktree-dirty check (defaults to `git status`). ### ImproveSkillsOptions -Defined in: [improvement/improve.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L143) +Defined in: [improvement/improve.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L151) #### Properties @@ -5696,15 +5696,15 @@ Defined in: [improvement/improve.ts:143](https://github.com/tangle-network/agent > **document**: `string` -Defined in: [improvement/improve.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L145) +Defined in: [improvement/improve.ts:153](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L153) The skill document's current text — the baseline `skillOptProposer` patches. ##### writeBack? -> `optional` **writeBack?**: (`winnerDocument`) => `void` +> `optional` **writeBack?**: (`winnerDocument`) => `void` \| `Promise`\<`void`\> -Defined in: [improvement/improve.ts:149](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L149) +Defined in: [improvement/improve.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L157) Persist the shipped winner document (write the file the profile ref points at). Called only on a ship verdict. When omitted, the winner is still returned in @@ -5718,13 +5718,47 @@ Persist the shipped winner document (write the file the profile ref points at). ###### Returns -`void` +`void` \| `Promise`\<`void`\> + +*** + +### ImproveMemoryOptions + +Defined in: [improvement/improve.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L160) + +#### Properties + +##### document + +> **document**: `string` + +Defined in: [improvement/improve.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L162) + +Current durable memory text used as the measured baseline. + +##### writeBack? + +> `optional` **writeBack?**: (`winnerDocument`) => `void` \| `Promise`\<`void`\> + +Defined in: [improvement/improve.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L164) + +Persist the promoted memory document. Never called on hold or error. + +###### Parameters + +###### winnerDocument + +`string` + +###### Returns + +`void` \| `Promise`\<`void`\> *** ### ImproveCodeOptions -Defined in: [improvement/improve.ts:152](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L152) +Defined in: [improvement/improve.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L167) #### Properties @@ -5732,7 +5766,7 @@ Defined in: [improvement/improve.ts:152](https://github.com/tangle-network/agent > **repoRoot**: `string` -Defined in: [improvement/improve.ts:154](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L154) +Defined in: [improvement/improve.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L169) Repo root candidate worktrees fork from. @@ -5740,7 +5774,7 @@ Repo root candidate worktrees fork from. > `optional` **baseRef?**: `string` -Defined in: [improvement/improve.ts:156](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L156) +Defined in: [improvement/improve.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L171) Base ref candidates fork from. Default `main`. @@ -5748,7 +5782,7 @@ Base ref candidates fork from. Default `main`. > `optional` **worktreeDir?**: `string` -Defined in: [improvement/improve.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L158) +Defined in: [improvement/improve.ts:173](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L173) Directory worktrees are created under. Default `/.worktrees`. @@ -5756,7 +5790,7 @@ Directory worktrees are created under. Default `/.worktrees`. > `optional` **harness?**: [`LocalHarness`](mcp.md#localharness) -Defined in: [improvement/improve.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L160) +Defined in: [improvement/improve.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L175) Coding harness the agentic generator runs in each worktree. Default `claude`. @@ -5764,7 +5798,7 @@ Coding harness the agentic generator runs in each worktree. Default `claude`. > `optional` **verify?**: [`Verifier`](#verifier) -Defined in: [improvement/improve.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L163) +Defined in: [improvement/improve.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L178) Verify a candidate worktree before it becomes a measurable surface; failures feed the next shot (see `agenticGenerator.verify` / `commandVerifier`). @@ -5773,7 +5807,7 @@ Verify a candidate worktree before it becomes a measurable surface; failures > `optional` **timeoutMs?**: `number` -Defined in: [improvement/improve.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L165) +Defined in: [improvement/improve.ts:180](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L180) Per-shot wall-clock timeout for the harness (ms). @@ -5781,7 +5815,7 @@ Per-shot wall-clock timeout for the harness (ms). > `optional` **generator?**: [`CandidateGenerator`](#candidategenerator) -Defined in: [improvement/improve.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L168) +Defined in: [improvement/improve.ts:183](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L183) Byte-producer override — the test seam and the escape hatch for custom candidate production. When set, `harness`/`verify`/`timeoutMs` are unused. @@ -5790,7 +5824,7 @@ Byte-producer override — the test seam and the escape hatch for custom ### ImproveResult -Defined in: [improvement/improve.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L171) +Defined in: [improvement/improve.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L186) #### Type Parameters @@ -5808,7 +5842,7 @@ Defined in: [improvement/improve.ts:171](https://github.com/tangle-network/agent > **profile**: `AgentProfile` -Defined in: [improvement/improve.ts:174](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L174) +Defined in: [improvement/improve.ts:189](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L189) The profile after improvement: the winner surface applied back into the matching field when the gate shipped, else the input profile unchanged. @@ -5817,7 +5851,7 @@ The profile after improvement: the winner surface applied back into the > **shipped**: `boolean` -Defined in: [improvement/improve.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L176) +Defined in: [improvement/improve.ts:191](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L191) True when `gateDecision === 'ship'`. @@ -5825,7 +5859,7 @@ True when `gateDecision === 'ship'`. > **lift**: `number` -Defined in: [improvement/improve.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L178) +Defined in: [improvement/improve.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L193) Held-out lift (`winner − baseline` composite). @@ -5833,7 +5867,7 @@ Held-out lift (`winner − baseline` composite). > **gateDecision**: `"ship"` \| `"hold"` \| `"need_more_work"` \| `"model_ceiling"` \| `"arch_ceiling"` -Defined in: [improvement/improve.ts:180](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L180) +Defined in: [improvement/improve.ts:195](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L195) The five-valued gate verdict from `selfImprove`. @@ -5841,7 +5875,7 @@ The five-valued gate verdict from `selfImprove`. > **raw**: `SelfImproveResult`\<`TScenario`, `TArtifact`\> -Defined in: [improvement/improve.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L182) +Defined in: [improvement/improve.ts:197](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L197) Full `selfImprove` result for advanced inspection. @@ -10632,9 +10666,9 @@ Verifies the edited worktree. Sync or async; throws only on a setup fault ### ImproveSurface -> **ImproveSurface** = `"prompt"` \| `"skills"` \| `"tools"` \| `"mcp"` \| `"hooks"` \| `"subagents"` \| `"workflow"` \| `"agent-profile"` \| `"code"` \| `"rollout-policy"` +> **ImproveSurface** = `"prompt"` \| `"skills"` \| `"tools"` \| `"mcp"` \| `"hooks"` \| `"subagents"` \| `"workflow"` \| `"agent-profile"` \| `"memory"` \| `"code"` \| `"rollout-policy"` -Defined in: [improvement/improve.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L75) +Defined in: [improvement/improve.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L78) The agent-profile lever `improve` optimizes. Mirrors the AgentProfile-law profile levers; `code` is the implementation-tier surface, `rollout-policy` @@ -10647,7 +10681,7 @@ The agent-profile lever `improve` optimizes. Mirrors the AgentProfile-law > **ImproveOptions**\<`TScenario`, `TArtifact`\> = `Omit`\<`SelfImproveOptions`\<`TScenario`, `TArtifact`\>, `"analyzeGeneration"` \| `"baselineSurface"` \| `"findings"` \| `"gate"` \| `"proposer"`\> & `object` -Defined in: [improvement/improve.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L87) +Defined in: [improvement/improve.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L91) #### Type Declaration @@ -10663,8 +10697,8 @@ Which profile lever to optimize. Default `'prompt'`. Selects the default > `optional` **generator?**: `SurfaceProposer` The `SurfaceProposer` that mutates the surface. When unset, the facade - picks the default for `surface` (`gepaProposer` for prompt, `skillOptProposer` - for skills); surfaces with no default REQUIRE this (fail-loud otherwise). + picks the default for prompt, skills, memory, and rollout policy; surfaces + with no default REQUIRE this (fail-loud otherwise). ##### gate? @@ -10730,6 +10764,14 @@ SKILLS-surface wiring for real skill-DOCUMENT optimization. Without this, shipped winner (the profile ref points at a file the caller owns). This is what makes skillOpt reachable through improve(). +##### memory? + +> `optional` **memory?**: [`ImproveMemoryOptions`](#improvememoryoptions) + +MEMORY-surface wiring for a curated durable memory document. The default + deterministic proposer deduplicates and ranks lessons from findings, then + replaces its managed block instead of growing memory without bound. + ##### promotionGate? > `optional` **promotionGate?**: `SelfImproveOptions`\<`TScenario`, `TArtifact`\>\[`"gate"`\] @@ -12592,7 +12634,7 @@ Build the starting instruction for a coder agent tasked with implementing a new > **improve**\<`TScenario`, `TArtifact`\>(`profile`, `findings`, `opts`): `Promise`\<[`ImproveResult`](#improveresult)\<`TScenario`, `TArtifact`\>\> -Defined in: [improvement/improve.ts:465](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L465) +Defined in: [improvement/improve.ts:510](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L510) Run the held-out-gated self-improvement loop on ONE profile surface. diff --git a/docs/api/intelligence.md b/docs/api/intelligence.md index 84684047..4bc0930e 100644 --- a/docs/api/intelligence.md +++ b/docs/api/intelligence.md @@ -3564,7 +3564,7 @@ Analyze one run and produce one measured, review-only improvement proposal. > **reviewAgentImprovementProposal**(`inputProposal`, `input`): [`AgentImprovementReview`](#agentimprovementreview) -Defined in: [intelligence/improvement-cycle.ts:201](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L201) +Defined in: [intelligence/improvement-cycle.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L208) Persist an approve/reject/change-request decision bound to one exact proposal. @@ -3588,7 +3588,7 @@ Persist an approve/reject/change-request decision bound to one exact proposal. > **executeApprovedAgentCandidate**(`options`): `Promise`\<[`ExecuteApprovedAgentCandidateResult`](#executeapprovedagentcandidateresult)\> -Defined in: [intelligence/improvement-cycle.ts:231](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L231) +Defined in: [intelligence/improvement-cycle.ts:238](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L238) Verify, materialize, run, grade, and receipt only the exact approved bundle. @@ -3608,7 +3608,7 @@ Verify, materialize, run, grade, and receipt only the exact approved bundle. > **verifyAgentImprovementProposal**(`input`): [`AgentImprovementProposal`](#agentimprovementproposal) -Defined in: [intelligence/improvement-cycle.ts:273](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L273) +Defined in: [intelligence/improvement-cycle.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L280) Validate a proposal's schema, profile, sealed bundle, and canonical digest. @@ -3628,7 +3628,7 @@ Validate a proposal's schema, profile, sealed bundle, and canonical digest. > **verifyAgentImprovementReview**(`input`): [`AgentImprovementReview`](#agentimprovementreview) -Defined in: [intelligence/improvement-cycle.ts:317](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L317) +Defined in: [intelligence/improvement-cycle.ts:324](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L324) Validate a review's decision fields and canonical digest. diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index bc6af84a..595c4d1c 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.92.1` and `@tangle-network/agent-eval@0.113.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.93.0` and `@tangle-network/agent-eval@0.114.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 330 exports. +Import from `@tangle-network/agent-runtime` — 331 exports. | Symbol | Kind | Summary | |---|---|---| @@ -212,7 +212,7 @@ Import from `@tangle-network/agent-runtime` — 330 exports. | `ToolLoopStopReason` | type | Why the loop stopped. `completed` = model finished naturally; `stuck-loop` = | | `Verifier` | type | Verifies the edited worktree. Sync or async; throws only on a setup fault | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateBenchmarkGraderIdentity`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorProfileFile`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateTaskExecution`, `AgentCandidateVerificationPorts`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `ChatTurnHooks`, `ChatTurnResult`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImprovementDriverOptions`, `ImproveResult`, `ImproveSkillsOptions`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `ManagedImprovementDriver`, `OtelAttribute`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RecoverExpiredAgentCandidateOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunChatTurnInput`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunHandle`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeTelemetryOptions`, `RunToolLoopOptions`, `SanitizedKnowledgeReadinessReport`, `StreamToolLoopOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `ToolLoopResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ControlDecision`, `ConversationStreamEvent`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveOptions`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeStreamEvent`, `StreamToolLoopYield`, `SupervisedKnowledgeUpdater`, `ToolLoopEvent`, `TurnOrder`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateBenchmarkGraderIdentity`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorProfileFile`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateTaskExecution`, `AgentCandidateVerificationPorts`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `ChatTurnHooks`, `ChatTurnResult`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImproveMemoryOptions`, `ImprovementDriverOptions`, `ImproveResult`, `ImproveSkillsOptions`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `ManagedImprovementDriver`, `OtelAttribute`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RecoverExpiredAgentCandidateOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunChatTurnInput`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunHandle`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeTelemetryOptions`, `RunToolLoopOptions`, `SanitizedKnowledgeReadinessReport`, `StreamToolLoopOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `ToolLoopResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ControlDecision`, `ConversationStreamEvent`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveOptions`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeStreamEvent`, `StreamToolLoopYield`, `SupervisedKnowledgeUpdater`, `ToolLoopEvent`, `TurnOrder`. ### Vertical agent — manifest + improvement adapter @@ -1105,7 +1105,7 @@ Import from `@tangle-network/agent-eval` — 10 exports. ### STATISTICS — significance, intervals, effect size -Import from `@tangle-network/agent-eval` — 50 exports. +Import from `@tangle-network/agent-eval` — 51 exports. | Symbol | Kind | Summary | |---|---|---| @@ -1128,6 +1128,7 @@ Import from `@tangle-network/agent-eval` — 50 exports. | `pairedEvalueSequence` | function | Run the paired e-value sequence over an in-order delta stream. | | `pairedMde` | function | Minimum detectable paired effect (standardised units) for a target paired | | `pairedRiskDifference` | function | Paired risk difference (the effect-size companion to {@link mcnemar}): the | +| `pairedSignTest` | function | Exact one-sided sign test over paired differences. | | `pairedTTest` | function | Paired t-test — before/after measurements on the SAME items. | | `partialCredit` | function | Partial credit: returns 0-1 ratio of current toward target | | `passAtK` | function | Unbiased pass@k for code generation (Chen et al. 2021, "Evaluating Large | diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 89d0c72d..c2056e0a 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -2,7 +2,7 @@ -> **Version 0.92.1.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned substrate is agent-eval `>=0.113.0 <1.0.0`; the sandbox substrate that materializes profiles into harness shapes is `@tangle-network/sandbox` (peer `>=0.8.0 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.25.0 <1.0.0`) — the single source of truth. Substrate primitives are re-exported through `@tangle-network/agent-eval/contract` (or `/campaign`), not local to this package — the catalog's §2 shows exactly which subpath each lives under. +> **Version 0.93.0.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned substrate is agent-eval `>=0.114.0 <1.0.0`; the sandbox substrate that materializes profiles into harness shapes is `@tangle-network/sandbox` (peer `>=0.8.0 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.25.0 <1.0.0`) — the single source of truth. Substrate primitives are re-exported through `@tangle-network/agent-eval/contract` (or `/campaign`), not local to this package — the catalog's §2 shows exactly which subpath each lives under. > > **`./loops` is the runtime barrel** — `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/loops` is the recursive-atom + loop-kernel surface. > @@ -86,7 +86,7 @@ A general "loop" primitive is the single most common modelling error in this rep | Pick the **chat backend an in-process turn runs on** (`router`/`tcloud`/`cli-bridge`/`sandbox`) from a product flag | `resolveAgentBackend({ backend })` — root `.` | the copy-pasted `backend-name → createOpenAICompatibleBackend` branch every eval product hand-rolled (the copies drift) | | Pick / register a leaf backend, or bring your own agent | `createExecutor({ backend })` / `createExecutorRegistry()` / implement `Executor` — `/loops` | a per-vendor adapter or closed `inline\|sandbox\|cli` switch (won't report through the `UsageEvent` channel) | | Evolve a **prompt/string** surface | `gepaProposer({ llm, model, target })` (default inside `selfImprove`; the skill-surface twin is `skillOptProposer`, same source) — `agent-eval/campaign` | a hand-rolled prompt-mutation reflection loop with its own Pareto bookkeeping | -| Self-improve a profile (one pluggable verb) — START HERE (self-improvement) | `improve(profile, findings, { surface, gate })` — root `.`; prompt and skill-document surfaces have built-in generators, tools/MCP/hooks/subagents/workflow/whole-profile take an explicit generator, rollout policy is deterministic, and code gets isolated incumbent/candidate worktrees | a bespoke optimize loop, calling `selfImprove`/a skill optimizer directly for the common case, or comparing code against an empty/string stand-in | +| Self-improve a profile (one pluggable verb) — START HERE (self-improvement) | `improve(profile, findings, { surface, gate })` — root `.`; prompt, skill-document, and curated-memory surfaces have built-in generators, tools/MCP/hooks/subagents/workflow/whole-profile take an explicit generator, rollout policy is deterministic, and code gets isolated incumbent/candidate worktrees | a bespoke optimize loop, calling `selfImprove`/a skill or memory optimizer directly for the common case, or comparing code against an empty/string stand-in | | Measure **one profile artifact's marginal lift** (with-vs-without, score+cost) / catalog artifacts | `measureMarginalLift(...)` / `ArtifactRegistry` (`applyArtifact` is the one `ArtifactKind`→`AgentProfile`-field bridge) — `/lifecycle` | a hand-rolled with/without ablation loop, or a per-kind `if kind==='skill'…` profile-field switch | | Run the **whole artifact lifecycle** — generate→measure→promote→store→compose, then drift-watch/dedupe the live set — over ANY profile surface (skill/prompt/tool/MCP) | `runLifecycle({ baseline, generators, evalRunner, gate })` then `composeProfile(registry, base, query)`; maintain with `driftWatch(...)` / `dedupeArtifacts(...)` — `/lifecycle` | a per-surface improve loop, a hand-rolled promote→compose step, or re-running `measureMarginalLift` without the registry/gate spine. The ONLY per-surface code is a thin `CandidateGenerator` (`skillGenerator` distills, `promptGenerator`/`buildableGenerator` for the rest) | | Run the self-improvement loop with full substrate control | `selfImprove({ agent, scenarios, judge, baselineSurface })` — `agent-eval/contract` | a bespoke optimize loop or a parallel skill-optimizer | diff --git a/package.json b/package.json index 2b9a0d5d..ba60e5aa 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.92.1", + "version": "0.93.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { @@ -104,7 +104,7 @@ }, "devDependencies": { "@biomejs/biome": "^2.4.15", - "@tangle-network/agent-eval": "^0.113.0", + "@tangle-network/agent-eval": "^0.114.0", "@tangle-network/agent-interface": "^0.25.0", "@tangle-network/sandbox": "^0.9.7", "@types/node": "^25.9.3", @@ -134,7 +134,7 @@ "license": "MIT", "packageManager": "pnpm@10.28.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.113.0 <1.0.0", + "@tangle-network/agent-eval": ">=0.114.0 <1.0.0", "@tangle-network/agent-interface": ">=0.25.0 <1.0.0", "@tangle-network/sandbox": ">=0.8.0 <1.0.0", "playwright": "^1.40.0" @@ -149,6 +149,6 @@ }, "dependencies": { "@tangle-network/agent-knowledge": "^1.11.2", - "@tangle-network/agent-profile-materialize": "0.3.1" + "@tangle-network/agent-profile-materialize": "0.3.2" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 38d4b547..e771d8d9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,15 +12,15 @@ importers: specifier: ^1.11.2 version: 1.11.2(typescript@5.9.3) '@tangle-network/agent-profile-materialize': - specifier: 0.3.1 - version: 0.3.1 + specifier: 0.3.2 + version: 0.3.2 devDependencies: '@biomejs/biome': specifier: ^2.4.15 version: 2.4.15 '@tangle-network/agent-eval': - specifier: ^0.113.0 - version: 0.113.0(typescript@5.9.3) + specifier: ^0.114.0 + version: 0.114.0(typescript@5.9.3) '@tangle-network/agent-interface': specifier: ^0.25.0 version: 0.25.0 @@ -648,6 +648,11 @@ packages: engines: {node: '>=20'} hasBin: true + '@tangle-network/agent-eval@0.114.0': + resolution: {integrity: sha512-pIZCLkKsHeKDpLOpRXRhnc2HaREcHduHm4u3jw0nN73B1BWe6Jaqh2wlNHTPnPUA1qQl5sCl1K6XF1p9R7eTZw==} + engines: {node: '>=20'} + hasBin: true + '@tangle-network/agent-interface@0.13.0': resolution: {integrity: sha512-CeTPGRLoXqpt0h+BCyFgZPkfU1zyRpWmqfD+85i/uk+uvbqxkfI+JprfKVf3tBsQuCgJPSjPt5qjdW8n3h2BVg==} @@ -665,8 +670,8 @@ packages: engines: {node: '>=20'} hasBin: true - '@tangle-network/agent-profile-materialize@0.3.1': - resolution: {integrity: sha512-yA/DaxmC+DsHdPl00iovR3tg80lQY1ukkHgGViLtzFYY7O46yZR9Ykhw/tMGTFYWFi5bzqjmr0NShSxdNmSzcQ==} + '@tangle-network/agent-profile-materialize@0.3.2': + resolution: {integrity: sha512-jCj1Hc/brPQ5eS7or9A+Klv0FAZcGtyFr7kx2cKfk+wCj0/4Di3k1pMjayrmRNgI/7Oc47gt6/t+qvdtZxBkAw==} '@tangle-network/sandbox@0.9.7': resolution: {integrity: sha512-9pCwJ5MlF7RUpp0AQKQDFyR0yu+E0udEhWkqhrlb/RuoJxlt72zVPuzO4FnMb1MZTkfjStmomC3k5xQyqi1YSA==} @@ -1640,6 +1645,24 @@ snapshots: - typescript - utf-8-validate + '@tangle-network/agent-eval@0.114.0(typescript@5.9.3)': + dependencies: + '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) + '@ax-llm/ax': 19.0.45(zod@4.4.3) + '@hono/node-server': 2.0.8(hono@4.12.28) + '@tangle-network/agent-interface': 0.22.0 + '@tangle-network/tcloud': 0.4.14(typescript@5.9.3)(zod@4.4.3) + hono: 4.12.28 + zod: 4.4.3 + transitivePeerDependencies: + - '@mastra/core' + - '@modelcontextprotocol/sdk' + - ai + - bufferutil + - openai + - typescript + - utf-8-validate + '@tangle-network/agent-interface@0.13.0': dependencies: zod: 4.4.3 @@ -1669,9 +1692,9 @@ snapshots: - typescript - utf-8-validate - '@tangle-network/agent-profile-materialize@0.3.1': + '@tangle-network/agent-profile-materialize@0.3.2': dependencies: - '@tangle-network/agent-interface': 0.22.0 + '@tangle-network/agent-interface': 0.25.0 '@tangle-network/sandbox@0.9.7(viem@2.54.6(typescript@5.9.3)(zod@4.4.3))': dependencies: diff --git a/skills/build-with-agent-runtime/SKILL.md b/skills/build-with-agent-runtime/SKILL.md index 8b1d51c7..0761921c 100644 --- a/skills/build-with-agent-runtime/SKILL.md +++ b/skills/build-with-agent-runtime/SKILL.md @@ -93,7 +93,7 @@ to its native default (`HARNESS_NATIVE_MODEL`) — never silently dropped. | **Sandbox coding rollout** (fresh box/round, or persistent+resume) | `runLoop(options)` / `openSandboxRun(client, opts, deliverable)` — `/loops` | canonical-api §3.1 | | **Optimize a CODE surface** in a gated loop | `improvementDriver({ worktree, generator })` — root `.` | canonical-api §3.4 | | **Freeze a measured profile/diff + code surface for execution** | `buildAgentCandidateBundle(...)` then `verifyAgentCandidateBundle(...)` — `/candidate-execution` | canonical-api §2 | -| **Optimize any agent/code surface** (one call) — START HERE | `improve(profile, findings, { surface, gate })` — root `.`; prompt and skill-document surfaces have built-in proposers, config/whole-profile surfaces accept a proposer, rollout policy enumerates bounded variants, and code gets isolated incumbent/candidate worktrees; drop to `selfImprove({ agent, scenarios, judge, baselineSurface })` only for lower-level control | canonical-api §3.4 | +| **Optimize any agent/code surface** (one call) — START HERE | `improve(profile, findings, { surface, gate })` — root `.`; prompt, skill-document, and curated-memory surfaces have built-in proposers, config/whole-profile surfaces accept a proposer, rollout policy enumerates bounded variants, and code gets isolated incumbent/candidate worktrees; drop to `selfImprove({ agent, scenarios, judge, baselineSurface })` only for lower-level control | canonical-api §3.4 | | **Gate: ship/hold a candidate** (campaign ctx) | `defaultProductionGate` / `heldOutGate` / `composeGate` — `agent-eval/contract`; `neutralizationGate` (footprint-matched PLACEBO gate — proves a held-out lift is CONTENT, not added prompt/mount footprint) — `agent-eval/campaign` | canonical-api §3.4 | | **Gate: ship/hold from a `BenchmarkReport`** (per-task cells) | `promotionGate({ report, incumbent, candidate })` — `/loops` | canonical-api §3.4 | | **Run the full multi-generation flywheel + certify** | `runStrategyEvolution(config)` — `/loops` | canonical-api §3.4 | diff --git a/src/improvement/improve.test.ts b/src/improvement/improve.test.ts index 8c030407..c3f0e456 100644 --- a/src/improvement/improve.test.ts +++ b/src/improvement/improve.test.ts @@ -2,8 +2,8 @@ * `improve()` default-proposer resolution proof. * * The regression this guards: `improve()` maps each surface to a default - * `SurfaceProposer` — `prompt → gepaProposer`, `skills → skillOptProposer`. - * Both proposers are factories exported from `@tangle-network/agent-eval/campaign`. + * `SurfaceProposer` — `prompt → gepaProposer`, `skills → skillOptProposer`, + * `memory → memoryCurationProposer`. * If either import resolves to `undefined` (a substrate export drift), the facade * does not fail at module load — it fails at CALL time, the first time a caller * names that surface. So a green typecheck is not enough; this test drives the @@ -17,7 +17,13 @@ * sees a real backend rather than a silent-zero stub. */ -import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' +import type { + CodeSurface, + DispatchContext, + JudgeConfig, + Scenario, + SurfaceProposer, +} from '@tangle-network/agent-eval/contract' import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { ConfigError } from '../errors' @@ -117,6 +123,145 @@ describe('improve() — default proposer resolution (substrate export drift guar ).rejects.toThrow(/requires opts\.skills\.document/) }) + it("surface 'memory' resolves memoryCurationProposer and requires a real baseline", async () => { + const result = await improve(promptProfile(), [], { + surface: 'memory', + gate: 'none', + scenarios, + judge, + agent: stubAgent, + memory: { document: '# Durable memory\n' }, + }) + + expect(result.gateDecision).toBe('hold') + expect(result.shipped).toBe(false) + await expect( + improve(promptProfile(), [], { + surface: 'memory', + gate: 'none', + scenarios, + judge, + agent: stubAgent, + }), + ).rejects.toThrow(/requires opts\.memory\.document/) + }) + + it("surface 'memory' curates seed findings and awaits persistence of the promoted document", async () => { + const baseline = '# Durable memory\n' + let writtenBack: string | null = null + const result = await improve(promptProfile(), [{ claim: 'improved verification lesson' }], { + surface: 'memory', + scenarios, + judge: improvementJudge, + agent: stubAgent, + memory: { + document: baseline, + writeBack: async (winner) => { + await Promise.resolve() + writtenBack = winner + }, + }, + promotionGate: { + name: 'test-ship', + decide: async () => ({ + decision: 'ship', + reasons: ['test candidate'], + contributingGates: [], + }), + }, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + }) + + expect(result.shipped).toBe(true) + expect(writtenBack).toContain('improved verification lesson') + expect(result.profile).toEqual(promptProfile()) + }) + + it("surface 'memory' rejects a shipped non-text winner before persistence", async () => { + let writeCalls = 0 + const nonTextSurface: CodeSurface = { + kind: 'code', + worktreeRef: 'not-a-memory-document', + baseRef: 'main', + baseCommit: 'a'.repeat(40), + baseTree: 'b'.repeat(40), + candidateCommit: 'c'.repeat(40), + candidateTree: 'd'.repeat(40), + patch: { + format: 'git-diff-binary', + sha256: `sha256:${'e'.repeat(64)}`, + byteLength: 1, + }, + } + const surfaceKindJudge: JudgeConfig<{ text: string }, Scenario> = { + name: 'surface-kind-judge', + dimensions: [{ key: 'q', description: 'candidate is code-tier' }], + score: ({ artifact }) => { + const score = artifact.text === 'code' ? 1 : 0 + return { dimensions: { q: score }, composite: score, notes: '' } + }, + } + const malformedMemory: SurfaceProposer = { + kind: 'malformed-memory', + propose: async () => [ + { + surface: nonTextSurface, + label: 'wrong-tier', + rationale: 'test malformed winner', + }, + ], + } + await expect( + improve(promptProfile(), [], { + surface: 'memory', + scenarios, + judge: surfaceKindJudge, + agent: async (surface, _scenario, ctx) => { + ctx.cost.observe(0.0001, 'stub-agent') + ctx.cost.observeTokens({ input: 1, output: 1 }) + return { text: typeof surface === 'string' ? 'text' : surface.kind } + }, + memory: { + document: '# Durable memory\n', + writeBack: () => { + writeCalls += 1 + }, + }, + generator: malformedMemory, + promotionGate: { + name: 'test-ship', + decide: async () => ({ + decision: 'ship', + reasons: ['test candidate'], + contributingGates: [], + }), + }, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + }), + ).rejects.toThrow(/winner must be text/) + expect(writeCalls).toBe(0) + }) + + it("surface 'memory' never writes back a held candidate", async () => { + let writeCalls = 0 + const result = await improve(promptProfile(), [], { + surface: 'memory', + gate: 'none', + scenarios, + judge, + agent: stubAgent, + memory: { + document: '# Durable memory\n', + writeBack: () => { + writeCalls += 1 + }, + }, + }) + + expect(result.shipped).toBe(false) + expect(writeCalls).toBe(0) + }) + it('a real runDir makes the loop durable: provenance lands on the filesystem', async () => { const { mkdtempSync, existsSync, rmSync } = await import('node:fs') const { tmpdir } = await import('node:os') @@ -329,8 +474,8 @@ describe('improve() — default proposer resolution (substrate export drift guar }) it('a surface with no zero-config default still fails loud with ConfigError', async () => { - // The default-proposer map covers prompt + skills only; the config surfaces - // (tools/mcp/hooks/code) require a caller-supplied generator. This is the + // Prompt, skills, memory, and rollout policy have defaults; config surfaces + // require a caller-supplied generator. This is the // designed boundary the proposer migration must NOT erase. const configSurfaces: ImproveSurface[] = [ 'tools', diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index c52bc789..7295fb23 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -12,6 +12,8 @@ * * - `surface: 'prompt'` → `gepaProposer` mutates `profile.prompt.systemPrompt`. * - `surface: 'skills'` → `skillOptProposer` mutates a skills document string. + * - `surface: 'memory'` → `memoryCurationProposer` curates a bounded durable + * lesson document supplied through `opts.memory`. * - `surface: 'agent-profile'` → caller-supplied proposer mutates the complete * canonical AgentProfile JSON in one candidate. * - `surface: 'rollout-policy'` → `rolloutPolicyProposer` mutates the @@ -36,6 +38,7 @@ import { gepaProposer, gitWorktreeAdapter, + memoryCurationProposer, skillOptProposer, } from '@tangle-network/agent-eval/campaign' import { @@ -81,6 +84,7 @@ export type ImproveSurface = | 'subagents' | 'workflow' | 'agent-profile' + | 'memory' | 'code' | 'rollout-policy' @@ -92,8 +96,8 @@ export type ImproveOptions = Omit< * generator + the baseline-surface extraction shape. */ surface?: ImproveSurface /** The `SurfaceProposer` that mutates the surface. When unset, the facade - * picks the default for `surface` (`gepaProposer` for prompt, `skillOptProposer` - * for skills); surfaces with no default REQUIRE this (fail-loud otherwise). */ + * picks the default for prompt, skills, memory, and rollout policy; surfaces + * with no default REQUIRE this (fail-loud otherwise). */ generator?: SurfaceProposer /** Gate mode. `'holdout'` (default) runs the held-out promotion gate; * `'none'` is a baseline-only run (`budget.generations = 0`). */ @@ -135,6 +139,10 @@ export type ImproveOptions = Omit< * shipped winner (the profile ref points at a file the caller owns). This is * what makes skillOpt reachable through improve(). */ skills?: ImproveSkillsOptions + /** MEMORY-surface wiring for a curated durable memory document. The default + * deterministic proposer deduplicates and ranks lessons from findings, then + * replaces its managed block instead of growing memory without bound. */ + memory?: ImproveMemoryOptions /** Custom held-back-exam decision. The string `gate` above controls whether * the exam runs; this callback controls how its evidence decides promotion. */ promotionGate?: SelfImproveOptions['gate'] @@ -146,7 +154,14 @@ export interface ImproveSkillsOptions { /** Persist the shipped winner document (write the file the profile ref points at). * Called only on a ship verdict. When omitted, the winner is still returned in * `result.raw.winner.surface` for the caller to materialize. */ - writeBack?: (winnerDocument: string) => void + writeBack?: (winnerDocument: string) => void | Promise +} + +export interface ImproveMemoryOptions { + /** Current durable memory text used as the measured baseline. */ + document: string + /** Persist the promoted memory document. Never called on hold or error. */ + writeBack?: (winnerDocument: string) => void | Promise } export interface ImproveCodeOptions { @@ -205,6 +220,8 @@ function defaultGeneratorFor( return gepaProposer({ llm: llmClientOptions(llm), model, target: 'agent system prompt' }) case 'skills': return skillOptProposer({ llm: llmClientOptions(llm), model, target: 'agent skill document' }) + case 'memory': + return memoryCurationProposer() case 'rollout-policy': // Deterministic bounded enumeration — no LLM, so `llm` is unused here. return rolloutPolicyProposer() @@ -214,12 +231,13 @@ function defaultGeneratorFor( } /** Extract the baseline surface a driver mutates from the profile field that - * backs `surface`. `prompt`/`skills` are string surfaces; the config surfaces - * serialize the matching profile record. */ + * backs `surface`. Prompt, skills, and memory are text surfaces; config + * surfaces serialize the matching profile record. */ function baselineSurfaceFor( profile: AgentProfile, surface: ImproveSurface, skills?: ImproveSkillsOptions, + memory?: ImproveMemoryOptions, ): MutableSurface { switch (surface) { case 'prompt': @@ -240,6 +258,11 @@ function baselineSurfaceFor( return JSON.stringify(profile.extensions?.[workflowExtension] ?? {}) case 'agent-profile': return JSON.stringify(profile) + case 'memory': + if (!memory) { + throw new ConfigError("improve(): surface 'memory' requires opts.memory.document") + } + return memory.document case 'rollout-policy': { // Empty surface when the profile never opted into structural rollout: the // proposer reads it as "propose nothing", so the loop runs baseline-only and @@ -265,8 +288,13 @@ function generationFailureDistiller( ): NonNullable['analyzeGeneration']> { const CAP = 12 return async (input) => { - const failures: Array<{ scenario: string; composite: number; notes: string; error?: string }> = - [] + const failures: Array<{ + scenario: string + composite: number + notes: string + claim?: string + error?: string + }> = [] for (const candidate of input.candidates) { for (const rawCell of candidate.campaign.cells) { const cell = rawCell as unknown as Record @@ -288,10 +316,12 @@ function generationFailureDistiller( .filter((n): n is string => typeof n === 'string' && n.length > 0) .join('; ') .slice(0, 400) + const claim = notes || (error ? `Scenario ${scenario} failed: ${error.slice(0, 200)}` : '') failures.push({ scenario, composite: Number(composite.toFixed(3)), notes, + ...(claim ? { claim } : {}), ...(error ? { error: error.slice(0, 200) } : {}), }) } @@ -302,6 +332,19 @@ function generationFailureDistiller( } } +/** Memory accumulates durable lessons, so keep the caller's seed findings while + * adding fresh judge failures. Curator proposers consume `claim`; the generic + * distiller retains the richer diagnostic fields for reflective proposers. */ +function memoryGenerationDistiller( + staticFindings: unknown[], +): NonNullable['analyzeGeneration']> { + const distillFailures = generationFailureDistiller(staticFindings) + return async (input) => { + const fresh = await distillFailures(input) + return fresh === staticFindings ? staticFindings : [...staticFindings, ...fresh] + } +} + interface PreparedCodeRun { baseline: CodeSurface proposer: SurfaceProposer @@ -424,6 +467,8 @@ function applyWinnerToProfile( case 'agent-profile': candidate = parseWinnerJson(winner, surface) break + case 'memory': + return profile case 'rollout-policy': { // Parse + re-validate the winner against the policy's own invariants — a // custom generator's malformed dial must fail loud, not persist silently. @@ -475,6 +520,7 @@ export async function improve( rawTraceContext, code, skills, + memory, promotionGate, analyzeGeneration, ...sharedOptions @@ -491,6 +537,9 @@ export async function improve( 'improve(): the default skills optimizer requires opts.skills.document; pass the skill text or an explicit generator that understands resource refs', ) } + if (surface === 'memory' && !memory) { + throw new ConfigError("improve(): surface 'memory' requires opts.memory.document") + } const usesReflectionModel = !generator && (surface === 'prompt' || surface === 'skills') if (usesReflectionModel) { assertModelAllowed(sharedOptions.llm?.model ?? defaultReflectionModel, allowedModels) @@ -520,7 +569,8 @@ export async function improve( try { raw = await selfImprove({ ...sharedOptions, - baselineSurface: preparedCode?.baseline ?? baselineSurfaceFor(profile, surface, skills), + baselineSurface: + preparedCode?.baseline ?? baselineSurfaceFor(profile, surface, skills, memory), proposer, budget, findings, @@ -532,7 +582,9 @@ export async function improve( analyzeGeneration ?? (rawTraceContext ? rawTraceDistiller({ fallbackFindings: findings }) - : generationFailureDistiller(findings)), + : surface === 'memory' + ? memoryGenerationDistiller(findings) + : generationFailureDistiller(findings)), }), }) } catch (cause) { @@ -549,18 +601,23 @@ export async function improve( } const shipped = raw.gateDecision === 'ship' - await preparedCode?.cleanup(shipped ? raw.winner.surface : undefined) + const winnerSurface = raw.winner.surface + await preparedCode?.cleanup(shipped ? winnerSurface : undefined) // When a skill DOCUMENT was optimized, the winner is document text — persist it // via writeBack (the profile ref points at the caller's file, unchanged) rather // than parsing it as a refs array. Otherwise use the standard field write-back. - const usedSkillDocument = surface === 'skills' && skills !== undefined - if (shipped && usedSkillDocument && typeof raw.winner.surface === 'string') { - skills?.writeBack?.(raw.winner.surface) + const externalDocument = + surface === 'skills' && skills ? skills : surface === 'memory' && memory ? memory : undefined + if (shipped && externalDocument) { + if (typeof winnerSurface !== 'string') { + throw new ConfigError( + `improve(): the shipped '${surface}' winner must be text before it can be persisted`, + ) + } + await externalDocument.writeBack?.(winnerSurface) } const nextProfile = - shipped && !usedSkillDocument - ? applyWinnerToProfile(profile, surface, raw.winner.surface) - : profile + shipped && !externalDocument ? applyWinnerToProfile(profile, surface, winnerSurface) : profile return { profile: nextProfile, shipped, lift: raw.lift, gateDecision: raw.gateDecision, raw } } diff --git a/src/improvement/index.ts b/src/improvement/index.ts index e4547308..be6c7e7b 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -20,6 +20,7 @@ export { export { mcpBuildPrompt, toolBuildPrompt } from './build-prompts' export { type ImproveCodeOptions, + type ImproveMemoryOptions, type ImproveOptions, type ImproveResult, type ImproveSkillsOptions, diff --git a/src/intelligence/improvement-cycle.ts b/src/intelligence/improvement-cycle.ts index a53df5dd..ab9f2d21 100644 --- a/src/intelligence/improvement-cycle.ts +++ b/src/intelligence/improvement-cycle.ts @@ -159,8 +159,15 @@ export interface ExecuteApprovedAgentCandidateResult { export async function proposeAgentImprovement( options: ProposeAgentImprovementOptions, ): Promise> { - if (options.improvement.skills?.writeBack) { - throw new Error('proposeAgentImprovement cannot write a skill before human approval') + const writeBackSurface = options.improvement.skills?.writeBack + ? 'skill' + : options.improvement.memory?.writeBack + ? 'memory' + : null + if (writeBackSurface) { + throw new Error( + `proposeAgentImprovement cannot write ${writeBackSurface} before human approval`, + ) } const analysis = await runAnalystLoop({ ...options.analysis, diff --git a/tests/improvement-cycle.test.ts b/tests/improvement-cycle.test.ts index 1c24d88b..9bb705a6 100644 --- a/tests/improvement-cycle.test.ts +++ b/tests/improvement-cycle.test.ts @@ -151,6 +151,30 @@ afterEach(() => { }) describe('agent improvement lifecycle', () => { + it('refuses memory persistence before human approval', async () => { + const writeBack = vi.fn() + await expect( + proposeAgentImprovement({ + runId: 'analysis-run-memory-writeback', + profile: fixtureProfile(), + analysis: { registry: registry([finding]), inputs: {}, findingsStore: null, log: () => {} }, + improvement: { + surface: 'memory', + memory: { + document: '# Durable memory\n', + writeBack, + }, + generator: proposer, + scenarios, + judge, + agent, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.5 }, + }, + }), + ).rejects.toThrow('cannot write memory before human approval') + expect(writeBack).not.toHaveBeenCalled() + }) + it('binds typed candidate hooks to their equivalent measured profile commands', () => { expect(() => assertCandidateProfileBinding(