Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions pebrel_app/src/session/recovery_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -77,3 +77,19 @@ fn registered_clis_keep_native_ids_in_their_own_split_across_saves() {
}
}
}

#[test]
fn live_recovery_cases_cover_the_registered_cli_entry_points() {
use std::collections::BTreeMap;
let cases: BTreeMap<String, Vec<String>> =
toml::from_str(include_str!("../../../scripts/conformance/agent_recovery/entries.toml"))
.unwrap();
assert_eq!(cases.len(), AgentKind::ALL.len());
for kind in AgentKind::ALL {
let commands = &cases[kind.slug()];
assert!((1..=2).contains(&commands.len()), "{kind:?}");
for command in commands {
assert_eq!(AgentKind::parse_command(command), Some(kind), "{command}");
}
}
}
22 changes: 22 additions & 0 deletions scripts/conformance/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,3 +92,25 @@ Each connection sends one UTF-8 JSON request line using protocol
`nebula.runtime` version 1 and reads one response line. This suite deliberately
does not use Windows named pipes or Unix-domain sockets because Nebula's actual
Runtime API transport is loopback TCP on every platform.

## Native AI conversation recovery

On a Linux desktop the offline test opens two native clients in one tab and working directory. It checks the saved IDs against each provider's own transcript. After each of three process restarts, a new user message must reach the same native conversation with that pane's earlier message in its model request. The other pane's marker must be absent. Install the CLI under test on PATH. The runner creates its own provider home, enables that provider's hook and serves model responses on loopback.

```sh
python3 scripts/conformance/agent_recovery/run.py --app /path/to/pebrel \
--agent claude --agent pi --agent omp --agent grok --agent codebuddy
```

The CodeBuddy case launches `codebuddy` in the upper pane and `cbc` in the lower pane. For headless Linux, run the command under `dbus-run-session -- xvfb-run -a`.

Clients that require a signed-in service can run the conversation test on a dedicated Linux desktop. Sign in to the provider with a test account in a separate test home. The selected provider may bill the test turns. This test starts two fresh chats in a disposable project, asks each to retain a different random word, and checks the native ID reported by Pebrel and a fresh response containing the right word after every restart. The second question omits the word. Check PATH when using `agent`, which is also an entry point of Grok Build.

```sh
python3 scripts/conformance/agent_recovery/interactive.py --app /path/to/pebrel \
--source cursor --test-home /path/to/cursor-test-profile
python3 scripts/conformance/agent_recovery/interactive.py --app /path/to/pebrel \
--source antigravity --test-home /path/to/antigravity-test-profile
```

The same command accepts any source in [the live test registry](agent_recovery/entries.toml). Missing identity, an unanswered challenge, and a missing executable fail that case. The exhaustive Rust session test separately checks file persistence and supported resume-command selection for every registered source.
29 changes: 29 additions & 0 deletions scripts/conformance/agent_recovery/entries.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# Each pair opens one same-directory split per entry point.
aider = ["aider"]
amp = ["amp"]
antigravity = ["agy"]
auggie = ["auggie"]
claude = ["claude"]
cline = ["cline"]
codebuddy = ["codebuddy", "cbc"]
codex = ["codex"]
command-code = ["command-code"]
copilot = ["copilot"]
cursor = ["cursor-agent", "agent"]
devin = ["devin"]
droid = ["droid"]
gemini = ["gemini"]
goose = ["goose"]
grok = ["grok"]
hermes = ["hermes"]
kilo = ["kilo"]
kimi = ["kimi"]
kiro = ["kiro-cli"]
maki = ["maki"]
omp = ["omp"]
opencode = ["opencode"]
pi = ["pi"]
qodercli = ["qodercli"]
qwen = ["qwen"]
trae-cli = ["trae-cli"]
vibe = ["vibe"]
99 changes: 99 additions & 0 deletions scripts/conformance/agent_recovery/interactive.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
"""Exercise a signed-in CLI on a disposable Linux desktop with two independent chats."""
import argparse
from contextlib import ExitStack
import os
from pathlib import Path
import shutil
import sys
import tomllib
from tempfile import TemporaryDirectory
from unittest.mock import patch
from uuid import uuid4

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from conformance.agent_recovery.run import live_id, panes, prepare_split, runtime_pane, saved_id, shell_environment, wait_for_editor
from conformance.harness import ConformanceContext, ResolvedApp


def token():
return uuid4().hex[:12]


def check_agent(ctx, agent, commands, rounds, timeout):
ids = []
secrets = [token(), token()]
sides = ("first", "second")
ctx.start(explicit_working_directory=False)
for index, pane in enumerate(panes(ctx)):
ctx.poll(lambda: "$" in ctx.read(pane)["text"], bool, "test shell is not ready")
ctx.prompt(commands[index], pane)
# An external login, trust prompt or source-only identity cannot satisfy
# the native-ID check. Leave time for interactive setup on this desktop.
def ready():
row = runtime_pane(ctx, pane)
status = row.get("agent") or {}
return status.get("kind") == agent and row.get("task_state") in ("idle", "finished", "waiting_input")
ctx.poll(ready, bool, f"{agent} is not ready for a user turn", timeout=timeout)
wait_for_editor(ctx, pane, agent)
prefix = token()
secret = secrets[index]
prompt = (f"Our test word is {secret}. Keep it in this conversation only. Do not use tools. "
f"Reply with {prefix}, a slash, and the test word joined with no spaces.")
ctx.prompt(prompt, pane)
ctx.poll(lambda: f"{prefix}/{secret}" in ctx.read(pane)["text"], bool,
f"{agent} did not answer the first challenge", timeout=timeout)
ids.append(ctx.poll(lambda: saved_id(ctx, sides[index], agent), bool,
f"{agent} did not save the conversation ID", timeout=timeout))
assert ids[0] != ids[1], "both split leaves saved the same conversation ID"
for attempt in range(rounds):
ctx.restart()
for index, pane in enumerate(panes(ctx)):
# Each challenge prefix is new after restart. Terminal scrollback
# cannot supply this answer, and the question omits both secrets.
prefix = token()
ctx.poll(lambda: live_id(ctx, pane, agent) == ids[index], bool,
"provider did not acknowledge the same ID", timeout=timeout)
wait_for_editor(ctx, pane, agent)
ctx.prompt(f"Without tools, recall our test word. Reply with {prefix}, a slash, "
"and that word joined with no spaces.", pane)
ctx.poll(lambda: f"{prefix}/{secrets[index]}" in ctx.read(pane)["text"], bool,
f"{agent} did not recall its own conversation", timeout=timeout)
assert f"{prefix}/{secrets[1-index]}" not in ctx.read(pane)["text"]
assert saved_id(ctx, sides[index], agent) == ids[index]
print(f"{agent} round {attempt + 1}/{rounds} continued both chats", flush=True)


def main():
parser = argparse.ArgumentParser(description=__doc__)
entries_by_source = tomllib.loads(Path(__file__).with_name("entries.toml").read_text(encoding="utf-8"))
parser.add_argument("--app", required=True)
parser.add_argument("--source", required=True, choices=entries_by_source)
parser.add_argument("--test-home", required=True, type=Path, help="Dedicated signed-in test home")
parser.add_argument("--rounds", type=int, default=3)
parser.add_argument("--timeout", type=int, default=90)
args = parser.parse_args()
home = args.test_home.resolve()
if not sys.platform.startswith("linux") or not os.environ.get("DISPLAY"):
parser.error("requires a Linux desktop")
if not home.is_dir() or home == Path.home().resolve():
parser.error("supply a separate test home containing the provider's test profile")
if args.rounds < 1 or args.timeout < 1:
parser.error("rounds and timeout must be positive")
commands = entries_by_source[args.source]
for entry in commands:
if not shutil.which(entry):
parser.error(f"{entry} is missing from PATH")
with ExitStack() as cleanup:
test = Path(cleanup.enter_context(TemporaryDirectory(prefix="pebrel-agent-recall-")))
app = ResolvedApp(args.app)
cleanup.callback(app.close)
ctx = ConformanceContext(app, "linux", test / "config", test / "project", test / "artifacts")
prepare_split(ctx, args.source)
cleanup.enter_context(patch.dict(os.environ, shell_environment(home), clear=True))
cleanup.callback(ctx.stop, force=True)
entries = commands * 2 if len(commands) == 1 else commands
check_agent(ctx, args.source, entries, args.rounds, args.timeout)


if __name__ == "__main__":
main()
79 changes: 79 additions & 0 deletions scripts/conformance/agent_recovery/model.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
"""Loopback model for native CLI recovery conformance."""
from contextlib import contextmanager, suppress
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import re
from threading import Thread
from urllib.parse import urlsplit


class FixtureModel(BaseHTTPRequestHandler):
def send_body(self, body, content_type):
data = body.encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(data)))
self.end_headers()
with suppress(BrokenPipeError, ConnectionResetError):
self.wfile.write(data)

def do_GET(self):
if urlsplit(self.path).path.endswith("/models"):
self.send_body(json.dumps({"object": "list", "data": [{"object": "model", "id": "fixture"}]}),
"application/json")
else:
self.send_error(404)

def do_POST(self):
path = urlsplit(self.path).path
request = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
if path.endswith("/messages/count_tokens"):
self.send_body('{"input_tokens":10}', "application/json")
return
if not path.endswith(("/messages", "/chat/completions")):
self.send_error(404)
return
# Record only fixture markers from user turns. Diagnostics, summaries
# and assistant messages cannot attest which conversation was resumed.
markers = {
marker for message in request.get("messages", []) if message.get("role") == "user"
for marker in re.findall(r"pebrel-qa-[0-9a-f]{16}", json.dumps(message.get("content")))
}
self.server.turns.append(markers)
reply = "Native recovery fixture reply."
if path.endswith("/messages"):
message = {"id": "msg_qa", "type": "message", "role": "assistant",
"model": "fixture", "content": [], "usage": {"input_tokens": 10, "output_tokens": 1}}
events = [
("message_start", {"message": message}),
("content_block_start", {"index": 0, "content_block": {"type": "text", "text": ""}}),
("content_block_delta", {"index": 0, "delta": {"type": "text_delta", "text": reply}}),
("content_block_stop", {"index": 0}),
("message_delta", {"delta": {"stop_reason": "end_turn", "stop_sequence": None},
"usage": {"output_tokens": 5}}),
("message_stop", {}),
]
wire = "".join(f"event: {kind}\ndata: {json.dumps({'type': kind, **body})}\n\n"
for kind, body in events)
else:
chunks = [{"id": "chatcmpl-qa", "object": "chat.completion.chunk", "model": "fixture",
"created": 1, "choices": [{"index": 0, "delta": delta, "finish_reason": finish}]}
for delta, finish in [({"role": "assistant", "content": reply}, None), ({}, "stop")]]
wire = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n"
self.send_body(wire, "text/event-stream")

def log_message(self, *_args):
pass


@contextmanager
def local_model():
with ThreadingHTTPServer(("127.0.0.1", 0), FixtureModel) as server:
server.turns = []
thread = Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
yield server
finally:
server.shutdown()
thread.join()
98 changes: 98 additions & 0 deletions scripts/conformance/agent_recovery/providers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
"""Native profile setup and independent transcript oracles for offline CLIs."""
import json


PROFILES = {"claude": ".claude", "pi": ".pi/agent", "omp": ".omp/agent",
"grok": ".grok", "codebuddy": ".codebuddy"}


def write_json(path, data):
path.write_text(json.dumps(data), encoding="utf-8")


def configure(agent, home, work_dir, url):
profile = home / PROFILES[agent]
profile.mkdir(parents=True)
if agent in ("pi", "omp"):
model = {"baseUrl": url + "/v1", "api": "openai-completions", "apiKey": "fixture",
"models": [{"id": "fixture", "name": "Fixture"}]}
models = "models.json" if agent == "pi" else "models.yml"
write_json(profile / models, {"providers": {"pebrelfixture": model}})
write_json(profile / "settings.json", {"defaultProvider": "pebrelfixture", "defaultModel": "fixture",
"defaultProjectTrust": "always"})
return {"PI_CODING_AGENT_DIR": str(profile), "PI_OFFLINE": "1",
"OMP_SKIP_SETUP": "1", "OMP_SKIP_VERSION_CHECK": "1"}
if agent == "claude":
onboarding = {"hasCompletedOnboarding": True, "theme": "dark",
"projects": {str(work_dir): {"hasTrustDialogAccepted": True,
"projectOnboardingSeenCount": 5}},
"customApiKeyResponses": {"approved": ["fixture-key"], "rejected": []}}
write_json(home / ".claude.json", onboarding)
write_json(profile / ".claude.json", onboarding)
return {"ANTHROPIC_BASE_URL": url, "ANTHROPIC_API_KEY": "fixture-key",
"CLAUDE_CONFIG_DIR": str(profile), "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1",
"DISABLE_AUTOUPDATER": "1"}
if agent == "grok":
(profile / "config.toml").write_text(
'[models]\ndefault="fixture"\n[model.fixture]\nmodel="fixture"\n'
f'base_url="{url}/v1"\napi_backend="chat_completions"\nenv_key="XAI_API_KEY"\n',
encoding="utf-8")
return {"GROK_HOME": str(profile), "XAI_API_KEY": "fixture-key",
"GROK_MODELS_BASE_URL": url + "/v1", "GROK_XAI_API_BASE_URL": url + "/v1"}
return {"CODEBUDDY_BASE_URL": url + "/v1", "CODEBUDDY_CONFIG_DIR": str(profile),
"CODEBUDDY_API_KEY": "fixture-key"}


def command_for(agent, pane_index):
if agent in ("pi", "omp"):
return f"{agent} --model pebrelfixture/fixture"
return "cbc" if agent == "codebuddy" and pane_index == 1 else agent


def has_text(content, marker):
if isinstance(content, str):
return content == marker
return isinstance(content, list) and any(
isinstance(part, dict) and part.get("text") == marker for part in content
)


def native_match(agent, file, rows, session_id, marker):
if agent in ("pi", "omp"):
header = next((row for row in rows if row.get("type") == "session"), {})
if header.get("id") != session_id:
return False
elif agent == "grok":
if session_id not in file.parts and session_id != file.stem:
return False
marker = f"<user_query>\n{marker}\n</user_query>"
seen = False
for row in rows:
if agent in ("claude", "codebuddy") and row.get("sessionId") != session_id:
continue
message = row.get("message") or row
role = message.get("role") or row.get("type")
content = message.get("content")
if role == "user":
seen = has_text(content, marker)
elif seen and role == "assistant" and has_text(content, "Native recovery fixture reply."):
return True
return False


def provider_confirmed(home, agent, session_id, marker):
if not session_id:
return False
profile = home / PROFILES[agent]
directory = profile / ("projects" if agent in ("claude", "codebuddy") else "sessions")
for file in directory.rglob("*.jsonl"):
if file.is_symlink():
continue
try:
rows = [json.loads(line) for line in file.read_text(encoding="utf-8").splitlines()
if line.strip()]
except (OSError, ValueError):
continue
if native_match(agent, file, rows, session_id, marker):
return True
return False
Loading
Loading