diff --git a/pebrel_app/src/session/recovery_tests.rs b/pebrel_app/src/session/recovery_tests.rs index 1df404bea..10eb2c30b 100644 --- a/pebrel_app/src/session/recovery_tests.rs +++ b/pebrel_app/src/session/recovery_tests.rs @@ -77,3 +77,19 @@ fn registered_clis_keep_native_ids_in_their_own_split_across_saves() { } } } + +#[test] +fn live_recovery_cases_cover_the_registered_cli_entry_points() { + use std::collections::BTreeMap; + let cases: BTreeMap> = + toml::from_str(include_str!("../../../scripts/conformance/agent_recovery/entries.toml")) + .unwrap(); + assert_eq!(cases.len(), AgentKind::ALL.len()); + for kind in AgentKind::ALL { + let commands = &cases[kind.slug()]; + assert!((1..=2).contains(&commands.len()), "{kind:?}"); + for command in commands { + assert_eq!(AgentKind::parse_command(command), Some(kind), "{command}"); + } + } +} diff --git a/scripts/conformance/README.md b/scripts/conformance/README.md index 28abcfaf0..cd04a8f0c 100644 --- a/scripts/conformance/README.md +++ b/scripts/conformance/README.md @@ -92,3 +92,25 @@ Each connection sends one UTF-8 JSON request line using protocol `nebula.runtime` version 1 and reads one response line. This suite deliberately does not use Windows named pipes or Unix-domain sockets because Nebula's actual Runtime API transport is loopback TCP on every platform. + +## Native AI conversation recovery + +On a Linux desktop the offline test opens two native clients in one tab and working directory. It checks the saved IDs against each provider's own transcript. After each of three process restarts, a new user message must reach the same native conversation with that pane's earlier message in its model request. The other pane's marker must be absent. Install the CLI under test on PATH. The runner creates its own provider home, enables that provider's hook and serves model responses on loopback. + +```sh +python3 scripts/conformance/agent_recovery/run.py --app /path/to/pebrel \ + --agent claude --agent pi --agent omp --agent grok --agent codebuddy +``` + +The CodeBuddy case launches `codebuddy` in the upper pane and `cbc` in the lower pane. For headless Linux, run the command under `dbus-run-session -- xvfb-run -a`. + +Clients that require a signed-in service can run the conversation test on a dedicated Linux desktop. Sign in to the provider with a test account in a separate test home. The selected provider may bill the test turns. This test starts two fresh chats in a disposable project, asks each to retain a different random word, and checks the native ID reported by Pebrel and a fresh response containing the right word after every restart. The second question omits the word. Check PATH when using `agent`, which is also an entry point of Grok Build. + +```sh +python3 scripts/conformance/agent_recovery/interactive.py --app /path/to/pebrel \ + --source cursor --test-home /path/to/cursor-test-profile +python3 scripts/conformance/agent_recovery/interactive.py --app /path/to/pebrel \ + --source antigravity --test-home /path/to/antigravity-test-profile +``` + +The same command accepts any source in [the live test registry](agent_recovery/entries.toml). Missing identity, an unanswered challenge, and a missing executable fail that case. The exhaustive Rust session test separately checks file persistence and supported resume-command selection for every registered source. diff --git a/scripts/conformance/agent_recovery/entries.toml b/scripts/conformance/agent_recovery/entries.toml new file mode 100644 index 000000000..3f25a9bf4 --- /dev/null +++ b/scripts/conformance/agent_recovery/entries.toml @@ -0,0 +1,29 @@ +# Each pair opens one same-directory split per entry point. +aider = ["aider"] +amp = ["amp"] +antigravity = ["agy"] +auggie = ["auggie"] +claude = ["claude"] +cline = ["cline"] +codebuddy = ["codebuddy", "cbc"] +codex = ["codex"] +command-code = ["command-code"] +copilot = ["copilot"] +cursor = ["cursor-agent", "agent"] +devin = ["devin"] +droid = ["droid"] +gemini = ["gemini"] +goose = ["goose"] +grok = ["grok"] +hermes = ["hermes"] +kilo = ["kilo"] +kimi = ["kimi"] +kiro = ["kiro-cli"] +maki = ["maki"] +omp = ["omp"] +opencode = ["opencode"] +pi = ["pi"] +qodercli = ["qodercli"] +qwen = ["qwen"] +trae-cli = ["trae-cli"] +vibe = ["vibe"] diff --git a/scripts/conformance/agent_recovery/interactive.py b/scripts/conformance/agent_recovery/interactive.py new file mode 100644 index 000000000..3e2211547 --- /dev/null +++ b/scripts/conformance/agent_recovery/interactive.py @@ -0,0 +1,99 @@ +"""Exercise a signed-in CLI on a disposable Linux desktop with two independent chats.""" +import argparse +from contextlib import ExitStack +import os +from pathlib import Path +import shutil +import sys +import tomllib +from tempfile import TemporaryDirectory +from unittest.mock import patch +from uuid import uuid4 + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from conformance.agent_recovery.run import live_id, panes, prepare_split, runtime_pane, saved_id, shell_environment, wait_for_editor +from conformance.harness import ConformanceContext, ResolvedApp + + +def token(): + return uuid4().hex[:12] + + +def check_agent(ctx, agent, commands, rounds, timeout): + ids = [] + secrets = [token(), token()] + sides = ("first", "second") + ctx.start(explicit_working_directory=False) + for index, pane in enumerate(panes(ctx)): + ctx.poll(lambda: "$" in ctx.read(pane)["text"], bool, "test shell is not ready") + ctx.prompt(commands[index], pane) + # An external login, trust prompt or source-only identity cannot satisfy + # the native-ID check. Leave time for interactive setup on this desktop. + def ready(): + row = runtime_pane(ctx, pane) + status = row.get("agent") or {} + return status.get("kind") == agent and row.get("task_state") in ("idle", "finished", "waiting_input") + ctx.poll(ready, bool, f"{agent} is not ready for a user turn", timeout=timeout) + wait_for_editor(ctx, pane, agent) + prefix = token() + secret = secrets[index] + prompt = (f"Our test word is {secret}. Keep it in this conversation only. Do not use tools. " + f"Reply with {prefix}, a slash, and the test word joined with no spaces.") + ctx.prompt(prompt, pane) + ctx.poll(lambda: f"{prefix}/{secret}" in ctx.read(pane)["text"], bool, + f"{agent} did not answer the first challenge", timeout=timeout) + ids.append(ctx.poll(lambda: saved_id(ctx, sides[index], agent), bool, + f"{agent} did not save the conversation ID", timeout=timeout)) + assert ids[0] != ids[1], "both split leaves saved the same conversation ID" + for attempt in range(rounds): + ctx.restart() + for index, pane in enumerate(panes(ctx)): + # Each challenge prefix is new after restart. Terminal scrollback + # cannot supply this answer, and the question omits both secrets. + prefix = token() + ctx.poll(lambda: live_id(ctx, pane, agent) == ids[index], bool, + "provider did not acknowledge the same ID", timeout=timeout) + wait_for_editor(ctx, pane, agent) + ctx.prompt(f"Without tools, recall our test word. Reply with {prefix}, a slash, " + "and that word joined with no spaces.", pane) + ctx.poll(lambda: f"{prefix}/{secrets[index]}" in ctx.read(pane)["text"], bool, + f"{agent} did not recall its own conversation", timeout=timeout) + assert f"{prefix}/{secrets[1-index]}" not in ctx.read(pane)["text"] + assert saved_id(ctx, sides[index], agent) == ids[index] + print(f"{agent} round {attempt + 1}/{rounds} continued both chats", flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + entries_by_source = tomllib.loads(Path(__file__).with_name("entries.toml").read_text(encoding="utf-8")) + parser.add_argument("--app", required=True) + parser.add_argument("--source", required=True, choices=entries_by_source) + parser.add_argument("--test-home", required=True, type=Path, help="Dedicated signed-in test home") + parser.add_argument("--rounds", type=int, default=3) + parser.add_argument("--timeout", type=int, default=90) + args = parser.parse_args() + home = args.test_home.resolve() + if not sys.platform.startswith("linux") or not os.environ.get("DISPLAY"): + parser.error("requires a Linux desktop") + if not home.is_dir() or home == Path.home().resolve(): + parser.error("supply a separate test home containing the provider's test profile") + if args.rounds < 1 or args.timeout < 1: + parser.error("rounds and timeout must be positive") + commands = entries_by_source[args.source] + for entry in commands: + if not shutil.which(entry): + parser.error(f"{entry} is missing from PATH") + with ExitStack() as cleanup: + test = Path(cleanup.enter_context(TemporaryDirectory(prefix="pebrel-agent-recall-"))) + app = ResolvedApp(args.app) + cleanup.callback(app.close) + ctx = ConformanceContext(app, "linux", test / "config", test / "project", test / "artifacts") + prepare_split(ctx, args.source) + cleanup.enter_context(patch.dict(os.environ, shell_environment(home), clear=True)) + cleanup.callback(ctx.stop, force=True) + entries = commands * 2 if len(commands) == 1 else commands + check_agent(ctx, args.source, entries, args.rounds, args.timeout) + + +if __name__ == "__main__": + main() diff --git a/scripts/conformance/agent_recovery/model.py b/scripts/conformance/agent_recovery/model.py new file mode 100644 index 000000000..ad9a21f5d --- /dev/null +++ b/scripts/conformance/agent_recovery/model.py @@ -0,0 +1,79 @@ +"""Loopback model for native CLI recovery conformance.""" +from contextlib import contextmanager, suppress +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import re +from threading import Thread +from urllib.parse import urlsplit + + +class FixtureModel(BaseHTTPRequestHandler): + def send_body(self, body, content_type): + data = body.encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + with suppress(BrokenPipeError, ConnectionResetError): + self.wfile.write(data) + + def do_GET(self): + if urlsplit(self.path).path.endswith("/models"): + self.send_body(json.dumps({"object": "list", "data": [{"object": "model", "id": "fixture"}]}), + "application/json") + else: + self.send_error(404) + + def do_POST(self): + path = urlsplit(self.path).path + request = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + if path.endswith("/messages/count_tokens"): + self.send_body('{"input_tokens":10}', "application/json") + return + if not path.endswith(("/messages", "/chat/completions")): + self.send_error(404) + return + # Record only fixture markers from user turns. Diagnostics, summaries + # and assistant messages cannot attest which conversation was resumed. + markers = { + marker for message in request.get("messages", []) if message.get("role") == "user" + for marker in re.findall(r"pebrel-qa-[0-9a-f]{16}", json.dumps(message.get("content"))) + } + self.server.turns.append(markers) + reply = "Native recovery fixture reply." + if path.endswith("/messages"): + message = {"id": "msg_qa", "type": "message", "role": "assistant", + "model": "fixture", "content": [], "usage": {"input_tokens": 10, "output_tokens": 1}} + events = [ + ("message_start", {"message": message}), + ("content_block_start", {"index": 0, "content_block": {"type": "text", "text": ""}}), + ("content_block_delta", {"index": 0, "delta": {"type": "text_delta", "text": reply}}), + ("content_block_stop", {"index": 0}), + ("message_delta", {"delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 5}}), + ("message_stop", {}), + ] + wire = "".join(f"event: {kind}\ndata: {json.dumps({'type': kind, **body})}\n\n" + for kind, body in events) + else: + chunks = [{"id": "chatcmpl-qa", "object": "chat.completion.chunk", "model": "fixture", + "created": 1, "choices": [{"index": 0, "delta": delta, "finish_reason": finish}]} + for delta, finish in [({"role": "assistant", "content": reply}, None), ({}, "stop")]] + wire = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n" + self.send_body(wire, "text/event-stream") + + def log_message(self, *_args): + pass + + +@contextmanager +def local_model(): + with ThreadingHTTPServer(("127.0.0.1", 0), FixtureModel) as server: + server.turns = [] + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + thread.join() diff --git a/scripts/conformance/agent_recovery/providers.py b/scripts/conformance/agent_recovery/providers.py new file mode 100644 index 000000000..8cf93f485 --- /dev/null +++ b/scripts/conformance/agent_recovery/providers.py @@ -0,0 +1,98 @@ +"""Native profile setup and independent transcript oracles for offline CLIs.""" +import json + + +PROFILES = {"claude": ".claude", "pi": ".pi/agent", "omp": ".omp/agent", + "grok": ".grok", "codebuddy": ".codebuddy"} + + +def write_json(path, data): + path.write_text(json.dumps(data), encoding="utf-8") + + +def configure(agent, home, work_dir, url): + profile = home / PROFILES[agent] + profile.mkdir(parents=True) + if agent in ("pi", "omp"): + model = {"baseUrl": url + "/v1", "api": "openai-completions", "apiKey": "fixture", + "models": [{"id": "fixture", "name": "Fixture"}]} + models = "models.json" if agent == "pi" else "models.yml" + write_json(profile / models, {"providers": {"pebrelfixture": model}}) + write_json(profile / "settings.json", {"defaultProvider": "pebrelfixture", "defaultModel": "fixture", + "defaultProjectTrust": "always"}) + return {"PI_CODING_AGENT_DIR": str(profile), "PI_OFFLINE": "1", + "OMP_SKIP_SETUP": "1", "OMP_SKIP_VERSION_CHECK": "1"} + if agent == "claude": + onboarding = {"hasCompletedOnboarding": True, "theme": "dark", + "projects": {str(work_dir): {"hasTrustDialogAccepted": True, + "projectOnboardingSeenCount": 5}}, + "customApiKeyResponses": {"approved": ["fixture-key"], "rejected": []}} + write_json(home / ".claude.json", onboarding) + write_json(profile / ".claude.json", onboarding) + return {"ANTHROPIC_BASE_URL": url, "ANTHROPIC_API_KEY": "fixture-key", + "CLAUDE_CONFIG_DIR": str(profile), "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", + "DISABLE_AUTOUPDATER": "1"} + if agent == "grok": + (profile / "config.toml").write_text( + '[models]\ndefault="fixture"\n[model.fixture]\nmodel="fixture"\n' + f'base_url="{url}/v1"\napi_backend="chat_completions"\nenv_key="XAI_API_KEY"\n', + encoding="utf-8") + return {"GROK_HOME": str(profile), "XAI_API_KEY": "fixture-key", + "GROK_MODELS_BASE_URL": url + "/v1", "GROK_XAI_API_BASE_URL": url + "/v1"} + return {"CODEBUDDY_BASE_URL": url + "/v1", "CODEBUDDY_CONFIG_DIR": str(profile), + "CODEBUDDY_API_KEY": "fixture-key"} + + +def command_for(agent, pane_index): + if agent in ("pi", "omp"): + return f"{agent} --model pebrelfixture/fixture" + return "cbc" if agent == "codebuddy" and pane_index == 1 else agent + + +def has_text(content, marker): + if isinstance(content, str): + return content == marker + return isinstance(content, list) and any( + isinstance(part, dict) and part.get("text") == marker for part in content + ) + + +def native_match(agent, file, rows, session_id, marker): + if agent in ("pi", "omp"): + header = next((row for row in rows if row.get("type") == "session"), {}) + if header.get("id") != session_id: + return False + elif agent == "grok": + if session_id not in file.parts and session_id != file.stem: + return False + marker = f"\n{marker}\n" + seen = False + for row in rows: + if agent in ("claude", "codebuddy") and row.get("sessionId") != session_id: + continue + message = row.get("message") or row + role = message.get("role") or row.get("type") + content = message.get("content") + if role == "user": + seen = has_text(content, marker) + elif seen and role == "assistant" and has_text(content, "Native recovery fixture reply."): + return True + return False + + +def provider_confirmed(home, agent, session_id, marker): + if not session_id: + return False + profile = home / PROFILES[agent] + directory = profile / ("projects" if agent in ("claude", "codebuddy") else "sessions") + for file in directory.rglob("*.jsonl"): + if file.is_symlink(): + continue + try: + rows = [json.loads(line) for line in file.read_text(encoding="utf-8").splitlines() + if line.strip()] + except (OSError, ValueError): + continue + if native_match(agent, file, rows, session_id, marker): + return True + return False diff --git a/scripts/conformance/agent_recovery/run.py b/scripts/conformance/agent_recovery/run.py new file mode 100644 index 000000000..045db5fec --- /dev/null +++ b/scripts/conformance/agent_recovery/run.py @@ -0,0 +1,181 @@ +"""Cold-restart two same-directory native CLIs in a disposable Linux desktop profile.""" +import argparse +from contextlib import ExitStack +import json +import os +from pathlib import Path +import shutil +import sys +from tempfile import TemporaryDirectory +import time +from unittest.mock import patch +from uuid import uuid4 + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from conformance.agent_recovery.model import local_model +from conformance.agent_recovery.providers import PROFILES, command_for, configure, provider_confirmed +from conformance.harness import ConformanceContext, ResolvedApp + + +def marker(): + return "pebrel-qa-" + uuid4().hex[:16] + + +def shell_environment(home): + env = {name: os.environ[name] for name in + ("PATH", "DISPLAY", "XAUTHORITY", "DBUS_SESSION_BUS_ADDRESS", "XDG_RUNTIME_DIR") + if name in os.environ} + env.update(HOME=str(home), XDG_CONFIG_HOME=str(home / ".config"), + XDG_DATA_HOME=str(home / ".local/share"), XDG_STATE_HOME=str(home / ".local/state"), + XDG_CACHE_HOME=str(home / ".cache"), SHELL="/bin/bash", LANG="C.UTF-8", + TERM="xterm-256color") + return env + + +def prepare_split(ctx, agent): + ctx.prepare() + settings = ctx.config_dir / "pebrel_settings.txt" + content = settings.read_text(encoding="utf-8") + assert "resume_ai=0" in content + settings.write_text(content.replace("resume_ai=0", "resume_ai=1") + f"ai_hooks_{agent}=1\n", + encoding="utf-8") + shell = {"kind": "shell", "name": "Bash", "program": "/bin/bash", "args": []} + leaf = {"kind": "pane", "cwd": str(ctx.work_dir), "launch": shell} + layout = {"kind": "split", "axis": "top_bottom", "ratio_permille": 500, "first": leaf, "second": leaf} + tab = {"cwd": str(ctx.work_dir), "launch": shell, "layout": layout} + ctx.session_file.write_text(json.dumps({"version": 4, "tabs": [tab], "active_tab": 0, + "clean_exit": True}), encoding="utf-8") + + +def panes(ctx): + ctx.refresh_targets() + tabs = ctx.snapshot()["windows"][0]["tabs"] + assert len(tabs) == 1 and len(tabs[0]["panes"]) == 2 + tree = tabs[0]["layout"] + assert tree["direction"] == "top_bottom" + return [tree[side]["pane_id"] for side in ("first", "second")] + + +def runtime_pane(ctx, pane): + tab = ctx.tab_for_pane(ctx.snapshot(), pane) + return next(row for row in tab["panes"] if row["id"] == pane) + + +def live_id(ctx, pane, source): + agent = runtime_pane(ctx, pane).get("agent") or {} + return agent.get("session_id") if agent.get("kind") == source else None + + +def saved_id(ctx, side, source): + try: + with ctx.session_file.open(encoding="utf-8") as stream: + tree = json.load(stream)["tabs"][0]["layout"] + agent = tree[side].get("agent") or {} + return agent.get("session_id") if agent.get("source") == source else None + except (ValueError, OSError, KeyError, IndexError): + return None + + +def wait_for_editor(ctx, pane, agent): + if agent == "claude": + # SessionStart can arrive while the grid still shows the shell command. + ctx.poll(lambda: any(row.strip() == "❯" for row in ctx.read(pane)["text"].splitlines()[-8:]), + bool, "Claude did not display its empty composer", timeout=45) + + +def accept_project_prompt(ctx, pane, agent): + if agent not in ("grok", "codebuddy"): + return + # Only the disposable project is approved. Wait for the native prompt or + # an input footer if a sibling has already trusted this directory. + deadline = time.monotonic() + 12 + while time.monotonic() < deadline: + screen = ctx.read(pane)["text"].lower() + if "yes, proceed" in screen and "contents of this directory" in screen: + ctx.prompt("y", pane, submit=False) + return + if "trust parent folder" in screen and "enter to confirm" in screen: + ctx.send_key("enter", pane) + return + if "for shortcuts" in screen or "ctrl+" in screen: + return + time.sleep(0.2) + + +def run_agent(agent, executable, rounds): + entries = [agent, "cbc"] if agent == "codebuddy" else [agent] + for binary in entries: + if not shutil.which(binary): + raise RuntimeError(f"{binary} is missing from PATH") + with ExitStack() as cleanup: + test = Path(cleanup.enter_context(TemporaryDirectory(prefix="pebrel-native-recovery-"))) + home = test / "home" + app = ResolvedApp(executable) + cleanup.callback(app.close) + ctx = ConformanceContext(app, "linux", test / "config", test / "project", test / "artifacts") + prepare_split(ctx, agent) + model = cleanup.enter_context(local_model()) + env = shell_environment(home) + env.update(configure(agent, home, ctx.work_dir, f"http://127.0.0.1:{model.server_port}")) + cleanup.enter_context(patch.dict(os.environ, env, clear=True)) + cleanup.callback(ctx.stop, force=True) + ctx.start(explicit_working_directory=False) + first = [marker(), marker()] + expected = [] + sides = ("first", "second") + for index, pane in enumerate(panes(ctx)): + ctx.poll(lambda: "$" in ctx.read(pane)["text"], bool, "shell is not ready") + command = command_for(agent, index) + if agent in ("pi", "omp"): + command += " " + first[index] + ctx.prompt(command, pane) + # A source-only live status identifies the TUI before any ID exists. + def running(): + return (runtime_pane(ctx, pane).get("agent") or {}).get("kind") == agent + ctx.poll(running, bool, f"{agent} did not start", timeout=30) + accept_project_prompt(ctx, pane, agent) + wait_for_editor(ctx, pane, agent) + if agent not in ("pi", "omp"): + ctx.prompt(first[index], pane) + ctx.poll(lambda: any(first[index] in turn for turn in model.turns), bool, + f"{agent} initial turn did not reach the local model", timeout=40) + def initial_id(): + native_id = saved_id(ctx, sides[index], agent) + return native_id if provider_confirmed(home, agent, native_id, first[index]) else None + expected.append(ctx.poll(initial_id, bool, f"{agent} native conversation did not match", timeout=40)) + assert expected[0] != expected[1] + for round_index in range(rounds): + ctx.restart() + for index, pane in enumerate(panes(ctx)): + ctx.poll(lambda: live_id(ctx, pane, agent) == expected[index], bool, + f"{agent} did not acknowledge its saved ID in {sides[index]}", timeout=45) + wait_for_editor(ctx, pane, agent) + after = marker() + ctx.prompt(after, pane) + turns = ctx.poll(lambda: [turn for turn in model.turns if after in turn], bool, + f"{agent} did not submit the restored turn in {sides[index]}", timeout=40) + assert any(first[index] in turn and first[1-index] not in turn for turn in turns), ( + f"{agent} restored turn has the wrong history in {sides[index]}" + ) + ctx.poll(lambda: saved_id(ctx, sides[index], agent) == expected[index] + and provider_confirmed(home, agent, expected[index], after), bool, + "new message did not enter the same native conversation", timeout=40) + print(f"{agent} round {round_index + 1}/{rounds} passed for both panes", flush=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--app", required=True, help="Packaged Pebrel executable") + parser.add_argument("--agent", choices=PROFILES, action="append", help="Repeat to select several CLIs") + parser.add_argument("--rounds", type=int, default=3) + args = parser.parse_args() + if not sys.platform.startswith("linux") or not os.environ.get("DISPLAY") or args.rounds < 1: + parser.error("requires a Linux desktop and at least one cold restart") + failures = [] + for source in args.agent or list(PROFILES): + try: + run_agent(source, args.app, args.rounds) + except (OSError, RuntimeError, AssertionError) as error: + failures.append(source) + print(f"{source}: {type(error).__name__}: {error}", file=sys.stderr) + sys.exit(bool(failures)) diff --git a/scripts/conformance/tests/test_agent_recovery.py b/scripts/conformance/tests/test_agent_recovery.py new file mode 100644 index 000000000..77c20ba98 --- /dev/null +++ b/scripts/conformance/tests/test_agent_recovery.py @@ -0,0 +1,49 @@ +import json +from pathlib import Path +import tempfile +import unittest + +from scripts.conformance.agent_recovery.providers import PROFILES, provider_confirmed + + +class NativeTranscriptTest(unittest.TestCase): + def test_native_transcript_must_match_the_pane_and_new_user_message(self): + marker = "pebrel-qa-a91d9f0cba239e10" + wanted = "0f82b8ec-61ec-458a-a2df-01c32502540a" + other = "4423c29f-bdfa-4611-91b2-866287bfb5c8" + part = [{"type": "text", "text": marker}] + for provider in PROFILES: + with self.subTest(provider=provider), tempfile.TemporaryDirectory() as directory: + home = Path(directory) + profile = home / PROFILES[provider] + root = profile / ("projects" if provider in ("claude", "codebuddy") else "sessions") + folder = root / wanted + folder.mkdir(parents=True) + if provider in ("pi", "omp"): + rows = [{"type": "session", "id": wanted}, + {"type": "message", "message": {"role": "user", "content": part}}] + if provider == "omp": + rows.insert(0, {"type": "title", "v": 1}) + elif provider == "claude": + rows = [{"type": "user", "sessionId": wanted, "message": {"content": part}}] + elif provider == "codebuddy": + rows = [{"type": "message", "role": "user", "content": part, "sessionId": wanted}] + else: + rows = [{"type": "user", "content": [{"text": f"\n{marker}\n"}]}] + path = folder / "chat_history.jsonl" + content = "".join(json.dumps(row) + "\n" for row in rows) + path.write_text(content, encoding="utf-8") + self.assertFalse(provider_confirmed(home, provider, wanted, marker)) + reply = "Native recovery fixture reply." + if provider in ("pi", "omp", "claude"): + assistant = {"type": "assistant" if provider == "claude" else "message", + "sessionId": wanted, "message": {"role": "assistant", "content": reply}} + else: + assistant = {"type": "assistant", "role": "assistant", "content": reply, "sessionId": wanted} + content += json.dumps(assistant) + "\n" + path.write_text(content, encoding="utf-8") + self.assertTrue(provider_confirmed(home, provider, wanted, marker)) + self.assertFalse(provider_confirmed(home, provider, other, marker)) + self.assertFalse(provider_confirmed(home, provider, wanted, "different-message")) + path.write_text(content.replace('"user"', '"assistant"'), encoding="utf-8") + self.assertFalse(provider_confirmed(home, provider, wanted, marker))