diff --git a/codewiki/cli/adapters/doc_generator.py b/codewiki/cli/adapters/doc_generator.py index 2a012d3a..dedaf735 100644 --- a/codewiki/cli/adapters/doc_generator.py +++ b/codewiki/cli/adapters/doc_generator.py @@ -365,6 +365,9 @@ async def _run_backend_generation(self, backend_config: BackendConfig): validate_module_file_coverage(module_tree, components) file_manager.save_json(module_tree, first_module_tree_path) file_manager.save_json(module_tree, module_tree_path) + from codewiki.src.be.cluster_batch_cache import delete_clustering_batch_cache + + delete_clustering_batch_cache(working_dir) self.job.module_count = len(module_tree) diff --git a/codewiki/src/be/cluster_batch_cache.py b/codewiki/src/be/cluster_batch_cache.py new file mode 100644 index 00000000..7fcd54bb --- /dev/null +++ b/codewiki/src/be/cluster_batch_cache.py @@ -0,0 +1,271 @@ +"""Disk cache for per-batch LLM clustering partial module trees.""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import tempfile +from copy import deepcopy +from typing import Any + +from codewiki.src.config import CLUSTERING_BATCH_CACHE_FILENAME, Config +from codewiki.src.utils import file_manager + +logger = logging.getLogger(__name__) + +CACHE_VERSION = 4 + + +def clustering_cache_scope(module_path: list[str]) -> str: + """Stable scope id for one ``cluster_modules`` invocation.""" + if not module_path: + return "__root__" + return "/".join(module_path) + + +def clustering_request_fingerprint(config: Config, prompt: str) -> str: + """Identify the rendered request and configured routing without persisting secrets.""" + routing = { + name: getattr(config, name, None) + for name in ( + "provider", + "cluster_model", + "main_model", + "llm_base_url", + "azure_deployment", + "api_version", + "aws_region", + "google_project", + "google_location", + "max_tokens", + ) + } + if routing["provider"] == "google-vertex": + routing["google_project"] = routing["google_project"] or os.getenv( + "GOOGLE_CLOUD_PROJECT", "" + ) + routing["google_location"] = ( + routing["google_location"] or os.getenv("GOOGLE_CLOUD_LOCATION") or "global" + ) + context = { + "repo": os.path.realpath(config.repo_path), + "routing": routing, + "prompt": prompt, + } + canonical = json.dumps(context, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def batch_entry_key( + scope: str, batch_index: int, node_ids: list[str], request_fingerprint: str +) -> str: + """Scope, batch index, component set, and exact request identity.""" + canonical = json.dumps(sorted(node_ids), ensure_ascii=False, separators=(",", ":")) + digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:16] + return f"{scope}|{batch_index}|{digest}|{request_fingerprint}" + + +def clustering_batch_cache_path(docs_dir: str) -> str: + return os.path.join(os.path.abspath(docs_dir), "temp", CLUSTERING_BATCH_CACHE_FILENAME) + + +def delete_clustering_batch_cache(docs_dir: str, *, scopes: list[str] | None = None) -> None: + """Remove committed scopes and descendants, or the whole cache after a full build.""" + if not docs_dir: + return + if scopes is not None: + if scopes: + ClusterBatchCache(docs_dir).delete_scopes(scopes) + return + _delete_cache_file(clustering_batch_cache_path(docs_dir)) + + +def _delete_cache_file(path: str) -> None: + try: + os.remove(path) + except FileNotFoundError: + return + except OSError as exc: + logger.warning("Could not remove clustering batch cache at %s: %s", path, exc) + return + logger.info("Removed clustering batch cache at %s", path) + + +def _valid_entry(entry: Any) -> bool: + if not isinstance(entry, dict): + return False + ids = entry.get("node_ids") + partial = entry.get("partial_tree") + return ( + isinstance(entry.get("scope"), str) + and isinstance(entry.get("request_fingerprint"), str) + and bool(entry["request_fingerprint"]) + and isinstance(entry.get("batch_index"), int) + and not isinstance(entry["batch_index"], bool) + and entry["batch_index"] > 0 + and isinstance(ids, list) + and all(isinstance(cid, str) for cid in ids) + and isinstance(partial, dict) + and bool(partial) + and all( + isinstance(name, str) + and isinstance(info, dict) + and isinstance(info.get("path", ""), str) + and isinstance(info.get("components"), list) + and all(isinstance(cid, str) and cid in ids for cid in info["components"]) + for name, info in partial.items() + ) + ) + + +class ClusterBatchCache: + """Read/write partial clustering results keyed by scope, batch index, and node set.""" + + def __init__(self, docs_dir: str, *, enabled: bool = True) -> None: + self._enabled = enabled and bool(docs_dir) + self._path = clustering_batch_cache_path(docs_dir) if self._enabled else "" + self._data: dict[str, Any] = {"version": CACHE_VERSION, "entries": {}} + if self._enabled: + self._load() + + def _load(self) -> None: + try: + raw = file_manager.load_json(self._path) + except (OSError, ValueError, UnicodeError) as exc: + logger.warning("Ignoring unreadable clustering batch cache at %s: %s", self._path, exc) + return + if not isinstance(raw, dict) or raw.get("version") != CACHE_VERSION: + return + entries = raw.get("entries") + if isinstance(entries, dict): + entries = {key: entry for key, entry in entries.items() if _valid_entry(entry)} + self._data = {"version": CACHE_VERSION, "entries": entries} + logger.info( + "Loaded clustering batch cache: %d entries from %s", + len(entries), + self._path, + ) + + def _save(self) -> bool: + """Publish a complete checkpoint while keeping the previous one on interruption.""" + temporary_path = None + try: + directory = os.path.dirname(self._path) + file_manager.ensure_directory(directory) + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=directory, prefix=".clustering-cache-", delete=False + ) as stream: + temporary_path = stream.name + json.dump(self._data, stream, indent=4, ensure_ascii=False) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary_path, self._path) + return True + except OSError as exc: + logger.warning("Could not save clustering batch cache at %s: %s", self._path, exc) + return False + finally: + if temporary_path is not None: + try: + os.remove(temporary_path) + except FileNotFoundError: + pass + except OSError as exc: + logger.warning("Could not remove temporary clustering cache: %s", exc) + + def entry_count(self) -> int: + entries = self._data.get("entries") + return len(entries) if isinstance(entries, dict) else 0 + + def scope_entry_count(self, scope: str) -> int: + entries = self._data.get("entries") + if not isinstance(entries, dict): + return 0 + return sum(1 for entry in entries.values() if entry.get("scope") == scope) + + def delete_scopes(self, scopes: list[str]) -> None: + """Keep checkpoints for failed or unrelated subtrees available for retry.""" + if not self._enabled or not scopes: + return + selected = set(scopes) + prefixes = tuple(scope + "/" for scope in selected) + entries = self._data["entries"] + remaining = { + key: entry + for key, entry in entries.items() + if entry["scope"] not in selected and not entry["scope"].startswith(prefixes) + } + if len(remaining) == len(entries): + return + self._data["entries"] = remaining + if remaining: + self._save() + else: + _delete_cache_file(self._path) + + def lookup( + self, scope: str, batch_index: int, node_ids: list[str], *, request_fingerprint: str + ) -> dict[str, Any] | None: + if not self._enabled: + return None + key = batch_entry_key(scope, batch_index, node_ids, request_fingerprint) + entry = self._data["entries"].get(key) + if not _valid_entry(entry): + return None + stored_ids = entry.get("node_ids") + if ( + stored_ids != sorted(node_ids) + or entry["scope"] != scope + or entry["batch_index"] != batch_index + or entry["request_fingerprint"] != request_fingerprint + ): + return None + partial = entry.get("partial_tree") + if not isinstance(partial, dict) or not partial: + return None + logger.info( + "Using cached clustering result for scope %s batch %d (%d components).", + scope, + batch_index, + len(node_ids), + ) + return deepcopy(partial) + + def store( + self, + scope: str, + batch_index: int, + node_ids: list[str], + partial_tree: dict[str, Any], + *, + request_fingerprint: str, + ) -> None: + if not self._enabled or not partial_tree: + return + key = batch_entry_key(scope, batch_index, node_ids, request_fingerprint) + entry = { + "scope": scope, + "batch_index": batch_index, + "node_ids": sorted(node_ids), + "request_fingerprint": request_fingerprint, + "partial_tree": deepcopy(partial_tree), + } + if not _valid_entry(entry): + return + self._data["entries"][key] = entry + if not self._save(): + return + module_names = list(partial_tree.keys()) + logger.info( + "Stored clustering batch cache for scope %s batch %d (%d components, %d modules: %s); " + "%d total entries at %s", + scope, + batch_index, + len(node_ids), + len(module_names), + ", ".join(module_names[:5]) + ("..." if len(module_names) > 5 else ""), + self.entry_count(), + self._path, + ) diff --git a/codewiki/src/be/cluster_modules.py b/codewiki/src/be/cluster_modules.py index 12659baf..a1ac7bdd 100644 --- a/codewiki/src/be/cluster_modules.py +++ b/codewiki/src/be/cluster_modules.py @@ -5,6 +5,11 @@ from collections.abc import Callable from typing import Any +from codewiki.src.be.cluster_batch_cache import ( + ClusterBatchCache, + clustering_cache_scope, + clustering_request_fingerprint, +) from codewiki.src.be.dependency_analyzer.models.core import Node from codewiki.src.be.llm_services import call_llm from codewiki.src.be.module_naming import resolve_unique_name, sanitize_module_name @@ -170,16 +175,19 @@ def _cluster_via_llm( current_module_name: str | None, module_label: str, completer: Completer | None, + *, + prompt: str | None = None, ) -> dict[str, Any]: """Run one clustering LLM call over these nodes. Returns {} for any empty, malformed, or non-dict response instead of raising, so callers can fall back gracefully. """ - potential_core_components, _ = format_potential_core_components(leaf_nodes, components) - prompt = format_cluster_prompt( - potential_core_components, current_module_tree, current_module_name - ) + if prompt is None: + potential_core_components, _ = format_potential_core_components(leaf_nodes, components) + prompt = format_cluster_prompt( + potential_core_components, current_module_tree, current_module_name + ) if completer is not None: response = completer(prompt) else: @@ -428,9 +436,37 @@ def cluster_modules( module_label, len(batches), ) + batch_cache: ClusterBatchCache | None = None + cache_scope = clustering_cache_scope(current_module_path) + if getattr(config, "cluster_batch_cache", True) and config.docs_dir: + batch_cache = ClusterBatchCache(config.docs_dir) module_tree = {} + cache_hits = 0 + if batch_cache is not None: + logger.info( + "Clustering batch cache for %r: %d on-disk entries (%d for this scope), %d batches to run.", + cache_scope, + batch_cache.entry_count(), + batch_cache.scope_entry_count(cache_scope), + len(batches), + ) for i, batch in enumerate(batches, 1): batch_label = f"{module_label} (batch {i}/{len(batches)})" + potential_core_components, _ = format_potential_core_components(batch, components) + prompt = format_cluster_prompt( + potential_core_components, current_module_tree, current_module_name + ) + request_fingerprint = ( + clustering_request_fingerprint(config, prompt) if batch_cache is not None else "" + ) + if batch_cache is not None: + cached = batch_cache.lookup( + cache_scope, i, batch, request_fingerprint=request_fingerprint + ) + if cached is not None: + cache_hits += 1 + _merge_module_trees(module_tree, cached) + continue # Each batch gets the *unmodified* current_module_tree: passing the # accumulating merge would flip format_cluster_prompt into its # module-level variant mid-partition. @@ -442,7 +478,14 @@ def cluster_modules( current_module_name, batch_label, completer, + prompt=prompt, ) + # Only validated LLM responses are resumable. A structural fallback + # keeps this pass usable but must be retried after a later failure. + if batch_cache is not None and partial: + batch_cache.store( + cache_scope, i, batch, partial, request_fingerprint=request_fingerprint + ) if not partial: name = _batch_fallback_name(batch, components, module_tree) partial = { @@ -460,6 +503,14 @@ def cluster_modules( name, ) _merge_module_trees(module_tree, partial) + if batch_cache is not None: + logger.info( + "Clustering batch cache for %r: %d/%d batches from cache, %d LLM batch call(s) this pass.", + cache_scope, + cache_hits, + len(batches), + len(batches) - cache_hits, + ) module_tree = normalize_module_partition(module_tree, candidates, components) if len(module_tree) <= 1: diff --git a/codewiki/src/be/documentation_generator.py b/codewiki/src/be/documentation_generator.py index b920d10a..db2661da 100644 --- a/codewiki/src/be/documentation_generator.py +++ b/codewiki/src/be/documentation_generator.py @@ -545,6 +545,9 @@ async def run(self) -> None: validate_module_file_coverage(module_tree, components) file_manager.save_json(module_tree, first_module_tree_path) file_manager.save_json(module_tree, module_tree_path) + from codewiki.src.be.cluster_batch_cache import delete_clustering_batch_cache + + delete_clustering_batch_cache(working_dir) if len(module_tree) == 0: logger.info( diff --git a/codewiki/src/be/updater/orchestrator.py b/codewiki/src/be/updater/orchestrator.py index 4e149d97..46edd26f 100644 --- a/codewiki/src/be/updater/orchestrator.py +++ b/codewiki/src/be/updater/orchestrator.py @@ -9,6 +9,10 @@ from typing import Any from codewiki.src.be.backend import LLMBackend +from codewiki.src.be.cluster_batch_cache import ( + clustering_cache_scope, + delete_clustering_batch_cache, +) from codewiki.src.be.cluster_modules import cluster_modules, get_clustering_input_token_count from codewiki.src.be.module_scope import get_module_token_count from codewiki.src.be.dependency_analyzer.models.core import Node @@ -381,6 +385,10 @@ async def _run( # Persist the repaired tree so the normal pipeline and the agents see it. if not self.whole_repo: file_manager.save_json(new_tree, os.path.join(self.docs_dir, MODULE_TREE_FILENAME)) + delete_clustering_batch_cache( + self.docs_dir, + scopes=[clustering_cache_scope(path) for path in rec.reclustered], + ) # ---- Step 5: sequential leaf agents order = order_active(active, new_tree, new_graph) diff --git a/codewiki/src/config.py b/codewiki/src/config.py index e5a32d1e..38d9d762 100644 --- a/codewiki/src/config.py +++ b/codewiki/src/config.py @@ -15,6 +15,7 @@ DOCS_DIR = "docs" FIRST_MODULE_TREE_FILENAME = "first_module_tree.json" MODULE_TREE_FILENAME = "module_tree.json" +CLUSTERING_BATCH_CACHE_FILENAME = "clustering_batch_cache.json" OVERVIEW_FILENAME = "overview.md" # Docs layout: "hierarchical" mirrors the module tree in nested folders # (``auth.md`` + ``auth/login.md``); "flat" keeps every page in the docs root @@ -136,6 +137,8 @@ class Config: with_prose: bool = False # Docs layout (LAYOUT_HIERARCHICAL or LAYOUT_FLAT) layout: str = DEFAULT_LAYOUT + # Persist each successful LLM clustering batch under docs/temp for resume after failures. + cluster_batch_cache: bool = True def __post_init__(self): validate_min_module_tokens(self.min_token_per_module) diff --git a/tests/test_cluster_batch_cache.py b/tests/test_cluster_batch_cache.py new file mode 100644 index 00000000..34cf551b --- /dev/null +++ b/tests/test_cluster_batch_cache.py @@ -0,0 +1,538 @@ +"""Checkpoint and retry regressions for batched module clustering.""" + +from __future__ import annotations + +import json +import asyncio +import importlib +from pathlib import Path + +import pytest + +from codewiki.src.be.cluster_batch_cache import ( + CACHE_VERSION, + ClusterBatchCache, + clustering_batch_cache_path, + delete_clustering_batch_cache, + clustering_request_fingerprint, +) +from codewiki.src.be.cluster_modules import cluster_modules, format_potential_core_components +from codewiki.src.be.prompt_template import format_cluster_prompt +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.config import Config +from codewiki.src.be.documentation_generator import DocumentationGenerator +from codewiki.src.utils import file_manager + + +def _tree(cid="a"): + return {"Module": {"components": [cid]}} + + +def _inputs(tmp_path, directories=("alpha", "beta", "gamma")): + ids = [f"{directory}/f{i}.py::C{i}" for directory in directories for i in range(2)] + components = {} + for cid in ids: + path, name = cid.split("::") + components[cid] = Node( + id=cid, + name=name, + component_type="class", + file_path=path, + relative_path=path, + source_code="code", + ) + config = Config.from_cli( + repo_path=str(tmp_path / "repo"), + output_dir=str(tmp_path / "docs"), + llm_base_url="http://localhost", + llm_api_key="unused", + main_model="m", + cluster_model="m", + min_token_per_module=1, + max_token_per_module=2, + max_leaf_nodes_per_cluster=2, + ) + return ids, components, config + + +def _prompt_ids(prompt): + section = prompt.split("")[1].split("")[ + 0 + ] + return [line.strip() for line in section.splitlines() if line.startswith("\t")] + + +def _response(name, ids): + return f"{ {name: {'components': ids}}!r}" + + +def test_successful_batches_resume_after_failure(tmp_path): + ids, components, config = _inputs(tmp_path) + + def first(prompt): + batch = _prompt_ids(prompt) + if batch[0].startswith("beta/"): + raise RuntimeError("network failed") + return _response("Alpha", batch) + + with pytest.raises(RuntimeError, match="network failed"): + cluster_modules(ids, components, config, completer=first) + calls = [] + + def retry(prompt): + batch = _prompt_ids(prompt) + calls.append(batch) + return _response(batch[0].split("/")[0].title(), batch) + + tree = cluster_modules(ids, components, config, completer=retry) + assert calls == [ids[2:4], ids[4:6]] + assert set(tree) == {"Alpha", "Beta", "Gamma"} + + +def test_interrupted_write_preserves_previous_checkpoint(tmp_path, monkeypatch): + cache = ClusterBatchCache(str(tmp_path)) + cache.store("__root__", 1, ["a"], _tree(), request_fingerprint="test-request") + + def interrupted(data, stream, **kwargs): + stream.write('{"version":') + stream.flush() + raise KeyboardInterrupt + + with monkeypatch.context() as patch: + patch.setattr(json, "dump", interrupted) + with pytest.raises(KeyboardInterrupt): + cache.store("__root__", 2, ["b"], _tree("b"), request_fingerprint="test-request") + resumed = ClusterBatchCache(str(tmp_path)) + assert resumed.lookup("__root__", 1, ["a"], request_fingerprint="test-request") == _tree() + assert resumed.lookup("__root__", 2, ["b"], request_fingerprint="test-request") is None + assert sorted(p.name for p in (tmp_path / "temp").iterdir()) == ["clustering_batch_cache.json"] + + +def test_failed_replace_preserves_previous_checkpoint(tmp_path, monkeypatch, caplog): + cache = ClusterBatchCache(str(tmp_path)) + cache.store("__root__", 1, ["a"], _tree(), request_fingerprint="test-request") + + def denied(*args): + raise PermissionError("replace denied") + + monkeypatch.setattr("codewiki.src.be.cluster_batch_cache.os.replace", denied) + cache.store("__root__", 2, ["b"], _tree("b"), request_fingerprint="test-request") + resumed = ClusterBatchCache(str(tmp_path)) + assert resumed.lookup("__root__", 1, ["a"], request_fingerprint="test-request") == _tree() + assert resumed.lookup("__root__", 2, ["b"], request_fingerprint="test-request") is None + assert "replace denied" in caplog.text + assert len(list((tmp_path / "temp").iterdir())) == 1 + + +@pytest.mark.parametrize( + "content", + [ + "", + '{"version":', + "[]", + "null", + "42", + '"text"', + json.dumps({"version": -1, "entries": {}}), + json.dumps({"version": CACHE_VERSION, "entries": []}), + ], +) +def test_invalid_cache_is_a_miss(tmp_path, content): + path = Path(clustering_batch_cache_path(str(tmp_path))) + path.parent.mkdir() + path.write_text(content) + cache = ClusterBatchCache(str(tmp_path)) + assert cache.entry_count() == 0 + assert cache.lookup("__root__", 1, ["a"], request_fingerprint="test-request") is None + + +def test_malformed_entries_are_ignored_individually(tmp_path): + cache = ClusterBatchCache(str(tmp_path)) + cache.store("__root__", 1, ["a"], _tree(), request_fingerprint="test-request") + path = Path(clustering_batch_cache_path(str(tmp_path))) + data = json.loads(path.read_text()) + data["entries"].update({"null": None, "list": [], "bad": {"scope": "__root__"}}) + path.write_text(json.dumps(data)) + resumed = ClusterBatchCache(str(tmp_path)) + assert resumed.entry_count() == resumed.scope_entry_count("__root__") == 1 + assert resumed.lookup("__root__", 1, ["a"], request_fingerprint="test-request") == _tree() + + +def test_unreadable_cache_is_a_miss(tmp_path, monkeypatch, caplog): + def denied(path): + raise PermissionError("read denied") + + monkeypatch.setattr(file_manager, "load_json", denied) + assert ClusterBatchCache(str(tmp_path)).entry_count() == 0 + assert "read denied" in caplog.text + + +def test_cleanup_does_not_parse_cache(tmp_path): + path = Path(clustering_batch_cache_path(str(tmp_path))) + path.parent.mkdir() + path.write_text("broken JSON") + delete_clustering_batch_cache(str(tmp_path)) + delete_clustering_batch_cache(str(tmp_path)) + assert not path.exists() + + +def test_cache_write_failure_does_not_abort_clustering(tmp_path, monkeypatch): + ids, components, config = _inputs(tmp_path) + + def denied(*args): + raise PermissionError("cache unavailable") + + monkeypatch.setattr("codewiki.src.be.cluster_batch_cache.os.replace", denied) + tree = cluster_modules( + ids, components, config, completer=lambda p: _response(_prompt_ids(p)[0], _prompt_ids(p)) + ) + assert len(tree) == 3 + + +def test_disabled_cache_never_writes(tmp_path): + ids, components, config = _inputs(tmp_path) + config.cluster_batch_cache = False + cluster_modules( + ids, components, config, completer=lambda p: _response(_prompt_ids(p)[0], _prompt_ids(p)) + ) + assert not Path(clustering_batch_cache_path(config.docs_dir)).exists() + + +def test_store_and_lookup_do_not_share_mutable_trees(tmp_path): + cache = ClusterBatchCache(str(tmp_path)) + proposed = _tree() + cache.store("__root__", 1, ["a"], proposed, request_fingerprint="test-request") + proposed["Module"]["components"].append("outside") + cached = cache.lookup("__root__", 1, ["a"], request_fingerprint="test-request") + assert cached == _tree() + cached["Module"]["components"].append("another") + # A subsequent store persists the cache's own snapshot, not either mutated view. + cache.store("__root__", 2, ["b"], _tree("b"), request_fingerprint="test-request") + assert cache.lookup("__root__", 1, ["a"], request_fingerprint="test-request") == _tree() + assert ( + ClusterBatchCache(str(tmp_path)).lookup( + "__root__", 1, ["a"], request_fingerprint="test-request" + ) + == _tree() + ) + + +def test_cached_components_must_belong_to_the_recorded_batch(tmp_path): + cache = ClusterBatchCache(str(tmp_path)) + cache.store("__root__", 1, ["a"], _tree(), request_fingerprint="test-request") + path = Path(clustering_batch_cache_path(str(tmp_path))) + data = json.loads(path.read_text()) + entry = next(iter(data["entries"].values())) + entry["partial_tree"]["Module"]["components"].append("outside") + path.write_text(json.dumps(data)) + assert ( + ClusterBatchCache(str(tmp_path)).lookup( + "__root__", 1, ["a"], request_fingerprint="test-request" + ) + is None + ) + + +def test_collisions_do_not_pollute_saved_batches_or_changed_input(tmp_path): + ids, components, config = _inputs(tmp_path, ("alpha", "beta", "gamma", "omega")) + + def first(prompt): + batch = _prompt_ids(prompt) + if batch[0].startswith("omega/"): + raise RuntimeError("network failed") + name = "A" if batch[0].startswith(("alpha/", "beta/")) else "Gamma" + return _response(name, batch) + + with pytest.raises(RuntimeError, match="network failed"): + cluster_modules(ids, components, config, completer=first) + cache = ClusterBatchCache(config.docs_dir) + prompt = format_cluster_prompt(format_potential_core_components(ids[:2], components)[0]) + saved = cache.lookup( + "__root__", 1, ids[:2], request_fingerprint=clustering_request_fingerprint(config, prompt) + ) + assert saved["A"]["components"] == ids[:2] + + # Adding a file changes beta's batches, while alpha's cache key still matches. + cid = "beta/f-1.py::New" + ids.append(cid) + components[cid] = Node( + id=cid, + name="New", + component_type="class", + file_path="beta/f-1.py", + relative_path="beta/f-1.py", + source_code="code", + ) + + def retry(prompt): + batch = _prompt_ids(prompt) + name = "A" if batch[0].startswith("alpha/") else "Z" + batch[0].split("/")[0] + return _response(name, batch) + + resumed = cluster_modules(ids, components, config, completer=retry) + config.cluster_batch_cache = False + fresh = cluster_modules(ids, components, config, completer=retry) + assert resumed["A"]["components"] == fresh["A"]["components"] == ids[:2] + + +def test_pre_fix_cache_versions_are_not_reused(tmp_path): + cache = ClusterBatchCache(str(tmp_path)) + cache.store("__root__", 1, ["a"], _tree(), request_fingerprint="test-request") + path = Path(clustering_batch_cache_path(str(tmp_path))) + data = json.loads(path.read_text()) + data["version"] = CACHE_VERSION - 1 + path.write_text(json.dumps(data)) + assert ( + ClusterBatchCache(str(tmp_path)).lookup( + "__root__", 1, ["a"], request_fingerprint="test-request" + ) + is None + ) + + +@pytest.mark.parametrize( + "failed_response", + [ + None, + "", + "missing tags", + "{broken", + "[]", + "{'Unknown': {'components': ['missing::ID']}}", + ], +) +def test_retry_requeries_failed_responses_but_reuses_successes(tmp_path, failed_response): + ids, components, config = _inputs(tmp_path) + + def first(prompt): + batch = _prompt_ids(prompt) + if batch[0].startswith("alpha/"): + return failed_response + if batch[0].startswith("gamma/"): + raise RuntimeError("network failed") + return _response("Beta", batch) + + with pytest.raises(RuntimeError, match="network failed"): + cluster_modules(ids, components, config, completer=first) + cache = ClusterBatchCache(config.docs_dir) + assert cache.entry_count() == 1 + calls = [] + + def retry(prompt): + batch = _prompt_ids(prompt) + calls.append(batch) + return _response(batch[0].split("/")[0].title(), batch) + + tree = cluster_modules(ids, components, config, completer=retry) + assert calls == [ids[:2], ids[4:6]] + assert set(tree) == {"Alpha", "Beta", "Gamma"} + + +def test_structural_fallback_is_still_available_without_a_later_failure(tmp_path): + ids, components, config = _inputs(tmp_path) + tree = cluster_modules(ids, components, config, completer=lambda p: None) + assert {cid for info in tree.values() for cid in info["components"]} == set(ids) + assert not Path(clustering_batch_cache_path(config.docs_dir)).exists() + + +@pytest.mark.parametrize( + ("attribute", "value"), + [ + ("cluster_model", "new-model"), + ("main_model", "new-main"), + ("provider", "codex"), + ("llm_base_url", "http://another-provider"), + ("azure_deployment", "new-deployment"), + ("api_version", "new-version"), + ("aws_region", "eu-west-1"), + ("google_project", "new-project"), + ("google_location", "us"), + ("max_tokens", 64000), + ("repo_path", "/another/repo"), + ], +) +def test_changed_request_context_invalidates_successful_batches(tmp_path, attribute, value): + ids, components, config = _inputs(tmp_path) + + def first(prompt): + batch = _prompt_ids(prompt) + if not batch[0].startswith("alpha/"): + raise RuntimeError("network failed") + return _response("OldAlpha", batch) + + with pytest.raises(RuntimeError, match="network failed"): + cluster_modules(ids, components, config, completer=first) + setattr(config, attribute, value) + calls = [] + + def retry(prompt): + batch = _prompt_ids(prompt) + calls.append(batch) + return _response(batch[0].split("/")[0].title(), batch) + + tree = cluster_modules(ids, components, config, completer=retry) + assert calls == [ids[:2], ids[2:4], ids[4:6]] + assert "OldAlpha" not in tree + + +def test_main_model_change_invalidates_default_cluster_model(tmp_path): + _, _, config = _inputs(tmp_path) + config.provider = "codex" + config.cluster_model = "" + before = clustering_request_fingerprint(config, "same prompt") + config.main_model = "luna" + assert clustering_request_fingerprint(config, "same prompt") != before + + +def test_changed_recursive_prompt_is_not_reused(tmp_path): + ids, components, config = _inputs(tmp_path) + + def outer(sibling): + return { + "Parent": {"components": ids, "children": {}}, + sibling: {"components": [], "children": {}}, + } + + def first(prompt): + batch = _prompt_ids(prompt) + if not batch[0].startswith("alpha/"): + raise RuntimeError("network failed") + return _response("OldAlpha", batch) + + with pytest.raises(RuntimeError, match="network failed"): + cluster_modules( + ids, components, config, outer("OldSibling"), "Parent", ["Parent"], completer=first + ) + calls = [] + + def retry(prompt): + assert "NewSibling" in prompt + batch = _prompt_ids(prompt) + calls.append(batch) + return _response(batch[0].split("/")[0].title(), batch) + + tree = cluster_modules( + ids, components, config, outer("NewSibling"), "Parent", ["Parent"], completer=retry + ) + assert calls == [ids[:2], ids[2:4], ids[4:6]] + assert "OldAlpha" not in tree + + +def test_fingerprints_do_not_persist_prompts_or_credentials(tmp_path): + _, _, config = _inputs(tmp_path) + prompt = "raw-prompt-sentinel" + config.llm_api_key = "private-key-sentinel" + config.llm_base_url = "https://private-user:private-pass@example.invalid" + fingerprint = clustering_request_fingerprint(config, prompt) + cache = ClusterBatchCache(config.docs_dir) + cache.store("__root__", 1, ["a"], _tree(), request_fingerprint=fingerprint) + serialized = Path(clustering_batch_cache_path(config.docs_dir)).read_text() + assert fingerprint in serialized + for secret in (prompt, config.llm_api_key, config.llm_base_url, "private-pass"): + assert secret not in serialized + config.llm_api_key = "rotated-key" + assert clustering_request_fingerprint(config, prompt) == fingerprint + + +def test_vertex_environment_routing_is_fingerprinted(tmp_path, monkeypatch): + _, _, config = _inputs(tmp_path) + config.provider = "google-vertex" + monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "first-project") + monkeypatch.setenv("GOOGLE_CLOUD_LOCATION", "global") + before = clustering_request_fingerprint(config, "prompt") + monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "second-project") + assert clustering_request_fingerprint(config, "prompt") != before + monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "first-project") + monkeypatch.setenv("GOOGLE_CLOUD_LOCATION", "us") + assert clustering_request_fingerprint(config, "prompt") != before + + +def test_request_fingerprint_and_component_set_must_match(tmp_path): + cache = ClusterBatchCache(str(tmp_path)) + cache.store("__root__", 1, ["a", "b"], _tree(), request_fingerprint="request-one") + assert cache.lookup("__root__", 1, ["b", "a"], request_fingerprint="request-one") == _tree() + assert cache.lookup("__root__", 1, ["a", "b"], request_fingerprint="request-two") is None + assert cache.lookup("__root__", 1, ["a", "c"], request_fingerprint="request-one") is None + + +def test_scoped_cleanup_preserves_failed_and_unrelated_scopes(tmp_path): + cache = ClusterBatchCache(str(tmp_path)) + scopes = ["core", "core/child", "core_extra", "failed", "failed/child", "__root__"] + for scope in scopes: + cache.store(scope, 1, ["a"], _tree(), request_fingerprint="test-request") + delete_clustering_batch_cache(str(tmp_path), scopes=["core"]) + resumed = ClusterBatchCache(str(tmp_path)) + assert resumed.entry_count() == 4 + for scope in scopes: + result = resumed.lookup(scope, 1, ["a"], request_fingerprint="test-request") + assert (result is None) == (scope in {"core", "core/child"}) + delete_clustering_batch_cache(str(tmp_path), scopes=scopes[2:]) + assert not Path(clustering_batch_cache_path(str(tmp_path))).exists() + + +def test_empty_scope_cleanup_keeps_checkpoints(tmp_path): + cache = ClusterBatchCache(str(tmp_path)) + cache.store("failed", 1, ["a"], _tree(), request_fingerprint="test-request") + delete_clustering_batch_cache(str(tmp_path), scopes=[]) + assert ClusterBatchCache(str(tmp_path)).entry_count() == 1 + + +def test_failed_scoped_cleanup_preserves_disk_checkpoints(tmp_path, monkeypatch): + cache = ClusterBatchCache(str(tmp_path)) + for scope in ["core", "failed"]: + cache.store(scope, 1, ["a"], _tree(), request_fingerprint="test-request") + + def denied(*args): + raise PermissionError("replace denied") + + monkeypatch.setattr("codewiki.src.be.cluster_batch_cache.os.replace", denied) + delete_clustering_batch_cache(str(tmp_path), scopes=["core"]) + assert ClusterBatchCache(str(tmp_path)).entry_count() == 2 + + +@pytest.mark.parametrize("failed_save", [None, "first_module_tree.json", "module_tree.json"]) +def test_full_build_cleanup_follows_both_tree_writes(tmp_path, monkeypatch, failed_save): + ids, components, config = _inputs(tmp_path) + cache = ClusterBatchCache(config.docs_dir) + cache.store("__root__", 1, ["a"], _tree(), request_fingerprint="test-request") + cache_path = Path(clustering_batch_cache_path(config.docs_dir)) + tree = { + name: {"components": ids[i : i + 2], "children": {}} + for name, i in [("Alpha", 0), ("Beta", 2), ("Gamma", 4)] + } + generator = DocumentationGenerator(config, backend=object()) + monkeypatch.setattr( + generator.graph_builder, "build_dependency_graph", lambda: (components, ids) + ) + module = importlib.import_module("codewiki.src.be.documentation_generator") + monkeypatch.setattr(module, "cluster_modules", lambda *args, **kwargs: tree) + + async def generate_pages(*args): + for name in ["overview", *tree]: + (Path(config.docs_dir) / (name + ".md")).write_text( + f"# {name}\n\nSmoke documentation.\n" + ) + return config.docs_dir + + monkeypatch.setattr(generator, "generate_module_documentation", generate_pages) + original_save = file_manager.save_json + writes = [] + + def save(data, path): + filename = Path(path).name + if filename in {"first_module_tree.json", "module_tree.json"}: + assert cache_path.exists() + writes.append(filename) + if filename == failed_save: + raise PermissionError("tree save failed") + original_save(data, path) + + monkeypatch.setattr(file_manager, "save_json", save) + if failed_save: + with pytest.raises(PermissionError, match="tree save failed"): + asyncio.run(generator.run()) + assert cache_path.exists() + else: + asyncio.run(generator.run()) + assert writes == ["first_module_tree.json", "module_tree.json"] + assert not cache_path.exists() diff --git a/tests/test_cluster_partitioning.py b/tests/test_cluster_partitioning.py index 1d85ceb0..0dd743f2 100644 --- a/tests/test_cluster_partitioning.py +++ b/tests/test_cluster_partitioning.py @@ -52,7 +52,10 @@ def _config(**overrides) -> Config: cluster_model="m", min_token_per_module=1, ) - return Config.from_cli(**kwargs, **overrides) + config = Config.from_cli(**kwargs, **overrides) + # Persistent reuse is covered separately with isolated tmp_path directories. + config.cluster_batch_cache = False + return config def _max_count_fits(limit: int): diff --git a/tests/test_updater_orchestrator.py b/tests/test_updater_orchestrator.py index 7db16ed4..f5d90311 100644 --- a/tests/test_updater_orchestrator.py +++ b/tests/test_updater_orchestrator.py @@ -9,15 +9,19 @@ from pathlib import Path from types import SimpleNamespace +import pytest + from updater_toy import OAUTH, USER, graph_r1, graph_r2, node, tracked_r2, tree_r1, write_pages from codewiki.src.be.agent_tools.str_replace_editor import str_replace_editor from codewiki.src.be.backend import AgentReply from codewiki.src.be.documentation_generator import DocumentationGenerator +from codewiki.src.be.cluster_batch_cache import ClusterBatchCache, clustering_batch_cache_path from codewiki.src.be.updater.graph_store import save_graph from codewiki.src.be.updater.options import UpdateOptions from codewiki.src.be.updater.orchestrator import IncrementalUpdater from codewiki.src.be.updater.record import RECORD_FILENAME +from codewiki.src.utils import file_manager class FakeBackend: @@ -383,3 +387,76 @@ def test_ownership_closure_switched_off(tmp_path): backend = FakeBackend(route_to="api") docs, rec = _run_untracked(tmp_path / "r1", backend, UpdateOptions.from_rung(1)) assert backend.routing_prompts == [] and rec.ownership == [] and rec.active == [] + + +class BatchedReclusterBackend(FakeBackend): + def __init__(self, *, fail_second=False): + super().__init__() + self.cluster_calls = 0 + self.fail_second = fail_second + + def complete(self, prompt, *, model=None, system_prompt=None): + if "" not in prompt: + return super().complete(prompt, model=model, system_prompt=system_prompt) + self.cluster_calls += 1 + if self.fail_second and self.cluster_calls == 2: + raise RuntimeError("recluster interrupted") + section = prompt.split("")[1].split( + "" + )[0] + ids = [line.strip() for line in section.splitlines() if line.startswith("\t")] + tree = {f"Cluster{self.cluster_calls}": {"components": ids}} + return f"{tree!r}" + + +def _batched_update_setup(tmp_path): + docs, config, prev = _setup(tmp_path) + config.min_token_per_module = 1 + config.max_token_per_module = 1 + config.max_leaf_nodes_per_cluster = 2 + config.max_tokens = 32768 + opts = UpdateOptions(tau_full=2.0, tau_tree=100.0, tau_grow=0.01, use_stale_scan=False) + return docs, config, prev, opts + + +def test_incremental_recluster_cleans_only_committed_scopes(tmp_path): + docs, config, prev, opts = _batched_update_setup(tmp_path) + cache = ClusterBatchCache(str(docs)) + cache.store( + "failed", 1, ["a"], {"A": {"components": ["a"]}}, request_fingerprint="test-request" + ) + backend = BatchedReclusterBackend() + rec = _run(docs, config, prev, backend, opts) + assert rec.outcome == "incremental" + assert rec.reclustered == [["core"]] + assert not rec.errors + assert backend.cluster_calls >= 2 + resumed = ClusterBatchCache(str(docs)) + assert resumed.scope_entry_count("core") == 0 + assert resumed.scope_entry_count("failed") == resumed.entry_count() == 1 + assert (docs / "module_tree.json").exists() + + +def test_failed_incremental_recluster_keeps_successful_batch_checkpoints(tmp_path): + docs, config, prev, opts = _batched_update_setup(tmp_path) + rec = _run(docs, config, prev, BatchedReclusterBackend(fail_second=True), opts) + assert rec.outcome == "incremental" + assert any("recluster interrupted" in error for error in rec.errors) + assert rec.reclustered == [] + assert ClusterBatchCache(str(docs)).scope_entry_count("core") == 1 + + +def test_incremental_tree_save_failure_retains_checkpoints(tmp_path, monkeypatch): + docs, config, prev, opts = _batched_update_setup(tmp_path) + original_save = file_manager.save_json + + def fail_tree_save(data, path): + if Path(path).name == "module_tree.json": + raise PermissionError("tree save failed") + original_save(data, path) + + monkeypatch.setattr(file_manager, "save_json", fail_tree_save) + with pytest.raises(PermissionError, match="tree save failed"): + _run(docs, config, prev, BatchedReclusterBackend(), opts) + assert Path(clustering_batch_cache_path(str(docs))).exists() + assert ClusterBatchCache(str(docs)).scope_entry_count("core") >= 2