diff --git a/codewiki/cli/adapters/doc_generator.py b/codewiki/cli/adapters/doc_generator.py
index 2a012d3a..dedaf735 100644
--- a/codewiki/cli/adapters/doc_generator.py
+++ b/codewiki/cli/adapters/doc_generator.py
@@ -365,6 +365,9 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
validate_module_file_coverage(module_tree, components)
file_manager.save_json(module_tree, first_module_tree_path)
file_manager.save_json(module_tree, module_tree_path)
+ from codewiki.src.be.cluster_batch_cache import delete_clustering_batch_cache
+
+ delete_clustering_batch_cache(working_dir)
self.job.module_count = len(module_tree)
diff --git a/codewiki/src/be/cluster_batch_cache.py b/codewiki/src/be/cluster_batch_cache.py
new file mode 100644
index 00000000..7fcd54bb
--- /dev/null
+++ b/codewiki/src/be/cluster_batch_cache.py
@@ -0,0 +1,271 @@
+"""Disk cache for per-batch LLM clustering partial module trees."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import os
+import tempfile
+from copy import deepcopy
+from typing import Any
+
+from codewiki.src.config import CLUSTERING_BATCH_CACHE_FILENAME, Config
+from codewiki.src.utils import file_manager
+
+logger = logging.getLogger(__name__)
+
+CACHE_VERSION = 4
+
+
+def clustering_cache_scope(module_path: list[str]) -> str:
+ """Stable scope id for one ``cluster_modules`` invocation."""
+ if not module_path:
+ return "__root__"
+ return "/".join(module_path)
+
+
+def clustering_request_fingerprint(config: Config, prompt: str) -> str:
+ """Identify the rendered request and configured routing without persisting secrets."""
+ routing = {
+ name: getattr(config, name, None)
+ for name in (
+ "provider",
+ "cluster_model",
+ "main_model",
+ "llm_base_url",
+ "azure_deployment",
+ "api_version",
+ "aws_region",
+ "google_project",
+ "google_location",
+ "max_tokens",
+ )
+ }
+ if routing["provider"] == "google-vertex":
+ routing["google_project"] = routing["google_project"] or os.getenv(
+ "GOOGLE_CLOUD_PROJECT", ""
+ )
+ routing["google_location"] = (
+ routing["google_location"] or os.getenv("GOOGLE_CLOUD_LOCATION") or "global"
+ )
+ context = {
+ "repo": os.path.realpath(config.repo_path),
+ "routing": routing,
+ "prompt": prompt,
+ }
+ canonical = json.dumps(context, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
+ return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def batch_entry_key(
+ scope: str, batch_index: int, node_ids: list[str], request_fingerprint: str
+) -> str:
+ """Scope, batch index, component set, and exact request identity."""
+ canonical = json.dumps(sorted(node_ids), ensure_ascii=False, separators=(",", ":"))
+ digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:16]
+ return f"{scope}|{batch_index}|{digest}|{request_fingerprint}"
+
+
+def clustering_batch_cache_path(docs_dir: str) -> str:
+ return os.path.join(os.path.abspath(docs_dir), "temp", CLUSTERING_BATCH_CACHE_FILENAME)
+
+
+def delete_clustering_batch_cache(docs_dir: str, *, scopes: list[str] | None = None) -> None:
+ """Remove committed scopes and descendants, or the whole cache after a full build."""
+ if not docs_dir:
+ return
+ if scopes is not None:
+ if scopes:
+ ClusterBatchCache(docs_dir).delete_scopes(scopes)
+ return
+ _delete_cache_file(clustering_batch_cache_path(docs_dir))
+
+
+def _delete_cache_file(path: str) -> None:
+ try:
+ os.remove(path)
+ except FileNotFoundError:
+ return
+ except OSError as exc:
+ logger.warning("Could not remove clustering batch cache at %s: %s", path, exc)
+ return
+ logger.info("Removed clustering batch cache at %s", path)
+
+
+def _valid_entry(entry: Any) -> bool:
+ if not isinstance(entry, dict):
+ return False
+ ids = entry.get("node_ids")
+ partial = entry.get("partial_tree")
+ return (
+ isinstance(entry.get("scope"), str)
+ and isinstance(entry.get("request_fingerprint"), str)
+ and bool(entry["request_fingerprint"])
+ and isinstance(entry.get("batch_index"), int)
+ and not isinstance(entry["batch_index"], bool)
+ and entry["batch_index"] > 0
+ and isinstance(ids, list)
+ and all(isinstance(cid, str) for cid in ids)
+ and isinstance(partial, dict)
+ and bool(partial)
+ and all(
+ isinstance(name, str)
+ and isinstance(info, dict)
+ and isinstance(info.get("path", ""), str)
+ and isinstance(info.get("components"), list)
+ and all(isinstance(cid, str) and cid in ids for cid in info["components"])
+ for name, info in partial.items()
+ )
+ )
+
+
+class ClusterBatchCache:
+ """Read/write partial clustering results keyed by scope, batch index, and node set."""
+
+ def __init__(self, docs_dir: str, *, enabled: bool = True) -> None:
+ self._enabled = enabled and bool(docs_dir)
+ self._path = clustering_batch_cache_path(docs_dir) if self._enabled else ""
+ self._data: dict[str, Any] = {"version": CACHE_VERSION, "entries": {}}
+ if self._enabled:
+ self._load()
+
+ def _load(self) -> None:
+ try:
+ raw = file_manager.load_json(self._path)
+ except (OSError, ValueError, UnicodeError) as exc:
+ logger.warning("Ignoring unreadable clustering batch cache at %s: %s", self._path, exc)
+ return
+ if not isinstance(raw, dict) or raw.get("version") != CACHE_VERSION:
+ return
+ entries = raw.get("entries")
+ if isinstance(entries, dict):
+ entries = {key: entry for key, entry in entries.items() if _valid_entry(entry)}
+ self._data = {"version": CACHE_VERSION, "entries": entries}
+ logger.info(
+ "Loaded clustering batch cache: %d entries from %s",
+ len(entries),
+ self._path,
+ )
+
+ def _save(self) -> bool:
+ """Publish a complete checkpoint while keeping the previous one on interruption."""
+ temporary_path = None
+ try:
+ directory = os.path.dirname(self._path)
+ file_manager.ensure_directory(directory)
+ with tempfile.NamedTemporaryFile(
+ mode="w", encoding="utf-8", dir=directory, prefix=".clustering-cache-", delete=False
+ ) as stream:
+ temporary_path = stream.name
+ json.dump(self._data, stream, indent=4, ensure_ascii=False)
+ stream.flush()
+ os.fsync(stream.fileno())
+ os.replace(temporary_path, self._path)
+ return True
+ except OSError as exc:
+ logger.warning("Could not save clustering batch cache at %s: %s", self._path, exc)
+ return False
+ finally:
+ if temporary_path is not None:
+ try:
+ os.remove(temporary_path)
+ except FileNotFoundError:
+ pass
+ except OSError as exc:
+ logger.warning("Could not remove temporary clustering cache: %s", exc)
+
+ def entry_count(self) -> int:
+ entries = self._data.get("entries")
+ return len(entries) if isinstance(entries, dict) else 0
+
+ def scope_entry_count(self, scope: str) -> int:
+ entries = self._data.get("entries")
+ if not isinstance(entries, dict):
+ return 0
+ return sum(1 for entry in entries.values() if entry.get("scope") == scope)
+
+ def delete_scopes(self, scopes: list[str]) -> None:
+ """Keep checkpoints for failed or unrelated subtrees available for retry."""
+ if not self._enabled or not scopes:
+ return
+ selected = set(scopes)
+ prefixes = tuple(scope + "/" for scope in selected)
+ entries = self._data["entries"]
+ remaining = {
+ key: entry
+ for key, entry in entries.items()
+ if entry["scope"] not in selected and not entry["scope"].startswith(prefixes)
+ }
+ if len(remaining) == len(entries):
+ return
+ self._data["entries"] = remaining
+ if remaining:
+ self._save()
+ else:
+ _delete_cache_file(self._path)
+
+ def lookup(
+ self, scope: str, batch_index: int, node_ids: list[str], *, request_fingerprint: str
+ ) -> dict[str, Any] | None:
+ if not self._enabled:
+ return None
+ key = batch_entry_key(scope, batch_index, node_ids, request_fingerprint)
+ entry = self._data["entries"].get(key)
+ if not _valid_entry(entry):
+ return None
+ stored_ids = entry.get("node_ids")
+ if (
+ stored_ids != sorted(node_ids)
+ or entry["scope"] != scope
+ or entry["batch_index"] != batch_index
+ or entry["request_fingerprint"] != request_fingerprint
+ ):
+ return None
+ partial = entry.get("partial_tree")
+ if not isinstance(partial, dict) or not partial:
+ return None
+ logger.info(
+ "Using cached clustering result for scope %s batch %d (%d components).",
+ scope,
+ batch_index,
+ len(node_ids),
+ )
+ return deepcopy(partial)
+
+ def store(
+ self,
+ scope: str,
+ batch_index: int,
+ node_ids: list[str],
+ partial_tree: dict[str, Any],
+ *,
+ request_fingerprint: str,
+ ) -> None:
+ if not self._enabled or not partial_tree:
+ return
+ key = batch_entry_key(scope, batch_index, node_ids, request_fingerprint)
+ entry = {
+ "scope": scope,
+ "batch_index": batch_index,
+ "node_ids": sorted(node_ids),
+ "request_fingerprint": request_fingerprint,
+ "partial_tree": deepcopy(partial_tree),
+ }
+ if not _valid_entry(entry):
+ return
+ self._data["entries"][key] = entry
+ if not self._save():
+ return
+ module_names = list(partial_tree.keys())
+ logger.info(
+ "Stored clustering batch cache for scope %s batch %d (%d components, %d modules: %s); "
+ "%d total entries at %s",
+ scope,
+ batch_index,
+ len(node_ids),
+ len(module_names),
+ ", ".join(module_names[:5]) + ("..." if len(module_names) > 5 else ""),
+ self.entry_count(),
+ self._path,
+ )
diff --git a/codewiki/src/be/cluster_modules.py b/codewiki/src/be/cluster_modules.py
index 12659baf..a1ac7bdd 100644
--- a/codewiki/src/be/cluster_modules.py
+++ b/codewiki/src/be/cluster_modules.py
@@ -5,6 +5,11 @@
from collections.abc import Callable
from typing import Any
+from codewiki.src.be.cluster_batch_cache import (
+ ClusterBatchCache,
+ clustering_cache_scope,
+ clustering_request_fingerprint,
+)
from codewiki.src.be.dependency_analyzer.models.core import Node
from codewiki.src.be.llm_services import call_llm
from codewiki.src.be.module_naming import resolve_unique_name, sanitize_module_name
@@ -170,16 +175,19 @@ def _cluster_via_llm(
current_module_name: str | None,
module_label: str,
completer: Completer | None,
+ *,
+ prompt: str | None = None,
) -> dict[str, Any]:
"""Run one clustering LLM call over these nodes.
Returns {} for any empty, malformed, or non-dict response instead of
raising, so callers can fall back gracefully.
"""
- potential_core_components, _ = format_potential_core_components(leaf_nodes, components)
- prompt = format_cluster_prompt(
- potential_core_components, current_module_tree, current_module_name
- )
+ if prompt is None:
+ potential_core_components, _ = format_potential_core_components(leaf_nodes, components)
+ prompt = format_cluster_prompt(
+ potential_core_components, current_module_tree, current_module_name
+ )
if completer is not None:
response = completer(prompt)
else:
@@ -428,9 +436,37 @@ def cluster_modules(
module_label,
len(batches),
)
+ batch_cache: ClusterBatchCache | None = None
+ cache_scope = clustering_cache_scope(current_module_path)
+ if getattr(config, "cluster_batch_cache", True) and config.docs_dir:
+ batch_cache = ClusterBatchCache(config.docs_dir)
module_tree = {}
+ cache_hits = 0
+ if batch_cache is not None:
+ logger.info(
+ "Clustering batch cache for %r: %d on-disk entries (%d for this scope), %d batches to run.",
+ cache_scope,
+ batch_cache.entry_count(),
+ batch_cache.scope_entry_count(cache_scope),
+ len(batches),
+ )
for i, batch in enumerate(batches, 1):
batch_label = f"{module_label} (batch {i}/{len(batches)})"
+ potential_core_components, _ = format_potential_core_components(batch, components)
+ prompt = format_cluster_prompt(
+ potential_core_components, current_module_tree, current_module_name
+ )
+ request_fingerprint = (
+ clustering_request_fingerprint(config, prompt) if batch_cache is not None else ""
+ )
+ if batch_cache is not None:
+ cached = batch_cache.lookup(
+ cache_scope, i, batch, request_fingerprint=request_fingerprint
+ )
+ if cached is not None:
+ cache_hits += 1
+ _merge_module_trees(module_tree, cached)
+ continue
# Each batch gets the *unmodified* current_module_tree: passing the
# accumulating merge would flip format_cluster_prompt into its
# module-level variant mid-partition.
@@ -442,7 +478,14 @@ def cluster_modules(
current_module_name,
batch_label,
completer,
+ prompt=prompt,
)
+ # Only validated LLM responses are resumable. A structural fallback
+ # keeps this pass usable but must be retried after a later failure.
+ if batch_cache is not None and partial:
+ batch_cache.store(
+ cache_scope, i, batch, partial, request_fingerprint=request_fingerprint
+ )
if not partial:
name = _batch_fallback_name(batch, components, module_tree)
partial = {
@@ -460,6 +503,14 @@ def cluster_modules(
name,
)
_merge_module_trees(module_tree, partial)
+ if batch_cache is not None:
+ logger.info(
+ "Clustering batch cache for %r: %d/%d batches from cache, %d LLM batch call(s) this pass.",
+ cache_scope,
+ cache_hits,
+ len(batches),
+ len(batches) - cache_hits,
+ )
module_tree = normalize_module_partition(module_tree, candidates, components)
if len(module_tree) <= 1:
diff --git a/codewiki/src/be/documentation_generator.py b/codewiki/src/be/documentation_generator.py
index b920d10a..db2661da 100644
--- a/codewiki/src/be/documentation_generator.py
+++ b/codewiki/src/be/documentation_generator.py
@@ -545,6 +545,9 @@ async def run(self) -> None:
validate_module_file_coverage(module_tree, components)
file_manager.save_json(module_tree, first_module_tree_path)
file_manager.save_json(module_tree, module_tree_path)
+ from codewiki.src.be.cluster_batch_cache import delete_clustering_batch_cache
+
+ delete_clustering_batch_cache(working_dir)
if len(module_tree) == 0:
logger.info(
diff --git a/codewiki/src/be/updater/orchestrator.py b/codewiki/src/be/updater/orchestrator.py
index 4e149d97..46edd26f 100644
--- a/codewiki/src/be/updater/orchestrator.py
+++ b/codewiki/src/be/updater/orchestrator.py
@@ -9,6 +9,10 @@
from typing import Any
from codewiki.src.be.backend import LLMBackend
+from codewiki.src.be.cluster_batch_cache import (
+ clustering_cache_scope,
+ delete_clustering_batch_cache,
+)
from codewiki.src.be.cluster_modules import cluster_modules, get_clustering_input_token_count
from codewiki.src.be.module_scope import get_module_token_count
from codewiki.src.be.dependency_analyzer.models.core import Node
@@ -381,6 +385,10 @@ async def _run(
# Persist the repaired tree so the normal pipeline and the agents see it.
if not self.whole_repo:
file_manager.save_json(new_tree, os.path.join(self.docs_dir, MODULE_TREE_FILENAME))
+ delete_clustering_batch_cache(
+ self.docs_dir,
+ scopes=[clustering_cache_scope(path) for path in rec.reclustered],
+ )
# ---- Step 5: sequential leaf agents
order = order_active(active, new_tree, new_graph)
diff --git a/codewiki/src/config.py b/codewiki/src/config.py
index e5a32d1e..38d9d762 100644
--- a/codewiki/src/config.py
+++ b/codewiki/src/config.py
@@ -15,6 +15,7 @@
DOCS_DIR = "docs"
FIRST_MODULE_TREE_FILENAME = "first_module_tree.json"
MODULE_TREE_FILENAME = "module_tree.json"
+CLUSTERING_BATCH_CACHE_FILENAME = "clustering_batch_cache.json"
OVERVIEW_FILENAME = "overview.md"
# Docs layout: "hierarchical" mirrors the module tree in nested folders
# (``auth.md`` + ``auth/login.md``); "flat" keeps every page in the docs root
@@ -136,6 +137,8 @@ class Config:
with_prose: bool = False
# Docs layout (LAYOUT_HIERARCHICAL or LAYOUT_FLAT)
layout: str = DEFAULT_LAYOUT
+ # Persist each successful LLM clustering batch under docs/temp for resume after failures.
+ cluster_batch_cache: bool = True
def __post_init__(self):
validate_min_module_tokens(self.min_token_per_module)
diff --git a/tests/test_cluster_batch_cache.py b/tests/test_cluster_batch_cache.py
new file mode 100644
index 00000000..34cf551b
--- /dev/null
+++ b/tests/test_cluster_batch_cache.py
@@ -0,0 +1,538 @@
+"""Checkpoint and retry regressions for batched module clustering."""
+
+from __future__ import annotations
+
+import json
+import asyncio
+import importlib
+from pathlib import Path
+
+import pytest
+
+from codewiki.src.be.cluster_batch_cache import (
+ CACHE_VERSION,
+ ClusterBatchCache,
+ clustering_batch_cache_path,
+ delete_clustering_batch_cache,
+ clustering_request_fingerprint,
+)
+from codewiki.src.be.cluster_modules import cluster_modules, format_potential_core_components
+from codewiki.src.be.prompt_template import format_cluster_prompt
+from codewiki.src.be.dependency_analyzer.models.core import Node
+from codewiki.src.config import Config
+from codewiki.src.be.documentation_generator import DocumentationGenerator
+from codewiki.src.utils import file_manager
+
+
+def _tree(cid="a"):
+ return {"Module": {"components": [cid]}}
+
+
+def _inputs(tmp_path, directories=("alpha", "beta", "gamma")):
+ ids = [f"{directory}/f{i}.py::C{i}" for directory in directories for i in range(2)]
+ components = {}
+ for cid in ids:
+ path, name = cid.split("::")
+ components[cid] = Node(
+ id=cid,
+ name=name,
+ component_type="class",
+ file_path=path,
+ relative_path=path,
+ source_code="code",
+ )
+ config = Config.from_cli(
+ repo_path=str(tmp_path / "repo"),
+ output_dir=str(tmp_path / "docs"),
+ llm_base_url="http://localhost",
+ llm_api_key="unused",
+ main_model="m",
+ cluster_model="m",
+ min_token_per_module=1,
+ max_token_per_module=2,
+ max_leaf_nodes_per_cluster=2,
+ )
+ return ids, components, config
+
+
+def _prompt_ids(prompt):
+ section = prompt.split("")[1].split("")[
+ 0
+ ]
+ return [line.strip() for line in section.splitlines() if line.startswith("\t")]
+
+
+def _response(name, ids):
+ return f"{ {name: {'components': ids}}!r}"
+
+
+def test_successful_batches_resume_after_failure(tmp_path):
+ ids, components, config = _inputs(tmp_path)
+
+ def first(prompt):
+ batch = _prompt_ids(prompt)
+ if batch[0].startswith("beta/"):
+ raise RuntimeError("network failed")
+ return _response("Alpha", batch)
+
+ with pytest.raises(RuntimeError, match="network failed"):
+ cluster_modules(ids, components, config, completer=first)
+ calls = []
+
+ def retry(prompt):
+ batch = _prompt_ids(prompt)
+ calls.append(batch)
+ return _response(batch[0].split("/")[0].title(), batch)
+
+ tree = cluster_modules(ids, components, config, completer=retry)
+ assert calls == [ids[2:4], ids[4:6]]
+ assert set(tree) == {"Alpha", "Beta", "Gamma"}
+
+
+def test_interrupted_write_preserves_previous_checkpoint(tmp_path, monkeypatch):
+ cache = ClusterBatchCache(str(tmp_path))
+ cache.store("__root__", 1, ["a"], _tree(), request_fingerprint="test-request")
+
+ def interrupted(data, stream, **kwargs):
+ stream.write('{"version":')
+ stream.flush()
+ raise KeyboardInterrupt
+
+ with monkeypatch.context() as patch:
+ patch.setattr(json, "dump", interrupted)
+ with pytest.raises(KeyboardInterrupt):
+ cache.store("__root__", 2, ["b"], _tree("b"), request_fingerprint="test-request")
+ resumed = ClusterBatchCache(str(tmp_path))
+ assert resumed.lookup("__root__", 1, ["a"], request_fingerprint="test-request") == _tree()
+ assert resumed.lookup("__root__", 2, ["b"], request_fingerprint="test-request") is None
+ assert sorted(p.name for p in (tmp_path / "temp").iterdir()) == ["clustering_batch_cache.json"]
+
+
+def test_failed_replace_preserves_previous_checkpoint(tmp_path, monkeypatch, caplog):
+ cache = ClusterBatchCache(str(tmp_path))
+ cache.store("__root__", 1, ["a"], _tree(), request_fingerprint="test-request")
+
+ def denied(*args):
+ raise PermissionError("replace denied")
+
+ monkeypatch.setattr("codewiki.src.be.cluster_batch_cache.os.replace", denied)
+ cache.store("__root__", 2, ["b"], _tree("b"), request_fingerprint="test-request")
+ resumed = ClusterBatchCache(str(tmp_path))
+ assert resumed.lookup("__root__", 1, ["a"], request_fingerprint="test-request") == _tree()
+ assert resumed.lookup("__root__", 2, ["b"], request_fingerprint="test-request") is None
+ assert "replace denied" in caplog.text
+ assert len(list((tmp_path / "temp").iterdir())) == 1
+
+
+@pytest.mark.parametrize(
+ "content",
+ [
+ "",
+ '{"version":',
+ "[]",
+ "null",
+ "42",
+ '"text"',
+ json.dumps({"version": -1, "entries": {}}),
+ json.dumps({"version": CACHE_VERSION, "entries": []}),
+ ],
+)
+def test_invalid_cache_is_a_miss(tmp_path, content):
+ path = Path(clustering_batch_cache_path(str(tmp_path)))
+ path.parent.mkdir()
+ path.write_text(content)
+ cache = ClusterBatchCache(str(tmp_path))
+ assert cache.entry_count() == 0
+ assert cache.lookup("__root__", 1, ["a"], request_fingerprint="test-request") is None
+
+
+def test_malformed_entries_are_ignored_individually(tmp_path):
+ cache = ClusterBatchCache(str(tmp_path))
+ cache.store("__root__", 1, ["a"], _tree(), request_fingerprint="test-request")
+ path = Path(clustering_batch_cache_path(str(tmp_path)))
+ data = json.loads(path.read_text())
+ data["entries"].update({"null": None, "list": [], "bad": {"scope": "__root__"}})
+ path.write_text(json.dumps(data))
+ resumed = ClusterBatchCache(str(tmp_path))
+ assert resumed.entry_count() == resumed.scope_entry_count("__root__") == 1
+ assert resumed.lookup("__root__", 1, ["a"], request_fingerprint="test-request") == _tree()
+
+
+def test_unreadable_cache_is_a_miss(tmp_path, monkeypatch, caplog):
+ def denied(path):
+ raise PermissionError("read denied")
+
+ monkeypatch.setattr(file_manager, "load_json", denied)
+ assert ClusterBatchCache(str(tmp_path)).entry_count() == 0
+ assert "read denied" in caplog.text
+
+
+def test_cleanup_does_not_parse_cache(tmp_path):
+ path = Path(clustering_batch_cache_path(str(tmp_path)))
+ path.parent.mkdir()
+ path.write_text("broken JSON")
+ delete_clustering_batch_cache(str(tmp_path))
+ delete_clustering_batch_cache(str(tmp_path))
+ assert not path.exists()
+
+
+def test_cache_write_failure_does_not_abort_clustering(tmp_path, monkeypatch):
+ ids, components, config = _inputs(tmp_path)
+
+ def denied(*args):
+ raise PermissionError("cache unavailable")
+
+ monkeypatch.setattr("codewiki.src.be.cluster_batch_cache.os.replace", denied)
+ tree = cluster_modules(
+ ids, components, config, completer=lambda p: _response(_prompt_ids(p)[0], _prompt_ids(p))
+ )
+ assert len(tree) == 3
+
+
+def test_disabled_cache_never_writes(tmp_path):
+ ids, components, config = _inputs(tmp_path)
+ config.cluster_batch_cache = False
+ cluster_modules(
+ ids, components, config, completer=lambda p: _response(_prompt_ids(p)[0], _prompt_ids(p))
+ )
+ assert not Path(clustering_batch_cache_path(config.docs_dir)).exists()
+
+
+def test_store_and_lookup_do_not_share_mutable_trees(tmp_path):
+ cache = ClusterBatchCache(str(tmp_path))
+ proposed = _tree()
+ cache.store("__root__", 1, ["a"], proposed, request_fingerprint="test-request")
+ proposed["Module"]["components"].append("outside")
+ cached = cache.lookup("__root__", 1, ["a"], request_fingerprint="test-request")
+ assert cached == _tree()
+ cached["Module"]["components"].append("another")
+ # A subsequent store persists the cache's own snapshot, not either mutated view.
+ cache.store("__root__", 2, ["b"], _tree("b"), request_fingerprint="test-request")
+ assert cache.lookup("__root__", 1, ["a"], request_fingerprint="test-request") == _tree()
+ assert (
+ ClusterBatchCache(str(tmp_path)).lookup(
+ "__root__", 1, ["a"], request_fingerprint="test-request"
+ )
+ == _tree()
+ )
+
+
+def test_cached_components_must_belong_to_the_recorded_batch(tmp_path):
+ cache = ClusterBatchCache(str(tmp_path))
+ cache.store("__root__", 1, ["a"], _tree(), request_fingerprint="test-request")
+ path = Path(clustering_batch_cache_path(str(tmp_path)))
+ data = json.loads(path.read_text())
+ entry = next(iter(data["entries"].values()))
+ entry["partial_tree"]["Module"]["components"].append("outside")
+ path.write_text(json.dumps(data))
+ assert (
+ ClusterBatchCache(str(tmp_path)).lookup(
+ "__root__", 1, ["a"], request_fingerprint="test-request"
+ )
+ is None
+ )
+
+
+def test_collisions_do_not_pollute_saved_batches_or_changed_input(tmp_path):
+ ids, components, config = _inputs(tmp_path, ("alpha", "beta", "gamma", "omega"))
+
+ def first(prompt):
+ batch = _prompt_ids(prompt)
+ if batch[0].startswith("omega/"):
+ raise RuntimeError("network failed")
+ name = "A" if batch[0].startswith(("alpha/", "beta/")) else "Gamma"
+ return _response(name, batch)
+
+ with pytest.raises(RuntimeError, match="network failed"):
+ cluster_modules(ids, components, config, completer=first)
+ cache = ClusterBatchCache(config.docs_dir)
+ prompt = format_cluster_prompt(format_potential_core_components(ids[:2], components)[0])
+ saved = cache.lookup(
+ "__root__", 1, ids[:2], request_fingerprint=clustering_request_fingerprint(config, prompt)
+ )
+ assert saved["A"]["components"] == ids[:2]
+
+ # Adding a file changes beta's batches, while alpha's cache key still matches.
+ cid = "beta/f-1.py::New"
+ ids.append(cid)
+ components[cid] = Node(
+ id=cid,
+ name="New",
+ component_type="class",
+ file_path="beta/f-1.py",
+ relative_path="beta/f-1.py",
+ source_code="code",
+ )
+
+ def retry(prompt):
+ batch = _prompt_ids(prompt)
+ name = "A" if batch[0].startswith("alpha/") else "Z" + batch[0].split("/")[0]
+ return _response(name, batch)
+
+ resumed = cluster_modules(ids, components, config, completer=retry)
+ config.cluster_batch_cache = False
+ fresh = cluster_modules(ids, components, config, completer=retry)
+ assert resumed["A"]["components"] == fresh["A"]["components"] == ids[:2]
+
+
+def test_pre_fix_cache_versions_are_not_reused(tmp_path):
+ cache = ClusterBatchCache(str(tmp_path))
+ cache.store("__root__", 1, ["a"], _tree(), request_fingerprint="test-request")
+ path = Path(clustering_batch_cache_path(str(tmp_path)))
+ data = json.loads(path.read_text())
+ data["version"] = CACHE_VERSION - 1
+ path.write_text(json.dumps(data))
+ assert (
+ ClusterBatchCache(str(tmp_path)).lookup(
+ "__root__", 1, ["a"], request_fingerprint="test-request"
+ )
+ is None
+ )
+
+
+@pytest.mark.parametrize(
+ "failed_response",
+ [
+ None,
+ "",
+ "missing tags",
+ "{broken",
+ "[]",
+ "{'Unknown': {'components': ['missing::ID']}}",
+ ],
+)
+def test_retry_requeries_failed_responses_but_reuses_successes(tmp_path, failed_response):
+ ids, components, config = _inputs(tmp_path)
+
+ def first(prompt):
+ batch = _prompt_ids(prompt)
+ if batch[0].startswith("alpha/"):
+ return failed_response
+ if batch[0].startswith("gamma/"):
+ raise RuntimeError("network failed")
+ return _response("Beta", batch)
+
+ with pytest.raises(RuntimeError, match="network failed"):
+ cluster_modules(ids, components, config, completer=first)
+ cache = ClusterBatchCache(config.docs_dir)
+ assert cache.entry_count() == 1
+ calls = []
+
+ def retry(prompt):
+ batch = _prompt_ids(prompt)
+ calls.append(batch)
+ return _response(batch[0].split("/")[0].title(), batch)
+
+ tree = cluster_modules(ids, components, config, completer=retry)
+ assert calls == [ids[:2], ids[4:6]]
+ assert set(tree) == {"Alpha", "Beta", "Gamma"}
+
+
+def test_structural_fallback_is_still_available_without_a_later_failure(tmp_path):
+ ids, components, config = _inputs(tmp_path)
+ tree = cluster_modules(ids, components, config, completer=lambda p: None)
+ assert {cid for info in tree.values() for cid in info["components"]} == set(ids)
+ assert not Path(clustering_batch_cache_path(config.docs_dir)).exists()
+
+
+@pytest.mark.parametrize(
+ ("attribute", "value"),
+ [
+ ("cluster_model", "new-model"),
+ ("main_model", "new-main"),
+ ("provider", "codex"),
+ ("llm_base_url", "http://another-provider"),
+ ("azure_deployment", "new-deployment"),
+ ("api_version", "new-version"),
+ ("aws_region", "eu-west-1"),
+ ("google_project", "new-project"),
+ ("google_location", "us"),
+ ("max_tokens", 64000),
+ ("repo_path", "/another/repo"),
+ ],
+)
+def test_changed_request_context_invalidates_successful_batches(tmp_path, attribute, value):
+ ids, components, config = _inputs(tmp_path)
+
+ def first(prompt):
+ batch = _prompt_ids(prompt)
+ if not batch[0].startswith("alpha/"):
+ raise RuntimeError("network failed")
+ return _response("OldAlpha", batch)
+
+ with pytest.raises(RuntimeError, match="network failed"):
+ cluster_modules(ids, components, config, completer=first)
+ setattr(config, attribute, value)
+ calls = []
+
+ def retry(prompt):
+ batch = _prompt_ids(prompt)
+ calls.append(batch)
+ return _response(batch[0].split("/")[0].title(), batch)
+
+ tree = cluster_modules(ids, components, config, completer=retry)
+ assert calls == [ids[:2], ids[2:4], ids[4:6]]
+ assert "OldAlpha" not in tree
+
+
+def test_main_model_change_invalidates_default_cluster_model(tmp_path):
+ _, _, config = _inputs(tmp_path)
+ config.provider = "codex"
+ config.cluster_model = ""
+ before = clustering_request_fingerprint(config, "same prompt")
+ config.main_model = "luna"
+ assert clustering_request_fingerprint(config, "same prompt") != before
+
+
+def test_changed_recursive_prompt_is_not_reused(tmp_path):
+ ids, components, config = _inputs(tmp_path)
+
+ def outer(sibling):
+ return {
+ "Parent": {"components": ids, "children": {}},
+ sibling: {"components": [], "children": {}},
+ }
+
+ def first(prompt):
+ batch = _prompt_ids(prompt)
+ if not batch[0].startswith("alpha/"):
+ raise RuntimeError("network failed")
+ return _response("OldAlpha", batch)
+
+ with pytest.raises(RuntimeError, match="network failed"):
+ cluster_modules(
+ ids, components, config, outer("OldSibling"), "Parent", ["Parent"], completer=first
+ )
+ calls = []
+
+ def retry(prompt):
+ assert "NewSibling" in prompt
+ batch = _prompt_ids(prompt)
+ calls.append(batch)
+ return _response(batch[0].split("/")[0].title(), batch)
+
+ tree = cluster_modules(
+ ids, components, config, outer("NewSibling"), "Parent", ["Parent"], completer=retry
+ )
+ assert calls == [ids[:2], ids[2:4], ids[4:6]]
+ assert "OldAlpha" not in tree
+
+
+def test_fingerprints_do_not_persist_prompts_or_credentials(tmp_path):
+ _, _, config = _inputs(tmp_path)
+ prompt = "raw-prompt-sentinel"
+ config.llm_api_key = "private-key-sentinel"
+ config.llm_base_url = "https://private-user:private-pass@example.invalid"
+ fingerprint = clustering_request_fingerprint(config, prompt)
+ cache = ClusterBatchCache(config.docs_dir)
+ cache.store("__root__", 1, ["a"], _tree(), request_fingerprint=fingerprint)
+ serialized = Path(clustering_batch_cache_path(config.docs_dir)).read_text()
+ assert fingerprint in serialized
+ for secret in (prompt, config.llm_api_key, config.llm_base_url, "private-pass"):
+ assert secret not in serialized
+ config.llm_api_key = "rotated-key"
+ assert clustering_request_fingerprint(config, prompt) == fingerprint
+
+
+def test_vertex_environment_routing_is_fingerprinted(tmp_path, monkeypatch):
+ _, _, config = _inputs(tmp_path)
+ config.provider = "google-vertex"
+ monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "first-project")
+ monkeypatch.setenv("GOOGLE_CLOUD_LOCATION", "global")
+ before = clustering_request_fingerprint(config, "prompt")
+ monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "second-project")
+ assert clustering_request_fingerprint(config, "prompt") != before
+ monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "first-project")
+ monkeypatch.setenv("GOOGLE_CLOUD_LOCATION", "us")
+ assert clustering_request_fingerprint(config, "prompt") != before
+
+
+def test_request_fingerprint_and_component_set_must_match(tmp_path):
+ cache = ClusterBatchCache(str(tmp_path))
+ cache.store("__root__", 1, ["a", "b"], _tree(), request_fingerprint="request-one")
+ assert cache.lookup("__root__", 1, ["b", "a"], request_fingerprint="request-one") == _tree()
+ assert cache.lookup("__root__", 1, ["a", "b"], request_fingerprint="request-two") is None
+ assert cache.lookup("__root__", 1, ["a", "c"], request_fingerprint="request-one") is None
+
+
+def test_scoped_cleanup_preserves_failed_and_unrelated_scopes(tmp_path):
+ cache = ClusterBatchCache(str(tmp_path))
+ scopes = ["core", "core/child", "core_extra", "failed", "failed/child", "__root__"]
+ for scope in scopes:
+ cache.store(scope, 1, ["a"], _tree(), request_fingerprint="test-request")
+ delete_clustering_batch_cache(str(tmp_path), scopes=["core"])
+ resumed = ClusterBatchCache(str(tmp_path))
+ assert resumed.entry_count() == 4
+ for scope in scopes:
+ result = resumed.lookup(scope, 1, ["a"], request_fingerprint="test-request")
+ assert (result is None) == (scope in {"core", "core/child"})
+ delete_clustering_batch_cache(str(tmp_path), scopes=scopes[2:])
+ assert not Path(clustering_batch_cache_path(str(tmp_path))).exists()
+
+
+def test_empty_scope_cleanup_keeps_checkpoints(tmp_path):
+ cache = ClusterBatchCache(str(tmp_path))
+ cache.store("failed", 1, ["a"], _tree(), request_fingerprint="test-request")
+ delete_clustering_batch_cache(str(tmp_path), scopes=[])
+ assert ClusterBatchCache(str(tmp_path)).entry_count() == 1
+
+
+def test_failed_scoped_cleanup_preserves_disk_checkpoints(tmp_path, monkeypatch):
+ cache = ClusterBatchCache(str(tmp_path))
+ for scope in ["core", "failed"]:
+ cache.store(scope, 1, ["a"], _tree(), request_fingerprint="test-request")
+
+ def denied(*args):
+ raise PermissionError("replace denied")
+
+ monkeypatch.setattr("codewiki.src.be.cluster_batch_cache.os.replace", denied)
+ delete_clustering_batch_cache(str(tmp_path), scopes=["core"])
+ assert ClusterBatchCache(str(tmp_path)).entry_count() == 2
+
+
+@pytest.mark.parametrize("failed_save", [None, "first_module_tree.json", "module_tree.json"])
+def test_full_build_cleanup_follows_both_tree_writes(tmp_path, monkeypatch, failed_save):
+ ids, components, config = _inputs(tmp_path)
+ cache = ClusterBatchCache(config.docs_dir)
+ cache.store("__root__", 1, ["a"], _tree(), request_fingerprint="test-request")
+ cache_path = Path(clustering_batch_cache_path(config.docs_dir))
+ tree = {
+ name: {"components": ids[i : i + 2], "children": {}}
+ for name, i in [("Alpha", 0), ("Beta", 2), ("Gamma", 4)]
+ }
+ generator = DocumentationGenerator(config, backend=object())
+ monkeypatch.setattr(
+ generator.graph_builder, "build_dependency_graph", lambda: (components, ids)
+ )
+ module = importlib.import_module("codewiki.src.be.documentation_generator")
+ monkeypatch.setattr(module, "cluster_modules", lambda *args, **kwargs: tree)
+
+ async def generate_pages(*args):
+ for name in ["overview", *tree]:
+ (Path(config.docs_dir) / (name + ".md")).write_text(
+ f"# {name}\n\nSmoke documentation.\n"
+ )
+ return config.docs_dir
+
+ monkeypatch.setattr(generator, "generate_module_documentation", generate_pages)
+ original_save = file_manager.save_json
+ writes = []
+
+ def save(data, path):
+ filename = Path(path).name
+ if filename in {"first_module_tree.json", "module_tree.json"}:
+ assert cache_path.exists()
+ writes.append(filename)
+ if filename == failed_save:
+ raise PermissionError("tree save failed")
+ original_save(data, path)
+
+ monkeypatch.setattr(file_manager, "save_json", save)
+ if failed_save:
+ with pytest.raises(PermissionError, match="tree save failed"):
+ asyncio.run(generator.run())
+ assert cache_path.exists()
+ else:
+ asyncio.run(generator.run())
+ assert writes == ["first_module_tree.json", "module_tree.json"]
+ assert not cache_path.exists()
diff --git a/tests/test_cluster_partitioning.py b/tests/test_cluster_partitioning.py
index 1d85ceb0..0dd743f2 100644
--- a/tests/test_cluster_partitioning.py
+++ b/tests/test_cluster_partitioning.py
@@ -52,7 +52,10 @@ def _config(**overrides) -> Config:
cluster_model="m",
min_token_per_module=1,
)
- return Config.from_cli(**kwargs, **overrides)
+ config = Config.from_cli(**kwargs, **overrides)
+ # Persistent reuse is covered separately with isolated tmp_path directories.
+ config.cluster_batch_cache = False
+ return config
def _max_count_fits(limit: int):
diff --git a/tests/test_updater_orchestrator.py b/tests/test_updater_orchestrator.py
index 7db16ed4..f5d90311 100644
--- a/tests/test_updater_orchestrator.py
+++ b/tests/test_updater_orchestrator.py
@@ -9,15 +9,19 @@
from pathlib import Path
from types import SimpleNamespace
+import pytest
+
from updater_toy import OAUTH, USER, graph_r1, graph_r2, node, tracked_r2, tree_r1, write_pages
from codewiki.src.be.agent_tools.str_replace_editor import str_replace_editor
from codewiki.src.be.backend import AgentReply
from codewiki.src.be.documentation_generator import DocumentationGenerator
+from codewiki.src.be.cluster_batch_cache import ClusterBatchCache, clustering_batch_cache_path
from codewiki.src.be.updater.graph_store import save_graph
from codewiki.src.be.updater.options import UpdateOptions
from codewiki.src.be.updater.orchestrator import IncrementalUpdater
from codewiki.src.be.updater.record import RECORD_FILENAME
+from codewiki.src.utils import file_manager
class FakeBackend:
@@ -383,3 +387,76 @@ def test_ownership_closure_switched_off(tmp_path):
backend = FakeBackend(route_to="api")
docs, rec = _run_untracked(tmp_path / "r1", backend, UpdateOptions.from_rung(1))
assert backend.routing_prompts == [] and rec.ownership == [] and rec.active == []
+
+
+class BatchedReclusterBackend(FakeBackend):
+ def __init__(self, *, fail_second=False):
+ super().__init__()
+ self.cluster_calls = 0
+ self.fail_second = fail_second
+
+ def complete(self, prompt, *, model=None, system_prompt=None):
+ if "" not in prompt:
+ return super().complete(prompt, model=model, system_prompt=system_prompt)
+ self.cluster_calls += 1
+ if self.fail_second and self.cluster_calls == 2:
+ raise RuntimeError("recluster interrupted")
+ section = prompt.split("")[1].split(
+ ""
+ )[0]
+ ids = [line.strip() for line in section.splitlines() if line.startswith("\t")]
+ tree = {f"Cluster{self.cluster_calls}": {"components": ids}}
+ return f"{tree!r}"
+
+
+def _batched_update_setup(tmp_path):
+ docs, config, prev = _setup(tmp_path)
+ config.min_token_per_module = 1
+ config.max_token_per_module = 1
+ config.max_leaf_nodes_per_cluster = 2
+ config.max_tokens = 32768
+ opts = UpdateOptions(tau_full=2.0, tau_tree=100.0, tau_grow=0.01, use_stale_scan=False)
+ return docs, config, prev, opts
+
+
+def test_incremental_recluster_cleans_only_committed_scopes(tmp_path):
+ docs, config, prev, opts = _batched_update_setup(tmp_path)
+ cache = ClusterBatchCache(str(docs))
+ cache.store(
+ "failed", 1, ["a"], {"A": {"components": ["a"]}}, request_fingerprint="test-request"
+ )
+ backend = BatchedReclusterBackend()
+ rec = _run(docs, config, prev, backend, opts)
+ assert rec.outcome == "incremental"
+ assert rec.reclustered == [["core"]]
+ assert not rec.errors
+ assert backend.cluster_calls >= 2
+ resumed = ClusterBatchCache(str(docs))
+ assert resumed.scope_entry_count("core") == 0
+ assert resumed.scope_entry_count("failed") == resumed.entry_count() == 1
+ assert (docs / "module_tree.json").exists()
+
+
+def test_failed_incremental_recluster_keeps_successful_batch_checkpoints(tmp_path):
+ docs, config, prev, opts = _batched_update_setup(tmp_path)
+ rec = _run(docs, config, prev, BatchedReclusterBackend(fail_second=True), opts)
+ assert rec.outcome == "incremental"
+ assert any("recluster interrupted" in error for error in rec.errors)
+ assert rec.reclustered == []
+ assert ClusterBatchCache(str(docs)).scope_entry_count("core") == 1
+
+
+def test_incremental_tree_save_failure_retains_checkpoints(tmp_path, monkeypatch):
+ docs, config, prev, opts = _batched_update_setup(tmp_path)
+ original_save = file_manager.save_json
+
+ def fail_tree_save(data, path):
+ if Path(path).name == "module_tree.json":
+ raise PermissionError("tree save failed")
+ original_save(data, path)
+
+ monkeypatch.setattr(file_manager, "save_json", fail_tree_save)
+ with pytest.raises(PermissionError, match="tree save failed"):
+ _run(docs, config, prev, BatchedReclusterBackend(), opts)
+ assert Path(clustering_batch_cache_path(str(docs))).exists()
+ assert ClusterBatchCache(str(docs)).scope_entry_count("core") >= 2