Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ Thumbs.db
!tests/test_updater_write_guard.py
!tests/test_updater_orchestrator.py
!tests/test_usage_ledger.py
!tests/test_generation_metadata.py
!tests/test_pricing.py
!tests/smoke_test_costs.py

Expand Down
17 changes: 14 additions & 3 deletions codewiki/cli/adapters/doc_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
# Import backend modules
from codewiki.src.be.documentation_generator import DocumentationGenerator
from codewiki.src.be.doc_layout import list_doc_files
from codewiki.src.be.generation_metadata import GenerationArtifacts, save_metadata
from codewiki.src.config import DEFAULT_LAYOUT, DEFAULT_MAX_TOKEN_PER_LEAF_MODULE
from codewiki.src.config import Config as BackendConfig
from codewiki.src.config import set_cli_context
Expand Down Expand Up @@ -138,6 +139,7 @@ def generate(self) -> DocumentationJob:
Raises:
APIError: If LLM API call fails
"""
self._generation_artifacts = GenerationArtifacts(self.output_dir)
self.job.start()

try:
Expand Down Expand Up @@ -177,6 +179,8 @@ def generate(self) -> DocumentationJob:
layout=self.config.get("layout", DEFAULT_LAYOUT),
)

self._backend_config = backend_config

# Run backend documentation generation
asyncio.run(self._run_backend_generation(backend_config))
self._record_usage_statistics()
Expand All @@ -197,6 +201,10 @@ def generate(self) -> DocumentationJob:

return self.job

except (KeyboardInterrupt, asyncio.CancelledError):
self.job.fail("Generation interrupted")
self._finish_usage(incomplete=True)
raise
except APIError as e:
self.job.fail(str(e))
self._finish_usage(incomplete=True)
Expand Down Expand Up @@ -520,6 +528,7 @@ def _move_docs_aside(self) -> None:
n += 1
shutil.move(str(self.output_dir), str(target))
self.output_dir.mkdir(parents=True, exist_ok=True)
self._generation_artifacts = GenerationArtifacts(self.output_dir)
self._preserved_docs_dir = str(target)
record = getattr(self, "_last_update_record", None)
if record is not None:
Expand Down Expand Up @@ -664,6 +673,10 @@ def _finish_usage(self, incomplete: bool = False):
metadata = {}
if not isinstance(metadata, dict):
metadata = {}
artifacts = getattr(self, "_generation_artifacts", None)
backend_config = getattr(self, "_backend_config", None)
if artifacts is not None and backend_config is not None:
artifacts.add_generation_info(metadata, backend_config)
metadata["usage"] = report
record = getattr(self, "_last_update_record", None)
if record is not None:
Expand All @@ -680,9 +693,7 @@ def _finish_usage(self, incomplete: bool = False):
else:
history.append(summary)
metadata.update(last_update=summary, update_history=history)
from codewiki.cli.utils.fs import safe_write

safe_write(path, json.dumps(metadata, indent=2, ensure_ascii=False))
save_metadata(path, metadata)
except Exception:
logging.getLogger(__name__).warning("Could not save the run usage report")
if incomplete:
Expand Down
37 changes: 27 additions & 10 deletions codewiki/cli/commands/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,30 @@ def parse_patterns(patterns_str: str) -> list[str]:
return [p.strip() for p in patterns_str.split(",") if p.strip()]


def _read_stored_language(output_dir: Path) -> str | None:
"""Language recorded in metadata.json by the previous generation (None = English)."""
def _read_stored_language_state(output_dir: Path) -> tuple[bool, str | None]:
"""Return ``(known, language)`` recorded in ``metadata.json``.

``known`` is False when ``metadata.json`` has no ``generation_info`` or no
``language`` key (e.g., legacy docs or an incomplete run that only
persisted usage stats), so ``--update`` does not misread an unknown
language as English.
"""
import json

try:
metadata = json.loads((output_dir / "metadata.json").read_text(encoding="utf-8"))
return (metadata.get("generation_info") or {}).get("language")
info = metadata.get("generation_info")
if not isinstance(info, dict) or "language" not in info:
return False, None
return True, info.get("language")
except (OSError, json.JSONDecodeError, AttributeError):
return None
return False, None


def _read_stored_language(output_dir: Path) -> str | None:
"""Language recorded in metadata.json by the previous generation (None = English)."""
_, stored = _read_stored_language_state(output_dir)
return stored


def _detect_changed_files(
Expand Down Expand Up @@ -644,12 +659,14 @@ def generate_command(
# Incremental update: detect changed files and selectively regenerate
changed_files = None
update_language = None
stored_language_known = False
updating_existing_docs = update and (output_dir / "metadata.json").exists()
if updating_existing_docs:
# Keep updated pages in the language of the existing docs
stored_language_known, stored_language = _read_stored_language_state(output_dir)
try:
update_language = resolve_update_language(
_read_stored_language(output_dir), language
stored_language, language, known=stored_language_known
)
except ValueError as e:
raise ConfigurationError(str(e)) from None
Expand Down Expand Up @@ -802,16 +819,16 @@ def generate_command(
elif config.agent_instructions and not config.agent_instructions.is_empty():
agent_instructions_dict = config.agent_instructions.to_dict()

if updating_existing_docs:
# The stored language overrides the saved `config agent --language` default
if updating_existing_docs and (stored_language_known or update_language is not None):
# The stored language (or explicit --language when metadata has none)
# overrides the saved `config agent --language` default
agent_instructions_dict = dict(agent_instructions_dict or {})
agent_instructions_dict.pop("language", None)
if update_language:
agent_instructions_dict["language"] = update_language
if verbose:
logger.debug(
f"Output language (from metadata.json): {update_language or 'English'}"
)
source = "metadata.json" if stored_language_known else "--language"
logger.debug(f"Output language (from {source}): {update_language or 'English'}")

# Create generator
# Get commit_id early so it can be stored in metadata.json for --update support
Expand Down
9 changes: 7 additions & 2 deletions codewiki/src/be/doc_layout.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,12 +124,17 @@ def read_layout(docs_dir: str) -> str:
"""Layout recorded in ``metadata.json``.

Docs without a recorded layout predate hierarchical output, so they are
flat.
flat—unless nested module pages already exist on disk (e.g., an incomplete
run whose metadata.json only persisted usage stats).
"""
metadata = _load_json(os.path.join(docs_dir, METADATA_FILENAME))
info = metadata.get("generation_info") if isinstance(metadata, dict) else None
layout = info.get("layout") if isinstance(info, dict) else None
return layout if layout in LAYOUTS else LAYOUT_FLAT
if layout in LAYOUTS:
return layout
if any("/" in rel for rel in list_doc_files(docs_dir).values()):
return LAYOUT_HIERARCHICAL
return LAYOUT_FLAT


def docs_layout(docs_dir: str) -> str:
Expand Down
8 changes: 6 additions & 2 deletions codewiki/src/be/documentation_generator.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import asyncio
import json
import logging
import os
Expand All @@ -23,6 +24,7 @@
organize_docs,
relative_link,
)
from codewiki.src.be.generation_metadata import GenerationArtifacts, save_metadata
from codewiki.src.be.utils import validate_mermaid_content
from codewiki.src.be.overview_support import (
bounded_overview_prompt,
Expand Down Expand Up @@ -477,6 +479,7 @@ async def generate_parent_module_docs(

async def run(self) -> None:
"""Run the complete documentation generation process using dynamic programming."""
artifacts = GenerationArtifacts(self.config.docs_dir)
try:
# Build dependency graph
components, leaf_nodes = self.graph_builder.build_dependency_graph()
Expand Down Expand Up @@ -578,7 +581,7 @@ async def run(self) -> None:
logger.debug("Processing order: leaf modules → parent modules → repository overview")
logger.debug(f"Documentation saved to: {working_dir}")

except Exception as e:
except (Exception, KeyboardInterrupt, asyncio.CancelledError) as e:
logger.error(f"Documentation generation failed: {e!s}")
logger.error(f"Traceback: {traceback.format_exc()}")
self.usage_ledger.incomplete = True
Expand All @@ -587,9 +590,10 @@ async def run(self) -> None:
metadata = file_manager.load_json(path) if os.path.exists(path) else {}
if not isinstance(metadata, dict):
metadata = {}
artifacts.add_generation_info(metadata, self.config)
metadata["usage"] = self.usage_ledger.summary()
file_manager.ensure_directory(self.config.docs_dir)
file_manager.save_json(metadata, path)
save_metadata(path, metadata)
except Exception:
logger.warning("Could not save the incomplete run usage report")
raise
71 changes: 71 additions & 0 deletions codewiki/src/be/generation_metadata.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
"""Generation context must describe artifacts produced by the current run."""

from __future__ import annotations

import hashlib
import json
import os
from pathlib import Path
import tempfile
from typing import Any

from codewiki.src.be.doc_layout import config_layout, list_doc_files
from codewiki.src.config import MODULE_TREE_FILENAME


class GenerationArtifacts:
"""Remember existing pages so a failed attempt cannot relabel untouched docs."""

def __init__(self, docs_dir: str | Path):
self.docs_dir = Path(docs_dir)
self.pages = self._pages()
self.had_tree = (self.docs_dir / MODULE_TREE_FILENAME).exists()

def _pages(self) -> dict[str, bytes | None]:
pages = {}
for rel in list_doc_files(str(self.docs_dir)).values():
try:
pages[rel] = hashlib.sha256((self.docs_dir / rel).read_bytes()).digest()
except OSError:
# An unreadable page is not evidence of a write by this run.
pages[rel] = None
return pages

def produced_artifacts(self) -> bool:
if not self.pages and not self.had_tree and (self.docs_dir / MODULE_TREE_FILENAME).exists():
return True
return any(
digest is not None
and (
rel not in self.pages or (self.pages[rel] is not None and self.pages[rel] != digest)
)
for rel, digest in self._pages().items()
)

def add_generation_info(self, metadata: dict[str, Any], config: Any) -> None:
if not self.produced_artifacts():
return
info = metadata.get("generation_info")
if not isinstance(info, dict):
info = {}
# A recorded null language means English; only absent keys are unknown.
info.setdefault("language", getattr(config, "language", None))
info.setdefault("layout", config_layout(config))
metadata["generation_info"] = info


def save_metadata(path: str | Path, metadata: dict[str, Any]) -> None:
"""Replace metadata atomically, retaining the previous file on write failure."""
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
temporary = None
try:
with tempfile.NamedTemporaryFile(
mode="w", encoding="utf-8", dir=path.parent, prefix=".metadata-", delete=False
) as stream:
temporary = Path(stream.name)
json.dump(metadata, stream, indent=2, ensure_ascii=False)
os.replace(temporary, path)
finally:
if temporary is not None:
temporary.unlink(missing_ok=True)
10 changes: 8 additions & 2 deletions codewiki/src/language.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,14 +84,20 @@ def format_language_directive(language: str | None) -> str:
return LANGUAGE_DIRECTIVE.format(language=name) if name else ""


def resolve_update_language(stored: str | None, requested: str | None) -> str | None:
def resolve_update_language(
stored: str | None, requested: str | None, *, known: bool = True
) -> str | None:
"""
Language for an incremental update of docs generated in `stored`.

The stored language wins over any saved default, so updated pages match
the existing ones. An explicit `requested` language that differs raises
ValueError: switching language needs a full regeneration.
ValueError: switching language needs a full regeneration. When `known` is
False (the previous metadata did not record a language), any explicit
`requested` language is accepted rather than assumed to conflict with English.
"""
if not known:
return normalize_language(requested) if requested is not None else None
stored_name = normalize_language(stored)
if requested is not None and normalize_language(requested) != stored_name:
raise ValueError(
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ dependencies = [
"colorama>=0.4.6",
"logfire>=4.1.0",
"coding-agent-wrapper @ git+https://github.com/anhnh2002/caw@main",
"mcp>=1.0.0"
"mcp>=1.30.0,<2"
]

[external]
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ loguru==0.7.3
markdown-it-py==3.0.0
MarkupSafe==3.0.2
matplotlib-inline==0.1.7
mcp==1.12.3
mcp==1.30.0
mdurl==0.1.2
mermaid-parser-py==0.0.2
mermaid-py==0.8.0
Expand Down
7 changes: 7 additions & 0 deletions tests/test_doc_layout.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,13 @@ def test_read_layout_defaults_to_flat_for_old_docs(tmp_path):
assert L.read_layout(str(tmp_path)) == "hierarchical"


def test_read_layout_detects_nested_docs_when_metadata_lacks_generation_info(tmp_path):
(tmp_path / "module_tree.json").write_text(json.dumps(TREE))
_write(tmp_path, "auth/login.md")
(tmp_path / "metadata.json").write_text(json.dumps({"usage": {"incomplete": True}}))
assert L.read_layout(str(tmp_path)) == "hierarchical"


def test_docs_layout_defaults_to_hierarchical_for_new_docs(tmp_path):
assert L.docs_layout(str(tmp_path)) == "hierarchical"

Expand Down
Loading
Loading