Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ Thumbs.db
!tests/test_generation_metadata.py
!tests/test_pricing.py
!tests/smoke_test_costs.py
!tests/smoke_test_nested_checkouts.py

# Mermaid regression tests and development-only parser harness
!tests/test_mermaid_validation.py
Expand Down
119 changes: 111 additions & 8 deletions codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
detailed file tree representations with filtering capabilities.
"""

import configparser
import fnmatch
import logging
import os
Expand All @@ -23,6 +24,93 @@
logger = logging.getLogger(__name__)


class RepositoryBoundaryFilter:
"""Keep declared submodules, but do not descend into unrelated checkouts."""

def __init__(self, repo_dir: Path) -> None:
self.root = repo_dir.resolve()
self.owner = next(
(path for path in (self.root, *self.root.parents) if (path / ".git").exists()),
self.root,
)
self._submodules: dict[Path, set[Path]] = {}

@staticmethod
def _config_path(value: str) -> str:
"""Decode Git config quoting, escapes and trailing comments without Git."""
result = []
quoted = False
escaped = False
end = 0
escapes = {"n": "\n", "t": "\t", "b": "\b", '"': '"', "\\": "\\"}
for char in value:
if escaped:
if char not in escapes:
raise ValueError("Invalid escape in submodule path")
result.append(escapes[char])
end = len(result)
escaped = False
elif char == "\\":
escaped = True
elif char == '"':
quoted = not quoted
elif char in "#;" and not quoted:
break
else:
result.append(char)
if quoted or not char.isspace():
end = len(result)
if quoted or escaped:
raise ValueError("Unterminated submodule path")
return "".join(result[:end])

def _declared_submodules(self, owner: Path) -> set[Path]:
if owner in self._submodules:
return self._submodules[owner]
paths: set[Path] = set()
config_path = owner / ".gitmodules"
try:
# Never follow a declaration file outside its owning repository.
if config_path.is_symlink() or not config_path.exists():
self._submodules[owner] = paths
return paths
config = configparser.RawConfigParser(strict=False)
config.read_string(config_path.read_text(encoding="utf-8"))
for section in config.sections():
if not section.lower().startswith('submodule "') or not section.endswith('"'):
continue
value = self._config_path(config.get(section, "path", fallback=""))
if not value:
continue
relative = Path(value)
path = (owner / relative).resolve()
if relative.is_absolute() or path == owner or not path.is_relative_to(owner):
logger.warning("Skipping escaping submodule path in %s: %s", config_path, value)
continue
paths.add(path)
except (OSError, UnicodeError, ValueError, configparser.Error) as exc:
logger.warning("Could not read submodule declarations at %s: %s", config_path, exc)
paths.clear()
self._submodules[owner] = paths
return paths

def should_skip(self, path: Path) -> bool:
path = path.resolve()
if path == self.root or not (path / ".git").exists():
return False
owner = self.owner
for parent in path.parents:
if parent == self.root:
break
if (parent / ".git").exists():
owner = parent
break
if path in self._declared_submodules(owner):
return False
logger.debug("Skipping nested Git checkout: %s", path)
return True


class GitIgnoreFilter:
"""Evaluate Git ignore rules once per repository analysis.

Expand All @@ -31,8 +119,9 @@ class GitIgnoreFilter:
Plain directories fall back to pathspec-backed ``.gitignore`` evaluation.
"""

def __init__(self, repo_dir: Path) -> None:
def __init__(self, repo_dir: Path, boundaries: RepositoryBoundaryFilter | None = None) -> None:
self.repo_dir = repo_dir.resolve()
self._boundaries = boundaries or RepositoryBoundaryFilter(self.repo_dir)
self._ignored_files: set[str] = set()
self._ignored_dirs: set[str] = set()
self._ignore_all_untracked = False
Expand Down Expand Up @@ -116,12 +205,20 @@ def _load_git_ignored_paths(self) -> bool:

def _load_fallback_specs(self) -> None:
try:
candidates = sorted(
(
path
for path in self.repo_dir.rglob(".gitignore")
if path.is_file() and not path.is_symlink()
),
candidates = []
for directory, dirs, files in os.walk(self.repo_dir, followlinks=False):
base = Path(directory)
dirs[:] = [
name
for name in dirs
if name != ".git"
and not (base / name).is_symlink()
and not self._boundaries.should_skip(base / name)
]
gitignore = base / ".gitignore"
if ".gitignore" in files and not gitignore.is_symlink():
candidates.append(gitignore)
candidates.sort(
key=lambda path: (len(path.relative_to(self.repo_dir).parts), path.as_posix()),
)
except OSError as exc:
Expand Down Expand Up @@ -194,9 +291,13 @@ def __init__(
self.exclude_patterns = self.default_exclude_patterns + self.user_exclude_patterns
self.use_gitignore = use_gitignore
self._gitignore_filter: GitIgnoreFilter | None = None
self._boundaries: RepositoryBoundaryFilter | None = None

def analyze_repository_structure(self, repo_dir: str) -> dict:
self._gitignore_filter = GitIgnoreFilter(Path(repo_dir)) if self.use_gitignore else None
self._boundaries = RepositoryBoundaryFilter(Path(repo_dir))
self._gitignore_filter = (
GitIgnoreFilter(Path(repo_dir), self._boundaries) if self.use_gitignore else None
)
file_tree = self._build_file_tree(repo_dir)
return {
"file_tree": file_tree,
Expand Down Expand Up @@ -240,6 +341,8 @@ def build_tree(path: Path, base_path: Path) -> dict | None:
}

elif path.is_dir():
if self._boundaries and self._boundaries.should_skip(path):
return None
children = []
try:
for child in sorted(path.iterdir()):
Expand Down
11 changes: 11 additions & 0 deletions guides/cli-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,17 @@ Pattern rules:
- Git ignore rules apply before the dependency analysis. Tracked files stay
in, as in Git. Built-in and `--exclude` patterns still apply when Git
includes a path.
- Nested Git checkouts (including worktrees and clones) are skipped, even with
`--no-gitignore` or when Git is unavailable. Submodules declared in the owning
repository's `.gitmodules` remain included unless excluded by other filters.
To document a nested checkout itself, run generation from that checkout.
Ordinary untracked source files remain included.

For an opt-in live boundary check, run
`.venv/bin/python tests/smoke_test_nested_checkouts.py --provider codex --model gpt-6-luna`
from the project root with an authenticated Codex CLI. It generates a temporary
repository's documentation, checks the saved graph, and retains logs and artifacts
without changing your stored CodeWiki settings.

### Output language

Expand Down
209 changes: 209 additions & 0 deletions tests/smoke_test_nested_checkouts.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
"""Opt-in live checkout-boundary smoke test; never collected by pytest.

Run: .venv/bin/python tests/smoke_test_nested_checkouts.py --provider codex --model gpt-6-luna
Requires an authenticated Codex CLI. Logs and generated artifacts are retained.
"""

from __future__ import annotations

import argparse
from contextlib import ExitStack
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
from unittest.mock import patch

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from click.testing import CliRunner

import codewiki.cli.config_manager as config_module
from codewiki.cli.main import cli
from codewiki.src.be.dependency_analyzer.models.core import Node
from codewiki.src.be.module_scope import validate_module_file_coverage


def git(repo: Path, *args: str) -> None:
subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True)


def commit(repo: Path) -> None:
git(repo, "add", ".")
git(
repo,
"-c",
"user.name=Checkout Smoke",
"-c",
"user.email=smoke@example.invalid",
"commit",
"-qm",
"fixture",
)


def make_fixture(root: Path) -> Path:
repo, dependency = root / "repo", root / "dependency"
repo.mkdir()
dependency.mkdir()
git(dependency, "init", "-q")
(dependency / "support.py").write_text(
'"""Clamp a numeric value to a nonnegative integer."""\n'
"def clamp(value: int) -> int:\n return max(0, value)\n"
)
commit(dependency)
git(repo, "init", "-q")
(repo / "helper.py").write_text(
'"""Arithmetic helpers for a small calculator."""\n'
"def helper(value: int) -> int:\n return value + 1\n"
)
(repo / "app.py").write_text(
'"""Calculator entrypoint."""\nfrom helper import helper\n\n'
"def run(value: int) -> int:\n return helper(value)\n"
)
commit(repo)
git(repo, "worktree", "add", "-qb", "smoke-worktree", ".claude/worktrees/feature")
git(repo, "clone", "-q", ".", "copies/other")
for directory in (".claude/worktrees/feature", "copies/other"):
(repo / directory / "sentinel.py").write_text(
"def nested_checkout_sentinel():\n return 'must not be documented'\n"
)
git(
repo,
"-c",
"protocol.file.allow=always",
"submodule",
"add",
"-q",
str(dependency),
"deps/math_support",
)
# Commit only the intended parent files and submodule registration.
git(
repo,
"-c",
"user.name=Checkout Smoke",
"-c",
"user.email=smoke@example.invalid",
"commit",
"-qm",
"register dependency",
)
(repo / "untracked.py").write_text(
'"""Format a calculator result."""\n'
"def format_result(value: int) -> str:\n return str(value)\n"
)
return repo


def verify(output: Path, model: str) -> dict:
metadata = json.loads((output / "metadata.json").read_text())
usage = metadata["usage"]
assert not usage.get("incomplete"), "Generation was marked incomplete"
assert usage["n_calls"] > 0, "No real model activity recorded"
calls = usage["calls"]
assert all(call["provider"] == "codex" for call in calls), calls
assert all(call["model"] == model for call in calls), calls
assert any(call["usage_reported"] for call in calls), "No provider usage recorded"
assert usage["totals"].get("output_tokens", 0) > 0, "No model output recorded"

tree = json.loads((output / "module_tree.json").read_text())
# A small repository uses an empty tree and one whole-repository overview.
assert isinstance(tree, dict), "Invalid module tree"
overview = output / "overview.md"
assert overview.is_file() and overview.read_text().strip(), "Missing overview"
pages = [p for p in output.rglob("*.md") if "temp" not in p.relative_to(output).parts]
assert pages and all(p.read_text().strip() for p in pages), "Empty documentation page"
for page in pages:
text = page.read_text()
assert "nested_checkout_sentinel" not in text, page
assert ".claude/worktrees/feature" not in text and "copies/other" not in text, page

graphs = list((output / "temp/dependency_graphs").glob("*_dependency_graph.json"))
assert len(graphs) == 1, graphs
graph = json.loads(graphs[0].read_text())
paths = {node["relative_path"] for node in graph.values()}
assert paths == {"app.py", "helper.py", "untracked.py", "deps/math_support/support.py"}, paths
assert set(graph["app.py::run"]["depends_on"]) == {"helper.py::helper"}
if tree:
validate_module_file_coverage(
tree, {ident: Node.model_validate(node) for ident, node in graph.items()}
)
return {
"provider": "codex",
"requested_model": model,
"recorded_models": sorted({call["model"] for call in calls}),
"model_calls": usage["n_calls"],
"source_paths": sorted(paths),
"documentation_pages": [str(p.relative_to(output)) for p in pages],
}


def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--provider", choices=["codex"], default="codex")
parser.add_argument("--model", default="gpt-6-luna")
args = parser.parse_args()
if not shutil.which("codex") or not shutil.which("git"):
parser.error("Git and an authenticated Codex CLI must be on PATH")
root = Path(tempfile.mkdtemp(prefix="codewiki-checkout-smoke-"))
print(f"Smoke artifacts: {root}", flush=True)
repo = make_fixture(root)
output, config_dir = root / "docs", root / "config"
previous_cwd = Path.cwd()
try:
with ExitStack() as stack:
stack.enter_context(patch.dict(os.environ, {"CODEWIKI_NO_KEYRING": "1"}))
for name, value in {
"CONFIG_DIR": config_dir,
"CONFIG_FILE": config_dir / "config.json",
"CREDENTIALS_FILE": config_dir / "credentials.json",
}.items():
stack.enter_context(patch.object(config_module, name, value))
config_module.ConfigManager().save(
provider=args.provider,
main_model=args.model,
cluster_model=args.model,
fallback_model="",
)
os.chdir(repo)
print(f"Generating with provider={args.provider}, model={args.model}...", flush=True)
result = CliRunner().invoke(
cli,
[
"generate",
"--output",
str(output),
"--no-artifacts",
"--no-cache",
"--min-token-per-module",
"1",
"--max-tokens",
"8192",
"--request-limit",
"24",
"--verbose",
"--instructions",
"Keep documentation concise, under 300 words per page. "
"Do not include Mermaid diagrams.",
],
)
(root / "generate.log").write_text(result.output)
if result.exit_code:
raise RuntimeError(
f"Generation failed with exit {result.exit_code}; see {root / 'generate.log'}"
) from result.exception
report = verify(output, args.model)
(root / "report.json").write_text(json.dumps(report, indent=2))
print(json.dumps(report, indent=2), flush=True)
print("PASS: live generation preserved repository boundaries", flush=True)
finally:
os.chdir(previous_cwd)
shutil.rmtree(config_dir, ignore_errors=True)


if __name__ == "__main__":
main()
Loading
Loading