diff --git a/.gitignore b/.gitignore index 352d15ae..5b5c99da 100644 --- a/.gitignore +++ b/.gitignore @@ -99,6 +99,7 @@ Thumbs.db !tests/test_generation_metadata.py !tests/test_pricing.py !tests/smoke_test_costs.py +!tests/smoke_test_nested_checkouts.py # Mermaid regression tests and development-only parser harness !tests/test_mermaid_validation.py diff --git a/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py b/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py index 67b6cfe3..253ac5bc 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py +++ b/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py @@ -5,6 +5,7 @@ detailed file tree representations with filtering capabilities. """ +import configparser import fnmatch import logging import os @@ -23,6 +24,93 @@ logger = logging.getLogger(__name__) +class RepositoryBoundaryFilter: + """Keep declared submodules, but do not descend into unrelated checkouts.""" + + def __init__(self, repo_dir: Path) -> None: + self.root = repo_dir.resolve() + self.owner = next( + (path for path in (self.root, *self.root.parents) if (path / ".git").exists()), + self.root, + ) + self._submodules: dict[Path, set[Path]] = {} + + @staticmethod + def _config_path(value: str) -> str: + """Decode Git config quoting, escapes and trailing comments without Git.""" + result = [] + quoted = False + escaped = False + end = 0 + escapes = {"n": "\n", "t": "\t", "b": "\b", '"': '"', "\\": "\\"} + for char in value: + if escaped: + if char not in escapes: + raise ValueError("Invalid escape in submodule path") + result.append(escapes[char]) + end = len(result) + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + quoted = not quoted + elif char in "#;" and not quoted: + break + else: + result.append(char) + if quoted or not char.isspace(): + end = len(result) + if quoted or escaped: + raise ValueError("Unterminated submodule path") + return "".join(result[:end]) + + def _declared_submodules(self, owner: Path) -> set[Path]: + if owner in self._submodules: + return self._submodules[owner] + paths: set[Path] = set() + config_path = owner / ".gitmodules" + try: + # Never follow a declaration file outside its owning repository. + if config_path.is_symlink() or not config_path.exists(): + self._submodules[owner] = paths + return paths + config = configparser.RawConfigParser(strict=False) + config.read_string(config_path.read_text(encoding="utf-8")) + for section in config.sections(): + if not section.lower().startswith('submodule "') or not section.endswith('"'): + continue + value = self._config_path(config.get(section, "path", fallback="")) + if not value: + continue + relative = Path(value) + path = (owner / relative).resolve() + if relative.is_absolute() or path == owner or not path.is_relative_to(owner): + logger.warning("Skipping escaping submodule path in %s: %s", config_path, value) + continue + paths.add(path) + except (OSError, UnicodeError, ValueError, configparser.Error) as exc: + logger.warning("Could not read submodule declarations at %s: %s", config_path, exc) + paths.clear() + self._submodules[owner] = paths + return paths + + def should_skip(self, path: Path) -> bool: + path = path.resolve() + if path == self.root or not (path / ".git").exists(): + return False + owner = self.owner + for parent in path.parents: + if parent == self.root: + break + if (parent / ".git").exists(): + owner = parent + break + if path in self._declared_submodules(owner): + return False + logger.debug("Skipping nested Git checkout: %s", path) + return True + + class GitIgnoreFilter: """Evaluate Git ignore rules once per repository analysis. @@ -31,8 +119,9 @@ class GitIgnoreFilter: Plain directories fall back to pathspec-backed ``.gitignore`` evaluation. """ - def __init__(self, repo_dir: Path) -> None: + def __init__(self, repo_dir: Path, boundaries: RepositoryBoundaryFilter | None = None) -> None: self.repo_dir = repo_dir.resolve() + self._boundaries = boundaries or RepositoryBoundaryFilter(self.repo_dir) self._ignored_files: set[str] = set() self._ignored_dirs: set[str] = set() self._ignore_all_untracked = False @@ -116,12 +205,20 @@ def _load_git_ignored_paths(self) -> bool: def _load_fallback_specs(self) -> None: try: - candidates = sorted( - ( - path - for path in self.repo_dir.rglob(".gitignore") - if path.is_file() and not path.is_symlink() - ), + candidates = [] + for directory, dirs, files in os.walk(self.repo_dir, followlinks=False): + base = Path(directory) + dirs[:] = [ + name + for name in dirs + if name != ".git" + and not (base / name).is_symlink() + and not self._boundaries.should_skip(base / name) + ] + gitignore = base / ".gitignore" + if ".gitignore" in files and not gitignore.is_symlink(): + candidates.append(gitignore) + candidates.sort( key=lambda path: (len(path.relative_to(self.repo_dir).parts), path.as_posix()), ) except OSError as exc: @@ -194,9 +291,13 @@ def __init__( self.exclude_patterns = self.default_exclude_patterns + self.user_exclude_patterns self.use_gitignore = use_gitignore self._gitignore_filter: GitIgnoreFilter | None = None + self._boundaries: RepositoryBoundaryFilter | None = None def analyze_repository_structure(self, repo_dir: str) -> dict: - self._gitignore_filter = GitIgnoreFilter(Path(repo_dir)) if self.use_gitignore else None + self._boundaries = RepositoryBoundaryFilter(Path(repo_dir)) + self._gitignore_filter = ( + GitIgnoreFilter(Path(repo_dir), self._boundaries) if self.use_gitignore else None + ) file_tree = self._build_file_tree(repo_dir) return { "file_tree": file_tree, @@ -240,6 +341,8 @@ def build_tree(path: Path, base_path: Path) -> dict | None: } elif path.is_dir(): + if self._boundaries and self._boundaries.should_skip(path): + return None children = [] try: for child in sorted(path.iterdir()): diff --git a/guides/cli-reference.md b/guides/cli-reference.md index 04252e5b..f64e37c8 100644 --- a/guides/cli-reference.md +++ b/guides/cli-reference.md @@ -60,6 +60,17 @@ Pattern rules: - Git ignore rules apply before the dependency analysis. Tracked files stay in, as in Git. Built-in and `--exclude` patterns still apply when Git includes a path. +- Nested Git checkouts (including worktrees and clones) are skipped, even with + `--no-gitignore` or when Git is unavailable. Submodules declared in the owning + repository's `.gitmodules` remain included unless excluded by other filters. + To document a nested checkout itself, run generation from that checkout. + Ordinary untracked source files remain included. + +For an opt-in live boundary check, run +`.venv/bin/python tests/smoke_test_nested_checkouts.py --provider codex --model gpt-6-luna` +from the project root with an authenticated Codex CLI. It generates a temporary +repository's documentation, checks the saved graph, and retains logs and artifacts +without changing your stored CodeWiki settings. ### Output language diff --git a/tests/smoke_test_nested_checkouts.py b/tests/smoke_test_nested_checkouts.py new file mode 100644 index 00000000..f2c422ac --- /dev/null +++ b/tests/smoke_test_nested_checkouts.py @@ -0,0 +1,209 @@ +"""Opt-in live checkout-boundary smoke test; never collected by pytest. + +Run: .venv/bin/python tests/smoke_test_nested_checkouts.py --provider codex --model gpt-6-luna +Requires an authenticated Codex CLI. Logs and generated artifacts are retained. +""" + +from __future__ import annotations + +import argparse +from contextlib import ExitStack +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from click.testing import CliRunner + +import codewiki.cli.config_manager as config_module +from codewiki.cli.main import cli +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.be.module_scope import validate_module_file_coverage + + +def git(repo: Path, *args: str) -> None: + subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True) + + +def commit(repo: Path) -> None: + git(repo, "add", ".") + git( + repo, + "-c", + "user.name=Checkout Smoke", + "-c", + "user.email=smoke@example.invalid", + "commit", + "-qm", + "fixture", + ) + + +def make_fixture(root: Path) -> Path: + repo, dependency = root / "repo", root / "dependency" + repo.mkdir() + dependency.mkdir() + git(dependency, "init", "-q") + (dependency / "support.py").write_text( + '"""Clamp a numeric value to a nonnegative integer."""\n' + "def clamp(value: int) -> int:\n return max(0, value)\n" + ) + commit(dependency) + git(repo, "init", "-q") + (repo / "helper.py").write_text( + '"""Arithmetic helpers for a small calculator."""\n' + "def helper(value: int) -> int:\n return value + 1\n" + ) + (repo / "app.py").write_text( + '"""Calculator entrypoint."""\nfrom helper import helper\n\n' + "def run(value: int) -> int:\n return helper(value)\n" + ) + commit(repo) + git(repo, "worktree", "add", "-qb", "smoke-worktree", ".claude/worktrees/feature") + git(repo, "clone", "-q", ".", "copies/other") + for directory in (".claude/worktrees/feature", "copies/other"): + (repo / directory / "sentinel.py").write_text( + "def nested_checkout_sentinel():\n return 'must not be documented'\n" + ) + git( + repo, + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + str(dependency), + "deps/math_support", + ) + # Commit only the intended parent files and submodule registration. + git( + repo, + "-c", + "user.name=Checkout Smoke", + "-c", + "user.email=smoke@example.invalid", + "commit", + "-qm", + "register dependency", + ) + (repo / "untracked.py").write_text( + '"""Format a calculator result."""\n' + "def format_result(value: int) -> str:\n return str(value)\n" + ) + return repo + + +def verify(output: Path, model: str) -> dict: + metadata = json.loads((output / "metadata.json").read_text()) + usage = metadata["usage"] + assert not usage.get("incomplete"), "Generation was marked incomplete" + assert usage["n_calls"] > 0, "No real model activity recorded" + calls = usage["calls"] + assert all(call["provider"] == "codex" for call in calls), calls + assert all(call["model"] == model for call in calls), calls + assert any(call["usage_reported"] for call in calls), "No provider usage recorded" + assert usage["totals"].get("output_tokens", 0) > 0, "No model output recorded" + + tree = json.loads((output / "module_tree.json").read_text()) + # A small repository uses an empty tree and one whole-repository overview. + assert isinstance(tree, dict), "Invalid module tree" + overview = output / "overview.md" + assert overview.is_file() and overview.read_text().strip(), "Missing overview" + pages = [p for p in output.rglob("*.md") if "temp" not in p.relative_to(output).parts] + assert pages and all(p.read_text().strip() for p in pages), "Empty documentation page" + for page in pages: + text = page.read_text() + assert "nested_checkout_sentinel" not in text, page + assert ".claude/worktrees/feature" not in text and "copies/other" not in text, page + + graphs = list((output / "temp/dependency_graphs").glob("*_dependency_graph.json")) + assert len(graphs) == 1, graphs + graph = json.loads(graphs[0].read_text()) + paths = {node["relative_path"] for node in graph.values()} + assert paths == {"app.py", "helper.py", "untracked.py", "deps/math_support/support.py"}, paths + assert set(graph["app.py::run"]["depends_on"]) == {"helper.py::helper"} + if tree: + validate_module_file_coverage( + tree, {ident: Node.model_validate(node) for ident, node in graph.items()} + ) + return { + "provider": "codex", + "requested_model": model, + "recorded_models": sorted({call["model"] for call in calls}), + "model_calls": usage["n_calls"], + "source_paths": sorted(paths), + "documentation_pages": [str(p.relative_to(output)) for p in pages], + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--provider", choices=["codex"], default="codex") + parser.add_argument("--model", default="gpt-6-luna") + args = parser.parse_args() + if not shutil.which("codex") or not shutil.which("git"): + parser.error("Git and an authenticated Codex CLI must be on PATH") + root = Path(tempfile.mkdtemp(prefix="codewiki-checkout-smoke-")) + print(f"Smoke artifacts: {root}", flush=True) + repo = make_fixture(root) + output, config_dir = root / "docs", root / "config" + previous_cwd = Path.cwd() + try: + with ExitStack() as stack: + stack.enter_context(patch.dict(os.environ, {"CODEWIKI_NO_KEYRING": "1"})) + for name, value in { + "CONFIG_DIR": config_dir, + "CONFIG_FILE": config_dir / "config.json", + "CREDENTIALS_FILE": config_dir / "credentials.json", + }.items(): + stack.enter_context(patch.object(config_module, name, value)) + config_module.ConfigManager().save( + provider=args.provider, + main_model=args.model, + cluster_model=args.model, + fallback_model="", + ) + os.chdir(repo) + print(f"Generating with provider={args.provider}, model={args.model}...", flush=True) + result = CliRunner().invoke( + cli, + [ + "generate", + "--output", + str(output), + "--no-artifacts", + "--no-cache", + "--min-token-per-module", + "1", + "--max-tokens", + "8192", + "--request-limit", + "24", + "--verbose", + "--instructions", + "Keep documentation concise, under 300 words per page. " + "Do not include Mermaid diagrams.", + ], + ) + (root / "generate.log").write_text(result.output) + if result.exit_code: + raise RuntimeError( + f"Generation failed with exit {result.exit_code}; see {root / 'generate.log'}" + ) from result.exception + report = verify(output, args.model) + (root / "report.json").write_text(json.dumps(report, indent=2)) + print(json.dumps(report, indent=2), flush=True) + print("PASS: live generation preserved repository boundaries", flush=True) + finally: + os.chdir(previous_cwd) + shutil.rmtree(config_dir, ignore_errors=True) + + +if __name__ == "__main__": + main() diff --git a/tests/test_gitignore_filtering.py b/tests/test_gitignore_filtering.py index 120ba128..11f8126a 100644 --- a/tests/test_gitignore_filtering.py +++ b/tests/test_gitignore_filtering.py @@ -1,6 +1,7 @@ -"""Tests for automatic Git ignore filtering (issue #71).""" +"""Tests for Git ignore filtering and checkout boundaries (issues #71 and #157).""" import asyncio +import os import shutil import subprocess from pathlib import Path @@ -50,6 +51,217 @@ def _git(repo: Path, *args: str) -> None: ) +def _commit(repo: Path) -> None: + _git(repo, "add", ".") + _git( + repo, + "-c", + "user.name=Checkout Test", + "-c", + "user.email=checkout@example.invalid", + "commit", + "-qm", + "fixture", + ) + + +@pytest.fixture +def nested_checkouts(tmp_path: Path) -> Path: + if shutil.which("git") is None: + pytest.skip("Git is required to create the fixture") + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "-q") + (repo / "helper.py").write_text("def helper():\n return 1\n") + (repo / "app.py").write_text("from helper import helper\ndef run():\n return helper()\n") + _commit(repo) + _git(repo, "worktree", "add", "-qb", "fixture-worktree", ".claude/worktrees/feature") + _git(repo, "clone", "-q", ".", "copies/other") + _write_source(repo / "untracked.py") + for relative in (".claude/worktrees/feature", "copies/other"): + (repo / relative / ".gitignore").write_text("secret.py\n") + _write_source(repo / relative / "secret.py") + return repo + + +@pytest.mark.parametrize("use_gitignore", [True, False]) +@pytest.mark.parametrize("git_available", [True, False]) +def test_nested_checkouts_are_pruned_before_traversal( + nested_checkouts: Path, + monkeypatch: pytest.MonkeyPatch, + use_gitignore: bool, + git_available: bool, +) -> None: + repo = nested_checkouts + skipped = {repo / ".claude/worktrees/feature", repo / "copies/other"} + original_scandir = os.scandir + original_iterdir = Path.iterdir + + def scandir(path): + assert Path(path) not in skipped, f"Fallback entered nested checkout: {path}" + return original_scandir(path) + + def iterdir(path): + assert path not in skipped, f"Source walk entered nested checkout: {path}" + return original_iterdir(path) + + monkeypatch.setattr("os.scandir", scandir) + monkeypatch.setattr(Path, "iterdir", iterdir) + if not git_available: + monkeypatch.setattr( + "codewiki.src.be.dependency_analyzer.analysis.repo_analyzer.shutil.which", + lambda _name: None, + ) + assert _analyzed_paths(repo, use_gitignore=use_gitignore) == { + "app.py", + "helper.py", + "untracked.py", + } + + +@pytest.mark.parametrize("relative", [".claude/worktrees/feature", "copies/other"]) +def test_selected_nested_checkout_is_analyzable(nested_checkouts: Path, relative: str) -> None: + assert _analyzed_paths(nested_checkouts / relative) == {"app.py", "helper.py"} + + +def test_nested_checkout_nodes_do_not_reach_dependency_graph(nested_checkouts: Path) -> None: + components = DependencyParser(str(nested_checkouts)).parse_repository() + assert {node.relative_path for node in components.values()} == { + "app.py", + "helper.py", + "untracked.py", + } + assert components["app.py::run"].depends_on == {"helper.py::helper"} + + +def test_plain_directory_prunes_nested_checkout(nested_checkouts: Path, tmp_path: Path) -> None: + plain = tmp_path / "plain" + plain.mkdir() + _write_source(plain / "main.py") + _git(plain, "clone", "-q", str(nested_checkouts), "nested") + assert _analyzed_paths(plain) == {"main.py"} + + +@pytest.mark.parametrize("git_available", [True, False]) +def test_declared_submodules_and_nested_submodules_are_preserved( + nested_checkouts: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, git_available: bool +) -> None: + leaf = tmp_path / "leaf" + leaf.mkdir() + _git(leaf, "init", "-q") + _write_source(leaf / "leaf.py") + _commit(leaf) + dependency = tmp_path / "dependency" + dependency.mkdir() + _git(dependency, "init", "-q") + _write_source(dependency / "dep.py") + _git( + dependency, "-c", "protocol.file.allow=always", "submodule", "add", "-q", str(leaf), "child" + ) + _commit(dependency) + repo = nested_checkouts + submodule = repo / "packages" / "registered dependency" + _git( + repo, + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + str(dependency), + "packages/registered dependency", + ) + _git(repo, "-c", "protocol.file.allow=always", "submodule", "update", "--init", "--recursive") + _git(submodule, "clone", "-q", str(leaf), "accidental") + # Quoting and inline comments are valid in Git configuration files. + (repo / ".gitmodules").write_text( + '[submodule "registered"]\n path = "packages/registered dependency" # keep\n' + ) + if not git_available: + monkeypatch.setattr( + "codewiki.src.be.dependency_analyzer.analysis.repo_analyzer.shutil.which", + lambda _name: None, + ) + assert _analyzed_paths(repo) == { + "app.py", + "helper.py", + "untracked.py", + "packages/registered dependency/dep.py", + "packages/registered dependency/child/leaf.py", + } + assert _analyzed_paths(repo / "packages") == { + "registered dependency/dep.py", + "registered dependency/child/leaf.py", + } + assert _analyzed_paths(submodule) == {"dep.py", "child/leaf.py"} + assert _analyzed_paths(repo, exclude_patterns=["packages/registered dependency"]) == { + "app.py", + "helper.py", + "untracked.py", + } + + +@pytest.mark.parametrize( + "declarations", + [ + '[submodule "copy"]\n path = copies/other\n[broken', + '[submodule "copy"]\n path = "copies/other\n', + '[submodule "copy"]\n path = "copies/other\\q"\n', + ], +) +def test_malformed_submodule_declarations_do_not_exempt_checkouts( + nested_checkouts: Path, declarations: str, caplog: pytest.LogCaptureFixture +) -> None: + (nested_checkouts / ".gitmodules").write_text(declarations) + assert _analyzed_paths(nested_checkouts) == {"app.py", "helper.py", "untracked.py"} + assert "Could not read submodule declarations" in caplog.text + + +def test_unreadable_submodule_declarations_do_not_abort_analysis( + nested_checkouts: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + declarations = nested_checkouts / ".gitmodules" + declarations.write_text('[submodule "copy"]\n path = copies/other\n') + original_read = Path.read_text + + def read_text(path, *args, **kwargs): + if path == declarations: + raise PermissionError("fixture") + return original_read(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", read_text) + assert _analyzed_paths(nested_checkouts) == {"app.py", "helper.py", "untracked.py"} + assert "Could not read submodule declarations" in caplog.text + + +def test_escaping_submodule_declarations_are_rejected( + nested_checkouts: Path, caplog: pytest.LogCaptureFixture +) -> None: + repo = nested_checkouts + (repo / ".gitmodules").write_text( + '[submodule "outside"]\n path = ../outside\n' + f'[submodule "absolute"]\n path = "{repo / "copies/other"}"\n' + ) + assert _analyzed_paths(repo) == {"app.py", "helper.py", "untracked.py"} + assert "Skipping escaping submodule path" in caplog.text + + +@pytest.mark.skipif(os.name == "nt", reason="Windows does not support trailing spaces in paths") +def test_quoted_submodule_path_preserves_significant_whitespace(nested_checkouts: Path) -> None: + repo = nested_checkouts + (repo / "copies/other").rename(repo / " dependency ") + (repo / ".gitmodules").write_text('[submodule "copy"]\n path = " dependency " # keep\n') + assert _analyzed_paths(repo) == { + "app.py", + "helper.py", + "untracked.py", + " dependency /app.py", + " dependency /helper.py", + # The parent Git ignore query does not recurse into submodules. + " dependency /secret.py", + } + + @pytest.mark.skipif(shutil.which("git") is None, reason="Git is required") def test_gitignore_is_default_and_keeps_tracked_matches(tmp_path: Path) -> None: _git(tmp_path, "init", "-q")