Skip to content

Commit 7fcf935

Browse files
authored
Merge pull request #165 from FSoft-AI4Code/fix/157-nested-git-checkouts
fix: exclude nested Git checkouts from repository analysis
2 parents d22b4a8 + 090d3ef commit 7fcf935

5 files changed

Lines changed: 545 additions & 9 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ Thumbs.db
9999
!tests/test_generation_metadata.py
100100
!tests/test_pricing.py
101101
!tests/smoke_test_costs.py
102+
!tests/smoke_test_nested_checkouts.py
102103

103104
# Mermaid regression tests and development-only parser harness
104105
!tests/test_mermaid_validation.py

‎codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py‎

Lines changed: 111 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
detailed file tree representations with filtering capabilities.
66
"""
77

8+
import configparser
89
import fnmatch
910
import logging
1011
import os
@@ -23,6 +24,93 @@
2324
logger = logging.getLogger(__name__)
2425

2526

27+
class RepositoryBoundaryFilter:
28+
"""Keep declared submodules, but do not descend into unrelated checkouts."""
29+
30+
def __init__(self, repo_dir: Path) -> None:
31+
self.root = repo_dir.resolve()
32+
self.owner = next(
33+
(path for path in (self.root, *self.root.parents) if (path / ".git").exists()),
34+
self.root,
35+
)
36+
self._submodules: dict[Path, set[Path]] = {}
37+
38+
@staticmethod
39+
def _config_path(value: str) -> str:
40+
"""Decode Git config quoting, escapes and trailing comments without Git."""
41+
result = []
42+
quoted = False
43+
escaped = False
44+
end = 0
45+
escapes = {"n": "\n", "t": "\t", "b": "\b", '"': '"', "\\": "\\"}
46+
for char in value:
47+
if escaped:
48+
if char not in escapes:
49+
raise ValueError("Invalid escape in submodule path")
50+
result.append(escapes[char])
51+
end = len(result)
52+
escaped = False
53+
elif char == "\\":
54+
escaped = True
55+
elif char == '"':
56+
quoted = not quoted
57+
elif char in "#;" and not quoted:
58+
break
59+
else:
60+
result.append(char)
61+
if quoted or not char.isspace():
62+
end = len(result)
63+
if quoted or escaped:
64+
raise ValueError("Unterminated submodule path")
65+
return "".join(result[:end])
66+
67+
def _declared_submodules(self, owner: Path) -> set[Path]:
68+
if owner in self._submodules:
69+
return self._submodules[owner]
70+
paths: set[Path] = set()
71+
config_path = owner / ".gitmodules"
72+
try:
73+
# Never follow a declaration file outside its owning repository.
74+
if config_path.is_symlink() or not config_path.exists():
75+
self._submodules[owner] = paths
76+
return paths
77+
config = configparser.RawConfigParser(strict=False)
78+
config.read_string(config_path.read_text(encoding="utf-8"))
79+
for section in config.sections():
80+
if not section.lower().startswith('submodule "') or not section.endswith('"'):
81+
continue
82+
value = self._config_path(config.get(section, "path", fallback=""))
83+
if not value:
84+
continue
85+
relative = Path(value)
86+
path = (owner / relative).resolve()
87+
if relative.is_absolute() or path == owner or not path.is_relative_to(owner):
88+
logger.warning("Skipping escaping submodule path in %s: %s", config_path, value)
89+
continue
90+
paths.add(path)
91+
except (OSError, UnicodeError, ValueError, configparser.Error) as exc:
92+
logger.warning("Could not read submodule declarations at %s: %s", config_path, exc)
93+
paths.clear()
94+
self._submodules[owner] = paths
95+
return paths
96+
97+
def should_skip(self, path: Path) -> bool:
98+
path = path.resolve()
99+
if path == self.root or not (path / ".git").exists():
100+
return False
101+
owner = self.owner
102+
for parent in path.parents:
103+
if parent == self.root:
104+
break
105+
if (parent / ".git").exists():
106+
owner = parent
107+
break
108+
if path in self._declared_submodules(owner):
109+
return False
110+
logger.debug("Skipping nested Git checkout: %s", path)
111+
return True
112+
113+
26114
class GitIgnoreFilter:
27115
"""Evaluate Git ignore rules once per repository analysis.
28116
@@ -31,8 +119,9 @@ class GitIgnoreFilter:
31119
Plain directories fall back to pathspec-backed ``.gitignore`` evaluation.
32120
"""
33121

34-
def __init__(self, repo_dir: Path) -> None:
122+
def __init__(self, repo_dir: Path, boundaries: RepositoryBoundaryFilter | None = None) -> None:
35123
self.repo_dir = repo_dir.resolve()
124+
self._boundaries = boundaries or RepositoryBoundaryFilter(self.repo_dir)
36125
self._ignored_files: set[str] = set()
37126
self._ignored_dirs: set[str] = set()
38127
self._ignore_all_untracked = False
@@ -116,12 +205,20 @@ def _load_git_ignored_paths(self) -> bool:
116205

117206
def _load_fallback_specs(self) -> None:
118207
try:
119-
candidates = sorted(
120-
(
121-
path
122-
for path in self.repo_dir.rglob(".gitignore")
123-
if path.is_file() and not path.is_symlink()
124-
),
208+
candidates = []
209+
for directory, dirs, files in os.walk(self.repo_dir, followlinks=False):
210+
base = Path(directory)
211+
dirs[:] = [
212+
name
213+
for name in dirs
214+
if name != ".git"
215+
and not (base / name).is_symlink()
216+
and not self._boundaries.should_skip(base / name)
217+
]
218+
gitignore = base / ".gitignore"
219+
if ".gitignore" in files and not gitignore.is_symlink():
220+
candidates.append(gitignore)
221+
candidates.sort(
125222
key=lambda path: (len(path.relative_to(self.repo_dir).parts), path.as_posix()),
126223
)
127224
except OSError as exc:
@@ -194,9 +291,13 @@ def __init__(
194291
self.exclude_patterns = self.default_exclude_patterns + self.user_exclude_patterns
195292
self.use_gitignore = use_gitignore
196293
self._gitignore_filter: GitIgnoreFilter | None = None
294+
self._boundaries: RepositoryBoundaryFilter | None = None
197295

198296
def analyze_repository_structure(self, repo_dir: str) -> dict:
199-
self._gitignore_filter = GitIgnoreFilter(Path(repo_dir)) if self.use_gitignore else None
297+
self._boundaries = RepositoryBoundaryFilter(Path(repo_dir))
298+
self._gitignore_filter = (
299+
GitIgnoreFilter(Path(repo_dir), self._boundaries) if self.use_gitignore else None
300+
)
200301
file_tree = self._build_file_tree(repo_dir)
201302
return {
202303
"file_tree": file_tree,
@@ -240,6 +341,8 @@ def build_tree(path: Path, base_path: Path) -> dict | None:
240341
}
241342

242343
elif path.is_dir():
344+
if self._boundaries and self._boundaries.should_skip(path):
345+
return None
243346
children = []
244347
try:
245348
for child in sorted(path.iterdir()):

‎guides/cli-reference.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,17 @@ Pattern rules:
6060
- Git ignore rules apply before the dependency analysis. Tracked files stay
6161
in, as in Git. Built-in and `--exclude` patterns still apply when Git
6262
includes a path.
63+
- Nested Git checkouts (including worktrees and clones) are skipped, even with
64+
`--no-gitignore` or when Git is unavailable. Submodules declared in the owning
65+
repository's `.gitmodules` remain included unless excluded by other filters.
66+
To document a nested checkout itself, run generation from that checkout.
67+
Ordinary untracked source files remain included.
68+
69+
For an opt-in live boundary check, run
70+
`.venv/bin/python tests/smoke_test_nested_checkouts.py --provider codex --model gpt-6-luna`
71+
from the project root with an authenticated Codex CLI. It generates a temporary
72+
repository's documentation, checks the saved graph, and retains logs and artifacts
73+
without changing your stored CodeWiki settings.
6374

6475
### Output language
6576

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
"""Opt-in live checkout-boundary smoke test; never collected by pytest.
2+
3+
Run: .venv/bin/python tests/smoke_test_nested_checkouts.py --provider codex --model gpt-6-luna
4+
Requires an authenticated Codex CLI. Logs and generated artifacts are retained.
5+
"""
6+
7+
from __future__ import annotations
8+
9+
import argparse
10+
from contextlib import ExitStack
11+
import json
12+
import os
13+
from pathlib import Path
14+
import shutil
15+
import subprocess
16+
import sys
17+
import tempfile
18+
from unittest.mock import patch
19+
20+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
21+
22+
from click.testing import CliRunner
23+
24+
import codewiki.cli.config_manager as config_module
25+
from codewiki.cli.main import cli
26+
from codewiki.src.be.dependency_analyzer.models.core import Node
27+
from codewiki.src.be.module_scope import validate_module_file_coverage
28+
29+
30+
def git(repo: Path, *args: str) -> None:
31+
subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True)
32+
33+
34+
def commit(repo: Path) -> None:
35+
git(repo, "add", ".")
36+
git(
37+
repo,
38+
"-c",
39+
"user.name=Checkout Smoke",
40+
"-c",
41+
"user.email=smoke@example.invalid",
42+
"commit",
43+
"-qm",
44+
"fixture",
45+
)
46+
47+
48+
def make_fixture(root: Path) -> Path:
49+
repo, dependency = root / "repo", root / "dependency"
50+
repo.mkdir()
51+
dependency.mkdir()
52+
git(dependency, "init", "-q")
53+
(dependency / "support.py").write_text(
54+
'"""Clamp a numeric value to a nonnegative integer."""\n'
55+
"def clamp(value: int) -> int:\n return max(0, value)\n"
56+
)
57+
commit(dependency)
58+
git(repo, "init", "-q")
59+
(repo / "helper.py").write_text(
60+
'"""Arithmetic helpers for a small calculator."""\n'
61+
"def helper(value: int) -> int:\n return value + 1\n"
62+
)
63+
(repo / "app.py").write_text(
64+
'"""Calculator entrypoint."""\nfrom helper import helper\n\n'
65+
"def run(value: int) -> int:\n return helper(value)\n"
66+
)
67+
commit(repo)
68+
git(repo, "worktree", "add", "-qb", "smoke-worktree", ".claude/worktrees/feature")
69+
git(repo, "clone", "-q", ".", "copies/other")
70+
for directory in (".claude/worktrees/feature", "copies/other"):
71+
(repo / directory / "sentinel.py").write_text(
72+
"def nested_checkout_sentinel():\n return 'must not be documented'\n"
73+
)
74+
git(
75+
repo,
76+
"-c",
77+
"protocol.file.allow=always",
78+
"submodule",
79+
"add",
80+
"-q",
81+
str(dependency),
82+
"deps/math_support",
83+
)
84+
# Commit only the intended parent files and submodule registration.
85+
git(
86+
repo,
87+
"-c",
88+
"user.name=Checkout Smoke",
89+
"-c",
90+
"user.email=smoke@example.invalid",
91+
"commit",
92+
"-qm",
93+
"register dependency",
94+
)
95+
(repo / "untracked.py").write_text(
96+
'"""Format a calculator result."""\n'
97+
"def format_result(value: int) -> str:\n return str(value)\n"
98+
)
99+
return repo
100+
101+
102+
def verify(output: Path, model: str) -> dict:
103+
metadata = json.loads((output / "metadata.json").read_text())
104+
usage = metadata["usage"]
105+
assert not usage.get("incomplete"), "Generation was marked incomplete"
106+
assert usage["n_calls"] > 0, "No real model activity recorded"
107+
calls = usage["calls"]
108+
assert all(call["provider"] == "codex" for call in calls), calls
109+
assert all(call["model"] == model for call in calls), calls
110+
assert any(call["usage_reported"] for call in calls), "No provider usage recorded"
111+
assert usage["totals"].get("output_tokens", 0) > 0, "No model output recorded"
112+
113+
tree = json.loads((output / "module_tree.json").read_text())
114+
# A small repository uses an empty tree and one whole-repository overview.
115+
assert isinstance(tree, dict), "Invalid module tree"
116+
overview = output / "overview.md"
117+
assert overview.is_file() and overview.read_text().strip(), "Missing overview"
118+
pages = [p for p in output.rglob("*.md") if "temp" not in p.relative_to(output).parts]
119+
assert pages and all(p.read_text().strip() for p in pages), "Empty documentation page"
120+
for page in pages:
121+
text = page.read_text()
122+
assert "nested_checkout_sentinel" not in text, page
123+
assert ".claude/worktrees/feature" not in text and "copies/other" not in text, page
124+
125+
graphs = list((output / "temp/dependency_graphs").glob("*_dependency_graph.json"))
126+
assert len(graphs) == 1, graphs
127+
graph = json.loads(graphs[0].read_text())
128+
paths = {node["relative_path"] for node in graph.values()}
129+
assert paths == {"app.py", "helper.py", "untracked.py", "deps/math_support/support.py"}, paths
130+
assert set(graph["app.py::run"]["depends_on"]) == {"helper.py::helper"}
131+
if tree:
132+
validate_module_file_coverage(
133+
tree, {ident: Node.model_validate(node) for ident, node in graph.items()}
134+
)
135+
return {
136+
"provider": "codex",
137+
"requested_model": model,
138+
"recorded_models": sorted({call["model"] for call in calls}),
139+
"model_calls": usage["n_calls"],
140+
"source_paths": sorted(paths),
141+
"documentation_pages": [str(p.relative_to(output)) for p in pages],
142+
}
143+
144+
145+
def main() -> None:
146+
parser = argparse.ArgumentParser(description=__doc__)
147+
parser.add_argument("--provider", choices=["codex"], default="codex")
148+
parser.add_argument("--model", default="gpt-6-luna")
149+
args = parser.parse_args()
150+
if not shutil.which("codex") or not shutil.which("git"):
151+
parser.error("Git and an authenticated Codex CLI must be on PATH")
152+
root = Path(tempfile.mkdtemp(prefix="codewiki-checkout-smoke-"))
153+
print(f"Smoke artifacts: {root}", flush=True)
154+
repo = make_fixture(root)
155+
output, config_dir = root / "docs", root / "config"
156+
previous_cwd = Path.cwd()
157+
try:
158+
with ExitStack() as stack:
159+
stack.enter_context(patch.dict(os.environ, {"CODEWIKI_NO_KEYRING": "1"}))
160+
for name, value in {
161+
"CONFIG_DIR": config_dir,
162+
"CONFIG_FILE": config_dir / "config.json",
163+
"CREDENTIALS_FILE": config_dir / "credentials.json",
164+
}.items():
165+
stack.enter_context(patch.object(config_module, name, value))
166+
config_module.ConfigManager().save(
167+
provider=args.provider,
168+
main_model=args.model,
169+
cluster_model=args.model,
170+
fallback_model="",
171+
)
172+
os.chdir(repo)
173+
print(f"Generating with provider={args.provider}, model={args.model}...", flush=True)
174+
result = CliRunner().invoke(
175+
cli,
176+
[
177+
"generate",
178+
"--output",
179+
str(output),
180+
"--no-artifacts",
181+
"--no-cache",
182+
"--min-token-per-module",
183+
"1",
184+
"--max-tokens",
185+
"8192",
186+
"--request-limit",
187+
"24",
188+
"--verbose",
189+
"--instructions",
190+
"Keep documentation concise, under 300 words per page. "
191+
"Do not include Mermaid diagrams.",
192+
],
193+
)
194+
(root / "generate.log").write_text(result.output)
195+
if result.exit_code:
196+
raise RuntimeError(
197+
f"Generation failed with exit {result.exit_code}; see {root / 'generate.log'}"
198+
) from result.exception
199+
report = verify(output, args.model)
200+
(root / "report.json").write_text(json.dumps(report, indent=2))
201+
print(json.dumps(report, indent=2), flush=True)
202+
print("PASS: live generation preserved repository boundaries", flush=True)
203+
finally:
204+
os.chdir(previous_cwd)
205+
shutil.rmtree(config_dir, ignore_errors=True)
206+
207+
208+
if __name__ == "__main__":
209+
main()

0 commit comments

Comments
 (0)