Fix .gitignore precedence and pattern handling (fixes #144)

src/kit/code_searcher.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import logging
+import os
 import re
 from dataclasses import dataclass
 from pathlib import Path
@@ -35,16 +37,89 @@ def __init__(self, repo_path: str) -> None:
         self._gitignore_spec = self._load_gitignore()  # Load gitignore spec
 
     def _load_gitignore(self):
-        """Loads .gitignore rules from the repository root."""
-        gitignore_path = self.repo_path / ".gitignore"
-        if gitignore_path.exists():
+        """Load all .gitignore files in repository tree and merge them.
+
+        Returns a PathSpec that respects all .gitignore files, with proper
+        precedence (patterns from deeper directories can override root patterns).
+        """
+        gitignore_files = []
+
+        # Collect all .gitignore files
+        for dirpath, dirnames, filenames in os.walk(self.repo_path):
+            # Skip .git directory
+            if ".git" in Path(dirpath).parts:
+                continue
+
+            if ".gitignore" in filenames:
+                gitignore_path = Path(dirpath) / ".gitignore"
+                gitignore_files.append(gitignore_path)
+
+        if not gitignore_files:
+            return None
+
+        # Sort by depth (shallowest first) for correct precedence
+        # Git processes .gitignore files from root to leaf, so later patterns can override earlier ones
+        gitignore_files.sort(key=lambda p: len(p.parts))
+
+        # Collect all patterns with proper path prefixes
+        all_patterns = []
+        for gitignore_path in gitignore_files:
+            gitignore_dir = gitignore_path.parent
+
             try:
                 with open(gitignore_path, "r", encoding="utf-8") as f:
-                    return pathspec.PathSpec.from_lines("gitwildmatch", f)
+                    patterns = f.readlines()
+
+                # Calculate relative base path from repo root
+                try:
+                    rel_base = gitignore_dir.relative_to(self.repo_path)
+                except ValueError:
+                    # gitignore outside repo (shouldn't happen, but be safe)
+                    continue
+
+                # Process each pattern
+                for pattern in patterns:
+                    pattern = pattern.strip()
+
+                    # Skip empty lines and comments
+                    if not pattern or pattern.startswith("#"):
+                        continue
+
+                    # Handle negation patterns (must preserve ! at the beginning)
+                    is_negation = pattern.startswith("!")
+                    if is_negation:
+                        pattern = pattern[1:]  # Remove the ! temporarily
+
+                    # Adjust pattern to be relative to repo root
+                    if str(rel_base) != ".":
+                        # Pattern is in subdirectory - prefix with path
+                        if pattern.startswith("/"):
+                            # Absolute pattern (from gitignore dir) - make relative to repo
+                            adjusted = f"{rel_base}/{pattern[1:]}"
+                        else:
+                            # Relative pattern - applies to directory and all subdirectories
+                            # Use /** to match files at any depth under the directory
+                            adjusted = f"{rel_base}/**/{pattern}" if "*" in pattern else f"{rel_base}/{pattern}"
+                    else:
+                        # Pattern is in root .gitignore - use as-is
+                        adjusted = pattern
+
+                    # Re-add negation prefix if needed
+                    if is_negation:
+                        adjusted = f"!{adjusted}"
+
+                    all_patterns.append(adjusted)
+
             except Exception as e:
-                # Log this error if logging is set up, or print
-                print(f"Warning: Could not load .gitignore: {e}")
-        return None
+                # Log warning but continue processing other .gitignore files
+                logging.warning(f"Could not load {gitignore_path}: {e}")
+                continue
+
+        if not all_patterns:
+            return None
+
+        # Create single merged pathspec
+        return pathspec.PathSpec.from_lines("gitwildmatch", all_patterns)
 
     def _should_ignore(self, file: Path) -> bool:
         """Checks if a file should be ignored based on .gitignore rules."""

src/kit/repo_mapper.py

@@ -24,11 +24,89 @@ def __init__(self, repo_path: str) -> None:
         self._gitignore_spec = self._load_gitignore()
 
     def _load_gitignore(self):
-        gitignore_path = self.repo_path / ".gitignore"
-        if gitignore_path.exists():
-            with open(gitignore_path) as f:
-                return pathspec.PathSpec.from_lines("gitwildmatch", f)
-        return None
+        """Load all .gitignore files in repository tree and merge them.
+
+        Returns a PathSpec that respects all .gitignore files, with proper
+        precedence (patterns from deeper directories can override root patterns).
+        """
+        gitignore_files = []
+
+        # Collect all .gitignore files
+        for dirpath, dirnames, filenames in os.walk(self.repo_path):
+            # Skip .git directory
+            if ".git" in Path(dirpath).parts:
+                continue
+
+            if ".gitignore" in filenames:
+                gitignore_path = Path(dirpath) / ".gitignore"
+                gitignore_files.append(gitignore_path)
+
+        if not gitignore_files:
+            return None
+
+        # Sort by depth (shallowest first) for correct precedence
+        # Git processes .gitignore files from root to leaf, so later patterns can override earlier ones
+        gitignore_files.sort(key=lambda p: len(p.parts))
+
+        # Collect all patterns with proper path prefixes
+        all_patterns = []
+        for gitignore_path in gitignore_files:
+            gitignore_dir = gitignore_path.parent
+
+            try:
+                with open(gitignore_path, "r", encoding="utf-8") as f:
+                    patterns = f.readlines()
+
+                # Calculate relative base path from repo root
+                try:
+                    rel_base = gitignore_dir.relative_to(self.repo_path)
+                except ValueError:
+                    # gitignore outside repo (shouldn't happen, but be safe)
+                    continue
+
+                # Process each pattern
+                for pattern in patterns:
+                    pattern = pattern.strip()
+
+                    # Skip empty lines and comments
+                    if not pattern or pattern.startswith("#"):
+                        continue
+
+                    # Handle negation patterns (must preserve ! at the beginning)
+                    is_negation = pattern.startswith("!")
+                    if is_negation:
+                        pattern = pattern[1:]  # Remove the ! temporarily
+
+                    # Adjust pattern to be relative to repo root
+                    if str(rel_base) != ".":
+                        # Pattern is in subdirectory - prefix with path
+                        if pattern.startswith("/"):
+                            # Absolute pattern (from gitignore dir) - make relative to repo
+                            adjusted = f"{rel_base}/{pattern[1:]}"
+                        else:
+                            # Relative pattern - applies to directory and all subdirectories
+                            # Use /** to match files at any depth under the directory
+                            adjusted = f"{rel_base}/**/{pattern}" if "*" in pattern else f"{rel_base}/{pattern}"
+                    else:
+                        # Pattern is in root .gitignore - use as-is
+                        adjusted = pattern
+
+                    # Re-add negation prefix if needed
+                    if is_negation:
+                        adjusted = f"!{adjusted}"
+
+                    all_patterns.append(adjusted)
+
+            except Exception as e:
+                # Log warning but continue processing other .gitignore files
+                logging.warning(f"Could not load {gitignore_path}: {e}")
+                continue
+
+        if not all_patterns:
+            return None
+
+        # Create single merged pathspec
+        return pathspec.PathSpec.from_lines("gitwildmatch", all_patterns)
 
     def _should_ignore(self, file: Path) -> bool:
         # Handle potential symlink resolution mismatches

tests/integration/test_humanlayer_repo.py

@@ -0,0 +1,44 @@
+import pytest
+from pathlib import Path
+from kit.repo_mapper import RepoMapper
+import subprocess
+
+
+@pytest.mark.integration
+@pytest.mark.skipif(
+    not Path("/home/selman/dev/humanlayer").exists(),
+    reason="Requires humanlayer repository"
+)
+def test_humanlayer_repo_gitignore():
+    """Integration test: Verify fix works on actual humanlayer repo."""
+
+    # Get git's file count
+    result = subprocess.run(
+        ["git", "ls-files"],
+        cwd="/home/selman/dev/humanlayer",
+        capture_output=True,
+        text=True
+    )
+    git_files = set(result.stdout.strip().split("\n"))
+    git_count = len(git_files)
+
+    # Get kit's file count
+    mapper = RepoMapper("/home/selman/dev/humanlayer")
+    tree = mapper.get_file_tree()
+    kit_count = len(tree)
+    kit_paths = {item["path"] for item in tree}
+
+    # Should be approximately equal (within 10% tolerance for build artifacts)
+    tolerance = 0.1
+    assert abs(kit_count - git_count) / git_count < tolerance, \
+        f"Kit returned {kit_count} files, Git tracks {git_count} files"
+
+    # Should be well under token limit (assuming ~100 chars per file path)
+    estimated_tokens = kit_count * 100
+    assert estimated_tokens < 25000, \
+        f"Estimated {estimated_tokens} tokens (exceeds 25k limit)"
+
+    # Verify no node_modules files included
+    node_modules_files = [p for p in kit_paths if "node_modules" in p]
+    assert len(node_modules_files) == 0, \
+        f"Found {len(node_modules_files)} node_modules files (should be 0)"