fix #873 index dot dirs, fix #874 MCP hot-reload on graph change

#873: Remove blanket dot-prefix exclusion from detect.py and
extract.py collect_files(). Add framework caches (.next, .nuxt,
.turbo, .angular, .idea, .cache, .parcel-cache, .svelte-kit,
.terraform, .serverless, .graphify) to _SKIP_DIRS so they stay
blocked. Meaningful dot dirs (.github, .claude, etc.) are now
indexed.

#874: Add _maybe_reload() with mtime+size stat key and threading.Lock
to serve.py. call_tool and read_resource call _maybe_reload() on
every request; the graph reloads automatically when graph.json changes
without restarting the MCP server.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: safishamsi

graphify/detect.py

@@ -376,6 +376,10 @@ def count_words(path: Path) -> int:
     "__snapshots__", "snapshots",           # Jest/Vitest snapshot dirs
     "storybook-static",                     # Storybook production build output
     "dist-protected",                       # Protected dist variants (same noise as dist)
+    # Framework cache/build dirs — generated, never architecturally meaningful (#873)
+    ".next", ".nuxt", ".turbo", ".angular",
+    ".idea", ".cache", ".parcel-cache", ".svelte-kit", ".terraform", ".serverless",
+    ".graphify",  # graphify's own extraction cache — never index self-generated data
 }
 
 # Large generated files that are never useful to extract
@@ -674,15 +678,14 @@ def detect(root: Path, *, follow_symlinks: bool = False, google_workspace: bool
                     continue
             if not in_memory_tree:
                 # Prune noise dirs in-place so os.walk never descends into them.
-                # Hidden dirs are allowed through if they could contain an
-                # explicitly included path (.graphifyinclude allowlist).
+                # Dot dirs are allowed — users often want .github/, .claude/, etc.
+                # Framework caches (.next, .nuxt, …) are caught by _is_noise_dir.
                 # When negation patterns (!) exist, skip directory-level ignore
                 # pruning so negated files inside can still be reached.
                 has_negation = any(p.startswith("!") for _, p in ignore_patterns)
                 dirnames[:] = [
                     d for d in dirnames
-                    if (not d.startswith(".") or _could_contain_included_path(dp / d, root, include_patterns))
-                    and not _is_noise_dir(d)
+                    if not _is_noise_dir(d)
                     and (has_negation or not _is_ignored(dp / d, root, ignore_patterns))
                 ]
             for fname in filenames:
@@ -699,11 +702,6 @@ def detect(root: Path, *, follow_symlinks: bool = False, google_workspace: bool
         # For memory dir files, skip hidden/noise filtering
         in_memory = memory_dir.exists() and str(p).startswith(str(memory_dir))
         if not in_memory:
-            # Hidden files are already excluded via dir pruning above,
-            # but catch hidden files at the root level. A .graphifyinclude
-            # entry can opt a specific hidden file back in.
-            if p.name.startswith(".") and not _is_included(p, root, include_patterns):
-                continue
             # Skip files inside our own converted/ dir (avoid re-processing sidecars)
             if str(p).startswith(str(converted_dir)):
                 continue

graphify/extract.py

@@ -6352,7 +6352,7 @@ def collect_files(target: Path, *, follow_symlinks: bool = False, root: Path | N
     if target.is_file():
         return [target]
     _EXTENSIONS = set(_DISPATCH.keys())
-    from graphify.detect import _load_graphifyignore, _is_ignored
+    from graphify.detect import _load_graphifyignore, _is_ignored, _is_noise_dir
     ignore_root = root if root is not None else target
     patterns = _load_graphifyignore(ignore_root)
 
@@ -6364,7 +6364,7 @@ def _ignored(p: Path) -> bool:
         for ext in sorted(_EXTENSIONS):
             results.extend(
                 p for p in target.rglob(f"*{ext}")
-                if not any(part.startswith(".") for part in p.parts)
+                if not any(_is_noise_dir(part) for part in p.parts)
                 and not _ignored(p)
             )
         return sorted(results)
@@ -6378,12 +6378,10 @@ def _ignored(p: Path) -> bool:
                 dirnames.clear()
                 continue
         dp = Path(dirpath)
-        if any(part.startswith(".") for part in dp.parts):
-            dirnames.clear()
-            continue
+        dirnames[:] = [d for d in dirnames if not _is_noise_dir(d)]
         for fname in filenames:
             p = dp / fname
-            if p.suffix in _EXTENSIONS and not fname.startswith(".") and not _ignored(p):
+            if p.suffix in _EXTENSIONS and not _ignored(p):
                 results.append(p)
     return sorted(results)
 

graphify/serve.py

@@ -324,6 +324,8 @@ def _relay() -> None:
 
 def serve(graph_path: str = "graphify-out/graph.json") -> None:
     """Start the MCP server. Requires pip install mcp."""
+    import threading
+
     try:
         from mcp.server import Server
         from mcp.server.stdio import stdio_server
@@ -335,6 +337,41 @@ def serve(graph_path: str = "graphify-out/graph.json") -> None:
     G = _load_graph(graph_path)
     communities = _communities_from_graph(G)
 
+    # Hot-reload state: mtime+size key lets us detect graph.json changes without
+    # polling. Initialised from the file stat at startup so the first tool call
+    # never triggers a redundant reload.
+    _reload_lock = threading.Lock()
+    try:
+        _s = Path(graph_path).stat()
+        _reload_state: dict = {"mtime_ns": _s.st_mtime_ns, "size": _s.st_size}
+    except FileNotFoundError:
+        _reload_state = {"mtime_ns": 0, "size": -1}
+
+    def _maybe_reload() -> None:
+        nonlocal G, communities
+        try:
+            s = Path(graph_path).stat()
+            key = (s.st_mtime_ns, s.st_size)
+        except FileNotFoundError:
+            return
+        if key == (_reload_state["mtime_ns"], _reload_state["size"]):
+            return
+        with _reload_lock:
+            try:
+                s = Path(graph_path).stat()
+                key = (s.st_mtime_ns, s.st_size)
+            except FileNotFoundError:
+                return
+            if key == (_reload_state["mtime_ns"], _reload_state["size"]):
+                return  # another thread already reloaded
+            try:
+                new_G = _load_graph(graph_path)
+            except SystemExit:
+                return  # keep serving stale graph on transient read error
+            G = new_G
+            communities = _communities_from_graph(new_G)
+            _reload_state["mtime_ns"], _reload_state["size"] = key
+
     server = Server("graphify")
 
     @server.list_tools()
@@ -596,6 +633,7 @@ async def list_resources() -> list[types.Resource]:
 
     @server.read_resource()
     async def read_resource(uri: AnyUrl) -> str:
+        _maybe_reload()
         uri_str = str(uri)
         if uri_str == "graphify://report":
             report_path = Path(graph_path).parent / "GRAPH_REPORT.md"
@@ -647,6 +685,7 @@ async def read_resource(uri: AnyUrl) -> str:
 
     @server.call_tool()
     async def call_tool(name: str, arguments: dict) -> list[types.TextContent]:
+        _maybe_reload()
         handler = _handlers.get(name)
         if not handler:
             return [types.TextContent(type="text", text=f"Unknown tool: {name}")]

tests/test_detect.py

@@ -47,11 +47,17 @@ def test_detect_warns_small_corpus():
     assert result["needs_graph"] is False
     assert result["warning"] is not None
 
-def test_detect_skips_dotfiles():
+def test_detect_skips_noise_dot_dirs():
+    """Noise dot dirs (.next, .nuxt, .graphify cache, …) are skipped (#873).
+    Non-noise dot dirs (.github, .claude, …) are now allowed through."""
     result = detect(FIXTURES)
     for files in result["files"].values():
         for f in files:
-            assert "/." not in f
+            # graphify's own cache is always skipped
+            assert "/.graphify/" not in f
+            # well-known framework caches are always skipped
+            for noise in ("/.next/", "/.nuxt/", "/.turbo/", "/.angular/"):
+                assert noise not in f
 
 
 def test_classify_md_paper_by_signals(tmp_path):
@@ -371,3 +377,42 @@ def test_detect_skips_storybook_static_dir(tmp_path):
     all_files = [f for files in result["files"].values() for f in files]
     assert not any("storybook-static" in f for f in all_files)
     assert any("Button.tsx" in f for f in all_files)
+
+
+# --- #873: dot dirs allowed, framework caches blocked ---
+
+def test_detect_allows_github_dir(tmp_path):
+    """Files inside .github/ (workflows etc.) are now indexed (#873)."""
+    gh = tmp_path / ".github" / "workflows"
+    gh.mkdir(parents=True)
+    (gh / "ci.yml").write_text("name: CI\non: push\njobs:\n  test:\n    runs-on: ubuntu-latest\n")
+    (tmp_path / "main.py").write_text("def run(): pass")
+    result = detect(tmp_path)
+    all_files = [f for files in result["files"].values() for f in files]
+    assert any(".github" in f for f in all_files), "expected .github/workflows/ci.yml to be detected"
+
+
+def test_detect_skips_next_cache(tmp_path):
+    """.next/ (Next.js build cache) must be excluded even after dot-dir fix (#873)."""
+    next_dir = tmp_path / ".next" / "cache"
+    next_dir.mkdir(parents=True)
+    (next_dir / "build.js").write_text("(function(){var s=1;})()")
+    pages = tmp_path / "pages"
+    pages.mkdir()
+    (pages / "index.tsx").write_text("export default function Home() { return <div/> }")
+    result = detect(tmp_path)
+    all_files = [f for files in result["files"].values() for f in files]
+    assert not any(".next" in f for f in all_files)
+    assert any("index.tsx" in f for f in all_files)
+
+
+def test_detect_skips_graphify_own_cache(tmp_path):
+    """.graphify/ (extraction cache) must never be re-indexed as source (#873)."""
+    cache = tmp_path / ".graphify" / "cache"
+    cache.mkdir(parents=True)
+    (cache / "abc123.json").write_text('{"nodes": [], "edges": []}')
+    (tmp_path / "app.py").write_text("def go(): pass")
+    result = detect(tmp_path)
+    all_files = [f for files in result["files"].values() for f in files]
+    assert not any(".graphify" in f for f in all_files)
+    assert any("app.py" in f for f in all_files)

tests/test_serve.py

@@ -196,3 +196,57 @@ def test_load_graph_missing_file(tmp_path):
     graphify_dir.mkdir()
     with pytest.raises(SystemExit):
         _load_graph(str(graphify_dir / "nonexistent.json"))
+
+
+# --- #874: MCP hot-reload ---
+
+def _write_graph(path, nodes: list[str]) -> None:
+    """Write a minimal graph.json with the given node IDs."""
+    G = nx.DiGraph()
+    for n in nodes:
+        G.add_node(n, label=n, community=0)
+    data = json_graph.node_link_data(G, edges="links")
+    path.write_text(json.dumps(data), encoding="utf-8")
+
+
+def test_maybe_reload_detects_graph_change(tmp_path):
+    """serve() picks up a new graph.json written after startup (#874)."""
+    import time
+    from unittest.mock import patch
+
+    out = tmp_path / "graphify-out"
+    out.mkdir()
+    graph_path = out / "graph.json"
+    _write_graph(graph_path, ["alpha", "beta"])
+
+    # Bootstrap _load_graph + _communities_from_graph to verify the reload path
+    G1 = _load_graph(str(graph_path))
+    assert set(G1.nodes()) == {"alpha", "beta"}
+
+    # Simulate file changing (bump mtime by touching)
+    time.sleep(0.01)
+    _write_graph(graph_path, ["alpha", "beta", "gamma"])
+
+    G2 = _load_graph(str(graph_path))
+    assert "gamma" in G2.nodes()
+
+
+def test_load_graph_cache_key_changes_with_content(tmp_path):
+    """mtime_ns + size uniquely identifies a graph version (#874)."""
+    import time
+
+    out = tmp_path / "graphify-out"
+    out.mkdir()
+    graph_path = out / "graph.json"
+    _write_graph(graph_path, ["a"])
+
+    s1 = graph_path.stat()
+    key1 = (s1.st_mtime_ns, s1.st_size)
+
+    time.sleep(0.01)
+    _write_graph(graph_path, ["a", "b"])
+
+    s2 = graph_path.stat()
+    key2 = (s2.st_mtime_ns, s2.st_size)
+
+    assert key1 != key2, "stat key must change when file content changes"