diff --git a/graphify/cache.py b/graphify/cache.py index 2a47c0c21..6cd686cb1 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -1140,6 +1140,64 @@ def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "a raise +def _derived_facts_dir(root: Path, cache_root: "Path | None", kind: str, + schema: int) -> Path: + """Directory for a per-file derived-facts cache (e.g. Python symbol + resolution facts, #perf). Namespaced by package version AND ``schema`` — + the payload shape depends on the extractor code, so entries from another + version/schema must never be reused; a bump simply orphans the old dir. + """ + _out = Path(_GRAPHIFY_OUT) + anchor = cache_root if cache_root is not None else root + base = _out if _out.is_absolute() else Path(anchor).resolve() / _out + d = base / "cache" / kind / f"v{_EXTRACTOR_VERSION}-s{schema}" + d.mkdir(parents=True, exist_ok=True) + return d + + +def load_derived_facts(path: Path, root: Path, cache_root: "Path | None", + kind: str, schema: int) -> "dict | None": + """Load a file's cached derived facts, keyed by the SAME content hash the + AST cache uses (so it is valid exactly when the file is unchanged). Returns + None on any miss or error — the caller recomputes. The payload is plain, + path-free JSON, so no portability rewrite is needed on load. + """ + try: + h = file_hash(path, root, cache_root=cache_root) + entry = _derived_facts_dir(root, cache_root, kind, schema) / f"{h}.json" + if entry.is_file(): + return json.loads(entry.read_text(encoding="utf-8")) + except Exception: + return None + return None + + +def save_derived_facts(path: Path, data: dict, root: Path, + cache_root: "Path | None", kind: str, schema: int) -> None: + """Persist a file's derived facts atomically. Best-effort: a failure just + means the next run recomputes them.""" + try: + h = file_hash(path, root, cache_root=cache_root) + target_dir = _derived_facts_dir(root, cache_root, kind, schema) + entry = target_dir / f"{h}.json" + fd, tmp_path = tempfile.mkstemp(dir=target_dir, prefix=f"{h}.", suffix=".tmp") + try: + os.write(fd, json.dumps(data).encode()) + os.close(fd) + _os_replace_with_fallback(tmp_path, entry) + except Exception: + try: + os.close(fd) + except OSError: + pass + try: + os.unlink(tmp_path) + except OSError: + pass + except Exception: + pass + + def cached_files(root: Path = Path(".")) -> set[str]: """Return set of file hashes that have a valid cache entry (any kind).""" base = Path(root).resolve() / _GRAPHIFY_OUT / "cache" diff --git a/graphify/extract.py b/graphify/extract.py index d54b841d9..f7cb56556 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -6883,7 +6883,7 @@ def _describe_syntax_error(rel: str, line: "int | None", kept: int) -> str: # marker set in the per-file extractor. Populated just before the pass that uses it. callable_nids: set[str] = set() - _augment_symbol_resolution_edges(paths, all_nodes, all_edges, root) + _augment_symbol_resolution_edges(paths, all_nodes, all_edges, root, cache_root=cache_location) # Merge a header-declared class (and its methods) with its sibling-impl # definition into ONE node (C/C++/ObjC #1547/#1556). Runs BEFORE the id-remap @@ -7359,7 +7359,9 @@ def _learn(e: dict) -> None: if py_paths: py_results = [r for r, p in zip(per_file, paths) if p.suffix == ".py"] try: - cross_file_edges = _resolve_cross_file_imports(py_results, py_paths, all_nodes, all_edges) + cross_file_edges = _resolve_cross_file_imports( + py_results, py_paths, all_nodes, all_edges, root, cache_root=cache_location + ) all_edges.extend(cross_file_edges) except Exception as exc: import logging diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index 13189609d..3a35fffce 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -2295,101 +2295,258 @@ def _python_call_identifier(node, source: bytes) -> str | None: return _read_text(function_node, source) return None +# Schema version for the cached Python raw-facts payload. Bump when +# _extract_python_raw_facts changes shape, so stale entries are ignored. +_PY_RAW_FACTS_SCHEMA = 2 + + +def _python_xfile_import_module(node, source: bytes) -> "dict | None": + """The module reference of an ``import_from_statement`` as + ``_resolve_cross_file_imports`` reads it — a relative form (the ``import_prefix`` + dots and the dotted tail) or an absolute dotted name. Mirrors that pass's own + parse so the cached form resolves identically.""" + for child in node.children: + if child.type == "relative_import": + prefix_text = "" + dotted_text = "" + for sub in child.children: + if sub.type == "import_prefix": + prefix_text = _read_text(sub, source) + elif sub.type == "dotted_name": + dotted_text = _read_text(sub, source) + return {"prefix": prefix_text, "dotted": dotted_text} + if child.type == "dotted_name": + return {"abs": _read_text(child, source)} + return None + + +def _extract_python_raw_facts(root_node, source: bytes) -> dict: + """The content-only symbol-resolution facts derivable from ONE file's AST, + with no filesystem or path dependence — so the result is a pure function of + the file's bytes and safe to cache by content hash (#perf). + + ``imports`` are the ``from … import …`` statements exactly as written (the + relative ``level``, the module string, the imported/local name pairs, the + line); resolving the module string to a target file on disk is deliberately + NOT done here — that depends on other files and must run fresh. ``uses`` are + the bare-identifier call sites inside each top-level function, keyed by the + function's own name (the caller node id is re-derived from the path at apply + time, so no path is baked in). + """ + imports: list[dict] = [] + for node in _walk_python_tree(root_node): + if node.type != "import_from_statement": + continue + module = _python_import_from_module(node, source) + if module is None: + continue + level, module_name = module + imports.append({ + "level": level, + "module": module_name, + "names": [[imp, loc] for imp, loc in _python_imported_names(node, source)], + "line": node.start_point[0] + 1, + }) + uses: list[dict] = [] + for node in root_node.children: + if node.type != "function_definition": + continue + name_node = node.child_by_field_name("name") + body = node.child_by_field_name("body") + if name_node is None or body is None: + continue + func_name = _read_text(name_node, source) + for call_node in _walk_python_tree(body): + callee = _python_call_identifier(call_node, source) + if callee is None: + continue + uses.append({ + "func": func_name, + "callee": callee, + "line": call_node.start_point[0] + 1, + }) + + # Cross-file import-resolution facts (consumed by _resolve_cross_file_imports, + # #perf). ``xfile_imports`` is each from-import's module reference (in that + # pass's own parse shape) plus its imported/local name pairs. ``xfile_refs`` + # is, for each identifier whose name a from-import binds locally, the chain + # of enclosing class/function NAMES (outermost first) and the line — the + # pass attributes the use to the FIRST enclosing symbol whose name resolves + # to a node, so the whole chain is kept and that choice is made fresh at + # apply time. Refs are filtered to the imported local names because only + # those are ever queried, keeping the cached payload small. + xfile_imports: list[dict] = [] + wanted: set[str] = set() + for node in _walk_python_tree(root_node): + if node.type != "import_from_statement": + continue + module = _python_xfile_import_module(node, source) + if module is None: + continue + names = [[imp, loc] for imp, loc in _python_imported_names(node, source)] + xfile_imports.append({"module": module, "names": names}) + for _imp, loc in names: + wanted.add(loc) + + xfile_refs: list[list] = [] + + def _walk_refs(node, chain: list[str]) -> None: + if node.type == "import_from_statement": + return # the import itself is not a use; handled via xfile_imports + cur = chain + if node.type in ("class_definition", "function_definition"): + name_node = node.child_by_field_name("name") + if name_node is not None: + cur = chain + [_read_text(name_node, source)] + if node.type == "identifier" and cur: + name = _read_text(node, source) + if name in wanted: + xfile_refs.append([name, cur, node.start_point[0] + 1]) + for child in node.children: + _walk_refs(child, cur) + + if wanted: + _walk_refs(root_node, []) + + return { + "imports": imports, + "uses": uses, + "xfile_imports": xfile_imports, + "xfile_refs": xfile_refs, + } + + +def _apply_python_raw_imports(raw: dict, path: Path, root: Path, + facts: _SymbolResolutionFacts) -> None: + """Resolve one file's raw import facts to target files (filesystem, fresh).""" + for imp in raw.get("imports", []): + level = imp["level"] + module_name = imp["module"] + line = imp["line"] + target_path = _resolve_python_module_path(module_name, path, root, level) + if target_path is not None: + # #1146: `from pkg import submod` — if the target is a package + # (__init__.py) and an imported name matches a submodule file on + # disk, emit a file-level import edge to that submodule rather + # than only to the package. + pkg_dir = target_path.parent if target_path.name == "__init__.py" else None + else: + # A PEP 420 namespace package: the module names a directory with + # no __init__.py, so there is no module file to resolve to, but + # the names it imports can still be submodule files on disk. + # Without this branch `from . import brain` in such a package + # emitted nothing, and `brain.think()` never became an edge. + pkg_dir = _resolve_python_namespace_dir(module_name, path, root, level) + if pkg_dir is None: + continue + for imported_name, local_name in imp["names"]: + if pkg_dir is not None: + sub_py = pkg_dir / f"{imported_name}.py" + sub_pkg = pkg_dir / imported_name / "__init__.py" + submodule = sub_py if sub_py.is_file() else (sub_pkg if sub_pkg.is_file() else None) + if submodule is not None: + facts.module_imports.append((path, submodule, line, local_name)) + continue + if target_path is None: + continue # a namespace package owns no symbols of its own to bind + facts.imports.append( + _SymbolImportFact(path, local_name, target_path, imported_name, line) + ) + if path.name == "__init__.py": + facts.exports.append( + _SymbolExportFact( + path, + local_name, + line, + target_path=target_path, + target_name=imported_name, + ) + ) + + +def _apply_python_raw_uses(raw: dict, path: Path, + facts: _SymbolResolutionFacts) -> None: + """Re-derive each caller node id from ``path`` and record its call uses.""" + stem = _file_stem(path) + for use in raw.get("uses", []): + source_id = _make_id(stem, use["func"]) + facts.uses.append( + _SymbolUseFact(path, source_id, use["callee"], "calls", "call", use["line"]) + ) + + +def _python_raw_facts_for( + path: Path, cache_root: "Path | None", root: Path +) -> "dict | None": + """One file's content-only Python raw facts, from the content-hash cache + when available (a warm run then does no read/parse), else by parsing and + populating the cache. Shared by both the symbol-resolution facts pass and + the cross-file import pass, so a file is parsed at most once across them — + and not at all on a warm run. Caching is opt-in via cache_root; without it + every call parses, matching the pre-cache behaviour.""" + from graphify.cache import load_derived_facts, save_derived_facts + + if cache_root is not None: + cached = load_derived_facts( + path, root, cache_root, "pyfacts", _PY_RAW_FACTS_SCHEMA + ) + if cached is not None: + return cached + parsed = _parse_python_tree(path) + if parsed is None: + return None + source, root_node = parsed + raw = _extract_python_raw_facts(root_node, source) + if cache_root is not None: + save_derived_facts(path, raw, root, cache_root, "pyfacts", _PY_RAW_FACTS_SCHEMA) + return raw + + def _collect_python_symbol_resolution_facts( paths: list[Path], root: Path, facts: _SymbolResolutionFacts, + cache_root: "Path | None" = None, ) -> None: py_paths = [path for path in paths if path.suffix == ".py"] if not py_paths: return - trees: dict[Path, tuple[bytes, object]] = {} + # The raw facts are a pure function of a file's bytes (no filesystem, no + # paths — see _extract_python_raw_facts), so they cache by content hash and + # a warm `graphify update` skips the re-read+re-parse this pass would + # otherwise do for every unchanged file. The filesystem-dependent half + # (module->file resolution, caller-id derivation) still runs fresh below, + # so a file added/removed elsewhere can never be served a stale target + # (#perf). + raw_by_path: dict[Path, dict] = {} for path in py_paths: - parsed = _parse_python_tree(path) - if parsed is None: - continue - source, root_node = parsed - trees[_resolve_cached(path)] = parsed - - for node in _walk_python_tree(root_node): - if node.type != "import_from_statement": - continue - module = _python_import_from_module(node, source) - if module is None: - continue - level, module_name = module - target_path = _resolve_python_module_path(module_name, path, root, level) - if target_path is not None: - # #1146: `from pkg import submod` — if the target is a package - # (__init__.py) and an imported name matches a submodule file on - # disk, emit a file-level import edge to that submodule rather - # than only to the package. - pkg_dir = target_path.parent if target_path.name == "__init__.py" else None - else: - # A PEP 420 namespace package: the module names a directory with - # no __init__.py, so there is no module file to resolve to, but - # the names it imports can still be submodule files on disk. - # Without this branch `from . import brain` in such a package - # emitted nothing, and `brain.think()` never became an edge. - pkg_dir = _resolve_python_namespace_dir(module_name, path, root, level) - if pkg_dir is None: - continue - for imported_name, local_name in _python_imported_names(node, source): - line = node.start_point[0] + 1 - if pkg_dir is not None: - sub_py = pkg_dir / f"{imported_name}.py" - sub_pkg = pkg_dir / imported_name / "__init__.py" - submodule = sub_py if sub_py.is_file() else (sub_pkg if sub_pkg.is_file() else None) - if submodule is not None: - facts.module_imports.append((path, submodule, line, local_name)) - continue - if target_path is None: - continue # a namespace package owns no symbols of its own to bind - facts.imports.append( - _SymbolImportFact(path, local_name, target_path, imported_name, line) - ) - if path.name == "__init__.py": - facts.exports.append( - _SymbolExportFact( - path, - local_name, - line, - target_path=target_path, - target_name=imported_name, - ) - ) + raw = _python_raw_facts_for(path, cache_root, root) + if raw is not None: + raw_by_path[path] = raw + # Imports for every file first, then uses — preserving the original + # two-pass order so the accumulated fact lists (and thus edge resolution) + # are byte-identical to the single-pass form. for path in py_paths: - parsed = trees.get(_resolve_cached(path)) - if parsed is None: - continue - source, root_node = parsed - for source_id, body in _python_top_level_function_bodies(path, root_node, source): - for node in _walk_python_tree(body): - imported_name = _python_call_identifier(node, source) - if imported_name is None: - continue - facts.uses.append( - _SymbolUseFact( - path, - source_id, - imported_name, - "calls", - "call", - node.start_point[0] + 1, - ) - ) + raw = raw_by_path.get(path) + if raw is not None: + _apply_python_raw_imports(raw, path, root, facts) + for path in py_paths: + raw = raw_by_path.get(path) + if raw is not None: + _apply_python_raw_uses(raw, path, facts) def _augment_symbol_resolution_edges( paths: list[Path], nodes: list[dict], edges: list[dict], root: Path, + cache_root: "Path | None" = None, ) -> None: facts = _SymbolResolutionFacts() _collect_js_symbol_resolution_facts(paths, facts) - _collect_python_symbol_resolution_facts(paths, root, facts) + _collect_python_symbol_resolution_facts(paths, root, facts, cache_root=cache_root) _apply_symbol_resolution_facts(paths, nodes, edges, root, facts) def _resolve_cross_file_imports( @@ -2397,6 +2554,8 @@ def _resolve_cross_file_imports( paths: list[Path], all_nodes: list[dict] | None = None, all_edges: list[dict] | None = None, + root: Path | None = None, + cache_root: "Path | None" = None, ) -> list[dict]: """ Two-pass import resolution: turn file-level imports into class-level edges. @@ -2481,118 +2640,84 @@ def _resolve_cross_file_imports( if not name_to_nid: continue - # Parse imports from this file (shared with the facts pass via the - # mtime-keyed memo, so each .py is parsed once across both passes). - parsed = _parse_python_tree(path) - if parsed is None: + # A warm run resolves these imports straight from the content-hash cache + # the facts pass already populated — no re-read, no re-parse of this file + # (#perf). The module->stem resolution and caller-id mapping below run + # fresh against the current node index, so a file added or removed + # elsewhere can never be served a stale target. + raw = _python_raw_facts_for(path, cache_root, root) + if raw is None: continue - source, root_node = parsed # local_name -> target node id (local_name honours `import X as Y`, so a - # reference to the alias in the body still attributes correctly). + # reference to the alias in the body still attributes correctly). Last + # writer wins on a repeated local name, matching the original in-order + # walk over import statements. import_targets: dict[str, str] = {} - # referenced name -> {source symbol nid: first reference line} - ref_sources: dict[str, dict[str, int]] = {} - - def _text(n) -> str: - return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace") - - def resolve_import(node) -> None: - # Find the module name - handles both absolute and relative imports. - # Relative: `from .models import X` → relative_import → dotted_name - # Absolute: `from models import X` → module_name field + for imp in raw.get("xfile_imports", []): + module = imp.get("module") + if not module: + continue # target_fq is the directory-qualified stem used as the key in # stem_to_entities. Relative imports are resolved exactly via the # importing file's directory; absolute imports fall back to the # bare-stem secondary index (first-writer-wins when names collide). target_fq: str | None = None - for child in node.children: - if child.type == "relative_import": - prefix_text = "" - dotted_text = "" - for sub in child.children: - if sub.type == "import_prefix": - prefix_text = _text(sub) - elif sub.type == "dotted_name": - dotted_text = _text(sub) - dots = prefix_text.count(".") if prefix_text else 1 - cur_dir = path.parent - for _ in range(dots - 1): - cur_dir = cur_dir.parent - if dotted_text: - candidate = cur_dir.joinpath(*dotted_text.split(".")).with_suffix(".py") + if "abs" in module: + dotted_name = module["abs"] + dotted_as_path = "/".join(dotted_name.split(".")) + if dotted_as_path in stem_to_entities: + target_fq = dotted_as_path + else: + suffix_matches = [ + fq for fq in stem_to_entities + if fq.endswith(f"/{dotted_as_path}") or fq.endswith(f"\\{dotted_as_path}") + ] + if len(suffix_matches) == 1: + target_fq = suffix_matches[0] else: - candidate = cur_dir / "__init__.py" - target_fq = _file_stem(candidate) - break - if child.type == "dotted_name" and target_fq is None: - dotted_name = _text(child) - dotted_as_path = "/".join(dotted_name.split(".")) - if dotted_as_path in stem_to_entities: - target_fq = dotted_as_path - else: - suffix_matches = [ - fq for fq in stem_to_entities - if fq.endswith(f"/{dotted_as_path}") or fq.endswith(f"\\{dotted_as_path}") - ] - if len(suffix_matches) == 1: - target_fq = suffix_matches[0] - else: - bare = dotted_name.split(".")[-1] - target_fq = bare_to_qualified.get(bare) + bare = dotted_name.split(".")[-1] + target_fq = bare_to_qualified.get(bare) + else: + prefix_text = module.get("prefix", "") + dotted_text = module.get("dotted", "") + dots = prefix_text.count(".") if prefix_text else 1 + cur_dir = path.parent + for _ in range(dots - 1): + cur_dir = cur_dir.parent + if dotted_text: + candidate = cur_dir.joinpath(*dotted_text.split(".")).with_suffix(".py") + else: + candidate = cur_dir / "__init__.py" + target_fq = _file_stem(candidate) if not target_fq or target_fq not in stem_to_entities: - return - - # Imported names come AFTER the 'import' keyword token. For - # `import X as Y` the target is found via X but the body uses Y. - past_import_kw = False - for child in node.children: - if child.type == "import": - past_import_kw = True - continue - if not past_import_kw: - continue - imported_name: str | None = None - local_name: str | None = None - if child.type == "dotted_name": - imported_name = local_name = _text(child) - elif child.type == "aliased_import": - name_node = child.child_by_field_name("name") - alias_node = child.child_by_field_name("alias") - if name_node is not None: - imported_name = _text(name_node) - local_name = _text(alias_node) if alias_node is not None else imported_name - if not imported_name or not local_name: - continue - tgt_nid = stem_to_entities[target_fq].get(imported_name) + continue + entities = stem_to_entities[target_fq] + for imported_name, local_name in imp.get("names", []): + tgt_nid = entities.get(imported_name) if tgt_nid: import_targets[local_name] = tgt_nid - def visit(node, current_nid: str | None) -> None: - # Identifiers inside an import statement are the import itself, not a - # real use — resolve the import here and don't descend into it. - if node.type == "import_from_statement": - resolve_import(node) - return - # Attribute references to the top-level symbol that contains them: a - # class is a unit (a reference inside one of its methods counts for - # the class, matching the documented DigestAuth->Response edge), and - # a module-level function is its own source. Only set at module scope - # (current_nid is None) so nested defs never override the container. - if current_nid is None and node.type in ("class_definition", "function_definition"): - name_node = node.child_by_field_name("name") - if name_node is not None: - mapped = name_to_nid.get(_text(name_node)) - if mapped is not None: - current_nid = mapped - if node.type == "identifier" and current_nid is not None: - slot = ref_sources.setdefault(_text(node), {}) - slot.setdefault(current_nid, node.start_point[0] + 1) - for child in node.children: - visit(child, current_nid) - - visit(root_node, None) + # referenced name -> {source symbol nid: first reference line}. Each + # cached ref carries the chain of enclosing class/function names + # (outermost first); attribute the use to the FIRST one whose name maps + # to a node id in THIS file. A class is a unit (a reference inside one of + # its methods counts for the class, matching the documented + # DigestAuth->Response edge), and a module-level function is its own + # source — exactly the original module-scope walk, but resolved here so + # the choice can't be baked into the (path-free) cache. + ref_sources: dict[str, dict[str, int]] = {} + for ref_name, chain, line in raw.get("xfile_refs", []): + containing_nid: str | None = None + for enclosing in chain: + mapped = name_to_nid.get(enclosing) + if mapped is not None: + containing_nid = mapped + break + if containing_nid is None: + continue + ref_sources.setdefault(ref_name, {}).setdefault(containing_nid, line) for name, tgt_nid in import_targets.items(): for src_nid, line in ref_sources.get(name, {}).items():