From ee3ef507b88850721273c8a86a66387876b0a8ec Mon Sep 17 00:00:00 2001 From: medagent26 <307346623+medagent26@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:24:06 +0100 Subject: [PATCH] perf: parse source files in parallel; make the parse timeout configurable Dependency analysis parsed files one at a time on a single core with a hard-coded 30 s per-file timeout. A file that times out is dropped from the graph with only a warning, so its components never reach the docs. On a ~680-file TypeScript monorepo the five central 6-8k line files timed out on every run (the largest alone takes ~91 s). - Parse files in a spawn-based process pool, largest first, and merge per-file state in the original file order. The output is identical to the sequential sweep, including order. - CODEWIKI_PARSE_TIMEOUT sets the per-file timeout (default 300 s). - CODEWIKI_PARSE_WORKERS sets the pool size. Default: one worker per usable CPU (cgroup CPU quota respected), per 8 files and per 400 MB of available memory (cgroup memory limit respected); 1 keeps the sequential path. Both values are validated before parsing starts. - Timed-out files count as failed in the analysis summary. - Each worker buffers its log records per file and returns them with the file's result; the parent replays them through its own loggers. No pipe, lock or thread, so a timeout or a killed worker cannot leave a half-written record or block the parent. - Ctrl-C stops at once: workers ignore SIGINT (an interrupted worker can die holding the call queue's lock and deadlock the pool), the parent cancels queued files and terminates the workers. - A worker that dies or fails to start (OOM kill, caller script without a __main__ guard) aborts with a hint instead of being reported as failed files. Measured on that monorepo (678 files, 32 CPUs): sequential, timeout raised to 900 s: 606.6 s wall, 11,623 functions, 32,918 relationships parallel, defaults: 108.6 s wall, 11,623 functions, 32,918 relationships Functions, relationships and visualization data are byte-identical. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VARC7X5n9RnBw6wLGrf4TW --- CHANGELOG.md | 12 + .../analysis/call_graph_analyzer.py | 470 ++++++++++++++++-- docs/Dependency_Analysis_Service.md | 22 +- ...aScript_TypeScript_Analyzers_typescript.md | 2 +- tests/test_pack_grammars.py | 4 +- tests/test_parallel_parse.py | 354 +++++++++++++ tests/test_parse_timeout.py | 9 +- 7 files changed, 833 insertions(+), 40 deletions(-) create mode 100644 tests/test_parallel_parse.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 77adf74a..76be550a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,18 @@ uses [Semantic Versioning](https://semver.org/). ### Added +- **Parallel source parsing.** Dependency analysis parses files in a process + pool, largest first, and merges the results in file order, so the graph is + identical to a sequential run (a 678-file TypeScript repository: 607 s to + 106 s on 32 CPUs). `CODEWIKI_PARSE_WORKERS` sets the pool size; the default + is one worker per usable CPU (cgroup CPU quota respected), per 8 files and + per 400 MB of available memory, and `1` restores the sequential path. +- **Configurable parse timeout.** `CODEWIKI_PARSE_TIMEOUT` sets the per-file + timeout. The default rises from 30 s to 300 s: large hand-written files + (6-8k lines of TypeScript) took 30-90 s and were dropped from the graph on + every run. Files that time out or whose analysis raises now count as + failed in the analysis summary (they used to count as analyzed), and the + latter are logged as warnings instead of at debug level. - **Backstage TechDocs output.** `codewiki generate --techdocs` (or `codewiki techdocs DOCS_DIR OUT_DIR` on existing docs, with no LLM calls) writes an MkDocs project that `techdocs-cli generate` builds as is: diff --git a/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py b/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py index 47bc73b4..d6e45c5a 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py +++ b/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py @@ -7,12 +7,20 @@ """ import logging +import logging.handlers +import math +import multiprocessing +import os import re import signal +import sys import threading import time import traceback from collections import defaultdict +from concurrent.futures import ProcessPoolExecutor, as_completed +from concurrent.futures import process as futures_process +from concurrent.futures.process import BrokenProcessPool from contextlib import contextmanager from pathlib import Path from typing import TYPE_CHECKING @@ -71,6 +79,269 @@ def signal_handler(signum, frame): signal.signal(signal.SIGALRM, old_handler) +# A 30 s limit dropped the central 6-8k line TypeScript files of real +# repositories (one took 91 s alone), leaving holes in the dependency graph. +# A dropped file costs documentation; a slow one only costs time. +DEFAULT_PARSE_TIMEOUT_S = 300 +# Each worker re-imports the parent's ``__main__`` (about 1.5 s and 150 MB for +# the CLI), so a worker only pays off with several files to parse. +FILES_PER_WORKER = 8 +# Memory to reserve per worker: the import plus parse trees (peak worker RSS +# was ~150 MB on a 678-file TypeScript repository), with headroom. +WORKER_MEMORY_BYTES = 400 * 1024 * 1024 +# ProcessPoolExecutor rejects more workers than this on Windows. +_WINDOWS_MAX_WORKERS = getattr(futures_process, "_MAX_WINDOWS_WORKERS", 61) + +_CGROUP_ROOT = Path("/sys/fs/cgroup") +_PROC_SELF_CGROUP = Path("/proc/self/cgroup") +_PROC_MEMINFO = Path("/proc/meminfo") + + +def _positive_int_env(name: str, default: int) -> int: + raw = os.environ.get(name) + if raw is None or raw.strip() == "": + return default + try: + value = int(raw) + except ValueError: + value = 0 + if value < 1: + raise ValueError(f"{name} must be a positive integer, got {raw!r}") + return value + + +def parse_timeout_seconds() -> int: + """Per-file parse timeout: ``CODEWIKI_PARSE_TIMEOUT`` or 300 s.""" + return _positive_int_env("CODEWIKI_PARSE_TIMEOUT", DEFAULT_PARSE_TIMEOUT_S) + + +def _read(path: Path) -> str | None: + try: + return path.read_text().strip() + except OSError: + return None + + +def _cgroup_dirs(v1_controller: str) -> list[Path]: + """This process's cgroup directories, innermost first up to the mount + root: the v2 hierarchy and the given v1 controller. In a container the + listed path may be a host path; walking up then reaches the mount root, + which is the container's own cgroup.""" + root = _CGROUP_ROOT + dirs: list[Path] = [] + for line in (_read(_PROC_SELF_CGROUP) or "").splitlines(): + parts = line.split(":", 2) + if len(parts) != 3: + continue + hierarchy, controllers, path = parts + if hierarchy == "0" and controllers == "": + bases = [root] + elif v1_controller in controllers.split(","): + bases = [root / controllers, root / v1_controller] + else: + continue + for base in bases: + current = base / path.lstrip("/") + while True: + if current.is_dir() and current not in dirs: + dirs.append(current) + if current == base or base not in current.parents: + break + current = current.parent + return dirs + + +def _cgroup_cpu_limit() -> int | None: + """Tightest CPU quota over this process's cgroups (v2 ``cpu.max`` or v1 + CFS quota), rounded up; ``None`` when unlimited or unknown. Containers + report every host CPU to ``sched_getaffinity`` even under ``--cpus``.""" + limits = [] + for directory in _cgroup_dirs("cpu"): + fields = (_read(directory / "cpu.max") or "").split() + quota = _read(directory / "cpu.cfs_quota_us") + period = _read(directory / "cpu.cfs_period_us") + candidates = [] + if len(fields) >= 2 and fields[0] != "max": + candidates.append((fields[0], fields[1])) + if quota is not None and period is not None: + candidates.append((quota, period)) + for raw_quota, raw_period in candidates: + try: + q, p = int(raw_quota), int(raw_period) + except ValueError: + continue + if q > 0 and p > 0: + limits.append(math.ceil(q / p)) + return max(1, min(limits)) if limits else None + + +def _available_memory_bytes() -> int | None: + """Memory this process can still use: ``MemAvailable``, capped by the + remaining room of every limited cgroup (v2 or v1) it is in.""" + available = [] + for line in (_read(_PROC_MEMINFO) or "").splitlines(): + if line.startswith("MemAvailable:"): + try: + available.append(int(line.split()[1]) * 1024) + except (IndexError, ValueError): + pass + for directory in _cgroup_dirs("memory"): + limit = _read(directory / "memory.max") or _read(directory / "memory.limit_in_bytes") + usage = _read(directory / "memory.current") or _read(directory / "memory.usage_in_bytes") + try: + limit_bytes, usage_bytes = int(limit or ""), int(usage or "") + except ValueError: + continue # "max" (v2 unlimited) or unreadable + if limit_bytes >= 1 << 60: # v1 reports "unlimited" as ~2**63 + continue + # Usage includes page cache the kernel reclaims on demand (a fresh + # clone fills it); subtract it as docker stats and the kubelet do. + stat = {} + for line in (_read(directory / "memory.stat") or "").splitlines(): + fields = line.split() + if len(fields) == 2: + stat[fields[0]] = fields[1] + # v1 ``inactive_file`` is this cgroup's own pages; ``total_`` covers the + # subtree that ``usage_in_bytes`` counts. v2 has only ``inactive_file``. + inactive = stat.get("total_inactive_file") or stat.get("inactive_file") or "0" + try: + usage_bytes -= min(usage_bytes, int(inactive)) + except ValueError: + pass + available.append(max(0, limit_bytes - usage_bytes)) + return min(available) if available else None + + +def parse_worker_count(file_count: int) -> int: + """Parser processes for ``file_count`` files: ``CODEWIKI_PARSE_WORKERS``, + else one per usable CPU (cgroup quota respected), per ``FILES_PER_WORKER`` + files and per ``WORKER_MEMORY_BYTES`` of available memory. 1 means the + sequential path.""" + if os.environ.get("CODEWIKI_PARSE_WORKERS", "").strip(): + workers = min(_positive_int_env("CODEWIKI_PARSE_WORKERS", 1), file_count) + else: + if hasattr(os, "sched_getaffinity"): + cpus = len(os.sched_getaffinity(0)) + else: + cpus = os.cpu_count() or 1 + cpu_limit = _cgroup_cpu_limit() + if cpu_limit is not None: + cpus = min(cpus, cpu_limit) + workers = min(cpus, file_count // FILES_PER_WORKER) + memory = _available_memory_bytes() + if memory is not None: + workers = min(workers, memory // WORKER_MEMORY_BYTES) + if _is_windows(): + workers = min(workers, _WINDOWS_MAX_WORKERS) + return max(1, workers) + + +def _is_windows() -> bool: + return sys.platform == "win32" + + +def _logger_levels() -> dict[str, int]: + """Every level set explicitly in this process, for workers to copy.""" + levels = {"": logging.getLogger().level} + # dict.copy() is atomic under the GIL; another thread may add loggers. + for name, candidate in logging.root.manager.loggerDict.copy().items(): + if isinstance(candidate, logging.Logger) and candidate.level != logging.NOTSET: + levels[name] = candidate.level + return levels + + +_STANDARD_RECORD_KEYS = frozenset(vars(logging.makeLogRecord({}))) | {"message"} +_MAX_RECORDS_PER_FILE = 10_000 + + +class _RecordBuffer(list): + """``QueueHandler`` target that keeps a worker's records for the current + file. They go back to the parent with the file's result, so logging needs + no pipe, lock or thread: a SIGALRM timeout or a killed worker cannot + leave a half-written record behind or block the parent. Records lose + their ``extra`` attributes, so an unpicklable one cannot sink the file's + result, and a file keeps at most ``_MAX_RECORDS_PER_FILE`` of them.""" + + dropped = 0 + + def put_nowait(self, record: logging.LogRecord) -> None: + if len(self) >= _MAX_RECORDS_PER_FILE: + self.dropped += 1 + return + self.append( + logging.makeLogRecord( + {k: v for k, v in vars(record).items() if k in _STANDARD_RECORD_KEYS} + ) + ) + + def take(self) -> list[logging.LogRecord]: + records = list(self) + if self.dropped: + records.append( + logging.makeLogRecord( + { + "name": __name__, + "levelno": logging.WARNING, + "levelname": "WARNING", + "msg": f"Dropped {self.dropped} further log records of this file", + } + ) + ) + self.clear() + self.dropped = 0 + return records + + +_worker_context: dict = {} + + +def _init_parse_worker( + levels: dict[str, int], base_dir: str, python_project_modules: set, timeout_s: int +) -> None: + # Ctrl-C reaches the whole foreground process group. A worker interrupted + # inside the call queue's Queue.get dies holding its shared lock and the + # other workers block on it forever, so only the parent handles SIGINT. + signal.signal(signal.SIGINT, signal.SIG_IGN) + records = _RecordBuffer() + root = logging.getLogger() + # QueueHandler.prepare() turns records into picklable ones. + root.handlers[:] = [logging.handlers.QueueHandler(records)] + for name, level in levels.items(): + logging.getLogger(name or None).setLevel(level) + # Handlers live in the parent; let every record reach the buffer. + for candidate in logging.root.manager.loggerDict.values(): + if isinstance(candidate, logging.Logger): + candidate.handlers.clear() + candidate.propagate = True + _worker_context["records"] = records + _worker_context["base_dir"] = base_dir + _worker_context["python_project_modules"] = python_project_modules + _worker_context["timeout_s"] = timeout_s + + +def _analyze_file_in_worker(file_info: dict) -> tuple[bool, dict, list[logging.LogRecord]]: + records = _worker_context["records"] + records.take() + analyzer = CallGraphAnalyzer() + analyzer._python_project_modules = _worker_context["python_project_modules"] + completed = analyzer._analyze_code_file( + _worker_context["base_dir"], file_info, _worker_context["timeout_s"] + ) + return completed, analyzer._per_file_state(), records.take() + + +def _replay_worker_records(records: list[logging.LogRecord]) -> None: + """Hand worker records to the parent's logger of the same name, so they go + through the levels and handlers the caller configured.""" + for record in records: + try: + target = logging.getLogger(record.name) + if target.isEnabledFor(record.levelno): + target.handle(record) + except Exception as e: # noqa: BLE001 — logging must not change a file's outcome + logger.warning(f"Could not replay a worker log record: {e!s}") + + class CallGraphAnalyzer: def __init__(self): """Initialize the call graph analyzer.""" @@ -116,29 +387,19 @@ def analyze_code_files(self, code_files: list[dict], base_dir: str) -> dict: self._go_index = None self._reset_lean_state() - files_analyzed = 0 - files_failed = 0 + # Read both settings before any parsing: a bad value must fail the + # run, not be swallowed by the per-file error handling. + timeout_s = parse_timeout_seconds() + workers = parse_worker_count(len(code_files)) start_time = time.time() - - for idx, file_info in enumerate(code_files, 1): - file_path = file_info["path"] - try: - # Log progress every file with elapsed time - if idx % max(1, len(code_files) // 10) == 0 or idx <= 5: - elapsed = time.time() - start_time - rate = idx / elapsed if elapsed > 0 else 0 - remaining = (len(code_files) - idx) / rate if rate > 0 else 0 - logger.info( - f" [{idx}/{len(code_files)}] {file_path} ({elapsed:.1f}s elapsed, ~{remaining:.1f}s remaining)" - ) - - self._analyze_code_file(base_dir, file_info) - files_analyzed += 1 - except Exception as e: # noqa: BLE001 — one unparsable file must not abort the sweep - files_failed += 1 - logger.warning( - f" ⚠️ [{idx}/{len(code_files)}] Failed to analyze {file_path}: {str(e)[:100]}" - ) + if workers > 1: + files_analyzed, files_failed = self._analyze_files_parallel( + code_files, base_dir, workers, timeout_s + ) + else: + files_analyzed, files_failed = self._analyze_files_sequential( + code_files, base_dir, timeout_s + ) elapsed_time = time.time() - start_time logger.info( @@ -167,6 +428,149 @@ def analyze_code_files(self, code_files: list[dict], base_dir: str) -> dict: "visualization": viz_data, } + def _analyze_files_sequential( + self, code_files: list[dict], base_dir: str, timeout_s: int + ) -> tuple[int, int]: + files_analyzed = 0 + files_failed = 0 + start_time = time.time() + + for idx, file_info in enumerate(code_files, 1): + file_path = file_info["path"] + # Log progress every file with elapsed time + if idx % max(1, len(code_files) // 10) == 0 or idx <= 5: + elapsed = time.time() - start_time + rate = idx / elapsed if elapsed > 0 else 0 + remaining = (len(code_files) - idx) / rate if rate > 0 else 0 + logger.info( + f" [{idx}/{len(code_files)}] {file_path} ({elapsed:.1f}s elapsed, ~{remaining:.1f}s remaining)" + ) + + # _analyze_code_file handles every per-file error itself. + if self._analyze_code_file(base_dir, file_info, timeout_s): + files_analyzed += 1 + else: + files_failed += 1 + return files_analyzed, files_failed + + def _analyze_files_parallel( + self, code_files: list[dict], base_dir: str, workers: int, timeout_s: int + ) -> tuple[int, int]: + """Parse files in worker processes, then merge per-file state in the + original file order, so the result equals the sequential sweep. + + Per-file analysis only adds to the accumulators of ``_per_file_state`` + and never reads another file's results, which is what makes this + safe. ``spawn`` because callers may hold threads (fork is unsafe then) + and each worker runs tasks on its main thread, where SIGALRM works. + """ + total = len(code_files) + logger.info(f" Parsing with {workers} worker processes, {timeout_s}s timeout per file") + + def size(file_info: dict) -> int: + try: + return (Path(base_dir) / file_info["path"]).stat().st_size + except OSError: + return 0 + + # Largest first, so the slow files do not start last and set the wall time. + schedule = sorted(range(total), key=lambda i: size(code_files[i]), reverse=True) + results: list[dict | None] = [None] * total + failed: set[int] = set() + + ctx = multiprocessing.get_context("spawn") + start_time = time.time() + pool = None + try: + pool = ProcessPoolExecutor( + max_workers=workers, + mp_context=ctx, + initializer=_init_parse_worker, + initargs=( + _logger_levels(), + base_dir, + self._python_project_modules, + timeout_s, + ), + ) + futures = {pool.submit(_analyze_file_in_worker, code_files[i]): i for i in schedule} + for done, future in enumerate(as_completed(futures), 1): + i = futures[future] + file_path = code_files[i]["path"] + try: + completed, results[i], records = future.result() + _replay_worker_records(records) + if not completed: + failed.add(i) + except BrokenProcessPool as e: + # A worker died or could not start. Its results are lost, + # so the graph cannot be completed: fail loudly. + raise RuntimeError( + f"A parser worker process died or failed to start ({workers} " + f"workers). Likely causes: killed for memory, or a calling script " + f"without an `if __name__ == '__main__':` guard (worker processes " + f"are spawned and re-import it). Retry with fewer workers, e.g. " + f"CODEWIKI_PARSE_WORKERS=1 for the sequential path." + ) from e + except Exception as e: # noqa: BLE001 — one unparsable file must not abort the sweep + failed.add(i) + logger.warning( + f"⚠️ Failed to analyze {Path(base_dir) / file_path}: {str(e)[:200]}" + ) + if done % max(1, total // 10) == 0 or done <= 5: + logger.info( + f" [{done}/{total}] {file_path} ({time.time() - start_time:.1f}s elapsed)" + ) + except BaseException: + # Ctrl-C or a fatal error: drop the queued files and stop the + # workers now instead of waiting for the files in flight (each up + # to the parse timeout). ``_processes`` is private but present in + # every CPython 3.12-3.14 (3.14 wraps it as terminate_workers()). + for process in list((getattr(pool, "_processes", None) or {}).values()): + process.terminate() + if pool is not None: + # The pool's manager thread sees the dead workers and reaps + # them; joining them here as well would race its waitpid. + pool.shutdown(wait=True, cancel_futures=True) + raise + else: + pool.shutdown(wait=True) + + for state in results: + if state is not None: + self._merge_per_file_state(state) + return total - len(failed), len(failed) + + def _per_file_state(self) -> dict: + """Everything ``_analyze_code_file`` accumulates, for a worker to return.""" + return { + "functions": self.functions, + "call_relationships": self.call_relationships, + "python_external_import_roots": self._python_external_import_roots, + "dart_directives": self._dart_directives, + "go_visible": self._go_visible, + "lean_files": self._lean_files, + "lean_scopes": self._lean_scopes, + "lean_aliases": dict(self._lean_aliases), + "lean_private": self._lean_private, + "lean_protected": self._lean_protected, + } + + def _merge_per_file_state(self, state: dict) -> None: + # dict.update keeps a key's first position and takes the last value: + # the same as the sequential per-key assignments. + self.functions.update(state["functions"]) + self.call_relationships.extend(state["call_relationships"]) + self._python_external_import_roots.update(state["python_external_import_roots"]) + self._dart_directives.update(state["dart_directives"]) + self._go_visible.update(state["go_visible"]) + self._lean_files.update(state["lean_files"]) + self._lean_scopes.update(state["lean_scopes"]) + for alias, node_ids in state["lean_aliases"].items(): + self._lean_aliases[alias].extend(node_ids) + self._lean_private.update(state["lean_private"]) + self._lean_protected.update(state["lean_protected"]) + def extract_code_files(self, file_tree: dict) -> list[dict]: """ Extract code files from file tree structure. @@ -251,7 +655,7 @@ def _header_has_cpp_signal(self, base_dir: str, relative_path: str) -> bool: return True return False - def _analyze_code_file(self, repo_dir: str, file_info: dict): + def _analyze_code_file(self, repo_dir: str, file_info: dict, timeout_s: int) -> bool: """ Analyze a single code file based on its language. @@ -260,14 +664,19 @@ def _analyze_code_file(self, repo_dir: str, file_info: dict): Args: repo_dir: Repository directory path file_info: File information dictionary - """ + timeout_s: Per-file timeout in seconds - base = Path(repo_dir) - file_path = base / file_info["path"] + Returns: + False when the file timed out or its analysis raised. + """ + file_path = file_info.get("path") try: - # Add timeout protection (30 seconds per file max) - with timeout(30): + # Inside the try: callers rely on every per-file error being + # handled here. + base = Path(repo_dir) + file_path = base / file_info["path"] + with timeout(timeout_s): content = safe_open_text(base, file_path) language = file_info["language"] if language == "python": @@ -311,9 +720,12 @@ def _analyze_code_file(self, repo_dir: str, file_info: dict): except TimeoutError as e: logger.warning(f"⏱️ Timeout analyzing {file_path}: {e!s}") + return False except Exception as e: # noqa: BLE001 — per-file analysis is best-effort - logger.debug(f"Error analyzing {file_path}: {e!s}") + logger.warning(f"⚠️ Failed to analyze {file_path}: {str(e)[:200]}") logger.debug(f"Traceback: {traceback.format_exc()}") + return False + return True def _analyze_python_file(self, file_path: str, content: str, base_dir: str): """ diff --git a/docs/Dependency_Analysis_Service.md b/docs/Dependency_Analysis_Service.md index dc8ae3f8..be936dfa 100644 --- a/docs/Dependency_Analysis_Service.md +++ b/docs/Dependency_Analysis_Service.md @@ -180,8 +180,24 @@ The multi-language orchestrator. Given a list of code files (already filtered to - `python` → `analyzers/python.py::analyze_python_file` - `javascript` / `typescript` → tree-sitter analyzers in `analyzers/javascript.py` / `typescript.py` - `java`, `kotlin`, `csharp`, `c`, `cpp`, `php` → respective tree-sitter analyzers - Each file is analyzed under a 30-second timeout guard (`timeout` context manager, POSIX `SIGALRM`-based, - no-op on Windows) so a pathological file cannot stall the whole run. + Each file is analyzed under a per-file timeout guard (`timeout` context manager, POSIX `SIGALRM`-based, + no-op on Windows) so a pathological file cannot stall the whole run. A file that times out is dropped + from the graph with a warning. The limit is `CODEWIKI_PARSE_TIMEOUT` seconds (default 300; large + generated or hand-written files can need well over 30 s). + + Files are parsed in a `spawn` process pool (`_analyze_files_parallel`), largest first, and each worker's + per-file state is merged in the original file order, so the result equals a sequential sweep. Pool size: + `CODEWIKI_PARSE_WORKERS`, else one worker per usable CPU (cgroup CPU quota respected), per 8 files and per + 400 MB of available memory (cgroup memory limits respected); `1` keeps the sequential path. Each worker + re-imports the parent's `__main__` (about 150 MB for the CLI), so a script that calls the analysis needs an + `if __name__ == "__main__":` guard. A worker that dies or fails to start aborts the analysis with a hint + to retry with fewer workers. Workers ignore SIGINT; on Ctrl-C the parent cancels queued files and + terminates the workers. + + Per-file state is the set of accumulators listed in `_per_file_state`; anything else a worker writes is + lost. A new accumulator, written by an `_analyze_*_file` method or a helper it calls, must be added there. + `tests/test_parallel_parse.py` flags direct writes in those methods for every language, and any write + for the languages in its fixture (Python, Go, Dart, Lean, TypeScript). 4. **Resolves call relationships** (`_resolve_call_relationships`): builds global and per-language exact/ simple-name lookup indexes (`_build_resolution_indexes`) from `id`, `component_id`, `qualified_name`, and `name`, then attempts to match each unresolved `CallRelationship.callee` — preferring matches within the @@ -200,7 +216,7 @@ It also exposes `generate_llm_format()` (a compact, LLM-friendly functions/relat flowchart TB Files["code_files (filtered by language)"] --> Route["_route_contextual_headers\n(.h -> c/cpp)"] Route --> PyMods["_collect_python_modules"] - PyMods --> Loop["For each file: _analyze_code_file\n(30s timeout guard)"] + PyMods --> Loop["For each file, in a process pool: _analyze_code_file\n(CODEWIKI_PARSE_TIMEOUT guard)"] Loop -->|python| PyA["analyzers.python"] Loop -->|js/ts| JsA["analyzers.javascript / typescript"] Loop -->|java/kotlin/csharp| JvmA["analyzers.java / kotlin / csharp"] diff --git a/docs/JavaScript_TypeScript_Analyzers_typescript.md b/docs/JavaScript_TypeScript_Analyzers_typescript.md index 550fc139..b97369c3 100644 --- a/docs/JavaScript_TypeScript_Analyzers_typescript.md +++ b/docs/JavaScript_TypeScript_Analyzers_typescript.md @@ -316,6 +316,6 @@ Both analyzers in [JavaScript_TypeScript_Analyzers](JavaScript_TypeScript_Analyz ## Error Handling & Robustness -- Parser initialization failures are caught and logged; `analyze()` becomes a no-op rather than raising, so a single bad file cannot abort the whole-repository analysis run performed by `CallGraphAnalyzer.analyze_code_files` (which itself wraps each file analysis in a try/except and a 30-second timeout — see [Dependency_Analysis_Service](Dependency_Analysis_Service.md)). +- Parser initialization failures are caught and logged; `analyze()` becomes a no-op rather than raising, so a single bad file cannot abort the whole-repository analysis run performed by `CallGraphAnalyzer.analyze_code_files` (which itself wraps each file analysis in a try/except and a per-file timeout, `CODEWIKI_PARSE_TIMEOUT`, default 300 s — see [Dependency_Analysis_Service](Dependency_Analysis_Service.md)). - Every extraction/relationship helper method is wrapped in its own `try/except`, logging at `debug` level and returning `None`/skipping — a malformed or unusual AST shape in one construct never prevents extraction of the rest of the file. - The top-level `analyze_typescript_file_treesitter` function itself catches all exceptions and returns `([], [])` on failure, guaranteeing a well-typed (if empty) result to its caller. diff --git a/tests/test_pack_grammars.py b/tests/test_pack_grammars.py index 88803fd9..0ca94e0a 100644 --- a/tests/test_pack_grammars.py +++ b/tests/test_pack_grammars.py @@ -64,7 +64,9 @@ def test_analysis_stops_before_the_first_file(offline, tmp_path, monkeypatch): (tmp_path / "main.go").write_text("package main\n\nfunc main() {}\n", encoding="utf-8") analyzed = [] monkeypatch.setattr( - CallGraphAnalyzer, "_analyze_code_file", lambda self, base, info: analyzed.append(info) + CallGraphAnalyzer, + "_analyze_code_file", + lambda self, base, info, timeout_s: analyzed.append(info), ) with pytest.raises(GrammarUnavailableError, match="'go'"): CallGraphAnalyzer().analyze_code_files( diff --git a/tests/test_parallel_parse.py b/tests/test_parallel_parse.py new file mode 100644 index 00000000..7b844037 --- /dev/null +++ b/tests/test_parallel_parse.py @@ -0,0 +1,354 @@ +"""Parallel parsing must give exactly the sequential result, and the parse +timeout / worker count come from the environment.""" + +import ast +import inspect +import logging +import multiprocessing +import textwrap +import time +from pathlib import Path + +import pytest + +pytest.importorskip("tree_sitter_language_pack") + +from codewiki.src.be.dependency_analyzer.analysis import call_graph_analyzer as cga +from codewiki.src.be.dependency_analyzer.analysis.call_graph_analyzer import ( + CallGraphAnalyzer, +) + +FILES = { + "pkg/__init__.py": "", + "pkg/a.py": "from pkg.b import helper\nimport requests\n\ndef run():\n return helper()\n", + "pkg/b.py": "def helper():\n return 1\n\nclass Thing:\n def go(self):\n return helper()\n", + "pkg/c.py": "from pkg.a import run\n\ndef main():\n run()\n", + "pkg/d.py": "from pkg.b import Thing\n\ndef make():\n return Thing().go()\n", + "go.mod": "module example.com/demo\n\ngo 1.22\n", + "geo/point.go": "package geo\n\nfunc Origin() int { return 0 }\n", + "geo/move.go": "package geo\n\nfunc Move() int { return Origin() + 1 }\n", + "app/main.go": 'package main\n\nimport "example.com/demo/geo"\n\nfunc main() { geo.Move() }\n', + "pubspec.yaml": "name: demo\n", + "lib/car.dart": "class Car {\n void drive() { honk(); }\n void honk() {}\n}\n", + "lib/garage.dart": "import 'car.dart';\n\nclass Garage {\n void open() { Car().drive(); }\n}\n", + "lib/main.dart": "import 'package:demo/garage.dart';\n\nvoid main() { Garage().open(); }\n", + "src/Geo/Point.lean": "namespace Geo\n\nstructure Point where\n x : Nat\n\ndef origin : Point := ⟨0⟩\n\nprivate def secret := 1\n\nprotected def guarded := 2\n\nend Geo\n", + "src/Geo/Use.lean": "import Geo.Point\n\nopen Geo\n\ndef useOrigin := origin\n", + "web/util.ts": "export function add(a: number, b: number) { return a + b; }\n", + "web/calc.ts": "import { add } from './util';\n\nexport class Calc {\n sum() { return add(1, 2); }\n}\n", + "web/index.ts": "import { Calc } from './calc';\n\nnew Calc().sum();\n", +} +LANG = {".py": "python", ".go": "go", ".dart": "dart", ".lean": "lean", ".ts": "typescript"} + + +def _repo(root: Path) -> list[dict]: + code_files = [] + for rel, text in FILES.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + if path.suffix in LANG: + code_files.append( + { + "path": rel, + "name": path.name, + "extension": path.suffix, + "language": LANG[path.suffix], + } + ) + return code_files + + +def _run(root: Path, code_files: list[dict]) -> tuple[dict, CallGraphAnalyzer]: + analyzer = CallGraphAnalyzer() + return analyzer.analyze_code_files(code_files, str(root)), analyzer + + +def _state(analyzer: CallGraphAnalyzer) -> dict: + return { + "functions": list(analyzer.functions), + "python_roots": sorted(analyzer._python_external_import_roots), + "dart": sorted(analyzer._dart_directives), + "go": {k: sorted(v) for k, v in analyzer._go_visible.items()}, + "lean_files": sorted(analyzer._lean_files), + "lean_scopes": sorted(analyzer._lean_scopes), + "lean_aliases": {k: list(v) for k, v in analyzer._lean_aliases.items()}, + "lean_private": sorted(analyzer._lean_private), + "lean_protected": sorted(analyzer._lean_protected), + } + + +def test_parallel_result_equals_sequential(tmp_path, monkeypatch): + code_files = _repo(tmp_path) + + monkeypatch.setenv("CODEWIKI_PARSE_WORKERS", "1") + sequential, seq_analyzer = _run(tmp_path, code_files) + + monkeypatch.setenv("CODEWIKI_PARSE_WORKERS", "4") + + def no_sequential(*args, **kwargs): + raise AssertionError("parallel path not taken") + + monkeypatch.setattr(CallGraphAnalyzer, "_analyze_files_sequential", no_sequential) + parallel, par_analyzer = _run(tmp_path, code_files) + + assert parallel == sequential + assert _state(par_analyzer) == _state(seq_analyzer) + + # Guard against a vacuous pass: every language produced components, and + # cross-file calls were resolved. + paths = {f["relative_path"] for f in sequential["functions"]} + for suffix in LANG: + assert any(p.endswith(suffix) for p in paths), suffix + assert sequential["call_graph"]["files_analyzed"] == len(code_files) + assert len(sequential["relationships"]) >= 5 + assert "requests" in _state(seq_analyzer)["python_roots"] + + +def test_parse_timeout_from_env(monkeypatch): + monkeypatch.delenv("CODEWIKI_PARSE_TIMEOUT", raising=False) + assert cga.parse_timeout_seconds() == cga.DEFAULT_PARSE_TIMEOUT_S == 300 + monkeypatch.setenv("CODEWIKI_PARSE_TIMEOUT", "45") + assert cga.parse_timeout_seconds() == 45 + for bad in ("0", "-3", "abc"): + monkeypatch.setenv("CODEWIKI_PARSE_TIMEOUT", bad) + with pytest.raises(ValueError): + cga.parse_timeout_seconds() + + +@pytest.mark.parametrize("workers", ["1", "4"]) +def test_invalid_timeout_fails_the_run_before_parsing(tmp_path, monkeypatch, workers): + # Must not be swallowed per file (which used to leave an empty graph). + code_files = _repo(tmp_path) + monkeypatch.setenv("CODEWIKI_PARSE_WORKERS", workers) + monkeypatch.setenv("CODEWIKI_PARSE_TIMEOUT", "abc") + with pytest.raises(ValueError, match="CODEWIKI_PARSE_TIMEOUT"): + _run(tmp_path, code_files) + + +def test_parse_worker_count(monkeypatch): + monkeypatch.setenv("CODEWIKI_PARSE_WORKERS", "3") + assert cga.parse_worker_count(100) == 3 + assert cga.parse_worker_count(2) == 2 + monkeypatch.setenv("CODEWIKI_PARSE_WORKERS", "0") + with pytest.raises(ValueError): + cga.parse_worker_count(100) + + monkeypatch.delenv("CODEWIKI_PARSE_WORKERS") + monkeypatch.setattr(cga, "_cgroup_cpu_limit", lambda: None) + assert cga.parse_worker_count(cga.FILES_PER_WORKER * 2 - 1) == 1 + assert 1 <= cga.parse_worker_count(10_000) + monkeypatch.setattr(cga, "_cgroup_cpu_limit", lambda: 2) + assert cga.parse_worker_count(10_000) <= 2 + monkeypatch.setattr(cga, "_cgroup_cpu_limit", lambda: None) + monkeypatch.setattr(cga, "_available_memory_bytes", lambda: 3 * cga.WORKER_MEMORY_BYTES + 1) + assert cga.parse_worker_count(10_000) <= 3 + monkeypatch.setattr(cga, "_available_memory_bytes", lambda: 0) + assert cga.parse_worker_count(10_000) == 1 + + # ProcessPoolExecutor rejects more than 61 workers on Windows. + monkeypatch.setattr(cga, "_available_memory_bytes", lambda: None) + monkeypatch.setattr(cga.os, "sched_getaffinity", lambda pid: set(range(128)), raising=False) + monkeypatch.setattr(cga, "_is_windows", lambda: True) + assert cga.parse_worker_count(10_000) == 61 + monkeypatch.setenv("CODEWIKI_PARSE_WORKERS", "100") + assert cga.parse_worker_count(10_000) == 61 + + +def _fake_system(monkeypatch, tmp_path: Path, cgroup: str, files: dict[str, str]) -> None: + root = tmp_path / "cgroup" + for rel, text in files.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + root.mkdir(exist_ok=True) + (tmp_path / "proc_cgroup").write_text(cgroup, encoding="utf-8") + (tmp_path / "meminfo").write_text("MemTotal: 64000000 kB\nMemAvailable: 32000000 kB\n") + monkeypatch.setattr(cga, "_CGROUP_ROOT", root) + monkeypatch.setattr(cga, "_PROC_SELF_CGROUP", tmp_path / "proc_cgroup") + monkeypatch.setattr(cga, "_PROC_MEMINFO", tmp_path / "meminfo") + + +def test_cgroup_v2_limits_at_an_ancestor(tmp_path, monkeypatch): + _fake_system( + monkeypatch, + tmp_path, + "garbage-line\n0::/kubepods/pod1/ctr\n", + { + "kubepods/cpu.max": "max 100000", + "kubepods/pod1/cpu.max": "250000 100000", + "kubepods/pod1/ctr/cpu.max": "max 100000", + "kubepods/pod1/memory.max": str(2 * 1024**3), + # 1.5 GiB in use, of which 0.5 GiB is reclaimable page cache. + "kubepods/pod1/memory.current": str(3 * 1024**3 // 2), + "kubepods/pod1/memory.stat": f"anon 1\ninactive_file {1024**3 // 2}\nmalformed\n", + "kubepods/pod1/ctr/memory.max": "max", + "kubepods/pod1/ctr/memory.current": "5", + }, + ) + assert cga._cgroup_cpu_limit() == 3 + assert cga._available_memory_bytes() == 1024**3 + + +def test_cgroup_v1_uses_own_path_and_ignores_unlimited(tmp_path, monkeypatch): + _fake_system( + monkeypatch, + tmp_path, + "4:memory:/system.slice/codewiki.service\n3:cpu,cpuacct:/system.slice/codewiki.service\n", + { + "cpu,cpuacct/cpu.cfs_quota_us": "-1", + "cpu,cpuacct/cpu.cfs_period_us": "100000", + "cpu,cpuacct/system.slice/codewiki.service/cpu.cfs_quota_us": "150000", + "cpu,cpuacct/system.slice/codewiki.service/cpu.cfs_period_us": "100000", + "memory/memory.limit_in_bytes": str(2**63 - 4096), + "memory/memory.usage_in_bytes": "1", + }, + ) + assert cga._cgroup_cpu_limit() == 2 + assert cga._available_memory_bytes() == 32_000_000 * 1024 + + +def test_no_cgroup_information(tmp_path, monkeypatch): + _fake_system(monkeypatch, tmp_path, "", {}) + assert cga._cgroup_cpu_limit() is None + assert cga._available_memory_bytes() == 32_000_000 * 1024 + + +def _add_slow_file(root: Path, code_files: list[dict], n: int) -> None: + # Superlinear in the TypeScript analyzer: 12,000 functions take about 5 s. + big = "".join( + f"export function f{i}(x: number): number {{ return f{max(i - 1, 0)}(x) + f{(i * 7) % n}(x); }}\n" + for i in range(n) + ) + (root / "web" / "big.ts").write_text(big, encoding="utf-8") + code_files.append( + {"path": "web/big.ts", "name": "big.ts", "extension": ".ts", "language": "typescript"} + ) + + +def test_interrupt_stops_workers_without_waiting_for_files_in_flight(tmp_path, monkeypatch): + code_files = _repo(tmp_path) + _add_slow_file(tmp_path, code_files, 30_000) # ~20 s, still parsing at the interrupt + monkeypatch.setenv("CODEWIKI_PARSE_WORKERS", "2") + real_as_completed = cga.as_completed + + def interrupted(futures): + for future in real_as_completed(futures): + yield future + raise KeyboardInterrupt # Ctrl-C after the first finished file + + monkeypatch.setattr(cga, "as_completed", interrupted) + started = time.monotonic() + with pytest.raises(KeyboardInterrupt): + _run(tmp_path, code_files) + assert time.monotonic() - started < 15 # well under the in-flight file's ~20 s + assert multiprocessing.active_children() == [] + + +def test_worker_timeout_drops_file_and_warning_reaches_parent(tmp_path, monkeypatch, caplog): + code_files = _repo(tmp_path) + _add_slow_file(tmp_path, code_files, 12_000) + monkeypatch.setenv("CODEWIKI_PARSE_WORKERS", "2") + monkeypatch.setenv("CODEWIKI_PARSE_TIMEOUT", "1") + caplog.set_level(logging.WARNING) + + result, _ = _run(tmp_path, code_files) + + timeouts = [r for r in caplog.records if "Timeout analyzing" in r.getMessage()] + assert len(timeouts) == 1 and "big.ts" in timeouts[0].getMessage() + assert result["call_graph"]["files_analyzed"] == len(code_files) - 1 + paths = {f["relative_path"] for f in result["functions"]} + assert not any(p.endswith("big.ts") for p in paths) + assert any(p.endswith("calc.ts") for p in paths) + + +def _accumulators_written_by_analyzers() -> set[str]: + """``self.`` that an ``_analyze_*_file`` method assigns, subscripts + into, or mutates through a method call.""" + mutators = {"append", "extend", "update", "add", "setdefault", "insert", "pop", "clear"} + written = set() + for name, method in inspect.getmembers(CallGraphAnalyzer, inspect.isfunction): + if not (name.startswith("_analyze_") and name.endswith("_file")): + continue + tree = ast.parse(textwrap.dedent(inspect.getsource(method))) + for node in ast.walk(tree): + target = None + if isinstance(node, (ast.Assign, ast.AugAssign, ast.AnnAssign)): + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + for t in targets: + while isinstance(t, ast.Subscript): + t = t.value + if isinstance(t, ast.Attribute): + target = t + if isinstance(t.value, ast.Name) and t.value.id == "self": + written.add(t.attr) + elif ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr in mutators + ): + target = node.func.value + while isinstance(target, ast.Subscript): + target = target.value + if ( + isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id == "self" + ): + written.add(target.attr) + return written + + +CARRIED = { + "functions", + "call_relationships", + "_python_external_import_roots", + "_dart_directives", + "_go_visible", + "_lean_files", + "_lean_scopes", + "_lean_aliases", + "_lean_private", + "_lean_protected", +} + + +def test_analyzing_a_file_changes_only_carried_state(tmp_path): + # Runtime counterpart of the static check below: also sees state written + # by helpers that the _analyze_*_file methods call. + code_files = _repo(tmp_path) + modules = CallGraphAnalyzer._collect_python_modules(code_files) + for file_info in code_files: + fresh = CallGraphAnalyzer() + fresh._python_project_modules = set(modules) + analyzer = CallGraphAnalyzer() + analyzer._python_project_modules = set(modules) + assert analyzer._analyze_code_file(str(tmp_path), file_info, 60) + changed = {k for k, v in vars(analyzer).items() if vars(fresh).get(k) != v} + assert changed <= CARRIED, (file_info["path"], changed - CARRIED) + + +def test_every_analyzer_accumulator_is_carried_back_from_workers(): + # A worker's state outside _per_file_state is thrown away, so the parallel + # path would silently lose an accumulator that is missing there. + carried = CARRIED + written = _accumulators_written_by_analyzers() + assert {"functions", "call_relationships", "_lean_aliases"} <= written + assert written <= carried, written - carried + assert set(CallGraphAnalyzer()._per_file_state()) == {k.lstrip("_") for k in carried} + + +def test_record_buffer_strips_extras_and_caps_records(monkeypatch): + import pickle + + monkeypatch.setattr(cga, "_MAX_RECORDS_PER_FILE", 3) + buffer = cga._RecordBuffer() + for i in range(5): + buffer.put_nowait( + logging.makeLogRecord({"name": "x", "msg": f"m{i}", "node": lambda: None}) + ) + records = buffer.take() + pickle.dumps(records) # an unpicklable ``extra`` must not sink the file's result + assert [r.getMessage() for r in records[:3]] == ["m0", "m1", "m2"] + assert "Dropped 2 further log records" in records[3].getMessage() + assert buffer.take() == [] diff --git a/tests/test_parse_timeout.py b/tests/test_parse_timeout.py index 205015c3..6f5ca419 100644 --- a/tests/test_parse_timeout.py +++ b/tests/test_parse_timeout.py @@ -6,7 +6,6 @@ import pytest -from codewiki.src.be.dependency_analyzer.analysis import call_graph_analyzer from codewiki.src.be.dependency_analyzer.analysis.call_graph_analyzer import ( CallGraphAnalyzer, ) @@ -31,11 +30,9 @@ def test_timeout_escapes_except_exception(tmp_path, monkeypatch): (tmp_path / "slow.js").write_text("f();\n", encoding="utf-8") monkeypatch.setattr(CallGraphAnalyzer, "_analyze_javascript_file", _swallowing_analyzer) - real_timeout = call_graph_analyzer.timeout - monkeypatch.setattr(call_graph_analyzer, "timeout", lambda seconds: real_timeout(1)) - started = time.monotonic() - CallGraphAnalyzer()._analyze_code_file( - str(tmp_path), {"path": "slow.js", "language": "javascript"} + completed = CallGraphAnalyzer()._analyze_code_file( + str(tmp_path), {"path": "slow.js", "language": "javascript"}, 1 ) assert time.monotonic() - started < 10 + assert completed is False