diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b73ae4591b..8286738c62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,10 +45,10 @@ jobs: echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" cachix-build-check: - # Mirror check in cachix-build to skip forks. - if: | - !cancelled() && - (github.event_name == 'push' || github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository) + # Holds no secrets, so it runs on fork PRs too: forks need its inputs-hash + # output to restore the base branch's publish marker and download the + # prebuilt binaries. Only cachix-build itself is fork-guarded. + if: ${{ !cancelled() }} timeout-minutes: 10 runs-on: ubuntu-latest permissions: @@ -149,14 +149,16 @@ jobs: CACHIX_CACHE_NAME: dimensionalos run: python3 bin/build-native-modules --verify-published - name: Record the published inputs manifest - # Forensics for the marker saved below: the manifest names every tree - # hash behind the inputs-hash this publish ran for. + # The manifest is forensics for the marker saved below; links.txt maps + # each result symlink to its store path so consumers can materialise + # the binaries with --link-results instead of re-running nix. env: CACHIX_CACHE_NAME: dimensionalos run: | mkdir -p .cachix-marker python3 bin/build-native-modules --inputs-hash \ > .cachix-marker/inputs-hash.txt 2> .cachix-marker/manifest.txt + python3 bin/build-native-modules --record-links > .cachix-marker/links.txt - name: Save publish marker uses: actions/cache/save@v6 with: @@ -548,6 +550,28 @@ jobs: - name: Set PYTHON_GIL=0 for free-threading builds if: ${{ endsWith(matrix.pyver, 't') }} run: echo "PYTHON_GIL=0" >> $GITHUB_ENV + # The published store paths are x86_64-linux, so the arm leg (and any + # future macOS leg) skips provisioning; binary-needing tests skip there. + # On a marker miss (fork PR, eviction) the binaries are simply absent — + # hosted runners never fall back to building the heavy C++ closures. + - name: Install Nix (with Cachix substituter) + if: runner.os == 'Linux' && runner.arch == 'X64' + env: + INPUT_EXTRA_NIX_CONFIG: | + extra-substituters = https://dimensionalos.cachix.org + extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI= + INPUT_SET_AS_TRUSTED_USER: "true" + run: bash docker/ros/install-nix.sh + - name: Restore publish marker + if: runner.os == 'Linux' && runner.arch == 'X64' + id: native-marker + uses: actions/cache/restore@v6 + with: + path: .cachix-marker + key: cachix-published-${{ runner.os }}-${{ runner.arch }}-${{ needs.cachix-build-check.outputs.inputs-hash }} + - name: Provision native modules from Cachix + if: steps.native-marker.outputs.cache-hit == 'true' + run: python3 bin/build-native-modules --link-results .cachix-marker/links.txt - name: Install dependencies run: uv sync --group tests --frozen - name: Run tests @@ -625,6 +649,11 @@ jobs: options: --memory=6g --memory-swap=6g volumes: - /var/cache/dimos-root-cache:/root/.cache + # Persistent Nix store — without it every run reinstalls nix + # and re-downloads the modules' full runtime closure (~600 MB) + # through the runner's uplink. GC'd at the end of the job; the + # provision step roots the live closure so GC keeps it. + - /var/cache/dimos-nix:/nix markers: "self_hosted or skipif_no_ros" experimental: false # macOS runner disabled for now — we don't want Mac tests. @@ -726,9 +755,35 @@ jobs: max-jobs = 0 EOF echo "/nix/var/nix/profiles/default/bin" >> "$GITHUB_PATH" - - name: Fetch native modules from Cachix + - name: Restore publish marker if: contains(matrix.markers, 'skipif_no_ros') - run: python3 bin/build-native-modules + id: native-marker + uses: actions/cache/restore@v6 + with: + path: .cachix-marker + key: cachix-published-${{ runner.os }}-${{ runner.arch }}-${{ needs.cachix-build-check.outputs.inputs-hash }} + - name: Provision native modules from Cachix + if: contains(matrix.markers, 'skipif_no_ros') + # The marker records every result -> store path mapping, so the warm + # path is symlink recreation against the runner's persistent /nix plus + # substitution of anything missing by exact path — no nix evaluation. + # A missed marker (evicted) falls back to the full nix build. + run: | + if [ "${{ steps.native-marker.outputs.cache-hit }}" = "true" ]; then + python3 bin/build-native-modules --link-results .cachix-marker/links.txt + else + python3 bin/build-native-modules + fi + # Root the out paths (and thereby their closures) so the end-of-job + # GC keeps them warm: the workspace result links die in the next + # run's `git clean`, so they cannot serve as roots themselves. + # Replacing the directory unroots superseded paths for GC to reap. + sudo mkdir -p /nix/var/nix/gcroots/dimos-native + sudo find /nix/var/nix/gcroots/dimos-native -maxdepth 1 -type l -delete + python3 bin/build-native-modules --record-links | cut -d' ' -f2 | sort -u | + while read -r p; do + sudo ln -sfn "$p" "/nix/var/nix/gcroots/dimos-native/$(basename "$p")" + done - name: Run tests run: uv run pytest --cov=dimos/ --junitxml=junit.xml -m '(${{ matrix.markers }}) and not mujoco' - name: Re-run the failing tests with maximum verbosity @@ -789,6 +844,13 @@ jobs: size_mb=$(du -sm "$UV_CACHE_DIR" | cut -f1) echo "uv cache size: ${size_mb} MB" if [ "$size_mb" -gt 25600 ]; then uv cache prune --ci; fi + - name: GC the persistent Nix store + # /nix persists on the runner (container volume above), so superseded + # module closures accumulate as inputs change. Unrooted paths go; the + # provision step's gcroots keep the live closure across runs. Not + # always(): a cancelled run can leave nix processes holding locks. + if: ${{ !cancelled() && matrix.os == 'Linux' }} + run: nix-collect-garbage --delete-older-than 3d self-hosted-large-tests: # Skip on PRs from forks which would expose the self-hosted runner to untrusted code from external contributors. diff --git a/bin/build-native-modules b/bin/build-native-modules index f1cba6d908..2545a05e79 100755 --- a/bin/build-native-modules +++ b/bin/build-native-modules @@ -24,6 +24,14 @@ Modes: while the push daemon drains. cachix-action reports push failures without failing the build, so CI must run this before trusting a build enough to save the publish marker. + --record-links Print `result-symlink store-path` per built module. The + publish job stores this in the marker cache entry so consumers + can materialise binaries without evaluating anything. + --link-results FILE + Recreate the symlinks a --record-links run captured. Store + paths missing locally are fetched with `nix copy` by exact + path — no evaluation — so NativeModule sees its executable + and skips building at test time. Why git object hashes are sound gate keys: a nix sandbox only sees store paths, and for these flakes every store path copied out of the repo is some tracked @@ -393,17 +401,48 @@ def _in_cachix_cache(cache: str, store_path: str) -> bool: return False -def verify_published(modules: tuple[DiscoveredModule, ...]) -> None: - cache = os.environ.get("CACHIX_CACHE_NAME") - if not cache: - raise SystemExit("--verify-published needs CACHIX_CACHE_NAME set") - missing: dict[str, str] = {} # store path -> module qualname, for the error message +def _result_links(modules: tuple[DiscoveredModule, ...]) -> list[tuple[str, str]]: + """(repo-relative result symlink, store path) for every built module.""" + entries = [] for module in modules: links = sorted((REPO_ROOT / module.build_dir).glob("result*")) if not links: raise SystemExit(f"{module.build_dir}: no result symlink — run the build first") for link in links: - missing[os.readlink(link)] = module.qualname + entries.append((link.relative_to(REPO_ROOT).as_posix(), os.readlink(str(link)))) + return entries + + +def link_results(links_file: str) -> None: + """Recreate the result symlinks a publish recorded, fetching any store + paths the local store lacks by exact path — no nix evaluation at all.""" + entries = [line.split(" ", 1) for line in Path(links_file).read_text().splitlines() if line] + if not entries: + raise SystemExit(f"{links_file}: no recorded result links") + missing = sorted({store for _, store in entries if not os.path.exists(store)}) + if missing: + # nix-store -r substitutes each path's closure from every configured + # substituter: the module paths come from Cachix, but their nixpkgs + # runtime deps (glibc, pcl, openmp, …) exist only in cache.nixos.org — + # Cachix holds just the locally-built paths, so a single-store + # `nix copy --from` cannot materialise the closure. + _log(f"Substituting {len(missing)} store path(s)") + subprocess.run(["nix-store", "--realise", *missing], check=True) + for link, store in entries: + target = REPO_ROOT / link + if target.is_symlink() or target.exists(): + target.unlink() + os.symlink(store, target) + _log(f"LINK: {link} -> {store}") + + +def verify_published(modules: tuple[DiscoveredModule, ...]) -> None: + cache = os.environ.get("CACHIX_CACHE_NAME") + if not cache: + raise SystemExit("--verify-published needs CACHIX_CACHE_NAME set") + missing: dict[str, str] = {} # store path -> result link, for the error message + for link, store in _result_links(modules): + missing[store] = link _log(f"Verifying {len(missing)} out path(s) against {cache}.cachix.org") deadline = time.monotonic() + int(os.environ.get("PUSH_VERIFY_TIMEOUT") or 600) while True: @@ -452,9 +491,27 @@ def main() -> None: action="store_true", help="check the built out paths are downloadable from the Cachix cache", ) + mode.add_argument( + "--record-links", + action="store_true", + help="print each built module's result symlink and its store path", + ) + mode.add_argument( + "--link-results", + metavar="FILE", + help="recreate the result symlinks a --record-links run captured," + " nix-copying missing store paths by exact path (no evaluation)", + ) args = parser.parse_args() modules = discover() + if args.link_results: + link_results(args.link_results) + return + if args.record_links: + for link, store in _result_links(modules): + print(f"{link} {store}") + return if args.verify_published: verify_published(modules) return