From 9dd67b56d6b927a6aa10e53d87797fb29b00ac02 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 18:34:44 +0000 Subject: [PATCH] Accept GH_TOKEN in vast GitHub token resolution Vast provisioning previously read GITHUB_TOKEN only, while Cloud Agents and RunPod paths inject GH_TOKEN. Align vast with the shared resolution order: CLI flag, GH_TOKEN, GITHUB_TOKEN, then gh auth token. Bootstrap normalizes GH_TOKEN onto GITHUB_TOKEN for clone/push, and redaction covers both env var names. Co-authored-by: Alex Vardakostas --- .cursor/skills/vast-provisioning/SKILL.md | 2 +- devops/vast/README.md | 4 +- devops/vast/bootstrap.sh | 5 ++- devops/vast/provision.py | 12 +++--- devops/vast/redaction.py | 1 + devops/vast/self_destruct.py | 1 + tests/test_infra_safeguards.py | 48 +++++++++++++++++++++++ 7 files changed, 63 insertions(+), 10 deletions(-) diff --git a/.cursor/skills/vast-provisioning/SKILL.md b/.cursor/skills/vast-provisioning/SKILL.md index fe9f1ec..7f077c4 100644 --- a/.cursor/skills/vast-provisioning/SKILL.md +++ b/.cursor/skills/vast-provisioning/SKILL.md @@ -47,7 +47,7 @@ run a command in `tmux`. (registered on the vast account automatically). Cursor Cloud images install `openssh-client` in the Dockerfile; bootstrap generates a key if missing. `provision up` refuses to rent when either is absent (avoids billed unready boxes). -- GitHub token (`--github-token` → `GITHUB_TOKEN` → `gh auth token`) when the +- GitHub token (`--github-token` → `GH_TOKEN` → `GITHUB_TOKEN` → `gh auth token`) when the experiment repo is private (needed for the initial clone) and/or when using `--self-destruct` (needed to push compact `experiments/` results). - Always run through the `devops` group so `vastai` never enters the training env: diff --git a/devops/vast/README.md b/devops/vast/README.md index 53fd879..8d18a7c 100644 --- a/devops/vast/README.md +++ b/devops/vast/README.md @@ -18,7 +18,7 @@ compact experiment `results/` back and self-destruct. - **SSH keypair** at `~/.ssh/id_rsa(.pub)`. The tool registers `id_rsa.pub` on your vast account so direct SSH works. - **`gh` CLI** authed (for `--self-destruct` result pushes): token resolution is - `--github-token` → `GITHUB_TOKEN` → `gh auth token`. + `--github-token` → `GH_TOKEN` → `GITHUB_TOKEN` → `gh auth token`. - The `devops` dependency group: `uv sync --group devops` (installs `vastai` locally only — it is never installed on the boxes). @@ -82,7 +82,7 @@ uv run --group devops python -m devops.vast.provision destroy --all | `--self-destruct` | inject teardown env + enable the training push+destroy hook | | `--run-name NAME` | per-shot results subdir + commit label | | `--results-branch NAME` | optional publication override (default: launch ref) | -| `--github-token TOK` | write token (else `GITHUB_TOKEN` / `gh auth token`) | +| `--github-token TOK` | write token (else `GH_TOKEN` / `GITHUB_TOKEN` / `gh auth token`) | | `--teardown-on-error` | also push+destroy if the run raises (off by default) | | `--max-age HOURS` | wall-clock lifetime cap (default `MAX_AGE_HOURS`=5; `0` disables) | | `--forward-b2` | inject local `B2_*` credentials for artifact upload (off by default; persists in Vast control-plane metadata) | diff --git a/devops/vast/bootstrap.sh b/devops/vast/bootstrap.sh index b6952e9..49d2048 100644 --- a/devops/vast/bootstrap.sh +++ b/devops/vast/bootstrap.sh @@ -13,7 +13,7 @@ # VAST_LIBRARY_GIT_REF branch or sha for the library (default: main) # VAST_RUN_CMD optional command run in the activated .venv in tmux # VAST_SELF_DESTRUCT "1" to arm the push-results teardown hook -# GITHUB_TOKEN write token for private clone and/or results push +# GH_TOKEN / GITHUB_TOKEN write token for private clone and/or results push # VAST_RESULTS_BRANCH branch the teardown hook pushes results to # VAST_RUN_NAME per-shot run label # GIT_USER_NAME/GIT_USER_EMAIL commit identity for the results push @@ -41,6 +41,7 @@ LIBRARY_REF="${VAST_LIBRARY_GIT_REF:-main}" EXPERIMENT_NAME="$(basename "${EXPERIMENT_URL%.git}")" EXPERIMENT_DIR="$WORK_DIR/${EXPERIMENT_NAME:-alex-rl-experiments}" export VAST_EXPERIMENT_DIR="$EXPERIMENT_DIR" +GITHUB_TOKEN="${GITHUB_TOKEN:-${GH_TOKEN:-}}" log() { echo "[bootstrap $(date -u +%H:%M:%S)] $*"; } fail() { log "ERROR: $*"; echo "$*" > "$FAIL_SENTINEL"; exit 1; } @@ -118,7 +119,7 @@ if [ -n "${GITHUB_TOKEN:-}" ] && [ -n "$EXPERIMENT_SLUG" ]; then git remote set-url origin \ "https://x-access-token:${GITHUB_TOKEN}@github.com/${EXPERIMENT_SLUG}.git" elif [ "${VAST_SELF_DESTRUCT:-0}" = "1" ]; then - log "WARNING: self-destruct set but GITHUB_TOKEN/VAST_EXPERIMENT_REPO_SLUG missing; push will be skipped" + log "WARNING: self-destruct set but GH_TOKEN/GITHUB_TOKEN/VAST_EXPERIMENT_REPO_SLUG missing; push will be skipped" fi # --- max-age watchdog --------------------------------------------------- diff --git a/devops/vast/provision.py b/devops/vast/provision.py index cec9931..9871f89 100644 --- a/devops/vast/provision.py +++ b/devops/vast/provision.py @@ -258,13 +258,15 @@ def resolve_library_ref(args, cfg: VastConfig, log=print) -> str: def resolve_github_token(args) -> Optional[str]: - """Token resolution: --github-token > GITHUB_TOKEN env > `gh auth token`.""" + """Token resolution: --github-token > GH_TOKEN > GITHUB_TOKEN > `gh auth token`.""" if getattr(args, "github_token", None): return args.github_token import os - if os.environ.get("GITHUB_TOKEN"): - return os.environ["GITHUB_TOKEN"] + for name in ("GH_TOKEN", "GITHUB_TOKEN"): + value = os.environ.get(name) + if value and value.strip(): + return value.strip() try: tok = subprocess.run(["gh", "auth", "token"], capture_output=True, text=True) if tok.returncode == 0 and tok.stdout.strip(): @@ -460,7 +462,7 @@ def cmd_up(args, cfg: VastConfig) -> int: github_token = resolve_github_token(args) if args.self_destruct and not github_token: log("--self-destruct requires a GitHub token with experiment-repo push access " - "(--github-token / GITHUB_TOKEN / `gh auth token`); refusing to rent.") + "(--github-token / GH_TOKEN / GITHUB_TOKEN / `gh auth token`); refusing to rent.") return 2 if args.offer_id is not None and args.count != 1: log("--offer-id selects one offer and requires --count 1.") @@ -944,7 +946,7 @@ def build_parser() -> argparse.ArgumentParser: up.add_argument("--run-name", default=None, help="per-shot results subdir + commit label") up.add_argument("--results-branch", default=None, help="branch the box pushes results to (default: 'results')") - up.add_argument("--github-token", default=None, help="write token (else GITHUB_TOKEN / gh auth token)") + up.add_argument("--github-token", default=None, help="write token (else GH_TOKEN / GITHUB_TOKEN / gh auth token)") up.add_argument("--teardown-on-error", action="store_true", help="also push+destroy if the run raises (off by default)") up.add_argument("--max-age", type=float, default=None, metavar="HOURS", diff --git a/devops/vast/redaction.py b/devops/vast/redaction.py index 1ba88bc..6442377 100644 --- a/devops/vast/redaction.py +++ b/devops/vast/redaction.py @@ -48,6 +48,7 @@ def _is_secret_env_key(key: object) -> bool: return True upper = name.upper() return upper in { + "GH_TOKEN", "GITHUB_TOKEN", "VAST_API_KEY", "B2_APPLICATION_KEY", diff --git a/devops/vast/self_destruct.py b/devops/vast/self_destruct.py index 476ab8a..e1861c1 100644 --- a/devops/vast/self_destruct.py +++ b/devops/vast/self_destruct.py @@ -92,6 +92,7 @@ def _log(msg: str, log=print, secrets: Iterable[str | None] = ()) -> None: known_secrets = ( *secrets, os.environ.get("VAST_API_KEY"), + os.environ.get("GH_TOKEN"), os.environ.get("GITHUB_TOKEN"), os.environ.get("B2_APPLICATION_KEY"), os.environ.get("B2_APPLICATION_KEY_ID"), diff --git a/tests/test_infra_safeguards.py b/tests/test_infra_safeguards.py index 5e5a2cb..3ebf877 100644 --- a/tests/test_infra_safeguards.py +++ b/tests/test_infra_safeguards.py @@ -19,6 +19,7 @@ cmd_up, load_state, record_instance, + resolve_github_token, unrecord_instance, ) from devops.vast.quarantine import active_exclusions, load_quarantine, record_failure @@ -322,6 +323,29 @@ def test_expand_git_ref_expands_short_commit_sha(tmp_path): assert _expand_git_ref(repo, "main") == "main" +def test_resolve_github_token_prefers_cli_then_gh_token(monkeypatch): + monkeypatch.delenv("GH_TOKEN", raising=False) + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + args = SimpleNamespace(github_token=None) + + monkeypatch.setenv("GH_TOKEN", "gh-from-gh-token") + assert resolve_github_token(args) == "gh-from-gh-token" + + monkeypatch.setenv("GITHUB_TOKEN", "gh-from-github-token") + assert resolve_github_token(args) == "gh-from-gh-token" + + args.github_token = "gh-from-cli" + assert resolve_github_token(args) == "gh-from-cli" + + +def test_resolve_github_token_falls_back_to_github_token(monkeypatch): + monkeypatch.delenv("GH_TOKEN", raising=False) + monkeypatch.setenv("GITHUB_TOKEN", "gh-from-github-token") + args = SimpleNamespace(github_token=None) + + assert resolve_github_token(args) == "gh-from-github-token" + + def test_self_destruct_refuses_to_rent_without_a_github_token(monkeypatch, capsys): monkeypatch.setattr("devops.vast.provision.resolve_github_token", lambda args: None) @@ -886,6 +910,30 @@ def test_redact_instance_metadata_hides_control_plane_secrets(): assert "ghp_should_hide" not in json.dumps(safe) +def test_redact_instance_metadata_hides_gh_token(): + from devops.vast.redaction import redact_instance_metadata + + safe = redact_instance_metadata( + { + "id": 10, + "extra_env": { + "GH_TOKEN": "ghp_should_hide", + "VAST_GIT_REF": "abc", + }, + } + ) + assert safe["extra_env"]["GH_TOKEN"] == "" + assert safe["extra_env"]["VAST_GIT_REF"] == "abc" + + +def test_bootstrap_normalizes_gh_token_to_github_token(): + bootstrap = ( + Path(__file__).resolve().parents[1] / "devops" / "vast" / "bootstrap.sh" + ).read_text() + + assert 'GITHUB_TOKEN="${GITHUB_TOKEN:-${GH_TOKEN:-}}"' in bootstrap + + def _record_instance_worker(state_path: str, instance_id: int) -> None: cfg = VastConfig(STATE_PATH=Path(state_path)) record_instance(cfg, {"id": instance_id, "label": f"box-{instance_id}"})