Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .cursor/skills/vast-provisioning/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ run a command in `tmux`.
(registered on the vast account automatically). Cursor Cloud images install
`openssh-client` in the Dockerfile; bootstrap generates a key if missing.
`provision up` refuses to rent when either is absent (avoids billed unready boxes).
- GitHub token (`--github-token` → `GITHUB_TOKEN` → `gh auth token`) when the
- GitHub token (`--github-token` → `GH_TOKEN` → `GITHUB_TOKEN` → `gh auth token`) when the
experiment repo is private (needed for the initial clone) and/or when using
`--self-destruct` (needed to push compact `experiments/` results).
- Always run through the `devops` group so `vastai` never enters the training env:
Expand Down
4 changes: 2 additions & 2 deletions devops/vast/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ compact experiment `results/` back and self-destruct.
- **SSH keypair** at `~/.ssh/id_rsa(.pub)`. The tool registers `id_rsa.pub` on
your vast account so direct SSH works.
- **`gh` CLI** authed (for `--self-destruct` result pushes): token resolution is
`--github-token` → `GITHUB_TOKEN` → `gh auth token`.
`--github-token` → `GH_TOKEN` → `GITHUB_TOKEN` → `gh auth token`.
- The `devops` dependency group: `uv sync --group devops` (installs `vastai`
locally only — it is never installed on the boxes).

Expand Down Expand Up @@ -82,7 +82,7 @@ uv run --group devops python -m devops.vast.provision destroy --all
| `--self-destruct` | inject teardown env + enable the training push+destroy hook |
| `--run-name NAME` | per-shot results subdir + commit label |
| `--results-branch NAME` | optional publication override (default: launch ref) |
| `--github-token TOK` | write token (else `GITHUB_TOKEN` / `gh auth token`) |
| `--github-token TOK` | write token (else `GH_TOKEN` / `GITHUB_TOKEN` / `gh auth token`) |
| `--teardown-on-error` | also push+destroy if the run raises (off by default) |
| `--max-age HOURS` | wall-clock lifetime cap (default `MAX_AGE_HOURS`=5; `0` disables) |
| `--forward-b2` | inject local `B2_*` credentials for artifact upload (off by default; persists in Vast control-plane metadata) |
Expand Down
5 changes: 3 additions & 2 deletions devops/vast/bootstrap.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# VAST_LIBRARY_GIT_REF branch or sha for the library (default: main)
# VAST_RUN_CMD optional command run in the activated .venv in tmux
# VAST_SELF_DESTRUCT "1" to arm the push-results teardown hook
# GITHUB_TOKEN write token for private clone and/or results push
# GH_TOKEN / GITHUB_TOKEN write token for private clone and/or results push
# VAST_RESULTS_BRANCH branch the teardown hook pushes results to
# VAST_RUN_NAME per-shot run label
# GIT_USER_NAME/GIT_USER_EMAIL commit identity for the results push
Expand Down Expand Up @@ -41,6 +41,7 @@ LIBRARY_REF="${VAST_LIBRARY_GIT_REF:-main}"
EXPERIMENT_NAME="$(basename "${EXPERIMENT_URL%.git}")"
EXPERIMENT_DIR="$WORK_DIR/${EXPERIMENT_NAME:-alex-rl-experiments}"
export VAST_EXPERIMENT_DIR="$EXPERIMENT_DIR"
GITHUB_TOKEN="${GITHUB_TOKEN:-${GH_TOKEN:-}}"

log() { echo "[bootstrap $(date -u +%H:%M:%S)] $*"; }
fail() { log "ERROR: $*"; echo "$*" > "$FAIL_SENTINEL"; exit 1; }
Expand Down Expand Up @@ -118,7 +119,7 @@ if [ -n "${GITHUB_TOKEN:-}" ] && [ -n "$EXPERIMENT_SLUG" ]; then
git remote set-url origin \
"https://x-access-token:${GITHUB_TOKEN}@github.com/${EXPERIMENT_SLUG}.git"
elif [ "${VAST_SELF_DESTRUCT:-0}" = "1" ]; then
log "WARNING: self-destruct set but GITHUB_TOKEN/VAST_EXPERIMENT_REPO_SLUG missing; push will be skipped"
log "WARNING: self-destruct set but GH_TOKEN/GITHUB_TOKEN/VAST_EXPERIMENT_REPO_SLUG missing; push will be skipped"
fi

# --- max-age watchdog ---------------------------------------------------
Expand Down
12 changes: 7 additions & 5 deletions devops/vast/provision.py
Original file line number Diff line number Diff line change
Expand Up @@ -258,13 +258,15 @@ def resolve_library_ref(args, cfg: VastConfig, log=print) -> str:


def resolve_github_token(args) -> Optional[str]:
"""Token resolution: --github-token > GITHUB_TOKEN env > `gh auth token`."""
"""Token resolution: --github-token > GH_TOKEN > GITHUB_TOKEN > `gh auth token`."""
if getattr(args, "github_token", None):
return args.github_token
import os

if os.environ.get("GITHUB_TOKEN"):
return os.environ["GITHUB_TOKEN"]
for name in ("GH_TOKEN", "GITHUB_TOKEN"):
value = os.environ.get(name)
if value and value.strip():
return value.strip()
try:
tok = subprocess.run(["gh", "auth", "token"], capture_output=True, text=True)
if tok.returncode == 0 and tok.stdout.strip():
Expand Down Expand Up @@ -460,7 +462,7 @@ def cmd_up(args, cfg: VastConfig) -> int:
github_token = resolve_github_token(args)
if args.self_destruct and not github_token:
log("--self-destruct requires a GitHub token with experiment-repo push access "
"(--github-token / GITHUB_TOKEN / `gh auth token`); refusing to rent.")
"(--github-token / GH_TOKEN / GITHUB_TOKEN / `gh auth token`); refusing to rent.")
return 2
if args.offer_id is not None and args.count != 1:
log("--offer-id selects one offer and requires --count 1.")
Expand Down Expand Up @@ -944,7 +946,7 @@ def build_parser() -> argparse.ArgumentParser:
up.add_argument("--run-name", default=None, help="per-shot results subdir + commit label")
up.add_argument("--results-branch", default=None,
help="branch the box pushes results to (default: 'results')")
up.add_argument("--github-token", default=None, help="write token (else GITHUB_TOKEN / gh auth token)")
up.add_argument("--github-token", default=None, help="write token (else GH_TOKEN / GITHUB_TOKEN / gh auth token)")
up.add_argument("--teardown-on-error", action="store_true",
help="also push+destroy if the run raises (off by default)")
up.add_argument("--max-age", type=float, default=None, metavar="HOURS",
Expand Down
1 change: 1 addition & 0 deletions devops/vast/redaction.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ def _is_secret_env_key(key: object) -> bool:
return True
upper = name.upper()
return upper in {
"GH_TOKEN",
"GITHUB_TOKEN",
"VAST_API_KEY",
"B2_APPLICATION_KEY",
Expand Down
1 change: 1 addition & 0 deletions devops/vast/self_destruct.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ def _log(msg: str, log=print, secrets: Iterable[str | None] = ()) -> None:
known_secrets = (
*secrets,
os.environ.get("VAST_API_KEY"),
os.environ.get("GH_TOKEN"),
os.environ.get("GITHUB_TOKEN"),
os.environ.get("B2_APPLICATION_KEY"),
os.environ.get("B2_APPLICATION_KEY_ID"),
Expand Down
48 changes: 48 additions & 0 deletions tests/test_infra_safeguards.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
cmd_up,
load_state,
record_instance,
resolve_github_token,
unrecord_instance,
)
from devops.vast.quarantine import active_exclusions, load_quarantine, record_failure
Expand Down Expand Up @@ -322,6 +323,29 @@ def test_expand_git_ref_expands_short_commit_sha(tmp_path):
assert _expand_git_ref(repo, "main") == "main"


def test_resolve_github_token_prefers_cli_then_gh_token(monkeypatch):
monkeypatch.delenv("GH_TOKEN", raising=False)
monkeypatch.delenv("GITHUB_TOKEN", raising=False)
args = SimpleNamespace(github_token=None)

monkeypatch.setenv("GH_TOKEN", "gh-from-gh-token")
assert resolve_github_token(args) == "gh-from-gh-token"

monkeypatch.setenv("GITHUB_TOKEN", "gh-from-github-token")
assert resolve_github_token(args) == "gh-from-gh-token"

args.github_token = "gh-from-cli"
assert resolve_github_token(args) == "gh-from-cli"


def test_resolve_github_token_falls_back_to_github_token(monkeypatch):
monkeypatch.delenv("GH_TOKEN", raising=False)
monkeypatch.setenv("GITHUB_TOKEN", "gh-from-github-token")
args = SimpleNamespace(github_token=None)

assert resolve_github_token(args) == "gh-from-github-token"


def test_self_destruct_refuses_to_rent_without_a_github_token(monkeypatch, capsys):
monkeypatch.setattr("devops.vast.provision.resolve_github_token", lambda args: None)

Expand Down Expand Up @@ -886,6 +910,30 @@ def test_redact_instance_metadata_hides_control_plane_secrets():
assert "ghp_should_hide" not in json.dumps(safe)


def test_redact_instance_metadata_hides_gh_token():
from devops.vast.redaction import redact_instance_metadata

safe = redact_instance_metadata(
{
"id": 10,
"extra_env": {
"GH_TOKEN": "ghp_should_hide",
"VAST_GIT_REF": "abc",
},
}
)
assert safe["extra_env"]["GH_TOKEN"] == "<REDACTED>"
assert safe["extra_env"]["VAST_GIT_REF"] == "abc"


def test_bootstrap_normalizes_gh_token_to_github_token():
bootstrap = (
Path(__file__).resolve().parents[1] / "devops" / "vast" / "bootstrap.sh"
).read_text()

assert 'GITHUB_TOKEN="${GITHUB_TOKEN:-${GH_TOKEN:-}}"' in bootstrap


def _record_instance_worker(state_path: str, instance_id: int) -> None:
cfg = VastConfig(STATE_PATH=Path(state_path))
record_instance(cfg, {"id": instance_id, "label": f"box-{instance_id}"})
Expand Down