ci.yml

# Continuous integration for Crewship.
#
# Runs the repository's real interfaces: locked Python dependency sync, the
# project's own lint/test commands, schema validation of both configuration
# documents through `./ship.sh inspect`, and a full worker image build plus the
# behavior smoke in tests/container-smoke.sh.
#
# Supply chain: least-privilege token, no write permissions anywhere, and every
# third-party action pinned to the commit SHA of its latest release, with the
# tag in a trailing comment (verified 2026-10-01 via the GitHub REST API).
# .github/dependabot.yml moves the pins to each new release weekly.
# uv and Bun are not pinned: like every ./ship.sh launch, CI installs their
# latest releases.
name: ci

on:
  push:
    branches: [main]
  # No labeled/unlabeled: a label added while opening a PR started a second run
  # that cancelled the first one (see changelog.yml for the label-aware check).
  pull_request:
    types: [opened, synchronize, reopened]
  workflow_dispatch:

# Read-only by default; no job requests more.
permissions:
  contents: read

# GitHub replaces a pending run in the same group, so pushes to main need
# their own group (by SHA); pull requests keep per-ref cancelling.
concurrency:
  group: ci-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.sha }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

defaults:
  run:
    shell: bash

jobs:
  changes:
    name: classify changes
    runs-on: ubuntu-24.04
    timeout-minutes: 5
    outputs:
      code: ${{ steps.diff.outputs.code }}
      scope: ${{ steps.diff.outputs.scope }}
    steps:
      - name: Checkout PR merge
        if: github.event_name == 'pull_request'
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          fetch-depth: 2

      - name: Classify changed paths
        id: diff
        env:
          EVENT_NAME: ${{ github.event_name }}
        run: |
          code=true scope=all
          if [[ "$EVENT_NAME" == pull_request ]]; then
            code=false scope=imessage
            # Disable rename detection so both the old and new paths count.
            git diff --name-only --no-renames -z HEAD^1 HEAD > "$RUNNER_TEMP/changed-files"
            while IFS= read -r -d '' path; do
              case "$path" in
                blueprint/skills/*|blueprint/rules/*|blueprint/config/*) code=true scope=all; break ;;
                README*|docs/*|changelog.d/*|*.md) ;;
                site/*) scope=all ;;
                # Only the iMessage bridge reads apps/imessage/; the Python tests that cover it are
                # tests/test_imessage.py, tests/test_imessage_smoke.py and tests/test_bluebubbles_relay.py.
                apps/imessage/*) code=true ;;
                *) code=true scope=all; break ;;
              esac
            done < "$RUNNER_TEMP/changed-files"
          fi
          [[ "$code" == true ]] || scope=all
          echo "code=$code" >> "$GITHUB_OUTPUT"
          echo "scope=$scope" >> "$GITHUB_OUTPUT"

  config-and-python:
    name: configuration and python checks
    needs: [changes, python_checks, runtime_smoke]
    if: >-
      always() &&
      (needs.changes.result != 'success' ||
       needs.changes.outputs.code != 'false')
    runs-on: ubuntu-24.04
    timeout-minutes: 2
    steps:
      - name: Require both validation jobs
        env:
          CHANGES_RESULT: ${{ needs.changes.result }}
          CODE: ${{ needs.changes.outputs.code }}
          PYTHON_RESULT: ${{ needs.python_checks.result }}
          RUNTIME_RESULT: ${{ needs.runtime_smoke.result }}
        run: |
          [[ "$CHANGES_RESULT" == success ]] &&
          [[ "$CODE" == true ]] &&
          [[ "$PYTHON_RESULT" == success ]] &&
          [[ "$RUNTIME_RESULT" == success ]]

  # The suite is sharded over four runners, one 4-vCPU machine each, because the audit of 100 runs
  # showed 1031 s of pytest on one runner was the critical path (waiting and setup were under 15 s).
  # The required "configuration and python checks" gate reads this job's aggregate result, so every
  # shard must pass. Shard 1 also runs the checks that are not pytest.
  python_checks:
    name: python and configuration tests (${{ matrix.shard }}/4)
    needs: changes
    if: >-
      always() &&
      (needs.changes.result != 'success' ||
       needs.changes.outputs.code != 'false')
    runs-on: ubuntu-24.04
    timeout-minutes: 35
    strategy:
      fail-fast: false
      matrix:
        shard: [1, 2, 3, 4]
    permissions:
      contents: read
    steps:
      - name: Check prerequisite results
        env:
          CHANGES_RESULT: ${{ needs.changes.result }}
          CODE: ${{ needs.changes.outputs.code }}
        run: |
          [[ "$CHANGES_RESULT" == success ]] &&
          [[ "$CODE" == true || "$CODE" == false ]]

      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Install latest uv
        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
        with:
          version: latest
          enable-cache: true

      - name: Sync locked dependencies
        run: uv sync --locked --group dev

      # Bun tracks its latest release, the same as every ./ship.sh launch.
      - name: Install latest Bun
        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
        with:
          bun-version: latest

      - name: Lint
        if: matrix.shard == 1
        run: uv run ruff check .

      # tests/test_configuration.py runs koncreet.yml under fakeroot when not root.
      - name: Install fakeroot
        timeout-minutes: 5
        run: |
          if ! command -v fakeroot >/dev/null 2>&1; then
            sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 update
            sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 install --yes --no-install-recommends fakeroot
          fi

      # pytest-xdist: -n auto is one worker per CPU of the runner, and worksteal lets an idle
      # worker take the tail of a busy one's queue, so the long node tests do not set the finish.
      # A pull request that changes only apps/imessage/ runs the iMessage tests and the core
      # (changelog, instructions, README, skills); every other change and every push runs all.
      - name: Python tests
        env:
          SCOPE: ${{ needs.changes.outputs.scope }}
          CREWSHIP_TEST_SHARD: ${{ matrix.shard }}/4
        run: |
          tests=(tests)
          if [[ "$SCOPE" == imessage ]]; then
            tests=(tests/test_imessage.py tests/test_imessage_smoke.py tests/test_bluebubbles_relay.py tests/test_changelog.py
              tests/test_instructions.py tests/test_readme.py tests/test_skills.py)
          fi
          uv run pytest -n auto --dist worksteal "${tests[@]}"

      - name: Token suggestion installation and runtime smoke
        if: matrix.shard == 1 && needs.changes.outputs.scope != 'imessage'
        run: |
          user=crewship-token-proof
          home=/home/$user
          sudo useradd --create-home --shell /bin/bash "$user"
          cleanup() {
            sudo loginctl disable-linger "$user" || true
            sudo loginctl terminate-user "$user" || true
            sudo userdel --remove "$user" || true
          }
          trap cleanup EXIT
          sudo loginctl enable-linger "$user"
          uid=$(id -u "$user")
          sudo install -d -o "$user" -g "$user" "$home/source"
          tar -cf - blueprint scripts tests pyproject.toml uv.lock | sudo tar -xf - -C "$home/source"
          sudo cp "$(command -v uv)" "$home/uv"
          sudo chown -R "$user:$user" "$home"
          sudo -u "$user" env HOME="$home" XDG_RUNTIME_DIR="/run/user/$uid" \
            DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$uid/bus" \
            CREWSHIP_TOKEN_SUGGESTIONS_VM=1 \
            "$home/uv" --directory "$home/source" run --locked --group dev \
            pytest -q tests/test_token_suggestions_install.py

      # herdr-patch also runs on the operator's computer: macOS ships Python 3.9.
      - name: herdr-patch tests on Python 3.9
        if: matrix.shard == 1
        run: uvx --python 3.9 pytest tests/test_herdr_patch.py

      # Offline: a stub omp records the argv omp-as-pi builds.
      - name: omp-as-pi tests
        if: matrix.shard == 1
        run: bash blueprint/config/omp-as-pi/test.sh

      - name: Validate default configuration
        if: matrix.shard == 1
        run: ./ship.sh inspect --config blueprint/config/default.yml

      - name: Validate container configuration
        if: matrix.shard == 1
        run: ./ship.sh inspect --config blueprint/containers/crewship.container.yml

  runtime_smoke:
    name: project database runtime smoke
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-24.04
    timeout-minutes: 20
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Project database disposable service smoke
        run: bash tests/project-databases-smoke.sh

  # apps/crewboard/ is a Rust crate; its Cargo.lock stays locked like uv.lock.
  # Rust tracks its latest stable release through the runner's rustup.
  crewboard:
    name: crewboard rust checks
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-24.04
    timeout-minutes: 15
    permissions:
      contents: read
    defaults:
      run:
        working-directory: apps/crewboard
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Install latest stable Rust
        run: rustup toolchain install stable --profile minimal --component clippy && rustup default stable

      - name: Build
        run: cargo build --locked

      - name: Test
        run: cargo test --locked

      - name: Clippy
        run: cargo clippy --locked --all-targets -- -D warnings

  # site/ (crewship.si) as an iPhone in WebKit: site/webkit-check.mjs fails on a crash,
  # an uncaught error, or text drawn in a way that grows iOS Safari until it kills the tab.
  # site/chromium-check.mjs loads it in desktop Chromium in a large DPR 2 window and fails on
  # the plain-page fallback, a black canvas after 3 s, or a console error.
  # Keep the Playwright package and official browser image on the same release.
  harbor-webkit:
    name: harbor webkit iphone check
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    container:
      image: mcr.microsoft.com/playwright:v1.64.1-noble
      options: --ipc=host
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Set up Python for the page build
        uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
        with:
          python-version: '3.x'

      - name: Build the page
        run: python3 site/build.py

      - name: Install Playwright
        working-directory: site
        run: npm install --no-save --no-package-lock --prefix . playwright@1.64.1

      - name: WebKit iPhone check
        run: node site/webkit-check.mjs

      - name: Chromium desktop high-DPR check
        run: node site/chromium-check.mjs

  # The repository's own quality gate (docs/omp.md#quality-gate): the same
  # blocking rule the omp extension applies at turn end, against the committed
  # .sentrux/baseline.json, plus fallow audit on the changed files as a warning.
  quality-gate:
    name: quality gate
    needs: changes
    if: github.event_name == 'pull_request' && needs.changes.outputs.code == 'true'
    runs-on: ubuntu-24.04
    timeout-minutes: 15
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          fetch-depth: 0

      - name: Install latest Bun
        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
        with:
          bun-version: latest

      # The recipe's own verified installer; the sentrux binary links GTK 3.
      - name: Install the latest sentrux and fallow
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          sudo apt-get update
          sudo apt-get install --yes --no-install-recommends libgtk-3-0t64
          python3 scripts/provisions.py --home "$HOME" --tools sentrux,fallow
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"

      - name: sentrux gate
        run: bun blueprint/config/omp-quality-gate.ts

      - name: fallow audit (advisory)
        env:
          BASE_REF: ${{ github.base_ref }}
        run: |
          fallow audit --changed-since "origin/${BASE_REF}" ||
            echo "::warning title=fallow audit::the changed JS/TS has findings (advisory, see the log)"

  container-smoke:
    name: worker image behavior smoke
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    permissions:
      contents: read
    env:
      DOCKER_BUILDKIT: "1"
      COMPOSE_DOCKER_CLI_BUILD: "1"
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Smoke script syntax
        run: bash -n tests/container-smoke.sh

      - name: Audit image bases in the Dockerfile
        run: |
          set -euo pipefail
          python3 - Dockerfile <<'PY'
          import re
          import sys

          text = open(sys.argv[1], encoding="utf-8").read()
          args = dict(re.findall(r"^ARG\s+([A-Za-z_][A-Za-z0-9_]*)=(\S+)", text, re.M))

          def expand(ref):
              return re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", lambda m: args.get(m.group(1), ""), ref)

          stages, problems = set(), []
          for line in text.splitlines():
              match = re.match(r"^FROM\s+(?:--platform=\S+\s+)?(\S+)(?:\s+[Aa][Ss]\s+(\S+))?", line)
              if not match:
                  continue
              ref, alias = expand(match.group(1)), match.group(2)
              # The base tracks the newest Ubuntu LTS from mirror.gcr.io (no pull limit);
              # the worker stage starts from scratch and copies the builder's filesystem.
              if ref not in stages and ref not in ("mirror.gcr.io/library/ubuntu:latest", "scratch"):
                  problems.append(f"FROM {ref} is neither a previous stage, scratch nor mirror.gcr.io/library/ubuntu:latest")
              if alias:
                  stages.add(alias)

          if not stages:
              problems.append("no build stage found; the audit would pass vacuously")
          if problems:
              print("Dockerfile audit failed:")
              for problem in problems:
                  print(f"  - {problem}")
              raise SystemExit(1)

          print("Dockerfile audit passed for stages: " + ", ".join(sorted(stages)))
          PY

      - name: Audit the devcontainer definition
        run: |
          set -euo pipefail
          python3 - .devcontainer/devcontainer.json <<'PY'
          import json
          import sys

          raw = open(sys.argv[1], encoding="utf-8").read()

          # devcontainer.json is JSONC: drop comments that are not inside strings.
          out, index, in_string = [], 0, False
          while index < len(raw):
              char = raw[index]
              if in_string:
                  out.append(char)
                  if char == "\\" and index + 1 < len(raw):
                      out.append(raw[index + 1])
                      index += 2
                      continue
                  in_string = char != '"'
                  index += 1
                  continue
              if char == '"':
                  in_string = True
                  out.append(char)
                  index += 1
                  continue
              if raw.startswith("//", index):
                  while index < len(raw) and raw[index] != "\n":
                      index += 1
                  continue
              if raw.startswith("/*", index):
                  end = raw.find("*/", index + 2)
                  index = len(raw) if end < 0 else end + 2
                  continue
              out.append(char)
              index += 1

          doc = json.loads("".join(out))
          problems = []

          if doc.get("build", {}).get("target") != "builder":
              problems.append("build.target must be the provisioned builder stage")
          for key in ("containerUser", "remoteUser"):
              if doc.get(key) in (None, "root"):
                  problems.append(f"{key} must name the non-root Crewship account")

          run_args = doc.get("runArgs", [])
          forbidden = ("--privileged", "--pid=host", "--network=host", "--ipc=host", "--userns=host")
          for arg in run_args:
              if arg in forbidden or arg.startswith(("--volume", "-v", "--device", "--mount")):
                  problems.append(f"runArgs entry {arg!r} breaks the container boundary")
          for cap in ("--memory=", "--cpus=", "--pids-limit="):
              if not any(arg.startswith(cap) for arg in run_args):
                  problems.append(f"runArgs is missing a {cap.rstrip('=')} cap")

          for mount in doc.get("mounts", []):
              problems.append(f"unexpected extra mount {mount!r}")

          workspace_mount = doc.get("workspaceMount", "")
          if "type=volume" not in workspace_mount or "type=bind" in workspace_mount or "${localWorkspaceFolder}" in workspace_mount:
              problems.append("workspaceMount must be a named volume, not a host checkout bind")

          if problems:
              print("devcontainer audit failed:")
              for problem in problems:
                  print(f"  - {problem}")
              raise SystemExit(1)

          print(f"devcontainer audit passed: {doc.get('name')!r} -> {doc['build']['target']} stage")
          PY

      - name: Create throwaway compose secret
        run: |
          set -euo pipefail
          secret_dir="$RUNNER_TEMP/crewship-secrets"
          install -d -m 700 "$secret_dir"
          echo "CREWSHIP_SECRET_DIR=$secret_dir" >> "$GITHUB_ENV"
          umask 077
          openssl rand -base64 32 | tr -d '\n' > "$secret_dir/postgres_password"

      - name: Render and audit compose definitions
        run: |
          set -euo pipefail
          docker compose --profile worker --profile data config --format json > /tmp/compose-rendered.json
          python3 - /tmp/compose-rendered.json <<'PY'
          import json
          import sys

          with open(sys.argv[1], encoding="utf-8") as fh:
              doc = json.load(fh)

          problems = []
          for name, svc in sorted(doc.get("services", {}).items()):
              if svc.get("privileged"):
                  problems.append(f"{name}: privileged container")
              if svc.get("network_mode") == "host":
                  problems.append(f"{name}: host network namespace")
              if svc.get("pid") == "host":
                  problems.append(f"{name}: host PID namespace")
              if svc.get("ipc") == "host":
                  problems.append(f"{name}: host IPC namespace")
              if not svc.get("build") and "@sha256:" not in str(svc.get("image", "")):
                  problems.append(f"{name}: image is not digest-pinned ({svc.get('image')!r})")

              for volume in svc.get("volumes", []):
                  if isinstance(volume, dict):
                      vtype, source = volume.get("type"), str(volume.get("source", ""))
                  else:
                      parts = str(volume).split(":")
                      vtype = "bind" if parts[0].startswith(("/", ".", "~")) else "volume"
                      source = parts[0]
                  if vtype == "bind":
                      problems.append(f"{name}: host bind mount {source}")
                  if "docker.sock" in source:
                      problems.append(f"{name}: docker socket mount {source}")

              for port in svc.get("ports", []):
                  host_ip = port.get("host_ip") if isinstance(port, dict) else None
                  if host_ip not in ("127.0.0.1", "::1"):
                      problems.append(f"{name}: port published on {host_ip or 'all interfaces'}")

              limits = {
                  "mem_limit": svc.get("mem_limit"),
                  "cpus": svc.get("cpus") or svc.get("deploy", {}).get("resources", {}).get("limits", {}).get("cpus"),
                  "pids_limit": svc.get("pids_limit"),
              }
              missing = [key for key, value in limits.items() if not value]
              if missing:
                  problems.append(f"{name}: missing resource caps {missing}")

          for secret_name, secret in sorted(doc.get("secrets", {}).items()):
              if "environment" in secret:
                  problems.append(f"secret {secret_name}: sourced from the environment, expected a file")

          if problems:
              print("compose audit failed:")
              for problem in problems:
                  print(f"  - {problem}")
              raise SystemExit(1)

          print(f"compose audit passed for {len(doc.get('services', {}))} service(s)")
          PY

      # The job's read-only GITHUB_TOKEN authenticates the latest-herdr lookup:
      # shared runner IPs exhaust the unauthenticated API budget. It enters
      # builds only as a BuildKit secret.
      - name: Build worker image
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: docker build --secret id=github_token,env=GITHUB_TOKEN --target worker --tag crewship/worker:ci .

      - name: Behavior smoke in the worker image
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          CF_SMOKE_MEMORY: 6g
          CF_SMOKE_CPUS: 2
          CF_SMOKE_TIMEOUT: "2700"
        run: tests/container-smoke.sh

      - name: Remove throwaway secret and images
        if: always()
        run: |
          rm -rf "${RUNNER_TEMP:?}/crewship-secrets"
          docker image rm -f crewship/worker:ci >/dev/null 2>&1 || true