diff --git a/.github/scripts/changed-files.sh b/.github/scripts/changed-files.sh new file mode 100644 index 00000000..0f4b9957 --- /dev/null +++ b/.github/scripts/changed-files.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +EVENT_NAME="${1:-${GITHUB_EVENT_NAME:-}}" +BASE_SHA="${2:-}" +HEAD_SHA="${3:-${GITHUB_SHA:-HEAD}}" + +ZERO_SHA="0000000000000000000000000000000000000000" + +if [[ -z "${BASE_SHA}" || "${BASE_SHA}" == "${ZERO_SHA}" ]]; then + git ls-files + exit 0 +fi + +if git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then + git diff --name-only "${BASE_SHA}" "${HEAD_SHA}" + exit 0 +fi + +echo "Base SHA ${BASE_SHA} is not available; using fallback diff." >&2 +if [[ "${EVENT_NAME}" == "pull_request" ]]; then + git diff --name-only HEAD^1 HEAD 2>/dev/null || git ls-files +else + git diff --name-only HEAD~1 HEAD 2>/dev/null || git ls-files +fi diff --git a/.github/scripts/cleanup-dev-squash.py b/.github/scripts/cleanup-dev-squash.py new file mode 100644 index 00000000..99881ce3 --- /dev/null +++ b/.github/scripts/cleanup-dev-squash.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2025 NVIDIA Corporation + +""" +Clean up old squash files in the dev/ subdirectory. + +Removes .sqsh files older than 7 days via SSH to the remote SLURM filesystem. +This is the GitHub Actions equivalent of GitLab's cleanup-dev-squash job. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from datetime import datetime + + +def log(level: str, message: str) -> None: + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + print(f"[{timestamp}] [{level}] {message}", flush=True) + + +def _require_env(name: str) -> str | None: + value = os.getenv(name, "").strip() + if not value: + log("WARN", f"{name} is not set") + return None + return value + + +def _first_env(names: list[str]) -> tuple[str | None, str | None]: + """Return the first non-empty env value and the env name used.""" + for name in names: + value = os.getenv(name, "").strip() + if value: + return value, name + return None, None + + +def make_ssh_cmd(command: str) -> str: + # Prefer generic GitHub names; keep legacy GitLab names as fallback. + ssh_key, key_name = _first_env( + ["DEV_SQUASH_SSH_KEY_B64", "SLURM_FRONTEND_USER_KEY"] + ) + ssh_user, user_name = _first_env(["DEV_SQUASH_SSH_USER", "SLURM_FRONTEND_USER"]) + ssh_host, host_name = _first_env(["DEV_SQUASH_SSH_HOST", "SLURM_ORD_HOST"]) + + if not ssh_key: + _require_env("DEV_SQUASH_SSH_KEY_B64") + _require_env("SLURM_FRONTEND_USER_KEY") + if not ssh_user: + _require_env("DEV_SQUASH_SSH_USER") + _require_env("SLURM_FRONTEND_USER") + if not ssh_host: + _require_env("DEV_SQUASH_SSH_HOST") + _require_env("SLURM_ORD_HOST") + + if not (ssh_key and ssh_user and ssh_host): + raise RuntimeError("Missing one or more required SSH environment variables") + + log( + "INFO", + f"Using SSH user from {user_name}, host from {host_name}, key from {key_name}", + ) + + return ( + "ssh-agent bash -c " + '"ssh-add <(echo ' + ssh_key + " | base64 -d) 2>/dev/null && " + "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null " + "-o LogLevel=QUIET -n " + ssh_user + "@" + ssh_host + " '" + command + "'\"" + ) + + +def main() -> int: + # Prefer generic GitHub name; keep legacy name as fallback. + squash_cache_dir = ( + os.getenv("DEV_SQUASH_CACHE_DIR", "").strip() + or os.getenv("SQUASH_CACHE_DIR", "").strip() + ) + if not squash_cache_dir: + log("WARN", "DEV_SQUASH_CACHE_DIR/SQUASH_CACHE_DIR is not set") + log("INFO", "Skipping cleanup because cache directory is not configured") + return 0 + + # For testability and safety in GitHub, allow dry-run mode. + dry_run = os.getenv("CLEANUP_DEV_SQUASH_DRY_RUN", "false").lower() == "true" + + log("INFO", f"DEV_SQUASH_CACHE_DIR={squash_cache_dir}") + log("INFO", f"Dry run mode: {dry_run}") + + dev_dir = os.path.join(squash_cache_dir, "dev") + log("INFO", f"Target directory: {dev_dir}") + + if not dev_dir.endswith("dev"): + log("ERROR", f"Safety check failed: path {dev_dir} does not end with dev") + return 1 + + # If SSH env isn't configured, skip rather than hard-fail. + missing_ssh = [] + if not ( + os.getenv("DEV_SQUASH_SSH_KEY_B64", "").strip() + or os.getenv("SLURM_FRONTEND_USER_KEY", "").strip() + ): + missing_ssh.append("DEV_SQUASH_SSH_KEY_B64") + if not ( + os.getenv("DEV_SQUASH_SSH_USER", "").strip() + or os.getenv("SLURM_FRONTEND_USER", "").strip() + ): + missing_ssh.append("DEV_SQUASH_SSH_USER") + if not ( + os.getenv("DEV_SQUASH_SSH_HOST", "").strip() + or os.getenv("SLURM_ORD_HOST", "").strip() + ): + missing_ssh.append("DEV_SQUASH_SSH_HOST") + if missing_ssh: + log("WARN", f"Missing SSH config env vars: {', '.join(missing_ssh)}") + log("INFO", "Skipping cleanup because remote SSH connection is not configured") + return 0 + + try: + check_cmd = make_ssh_cmd( + f"[ -d {dev_dir} ] && echo 'exists' || echo 'not_found'" + ) + result = subprocess.run( + check_cmd, shell=True, capture_output=True, text=True, timeout=30 + ) + if "not_found" in result.stdout: + log( + "WARN", + f"Dev directory {dev_dir} not found on remote. Nothing to clean.", + ) + return 0 + + list_cmd = make_ssh_cmd( + f"find {dev_dir} -type f -name '\\''*.sqsh'\\'' -mtime +7 2>/dev/null" + ) + result = subprocess.run( + list_cmd, shell=True, capture_output=True, text=True, timeout=60 + ) + paths_to_delete = [ + p.strip() for p in (result.stdout or "").strip().splitlines() if p.strip() + ] + + count_all_cmd = make_ssh_cmd( + f"find {dev_dir} -type f -name '\\''*.sqsh'\\'' 2>/dev/null | wc -l" + ) + result_all = subprocess.run( + count_all_cmd, shell=True, capture_output=True, text=True, timeout=60 + ) + file_count_all = ( + result_all.stdout.strip() if result_all.returncode == 0 else "?" + ) + + if not paths_to_delete: + log("INFO", "No old squash files to clean up") + log("INFO", f"Total files: {file_count_all}") + return 0 + + file_count = len(paths_to_delete) + log( + "INFO", + f"Found {file_count} of {file_count_all} squash files older than 7 days", + ) + log("INFO", "Sample files to be deleted:") + for path in paths_to_delete[:10]: + log("INFO", f" - {os.path.basename(path)}") + + if dry_run: + log("INFO", "Dry run enabled, skipping deletion") + return 0 + + quoted_paths = " ".join( + "'\\''" + p.replace("'", "'\\''") + "'\\''" for p in paths_to_delete + ) + delete_cmd = make_ssh_cmd(f"rm -- {quoted_paths} || true") + result = subprocess.run( + delete_cmd, shell=True, capture_output=True, text=True, timeout=300 + ) + if result.returncode != 0: + log("ERROR", f"Failed to delete old files (exit code {result.returncode})") + if result.stderr.strip(): + log("ERROR", f"stderr: {result.stderr.strip()}") + return 1 + + log("INFO", f"Cleanup complete - removed up to {file_count} old dev files") + + verify_cmd = make_ssh_cmd( + f"find {dev_dir} -type f -name '\\''*.sqsh'\\'' -mtime +7 2>/dev/null | wc -l" + ) + result = subprocess.run( + verify_cmd, shell=True, capture_output=True, text=True, timeout=60 + ) + if result.returncode == 0: + remaining = int(result.stdout.strip()) + if remaining > 0: + log("WARN", f"Verification shows {remaining} old files still remain") + else: + log("INFO", "Verification successful - all old files removed") + + except subprocess.TimeoutExpired: + log("ERROR", "SSH command timed out") + return 1 + except Exception as exc: # pragma: no cover + log("ERROR", f"Unexpected error during cleanup: {exc}") + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/cleanup-dev-squash.yml b/.github/workflows/cleanup-dev-squash.yml new file mode 100644 index 00000000..9dbd256d --- /dev/null +++ b/.github/workflows/cleanup-dev-squash.yml @@ -0,0 +1,52 @@ +name: Cleanup Dev Squash + +on: + pull_request: + schedule: + # Weekly on Sunday at 07:00 UTC. Adjust as needed. + - cron: "0 7 * * 0" + workflow_dispatch: + +permissions: + contents: read + +jobs: + cleanup-dev-squash: + # Temporarily disabled: + # this job requires a dedicated infra runner with access to internal cluster/lustre. + # Re-enable by replacing the condition below with: + # github.event_name != 'pull_request' || github.base_ref == github.event.repository.default_branch + if: ${{ false }} + runs-on: self-hosted + continue-on-error: true + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install SSH client + shell: bash + run: | + set -euo pipefail + if ! command -v ssh >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y openssh-client + fi + + - name: Cleanup old dev squash files + shell: bash + env: + # Generic, repo-neutral names. + DEV_SQUASH_CACHE_DIR: ${{ vars.DEV_SQUASH_CACHE_DIR }} + DEV_SQUASH_SSH_USER: ${{ vars.DEV_SQUASH_SSH_USER }} + DEV_SQUASH_SSH_HOST: ${{ vars.DEV_SQUASH_SSH_HOST }} + DEV_SQUASH_SSH_KEY_B64: ${{ secrets.DEV_SQUASH_SSH_KEY_B64 }} + # Legacy fallback names (optional, for migration). + SQUASH_CACHE_DIR: ${{ vars.SQUASH_CACHE_DIR }} + SLURM_FRONTEND_USER: ${{ vars.SLURM_FRONTEND_USER }} + SLURM_ORD_HOST: ${{ vars.SLURM_ORD_HOST }} + SLURM_FRONTEND_USER_KEY: ${{ secrets.SLURM_FRONTEND_USER_KEY }} + # Optional safety mode for first rollout. + CLEANUP_DEV_SQUASH_DRY_RUN: ${{ vars.CLEANUP_DEV_SQUASH_DRY_RUN }} + run: | + set -euo pipefail + python3 .github/scripts/cleanup-dev-squash.py diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..c38e1ef2 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,69 @@ +# Lint: run pre-commit for changed files. +# Skips insert-license to match CI behavior in GitLab. +name: Lint + +on: + push: + branches: [main, master] + pull_request: + # Run on all PRs (matches GitLab merge_request_event for lint) + +jobs: + pre-commit: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # Required to diff against base SHA for pre-commit ranges. + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install pre-commit + run: pip install pre-commit + + - name: Run pre-commit (pull_request) + if: github.event_name == 'pull_request' + run: | + BASE_SHA="${{ github.event.pull_request.base.sha }}" + HEAD_SHA="${{ github.event.pull_request.head.sha }}" + mapfile -t CHANGED_FILES < <(bash ./.github/scripts/changed-files.sh pull_request "${BASE_SHA}" "${HEAD_SHA}") + + EXISTING_FILES=() + for path in "${CHANGED_FILES[@]}"; do + if [[ -e "${path}" ]]; then + EXISTING_FILES+=("${path}") + fi + done + + if [[ ${#EXISTING_FILES[@]} -eq 0 ]]; then + echo "No changed files to lint." + exit 0 + fi + + SKIP=insert-license pre-commit run -c .pre-commit-config.yaml --files "${EXISTING_FILES[@]}" + + - name: Run pre-commit (push) + if: github.event_name == 'push' + run: | + BEFORE_SHA="${{ github.event.before }}" + HEAD_SHA="${{ github.sha }}" + mapfile -t CHANGED_FILES < <(bash ./.github/scripts/changed-files.sh push "${BEFORE_SHA}" "${HEAD_SHA}") + + EXISTING_FILES=() + for path in "${CHANGED_FILES[@]}"; do + if [[ -e "${path}" ]]; then + EXISTING_FILES+=("${path}") + fi + done + + if [[ ${#EXISTING_FILES[@]} -eq 0 ]]; then + echo "No changed files to lint." + exit 0 + fi + + SKIP=insert-license pre-commit run -c .pre-commit-config.yaml --files "${EXISTING_FILES[@]}" diff --git a/.github/workflows/pre.yml b/.github/workflows/pre.yml new file mode 100644 index 00000000..ad2525c9 --- /dev/null +++ b/.github/workflows/pre.yml @@ -0,0 +1,325 @@ +name: Pre Stage + +on: + pull_request: + push: + branches: [main, master] + workflow_dispatch: + inputs: + pipeline_target: + description: "Pipeline target (currently informational)" + required: false + default: "Alpasim Test" + runtime_image: + description: "Optional runtime image override" + required: false + physics_image: + description: "Optional physics image override" + required: false + controller_image: + description: "Optional controller image override" + required: false + driver_image: + description: "Optional driver image override" + required: false + +permissions: + contents: read + +jobs: + prepare-variables: + runs-on: ubuntu-22.04 + outputs: + changed_services: ${{ steps.prepare.outputs.changed_services }} + all_services: ${{ steps.prepare.outputs.all_services }} + docker_services: ${{ steps.prepare.outputs.docker_services }} + python_test_services: ${{ steps.prepare.outputs.python_test_services }} + docker_test_services: ${{ steps.prepare.outputs.docker_test_services }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Prepare variables and detect changes + id: prepare + shell: bash + run: | + set -euo pipefail + + write_env() { + local key="$1" + local value="$2" + # %q produces shell-escaped values so spaces survive `source build.env`. + printf "%s=%q\n" "${key}" "${value}" >> build.env + } + + ALL_SERVICES="runtime eval physics controller grpc utils driver" + DOCKER_SERVICES="runtime physics controller driver" + NON_BASE_DOCKER_SERVICES="" + PYTHON_TEST_SERVICES="grpc runtime wizard tools eval utils" + DOCKER_TEST_SERVICES="physics controller driver" + VALIDATION_GLOB_PATTERNS="rollouts/**/*.asl=ASL files not found, main job failed" + VALIDATION_EXPECTED_FILES="aggregate/metrics_results.txt=No aggregate metrics_results.txt found, aggregation did not complete successfully" + RCLONE_REMOTE_NAME="pbss-team-alpamayo" + RCLONE_REMOTE_TYPE="swift" + RCLONE_REMOTE_ENV_AUTH="false" + RCLONE_REMOTE_USER="team-alpamayo" + RCLONE_REMOTE_AUTH_URL="https://pdx.s8k.io/auth/v1.0" + RCLONE_REMOTE_AUTH_VERSION="1" + DEFAULT_ALPACKAGES_PATH="alpasim/artifacts/alpackages/25.10.08" + + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + BASE_SHA="${{ github.event.pull_request.base.sha }}" + HEAD_SHA="${{ github.event.pull_request.head.sha }}" + else + BASE_SHA="${{ github.event.before }}" + HEAD_SHA="${{ github.sha }}" + fi + + CHANGED_FILES="$(bash ./.github/scripts/changed-files.sh "${{ github.event_name }}" "${BASE_SHA}" "${HEAD_SHA}")" + + CHANGED_SERVICES="" + for service in ${DOCKER_SERVICES}; do + # Keep behavior aligned with GitLab detect-changed-services.sh: + # service considered changed only when src// changes. + if echo "${CHANGED_FILES}" | grep -Eq "^src/${service}/"; then + CHANGED_SERVICES="${CHANGED_SERVICES} ${service}" + fi + done + CHANGED_SERVICES="$(echo "${CHANGED_SERVICES}" | xargs || true)" + + : > build.env + write_env "ALPASIM_ALL_SERVICES" "${ALL_SERVICES}" + write_env "ALPASIM_DOCKER_SERVICES" "${DOCKER_SERVICES}" + write_env "ALPASIM_NON_BASE_DOCKER_SERVICES" "${NON_BASE_DOCKER_SERVICES}" + write_env "ALPASIM_PYTHON_TEST_SERVICES" "${PYTHON_TEST_SERVICES}" + write_env "ALPASIM_DOCKER_TEST_SERVICES" "${DOCKER_TEST_SERVICES}" + write_env "ALPASIM_VALIDATION_GLOB_PATTERNS" "${VALIDATION_GLOB_PATTERNS}" + write_env "ALPASIM_VALIDATION_EXPECTED_FILES" "${VALIDATION_EXPECTED_FILES}" + write_env "ALPASIM_RCLONE_REMOTE_NAME" "${RCLONE_REMOTE_NAME}" + write_env "ALPASIM_RCLONE_REMOTE_TYPE" "${RCLONE_REMOTE_TYPE}" + write_env "ALPASIM_RCLONE_REMOTE_ENV_AUTH" "${RCLONE_REMOTE_ENV_AUTH}" + write_env "ALPASIM_RCLONE_REMOTE_USER" "${RCLONE_REMOTE_USER}" + write_env "ALPASIM_RCLONE_REMOTE_AUTH_URL" "${RCLONE_REMOTE_AUTH_URL}" + write_env "ALPASIM_RCLONE_REMOTE_AUTH_VERSION" "${RCLONE_REMOTE_AUTH_VERSION}" + write_env "ALPASIM_DEFAULT_ALPACKAGES_PATH" "${DEFAULT_ALPACKAGES_PATH}" + write_env "CHANGED_SERVICES" "${CHANGED_SERVICES}" + + echo "all_services=${ALL_SERVICES}" >> "${GITHUB_OUTPUT}" + echo "docker_services=${DOCKER_SERVICES}" >> "${GITHUB_OUTPUT}" + echo "python_test_services=${PYTHON_TEST_SERVICES}" >> "${GITHUB_OUTPUT}" + echo "docker_test_services=${DOCKER_TEST_SERVICES}" >> "${GITHUB_OUTPUT}" + echo "changed_services=${CHANGED_SERVICES}" >> "${GITHUB_OUTPUT}" + + - name: Upload build.env + uses: actions/upload-artifact@v4 + with: + name: build-env + path: build.env + retention-days: 7 + + bump-versions-plan: + # Safe first step: report what would be bumped on PRs to default branch. + # We can replace with commit/push behavior later. + if: github.event_name == 'pull_request' && github.base_ref == github.event.repository.default_branch + runs-on: ubuntu-22.04 + needs: [prepare-variables] + steps: + - name: Download build.env + uses: actions/download-artifact@v4 + with: + name: build-env + + - name: Print bump plan + shell: bash + run: | + set -euo pipefail + source build.env + echo "Changed services: ${CHANGED_SERVICES:-}" + if [[ -z "${CHANGED_SERVICES:-}" ]]; then + echo "No changed docker services detected; nothing to bump." + exit 0 + fi + for service in ${CHANGED_SERVICES}; do + echo "Would bump minor version for: ${service}" + done + + prepare-images: + runs-on: ubuntu-22.04 + needs: [prepare-variables] + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + RUNTIME_IMAGE_OVERRIDE: ${{ github.event.inputs.runtime_image || '' }} + PHYSICS_IMAGE_OVERRIDE: ${{ github.event.inputs.physics_image || '' }} + CONTROLLER_IMAGE_OVERRIDE: ${{ github.event.inputs.controller_image || '' }} + DRIVER_IMAGE_OVERRIDE: ${{ github.event.inputs.driver_image || '' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download build.env + uses: actions/download-artifact@v4 + with: + name: build-env + + - name: Resolve image sources and tags + shell: bash + run: | + set -euo pipefail + source build.env + + write_env() { + local key="$1" + local value="$2" + printf "%s=%q\n" "${key}" "${value}" >> build.env + } + + SHORT_SHA="$(echo "${{ github.sha }}" | cut -c1-8)" + IS_RELEASE_PIPELINE="false" + if [[ "${{ github.event_name }}" == "push" && "${{ github.ref_name }}" == "${DEFAULT_BRANCH}" ]]; then + IS_RELEASE_PIPELINE="true" + fi + + get_source_and_tag() { + local service="$1" + local changed="${CHANGED_SERVICES:-}" + local source="stable" + local tag="stable" + + if echo "${changed}" | grep -Eq "(^| )${service}( |$)"; then + source="build" + if [[ "${IS_RELEASE_PIPELINE}" == "true" ]]; then + tag="release-${SHORT_SHA}" + else + tag="pr-${SHORT_SHA}" + fi + fi + + echo "${source}" "${tag}" + } + + append_service_vars() { + local service="$1" + local image_override="$2" + local service_upper + service_upper="$(echo "${service}" | tr '[:lower:]-' '[:upper:]_')" + read -r source tag < <(get_source_and_tag "${service}") + + if [[ -n "${image_override}" ]]; then + source="override" + image="${image_override}" + else + image="alpasim-${service}:${tag}" + fi + + write_env "${service_upper}_SOURCE" "${source}" + write_env "${service_upper}_IMAGE" "${image}" + } + + append_service_vars "runtime" "${RUNTIME_IMAGE_OVERRIDE}" + append_service_vars "physics" "${PHYSICS_IMAGE_OVERRIDE}" + append_service_vars "controller" "${CONTROLLER_IMAGE_OVERRIDE}" + append_service_vars "driver" "${DRIVER_IMAGE_OVERRIDE}" + + write_env "IS_RELEASE_PIPELINE" "${IS_RELEASE_PIPELINE}" + echo "Final build.env:" + cat build.env + + - name: Upload final build.env + uses: actions/upload-artifact@v4 + with: + name: build-env-final + path: build.env + retention-days: 7 + + build-health-check: + runs-on: ubuntu-22.04 + needs: [prepare-images] + steps: + - name: Health check + run: echo "Build stage jobs are enabled in pre.yml." + + build-services: + runs-on: self-hosted + needs: [prepare-images] + strategy: + fail-fast: false + matrix: + service: [runtime, physics, controller, driver] + steps: + - name: Fix workspace ownership on self-hosted runner + shell: bash + run: | + set -euo pipefail + if command -v sudo >/dev/null 2>&1; then + sudo chown -R "$(id -u):$(id -g)" "${GITHUB_WORKSPACE}" || true + else + chown -R "$(id -u):$(id -g)" "${GITHUB_WORKSPACE}" || true + fi + chmod -R u+rwX "${GITHUB_WORKSPACE}" || true + + - name: Ensure git-lfs is available for checkout + shell: bash + run: | + set -euo pipefail + if command -v git-lfs >/dev/null 2>&1; then + git-lfs version + exit 0 + fi + if command -v sudo >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y git-lfs + else + apt-get update + apt-get install -y git-lfs + fi + git-lfs version + + - name: Checkout + uses: actions/checkout@v4 + + - name: Download final build.env + uses: actions/download-artifact@v4 + with: + name: build-env-final + + - name: Determine service build source + id: source + shell: bash + run: | + set -euo pipefail + source build.env + SERVICE="${{ matrix.service }}" + SERVICE_UPPER="$(echo "${SERVICE}" | tr '[:lower:]-' '[:upper:]_')" + SOURCE_VAR="${SERVICE_UPPER}_SOURCE" + IMAGE_VAR="${SERVICE_UPPER}_IMAGE" + SOURCE="${!SOURCE_VAR:-stable}" + IMAGE="${!IMAGE_VAR:-}" + + echo "source=${SOURCE}" >> "${GITHUB_OUTPUT}" + echo "image=${IMAGE}" >> "${GITHUB_OUTPUT}" + + - name: Skip non-build sources + if: steps.source.outputs.source != 'build' + run: | + echo "Skipping ${{ matrix.service }} (source=${{ steps.source.outputs.source }})" + + - name: Build service image via wizard + if: steps.source.outputs.source == 'build' + shell: bash + run: | + set -euo pipefail + SERVICE="${{ matrix.service }}" + IMAGE="${{ steps.source.outputs.image }}" + LOG_DIR="${GITHUB_WORKSPACE}/wizard_logs" + + git lfs pull --include="src/**/*" || true + + cd "${GITHUB_WORKSPACE}/src/wizard" + python3 -m pip install uv + uv sync + uv run alpasim_wizard +deploy=docker_build_only services.${SERVICE}.image=${IMAGE} wizard.log_dir=${LOG_DIR} + + cd "${GITHUB_WORKSPACE}" + docker compose -f "${LOG_DIR}/docker-compose.yaml" build "${SERVICE}-0" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..1ef2db9f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,536 @@ +name: Test Stage + +on: + pull_request: + push: + branches: [main, master] + workflow_dispatch: + inputs: + enable_infra_tests: + description: "Enable infra-coupled tests (docker/gpu/fake_lustre)" + required: false + default: "false" + enable_slurm_tests: + description: "Enable SLURM smoke tests" + required: false + default: "false" + +permissions: + contents: read + +jobs: + test-services-native: + name: test-native-${{ matrix.service }} + runs-on: [self-hosted, type/docker] + container: + image: ubuntu:22.04 + strategy: + fail-fast: false + matrix: + service: [grpc, runtime, wizard, tools, eval, utils] + steps: + - name: Install git for checkout + shell: bash + run: | + set -euo pipefail + apt-get update + apt-get install -y git git-lfs + + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Mark checkout as safe for git + shell: bash + run: | + set -euo pipefail + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + + - name: Install test dependencies in job container + shell: bash + run: | + set -euo pipefail + apt-get update + apt-get install -y python3 python3-pip git-lfs + python3 -m pip install --upgrade pip setuptools wheel uv + git lfs pull --include="src/${{ matrix.service }}/tests/**/*" || true + + - name: Compile gRPC protos + shell: bash + run: | + set -euo pipefail + cd src/grpc + uv sync + uv run compile-protos + + - name: Run ${{ matrix.service }} tests + shell: bash + run: | + set -euo pipefail + cd "src/${{ matrix.service }}" + uv sync --all-extras + + set +e + uv run --all-extras pytest -vvs --tb=short + status=$? + set -e + + if [[ $status -eq 5 ]]; then + echo "No tests collected for ${{ matrix.service }}; treating as success." + exit 0 + fi + + exit $status + + # Equivalent to GitLab test-services-in-docker. + # Runs automatically on PR/push. On manual runs, can be disabled via input. + test-services-in-docker: + if: ${{ github.event_name != 'workflow_dispatch' || inputs.enable_infra_tests == 'true' }} + name: test-docker-${{ matrix.service }} + runs-on: [self-hosted, type/docker, type/gpu] + container: + image: ubuntu:22.04 + options: >- + --gpus all + -v /var/run/docker.sock:/var/run/docker.sock + strategy: + fail-fast: false + matrix: + service: [physics, controller, driver] + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + steps: + - name: Install git for checkout + shell: bash + run: | + set -euo pipefail + apt-get update + apt-get install -y git git-lfs + + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Mark checkout as safe for git + shell: bash + run: | + set -euo pipefail + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + + - name: Install dependencies in job container + shell: bash + run: | + set -euo pipefail + apt-get update + apt-get install -y python3 python3-pip git-lfs ca-certificates curl gnupg + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + . /etc/os-release + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu ${VERSION_CODENAME} stable" \ + > /etc/apt/sources.list.d/docker.list + apt-get update + apt-get install -y docker-ce-cli docker-buildx-plugin docker-compose-plugin + docker compose version + python3 -m pip install --upgrade pip setuptools wheel uv + + - name: Compute source/image + id: plan + shell: bash + run: | + set -euo pipefail + SERVICE="${{ matrix.service }}" + + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + BASE_SHA="${{ github.event.pull_request.base.sha }}" + HEAD_SHA="${{ github.event.pull_request.head.sha }}" + else + BASE_SHA="${{ github.event.before }}" + HEAD_SHA="${{ github.sha }}" + fi + + CHANGED_FILES="$(bash ./.github/scripts/changed-files.sh "${{ github.event_name }}" "${BASE_SHA}" "${HEAD_SHA}")" + + SOURCE="stable" + if echo "${CHANGED_FILES}" | grep -Eq "^src/${SERVICE}/"; then + SOURCE="build" + fi + + SHORT_SHA="$(echo "${{ github.sha }}" | cut -c1-8)" + TAG="pr-${SHORT_SHA}" + if [[ "${{ github.event_name }}" == "push" && "${{ github.ref_name }}" == "${DEFAULT_BRANCH}" ]]; then + TAG="release-${SHORT_SHA}" + fi + IMAGE="alpasim-${SERVICE}:${TAG}" + + echo "source=${SOURCE}" >> "${GITHUB_OUTPUT}" + echo "image=${IMAGE}" >> "${GITHUB_OUTPUT}" + + - name: Skip non-build sources + if: steps.plan.outputs.source != 'build' + run: | + echo "Skipping ${{ matrix.service }} (source=${{ steps.plan.outputs.source }})" + + - name: Build service image + if: steps.plan.outputs.source == 'build' + shell: bash + run: | + set -euo pipefail + SERVICE="${{ matrix.service }}" + IMAGE="${{ steps.plan.outputs.image }}" + LOG_DIR="${GITHUB_WORKSPACE}/wizard_logs" + + git lfs pull --include="src/**/*" || true + + cd "${GITHUB_WORKSPACE}/src/wizard" + uv sync + uv run alpasim_wizard +deploy=docker_build_only services.${SERVICE}.image=${IMAGE} wizard.log_dir=${LOG_DIR} + + cd "${GITHUB_WORKSPACE}" + docker compose -f "${LOG_DIR}/docker-compose.yaml" build "${SERVICE}-0" + + - name: Run tests in docker container + if: steps.plan.outputs.source == 'build' + shell: bash + run: | + set -euo pipefail + SERVICE="${{ matrix.service }}" + IMAGE="${{ steps.plan.outputs.image }}" + PYTEST_OPTS="-vvs --tb=short" + + set +e + docker run --rm --gpus all --entrypoint bash "${IMAGE}" -c \ + "cd /repo/src/${SERVICE} && uv sync --all-extras && uv run pytest ${PYTEST_OPTS}" + status=$? + set -e + + if [[ $status -eq 5 ]]; then + echo "No tests collected for ${SERVICE}; treating as success." + exit 0 + fi + exit $status + + # Equivalent to GitLab test-docker-compose (requires storage/fake_lustre). + # This must run only on runners that provide the fake lustre mount. + test-docker-compose: + if: ${{ github.event_name != 'workflow_dispatch' || inputs.enable_infra_tests == 'true' }} + name: test-docker-compose-${{ matrix.variant }} + runs-on: [self-hosted, type/docker, type/gpu, storage/fake_lustre] + timeout-minutes: 120 + container: + image: ubuntu:22.04 + options: >- + --gpus all + -v /var/run/docker.sock:/var/run/docker.sock + -v /raid/fake_lustre:/lustre + strategy: + fail-fast: false + matrix: + variant: [docker_oss] + env: + OUTPUT_PATH: /lustre/output/${{ github.run_id }}-${{ github.run_attempt }} + LOCAL_OUTPUT_PATH: /lustre/output/${{ github.run_id }}-${{ github.run_attempt }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} + HUGGINGFACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }} + steps: + - name: Install git for checkout + shell: bash + run: | + set -euo pipefail + apt-get update + apt-get install -y git git-lfs + + - name: Checkout + uses: actions/checkout@v4 + + - name: Install dependencies in job container + shell: bash + run: | + set -euo pipefail + apt-get update + apt-get install -y python3 python3-pip git-lfs ca-certificates curl gnupg + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + . /etc/os-release + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu ${VERSION_CODENAME} stable" \ + > /etc/apt/sources.list.d/docker.list + apt-get update + apt-get install -y docker-ce-cli docker-buildx-plugin docker-compose-plugin + docker compose version + python3 -m pip install --upgrade pip setuptools wheel uv + + - name: Verify Hugging Face token is available + shell: bash + run: | + set -euo pipefail + if [[ -z "${HF_TOKEN:-}" ]]; then + echo "HF_TOKEN is not available in this workflow context." + echo "If this is a trusted branch in the main repo, verify repo/org secret configuration." + echo "If this is an untrusted context (e.g. fork), GitHub will not expose secrets." + exit 1 + fi + + - name: Validate fake_lustre mount + shell: bash + run: | + set -euo pipefail + test -d /lustre + docker run --rm -v /lustre:/lustre alpine sh -c 'test -d /lustre && echo fake_lustre_mount_ok' + + - name: Generate and validate docker compose config + shell: bash + run: | + set -euo pipefail + mkdir -p \ + "${LOCAL_OUTPUT_PATH}" \ + "${LOCAL_OUTPUT_PATH}/data" \ + "${LOCAL_OUTPUT_PATH}/data/drivers" \ + "${LOCAL_OUTPUT_PATH}/data/alpackages" \ + "${LOCAL_OUTPUT_PATH}/data/nre-artifacts" \ + "${LOCAL_OUTPUT_PATH}/data/nre-artifacts/ego-hoods" \ + "${LOCAL_OUTPUT_PATH}/data/trafficsim/unified_data_cache" \ + "${LOCAL_OUTPUT_PATH}/data/huggingface" \ + "${LOCAL_OUTPUT_PATH}/src" \ + "${LOCAL_OUTPUT_PATH}/plugins" + + # Match local smoke setup: ensure VaVAM assets exist for OSS driver. + bash "${GITHUB_WORKSPACE}/data/download_vavam_assets.sh" --model VaVAM-B + + if [[ -d "${GITHUB_WORKSPACE}/data/drivers" ]]; then + cp -r "${GITHUB_WORKSPACE}/data/drivers/." "${LOCAL_OUTPUT_PATH}/data/drivers/" || true + fi + if [[ -d "${GITHUB_WORKSPACE}/data/nre-artifacts/ego-hoods" ]]; then + cp -r "${GITHUB_WORKSPACE}/data/nre-artifacts/ego-hoods" "${LOCAL_OUTPUT_PATH}/data/nre-artifacts/" || true + fi + if [[ -d "${GITHUB_WORKSPACE}/src" ]]; then + cp -r "${GITHUB_WORKSPACE}/src/." "${LOCAL_OUTPUT_PATH}/src/" || true + fi + if [[ -d "${GITHUB_WORKSPACE}/plugins" ]]; then + cp -r "${GITHUB_WORKSPACE}/plugins/." "${LOCAL_OUTPUT_PATH}/plugins/" || true + fi + + # Compose mounts the copied src tree; generate grpc python files there. + cd "${LOCAL_OUTPUT_PATH}/src/grpc" + uv sync + uv run compile-protos + + cd "${GITHUB_WORKSPACE}/src/wizard" + uv sync + # CI runners may expose only one GPU; pin all GPU-backed services to GPU 0. + HF_HOME="${LOCAL_OUTPUT_PATH}/data/huggingface" uv run alpasim_wizard \ + +deploy=local \ + wizard.log_dir="${LOCAL_OUTPUT_PATH}" \ + wizard.run_method=NONE \ + defines.filesystem="${LOCAL_OUTPUT_PATH}/data" \ + services.driver.workdir=/repo/src/driver \ + services.physics.workdir=/repo/src/physics \ + services.runtime.workdir=/repo/src/runtime \ + services.controller.workdir=/repo/src/controller \ + services.sensorsim.gpus=[0] \ + services.physics.gpus=[0] \ + services.driver.gpus=[0] + + test -f "${LOCAL_OUTPUT_PATH}/docker-compose.yaml" + + # In Docker-socket mode, compose mounts must use host-visible paths. + REPO_ROOT_SRC="$(realpath "${GITHUB_WORKSPACE}/src")" + sed -i "s|${LOCAL_OUTPUT_PATH}|${OUTPUT_PATH}|g" "${LOCAL_OUTPUT_PATH}/docker-compose.yaml" + sed -i "s|${REPO_ROOT_SRC}|${OUTPUT_PATH}/src|g" "${LOCAL_OUTPUT_PATH}/docker-compose.yaml" + if [[ -d "${GITHUB_WORKSPACE}/plugins" ]]; then + REPO_ROOT_PLUGINS="$(realpath "${GITHUB_WORKSPACE}/plugins")" + sed -i "s|${REPO_ROOT_PLUGINS}|${OUTPUT_PATH}/plugins|g" "${LOCAL_OUTPUT_PATH}/docker-compose.yaml" + fi + + docker compose -f "${LOCAL_OUTPUT_PATH}/docker-compose.yaml" config >/dev/null + echo "docker compose configuration validated for ${{ matrix.variant }}" + + - name: Run docker compose integration stack + shell: bash + run: | + set -euo pipefail + PROJECT="gha-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ matrix.variant }}" + echo "Using compose project: ${PROJECT}" + cd "${LOCAL_OUTPUT_PATH}" + docker compose -f docker-compose.yaml --project-name "${PROJECT}" up --build --abort-on-container-failure --remove-orphans + + - name: Collect compose logs and tear down + if: always() + shell: bash + run: | + set +e + PROJECT="gha-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ matrix.variant }}" + LOG_DIR="${LOCAL_OUTPUT_PATH}/txt-logs" + mkdir -p "${LOG_DIR}" + cd "${LOCAL_OUTPUT_PATH}" || exit 0 + + services="$(docker compose -f docker-compose.yaml --project-name "${PROJECT}" ps --services 2>/dev/null)" + for service in ${services}; do + docker compose -f docker-compose.yaml --project-name "${PROJECT}" logs --no-color "${service}" > "${LOG_DIR}/${service}.log" 2>&1 || true + done + + docker compose -f docker-compose.yaml --project-name "${PROJECT}" down -v --remove-orphans || true + + # Tutorial smoke test (OSS only): generate tutorial-style compose and run core sim services. + test-tutorial-smoke: + if: ${{ github.event_name != 'workflow_dispatch' || inputs.enable_infra_tests == 'true' }} + name: test-tutorial-smoke + runs-on: [self-hosted, type/docker, type/gpu, storage/fake_lustre] + timeout-minutes: 120 + container: + image: ubuntu:22.04 + options: >- + --gpus all + -v /var/run/docker.sock:/var/run/docker.sock + -v /raid/fake_lustre:/lustre + env: + OUTPUT_PATH: /lustre/output/${{ github.run_id }}-${{ github.run_attempt }}-tutorial + LOCAL_OUTPUT_PATH: /lustre/output/${{ github.run_id }}-${{ github.run_attempt }}-tutorial + HF_TOKEN: ${{ secrets.HF_TOKEN }} + HUGGINGFACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }} + steps: + - name: Install git for checkout + shell: bash + run: | + set -euo pipefail + apt-get update + apt-get install -y git git-lfs + + - name: Checkout + uses: actions/checkout@v4 + + - name: Install dependencies in job container + shell: bash + run: | + set -euo pipefail + apt-get update + apt-get install -y python3 python3-pip git-lfs ca-certificates curl gnupg + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + . /etc/os-release + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu ${VERSION_CODENAME} stable" \ + > /etc/apt/sources.list.d/docker.list + apt-get update + apt-get install -y docker-ce-cli docker-buildx-plugin docker-compose-plugin + docker compose version + python3 -m pip install --upgrade pip setuptools wheel uv + + - name: Verify Hugging Face token is available + shell: bash + run: | + set -euo pipefail + if [[ -z "${HF_TOKEN:-}" ]]; then + echo "HF_TOKEN is not available in this workflow context." + exit 1 + fi + + - name: Validate fake_lustre mount + shell: bash + run: | + set -euo pipefail + test -d /lustre + docker run --rm -v /lustre:/lustre alpine sh -c 'test -d /lustre && echo fake_lustre_mount_ok' + + - name: Generate tutorial compose config + shell: bash + run: | + set -euo pipefail + mkdir -p \ + "${LOCAL_OUTPUT_PATH}" \ + "${LOCAL_OUTPUT_PATH}/data/drivers" \ + "${LOCAL_OUTPUT_PATH}/data/alpackages" \ + "${LOCAL_OUTPUT_PATH}/data/nre-artifacts/ego-hoods" \ + "${LOCAL_OUTPUT_PATH}/data/trafficsim/unified_data_cache" \ + "${LOCAL_OUTPUT_PATH}/data/huggingface" \ + "${LOCAL_OUTPUT_PATH}/src" \ + "${LOCAL_OUTPUT_PATH}/plugins" + + # Match local smoke setup: ensure VaVAM assets exist for OSS driver. + bash "${GITHUB_WORKSPACE}/data/download_vavam_assets.sh" --model VaVAM-B + + if [[ -d "${GITHUB_WORKSPACE}/data/drivers" ]]; then + cp -r "${GITHUB_WORKSPACE}/data/drivers/." "${LOCAL_OUTPUT_PATH}/data/drivers/" || true + fi + if [[ -d "${GITHUB_WORKSPACE}/data/nre-artifacts/ego-hoods" ]]; then + cp -r "${GITHUB_WORKSPACE}/data/nre-artifacts/ego-hoods" "${LOCAL_OUTPUT_PATH}/data/nre-artifacts/" || true + fi + if [[ -d "${GITHUB_WORKSPACE}/src" ]]; then + cp -r "${GITHUB_WORKSPACE}/src/." "${LOCAL_OUTPUT_PATH}/src/" || true + fi + if [[ -d "${GITHUB_WORKSPACE}/plugins" ]]; then + cp -r "${GITHUB_WORKSPACE}/plugins/." "${LOCAL_OUTPUT_PATH}/plugins/" || true + fi + + # Compose mounts the copied src tree; generate grpc python files there. + cd "${LOCAL_OUTPUT_PATH}/src/grpc" + uv sync + uv run compile-protos + + cd "${GITHUB_WORKSPACE}/src/wizard" + uv sync + HF_HOME="${LOCAL_OUTPUT_PATH}/data/huggingface" uv run alpasim_wizard \ + +deploy=local \ + wizard.log_dir="${LOCAL_OUTPUT_PATH}" \ + wizard.run_method=NONE \ + defines.filesystem="${LOCAL_OUTPUT_PATH}/data" \ + services.driver.workdir=/repo/src/driver \ + services.physics.workdir=/repo/src/physics \ + services.runtime.workdir=/repo/src/runtime \ + services.controller.workdir=/repo/src/controller \ + runtime.simulation_config.n_rollouts=1 \ + runtime.simulation_config.n_sim_steps=60 + + test -f "${LOCAL_OUTPUT_PATH}/docker-compose.yaml" + test -f "${LOCAL_OUTPUT_PATH}/generated-user-config-0.yaml" + + REPO_ROOT_SRC="$(realpath "${GITHUB_WORKSPACE}/src")" + sed -i "s|${LOCAL_OUTPUT_PATH}|${OUTPUT_PATH}|g" "${LOCAL_OUTPUT_PATH}/docker-compose.yaml" + sed -i "s|${REPO_ROOT_SRC}|${OUTPUT_PATH}/src|g" "${LOCAL_OUTPUT_PATH}/docker-compose.yaml" + if [[ -d "${GITHUB_WORKSPACE}/plugins" ]]; then + REPO_ROOT_PLUGINS="$(realpath "${GITHUB_WORKSPACE}/plugins")" + sed -i "s|${REPO_ROOT_PLUGINS}|${OUTPUT_PATH}/plugins|g" "${LOCAL_OUTPUT_PATH}/docker-compose.yaml" + fi + + docker compose -f "${LOCAL_OUTPUT_PATH}/docker-compose.yaml" config >/dev/null + + - name: Run tutorial compose stack + shell: bash + run: | + set -euo pipefail + PROJECT="gha-tutorial-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + cd "${LOCAL_OUTPUT_PATH}" + docker compose -f docker-compose.yaml --project-name "${PROJECT}" --profile sim up --build --abort-on-container-failure --remove-orphans runtime-0 driver-0 physics-0 sensorsim-0 controller-0 + + - name: Collect tutorial logs and tear down + if: always() + shell: bash + run: | + set +e + PROJECT="gha-tutorial-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + LOG_DIR="${LOCAL_OUTPUT_PATH}/txt-logs" + mkdir -p "${LOG_DIR}" + cd "${LOCAL_OUTPUT_PATH}" || exit 0 + services="$(docker compose -f docker-compose.yaml --project-name "${PROJECT}" ps --services 2>/dev/null)" + for service in ${services}; do + docker compose -f docker-compose.yaml --project-name "${PROJECT}" logs --no-color "${service}" > "${LOG_DIR}/${service}.log" 2>&1 || true + done + docker compose -f docker-compose.yaml --project-name "${PROJECT}" down -v --remove-orphans || true + + # Equivalent to GitLab test-slurm. + # Explicitly disabled for now due SLURM infra dependency. + test-slurm: + if: ${{ false }} + name: test-slurm-${{ matrix.variant }} + runs-on: [self-hosted, slurm] + strategy: + fail-fast: false + matrix: + variant: [slurm_internal, slurm_oss] + steps: + - run: echo "SLURM tests are not yet migrated. Add run-test-slurm equivalent next." diff --git a/.gitignore b/.gitignore index 1663a79e..c66814b7 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,13 @@ asset_harvesting/ # which does not require uv.lock. **/uv.lock .worktrees/ + + +interactive_run +src/pdm_planner + +*node_modules + +.codex +.cache +.skills diff --git a/AGENTS.md b/AGENTS.md index 953a4a11..364d0d08 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,3 +39,11 @@ When asked to access any of the following services, check if you have access to - Linar - Gitlab (especially relevant for MRs) + +## Skills + +A skill is a set of local instructions stored in a `SKILL.md` file. + +### Project skills + +- alpasim-data-access: Comprehensive data access guide for AlpaSim scene/state/map access. Use when implementing or reviewing ego state queries, traffic object access, traffic rule extraction, or map/lane lookups. (file: `.codeartsdoer/skills/alpasim-data-access/SKILL.md`) diff --git a/MTGS b/MTGS new file mode 120000 index 00000000..0e279a98 --- /dev/null +++ b/MTGS @@ -0,0 +1 @@ +../MTGS/ \ No newline at end of file diff --git a/alpasim_controller_session_uuid.csv b/alpasim_controller_session_uuid.csv new file mode 100644 index 00000000..0ebdd77e --- /dev/null +++ b/alpasim_controller_session_uuid.csv @@ -0,0 +1,101 @@ +timestamp_us,x,y,z,qx,qy,qz,qw,vx,vy,wz,u_steering_angle,u_longitudinal_actuation,ref_traj_0_x,ref_traj_0_y,front_steering_angle,acceleration,x_ref_0,y_ref_0,yaw_ref_0 +100000,0.07500801980495453,0.5999936461448669,0.0,0.0,0.0,-0.0001576652139192447,1.0,0.7502248174841475,-0.012825098096396061,-0.008066099431695636,-0.11566527765984198,0.006100203054728386,0.0,-0.6000000238418579,-0.07303788232003759,0.003852028213254145,0.0,-0.6000000238418579,0.0 +200000,0.14261981844902039,0.5999305844306946,0.0,0.0,0.0,-0.0009022931917570531,0.9999996423721313,0.677980380280801,-0.0313171360530033,-0.0226690002113434,-0.21212970860614316,0.07426931273097559,0.00018117645231541246,-0.5999936461448669,-0.1608686699643645,0.04831765734551011,0.00018117645231541246,-0.5999936461448669,0.00031533045694231987 +300000,0.21075312793254852,0.5997101068496704,0.0,0.0,0.0,-0.0024907910265028477,0.9999969601631165,0.6854491860148167,-0.06366244178151403,-0.04113482900796971,-0.29434224819159727,0.11987116449901485,0.008462795056402683,-0.5999162793159485,-0.24515176422247478,0.09350076450436762,0.008462796919047832,-0.599916398525238,0.0018045870820060372 +400000,0.2798338830471039,0.5992015600204468,0.0,0.0,0.0,-0.005029636435210705,0.9999874234199524,0.6967694890868227,-0.09526284334196772,-0.060317495771481364,-0.3628282033002465,0.14696094132728232,0.017234187573194504,-0.599631667137146,-0.3194596125508191,0.12725867511158953,0.017234191298484802,-0.5996317863464355,0.004981588572263718 +500000,0.3501828908920288,0.5982593297958374,0.0,0.0,0.0,-0.008509155362844467,0.9999638795852661,0.7106781249378832,-0.12472044220372215,-0.07858933099269837,-0.41813161300878937,0.1593519398657329,0.026192527264356613,-0.5989683866500854,-0.3817669367845123,0.14752425646671696,0.026192540302872658,-0.5989686846733093,0.010059322230517864 +600000,0.42198893427848816,0.5967336893081665,0.0,0.0,0.0,-0.01286084484308958,0.9999173879623413,0.7259081022755902,-0.15116182178364876,-0.09512516754524072,-0.4608456432554582,0.16048215560651327,0.03499438613653183,-0.5977503657341003,-0.4317018988932924,0.15570663869615892,0.03499439358711243,-0.5977504849433899,0.017018521204590797 +700000,0.495320200920105,0.5944786071777344,0.0,0.0,0.0,-0.017985137179493904,0.9998383522033691,0.7413919216159487,-0.17401044497051,-0.10946074198689425,-0.4916356234530287,0.15335019713792936,0.04334910586476326,-0.595815896987915,-0.4695475895790407,0.15421864243050198,0.043349117040634155,-0.5958160161972046,0.025722408667206764 +800000,0.5701470375061035,0.5913587808609009,0.0,0.0,0.0,-0.023763813078403473,0.9997177124023438,0.7563079525543338,-0.19285391976174832,-0.1212992218543431,-0.5112485824652292,0.14049334930215712,0.051039546728134155,-0.5930266976356506,-0.49588005747841635,0.14555168234880653,0.05103955790400505,-0.5930268168449402,0.0359722301363945 +900000,0.6463664174079895,0.5872558951377869,0.0,0.0,0.0,-0.030066434293985367,0.9995478987693787,0.7700686932745635,-0.2074068991436585,-0.13044733570416633,-0.520510986604234,0.12399566694363286,0.057914551347494125,-0.589272677898407,-0.5114334797268397,0.1319399420901428,0.05791454389691353,-0.5892726182937622,0.0475321002304554 +1000000,0.7238245606422424,0.5820741653442383,0.0,0.0,0.0,-0.03675514832139015,0.999324381351471,0.7822889008239556,-0.21751444679767185,-0.13680255052907153,-0.520317648754787,0.10551719130539097,0.06387393176555634,-0.5844736695289612,-0.5170434683517995,0.1152550579016149,0.06387394666671753,-0.5844737887382507,0.06014195457100868 +1100000,0.8023363351821899,0.575744092464447,0.0,0.0,0.0,-0.043689217418432236,0.9990451335906982,0.7927485140438649,-0.2231604464664909,-0.14035285489353186,-0.5116142531716713,0.08633310310916631,0.0688546672463417,-0.5785797238349915,-0.513615141481029,0.09699202881168678,0.06885465979576111,-0.5785796642303467,0.07352685183286667 +1200000,0.8817011713981628,0.5682248473167419,0.0,0.0,0.0,-0.050729624927043915,0.9987124800682068,0.8013565933519413,-0.22446667103756868,-0.14117414434836908,-0.49537764092064573,0.06738545537766184,0.07282082736492157,-0.5715698003768921,-0.5020989073380719,0.07829669110858427,0.07282084226608276,-0.5715699195861816,0.08740627765655518 +1300000,0.961715817451477,0.5595048069953918,0.0,0.0,0.0,-0.057743269950151443,0.9983315467834473,0.8081188618366633,-0.22167998495857974,-0.13942142482771355,-0.47259478193713766,0.049333816691282964,0.07575753331184387,-0.5634493231773376,-0.4834682613690506,0.060007822952645666,0.07575754821300507,-0.5634494423866272,0.10150285065174103 +1400000,1.0421838760375977,0.5496003031730652,0.0,0.0,0.0,-0.0646066889166832,0.9979107975959778,0.8131098890692127,-0.21515040344618905,-0.13531473781143535,-0.4442434078827838,0.03260910464804631,0.07766743749380112,-0.5542471408843994,-0.45869937401656435,0.04270665527520043,0.07766743004322052,-0.5542470812797546,0.11555080115795135 +1500000,1.122922420501709,0.5385533571243286,0.0,0.0,0.0,-0.0712089091539383,0.9974614381790161,0.8164502458774421,-0.20530423397684835,-0.12912216661190629,-0.4112748282323817,0.017463635975435602,0.07856922596693039,-0.5440114736557007,-0.42875271704096823,0.02676672316834329,0.07856922596693039,-0.5440114736557007,0.129303440451622 +1600000,1.2037665843963623,0.52642822265625,0.0,0.0,0.0,-0.07745338976383209,0.9969959855079651,0.8182884085643546,-0.19261616556455877,-0.12114224502827263,-0.37459945667435374,0.004014582580761568,0.07849711924791336,-0.5328059792518616,-0.39455715258181356,0.012399678879980362,0.07849711924791336,-0.5328059792518616,0.14253845810890198 +1700000,1.2845722436904907,0.5133074522018433,0.0,0.0,0.0,-0.08325912058353424,0.9965279698371887,0.8187869816859817,-0.17758324266013056,-0.11168757493427435,-0.3350757855809254,-0.0077173446501297574,0.07750007510185242,-0.5207058787345886,-0.35699710715467864,-0.00030339698642323806,0.07750007510185242,-0.5207058787345886,0.15506207942962646 +1800000,1.3652164936065674,0.4992879033088684,0.0,0.0,0.0,-0.08856090158224106,0.9960708618164062,0.818112676132318,-0.16070254319641808,-0.10107078222882779,-0.2935023382876274,-0.017776782078509863,0.07564074546098709,-0.5077939033508301,-0.31690276293766917,-0.011337123528296356,0.0756407380104065,-0.5077939033508301,0.1667112410068512 +1900000,1.4455980062484741,0.4844765067100525,0.0,0.0,0.0,-0.09330900013446808,0.9956372380256653,0.8164293949505881,-0.14245339694085962,-0.08959333153622998,-0.2506124804344426,-0.02624913742950413,0.0729946717619896,-0.4941575825214386,-0.27504316643307614,-0.0207534491405014,0.0729946494102478,-0.49415746331214905,0.1773541122674942 +2000000,1.5256353616714478,0.4689865708351135,0.0,0.0,0.0,-0.09746836125850677,0.995238721370697,0.8138938089380157,-0.12328426367257755,-0.07753727279681086,-0.2070717145003304,-0.03324164224418339,0.06964779645204544,-0.47988638281822205,-0.23212198033619097,-0.028639231258960857,0.06964781880378723,-0.4798865020275116,0.18688993155956268 +2100000,1.6052660942077637,0.45293429493904114,0.0,0.0,0.0,-0.10101734846830368,0.9948848485946655,0.8106527761812392,-0.1036038755269511,-0.06515967015997026,-0.1634767175929403,-0.038871733108127325,0.06569496542215347,-0.46506962180137634,-0.18877531032849582,-0.03510063679932437,0.06569498032331467,-0.4650697410106659,0.19524678587913513 +2200000,1.6844444274902344,0.4364359676837921,0.0,0.0,0.0,-0.10394638776779175,0.994583010673523,0.8068421021566573,-0.08377604116295477,-0.052689334071102746,-0.12035622979550566,-0.04325740516486691,0.06123668700456619,-0.44979503750801086,-0.14557146511653996,-0.0402513017183707,0.061236701905727386,-0.4497951567173004,0.20237994194030762 +2300000,1.7631393671035767,0.41960540413856506,0.0,0.0,0.0,-0.10625650733709335,0.9943387508392334,0.8025862138156429,-0.06411740126069058,-0.04032540960025933,-0.07817296064828105,-0.046512699464447554,0.05637696012854576,-0.4341479241847992,-0.10301207186132173,-0.044205117772673816,0.056376952677965164,-0.4341478645801544,0.20826898515224457 +2400000,1.8413329124450684,0.40255206823349,0.0,0.0,0.0,-0.10795780271291733,0.9941555261611938,0.7979984123588613,-0.04489744511804935,-0.028237386867560346,-0.03732649968538256,-0.04874435998449734,0.05122055113315582,-0.4182101786136627,-0.06153432514445853,-0.04707146318935387,0.05122055113315582,-0.4182101786136627,0.21291494369506836 +2500000,1.9190181493759155,0.3853796124458313,0.0,0.0,0.0,-0.10906791687011719,0.9940343499183655,0.7931814121736402,-0.02634013532607498,-0.016566122846905958,0.0018435375384533823,-0.05005224065526918,0.04587031528353691,-0.40206030011177063,-0.021513802391352584,-0.04895370199241109,0.04587031900882721,-0.4020603895187378,0.21633726358413696 +2600000,1.99619722366333,0.3681846857070923,0.0,0.0,0.0,-0.10961073637008667,0.9939746856689453,0.7882279647069421,-0.008626644648938482,-0.005425562672405521,0.039053715223671834,-0.05052957197287487,0.04042497277259827,-0.38577306270599365,0.016732102632907236,-0.04994879929830512,0.04042498394846916,-0.3857731521129608,0.21857070922851562 +2700000,2.072880268096924,0.35105636715888977,0.0,0.0,0.0,-0.1096150353550911,0.9939740896224976,0.7832214855198394,0.008101188892323495,0.005095087353619395,0.07407288805876401,-0.05026353382579383,0.03497745841741562,-0.3694194555282593,0.05294045852678938,-0.050147541253072005,0.03497745469212532,-0.3694194257259369,0.2196628302335739 +2800000,2.1490836143493652,0.3340756297111511,0.0,0.0,0.0,-0.1091134175658226,0.9940293431282043,0.7782366397546407,0.023736992078262804,0.014928925835369498,0.1067181268745043,-0.049336007086196704,0.029612839221954346,-0.3530668020248413,0.08689885201958532,-0.049635090687283,0.029612839221954346,-0.3530668020248413,0.21967150270938873 +2900000,2.2248294353485107,0.3173151910305023,0.0,0.0,0.0,-0.10814116150140762,0.9941355586051941,0.7733398184486961,0.038205837775579184,0.024028828789666574,0.1368509745851386,-0.047825583318281364,0.02440749853849411,-0.336778849363327,0.1184415701403033,-0.048492460946116804,0.024407504126429558,-0.3367789387702942,0.21866227686405182 +3000000,2.3001434803009033,0.3008396327495575,0.0,0.0,0.0,-0.10673540830612183,0.9942874312400818,0.7685894717688305,0.05146023304937507,0.03236492644614567,0.1643735538949117,-0.04580892183333735,0.01942734606564045,-0.3206157982349396,0.1474457353666296,-0.04679791598080045,0.01942734606564045,-0.3206157982349396,0.21670611202716827 +3100000,2.3750550746917725,0.28470557928085327,0.0,0.0,0.0,-0.10493440181016922,0.9944791793823242,0.7640363850181089,0.06347620988754131,0.039922144583358514,0.18922473872446988,-0.04335990358672215,0.014728541485965252,-0.30463409423828125,0.17382746368160717,-0.04462695206030573,0.014728541485965252,-0.30463409423828125,0.21387825906276703 +3200000,2.449596405029297,0.26896199584007263,0.0,0.0,0.0,-0.10277673602104187,0.9947044849395752,0.7597239054411262,0.07424963090557567,0.04669788107268881,0.21137667033234295,-0.04055099754698017,0.010356385260820389,-0.288886696100235,0.19753824873356224,-0.04205315383745794,0.010356385260820389,-0.288886696100235,0.21025589108467102 +3300000,2.523801326751709,0.2536506950855255,0.0,0.0,0.0,-0.10030090808868408,0.9949571490287781,0.7556881170826614,0.08379276499726926,0.052699852199540306,0.23083128075195508,-0.037453203431461815,0.0063452571630477905,-0.2734229266643524,0.21856143394380162,-0.03914847368427312,0.006345252972096205,-0.2734229266643524,0.20591707527637482 +3400000,2.5977048873901367,0.2388067990541458,0.0,0.0,0.0,-0.0975448340177536,0.9952311515808105,0.7519580110608759,0.09213117624666273,0.05794413600419034,0.24761709276102534,-0.03413569751434023,0.002719507087022066,-0.25828829407691956,0.23690889164553805,-0.03598311098075745,0.002719507087022066,-0.25828829407691956,0.20093970000743866 +3500000,2.6713428497314453,0.2244592308998108,0.0,0.0,0.0,-0.09454546123743057,0.9955205321311951,0.7485556892370264,0.09930094226530965,0.06245342280837084,0.2617860537937639,-0.030665133271581182,-0.0005061662523075938,-0.24352450668811798,0.25261779995579275,-0.03262502601384388,-0.000506162759847939,-0.24352456629276276,0.1954004466533661 +3600000,2.744751453399658,0.21063129603862762,0.0,0.0,0.0,-0.09133853018283844,0.9958198666572571,0.7454965950678831,0.10534622365484068,0.0662554865753715,0.27341068019706516,-0.027105736691574046,-0.003326086327433586,-0.22916924953460693,0.2657476516354205,-0.029139821013984674,-0.003326086327433586,-0.22916924953460693,0.18937376141548157 +3700000,2.8179662227630615,0.197341188788414,0.0,0.0,0.0,-0.08795834332704544,0.9961242079734802,0.7427898079330896,0.11031717487477141,0.06938187099042227,0.28258121599986435,-0.023517787776660512,-0.005742971319705248,-0.2152562439441681,0.27637735761073323,-0.025589737442710918,-0.005742971319705248,-0.2152562439441681,0.1829320192337036 +3800000,2.891022205352783,0.18460246920585632,0.0,0.0,0.0,-0.08443757146596909,0.9964287877082825,0.7404384133401726,0.11426819105652763,0.07186678682800479,0.2894030398729496,-0.019957279727113135,-0.007766275200992823,-0.20181508362293243,0.2846025421039034,-0.02203307124065288,-0.007766277063637972,-0.20181512832641602,0.17614436149597168 +3900000,2.9639532566070557,0.17242459952831268,0.0,0.0,0.0,-0.08080722391605377,0.9967297315597534,0.7384399122959339,0.11725647609447463,0.0737462113801727,0.29399422922783264,-0.01647585811497884,-0.009411840699613094,-0.18887114524841309,0.2905330076059311,-0.01852391891324469,-0.009411840699613094,-0.18887114524841309,0.16907645761966705 +4000000,3.0367918014526367,0.16081330180168152,0.0,0.0,0.0,-0.07709651440382004,0.9970236420631409,0.7367867128931364,0.11934089149731318,0.07505716446371898,0.2964831631981685,-0.013119025109723032,-0.01070049125701189,-0.17644573748111725,0.29429028699629245,-0.015110949994993662,-0.010700494050979614,-0.17644578218460083,0.1617908924818039 +4100000,3.1095685958862305,0.14977101981639862,0.0,0.0,0.0,-0.07333281636238098,0.997307538986206,0.7354666639795591,0.12058107159005758,0.07583715194343242,0.2970064530057232,-0.009927100876870263,-0.011657146736979485,-0.16455596685409546,0.2960054345096361,-0.011837561736092,-0.011657146736979485,-0.16455596685409546,0.15434618294239044 +4200000,3.182312250137329,0.139297217130661,0.0,0.0,0.0,-0.06954168528318405,0.997579038143158,0.734463634332942,0.12103675060837826,0.07612374252099259,0.29570679437486397,-0.0069337237796768805,-0.012309868820011616,-0.1532149314880371,0.2958168555042437,-0.00874098904959826,-0.012309868820011616,-0.1532149314880371,0.14679740369319916 +4300000,3.255048990249634,0.129388689994812,0.0,0.0,0.0,-0.06574683636426926,0.9978364109992981,0.7337581361171561,0.12076727469074192,0.07595426081178736,0.29273113011951174,-0.0041661738699490455,-0.01268870197236538,-0.14243172109127045,0.2938683463917268,-0.00585218076168845,-0.012688704766333103,-0.14243175089359283,0.1391957700252533 +4400000,3.3278026580810547,0.12003985792398453,0.0,0.0,0.0,-0.061970144510269165,0.9980780482292175,0.7333279492664503,0.11983126505560042,0.07536557550666695,0.28822891188572985,-0.0016455597387112727,-0.012824775651097298,-0.1322115659713745,0.2903072746324743,-0.003195871993340805,-0.012824778445065022,-0.1322115957736969,0.1315886229276657 +4500000,3.400594711303711,0.11124298721551895,0.0,0.0,0.0,-0.058231696486473083,0.9983031749725342,0.7331487551112409,0.11828638871332381,0.07439395516561245,0.28235046899658495,0.0006135562630048421,-0.01274967659264803,-0.12255611270666122,0.28528287798176477,-0.0007903741782415429,-0.012749679386615753,-0.12255614250898361,0.12401977926492691 +4600000,3.4734437465667725,0.10298836976289749,0.0,0.0,0.0,-0.05454979091882706,0.9985111355781555,0.7331947633483301,0.11618920629949013,0.0730749725154026,0.2752455409761543,0.0026026176478905314,-0.012494860216975212,-0.11346352100372314,0.2789447110413082,0.0013521610987556251,-0.012494863010942936,-0.11346355080604553,0.11652934551239014 +4700000,3.546365737915039,0.09526453912258148,0.0,0.0,0.0,-0.05094098299741745,0.9987017512321472,0.7334393143712273,0.11359506849938737,0.07144343930779079,0.267061938726178,0.004318857274250287,-0.0120906513184309,-0.10492868721485138,0.27144122733774895,0.0032255081440315534,-0.012090654112398624,-0.10492870956659317,0.10915378481149673 +4800000,3.6193742752075195,0.08805840462446213,0.0,0.0,0.0,-0.047420114278793335,0.9988751411437988,0.7338554481950994,0.11055804279315272,0.06953336024726585,0.2579443933184631,0.005764791787374098,-0.011565989814698696,-0.09694356471300125,0.2629185298200657,0.004828961692684907,-0.01156599260866642,-0.09694358706474304,0.10192611068487167 +4900000,3.6924798488616943,0.08135542273521423,0.0,0.0,0.0,-0.044000349938869476,0.9990315437316895,0.7344164140141342,0.10713084762343934,0.06737789158706875,0.2480334920219092,0.00694730545847796,-0.010948242619633675,-0.08949737995862961,0.2535192385113264,0.006166608960816573,-0.0109482416883111,-0.08949737250804901,0.09487581253051758 +5000000,3.765691041946411,0.07513970881700516,0.0,0.0,0.0,-0.04069322720170021,0.9991717338562012,0.7350961589968564,0.1033647698763127,0.06500928923038533,0.23746467443917244,0.007878334094160033,-0.010262112133204937,-0.08257686346769333,0.24338143929339742,0.007247493227753291,-0.01026211492717266,-0.08257688581943512,0.08802913874387741 +5100000,3.8390138149261475,0.06939420849084854,0.0,0.0,0.0,-0.03750869259238243,0.9992963671684265,0.7358697309497402,0.09930958414956127,0.062458857955698915,0.22636760473133863,0.008572247654468526,-0.009530498646199703,-0.07616662234067917,0.2326379000766349,0.008084021353384284,-0.009530501440167427,-0.07616663724184036,0.08140894025564194 +5200000,3.91245174407959,0.06410080194473267,0.0,0.0,0.0,-0.03445513918995857,0.9994063377380371,0.7367136054348389,0.09501344879472039,0.05975688603441533,0.21486542022385702,0.00904652895009928,-0.008773510344326496,-0.07024933397769928,0.22141530745173424,0.00869180545249616,-0.008773512206971645,-0.07024934887886047,0.07503500580787659 +5300000,3.98600697517395,0.05924046039581299,0.0,0.0,0.0,-0.03153947368264198,0.9995024800300598,0.7376059628085261,0.09052277822906599,0.05693256492394087,0.20307420171615098,0.009320686076125918,-0.008008482865989208,-0.06480604410171509,0.20983365088687914,0.009088917791750669,-0.008008481003344059,-0.0648060292005539,0.06892391294240952 +5400000,4.05967903137207,0.05479336529970169,0.0,0.0,0.0,-0.02876717783510685,0.9995862245559692,0.7385269189029535,0.08588209951140867,0.05401389906377904,0.1911025394174522,0.009416271213722337,-0.007250736001878977,-0.05981649085879326,0.19800572168542188,0.009295628061200267,-0.007250736001878977,-0.05981649085879326,0.06308941543102264 +5500000,4.133466720581055,0.050739072263240814,0.0,0.0,0.0,-0.026142334565520287,0.9996582865715027,0.7394586846688344,0.08113390505203083,0.05102761324027097,0.17905127139491855,0.009355403224651428,-0.006512254010885954,-0.05525927245616913,0.18603676317195927,0.00933337362704455,-0.006512255407869816,-0.05525928735733032,0.057542309165000916 +5600000,4.207367420196533,0.04705662652850151,0.0,0.0,0.0,-0.02366773784160614,0.9997199773788452,0.7403856339864916,0.07631849943702415,0.04799905624970073,0.16701325732111127,0.009160040258161087,-0.005803483538329601,-0.05111221224069595,0.17402419890236964,0.009223920708633915,-0.005803484935313463,-0.05111222341656685,0.05229064077138901 +5700000,4.281376838684082,0.043724726885557175,0.0,0.0,0.0,-0.02134493738412857,0.999772310256958,0.7412943174356698,0.07147384609986371,0.0449521044653231,0.15507327361847523,0.008851950636120817,-0.005132551770657301,-0.04735252633690834,0.16205746628610873,0.008989036852973403,-0.005132553167641163,-0.047352537512779236,0.04733991622924805 +5800000,4.355489730834961,0.04072186350822449,0.0,0.0,0.0,-0.019174322485923767,0.9998161792755127,0.7421734419567557,0.06663542260816062,0.04190907082274252,0.14330797645897295,0.008452348532665866,-0.004504992626607418,-0.04395703598856926,0.1502179319049922,0.008650140174780671,-0.004504992626607418,-0.04395703598856926,0.0426931157708168 +5900000,4.4297003746032715,0.03802645206451416,0.0,0.0,0.0,-0.017155203968286514,0.9998528957366943,0.7430138338228176,0.061836091302360036,0.03889062346060378,0.1317859448663781,0.007982042122474478,-0.003924448974430561,-0.04090239480137825,0.13857888752202818,0.008228263636636781,-0.003924449905753136,-0.040902405977249146,0.03835100308060646 +6000000,4.504002571105957,0.035616982728242874,0.0,0.0,0.0,-0.015285893343389034,0.999883234500885,0.743808340053513,0.057105996465017725,0.03591572104718095,0.12056784668207834,0.007459824463418587,-0.0033931611105799675,-0.03816531226038933,0.12720565341111076,0.007743025793101099,-0.0033931618090718985,-0.038165319710969925,0.03431209921836853 +6100000,4.578390121459961,0.033472150564193726,0.0,0.0,0.0,-0.013563795015215874,0.9999080896377563,0.7445516846525192,0.05247248467626569,0.03300156268947528,0.10970659996212992,0.00690301120645016,-0.002911995630711317,-0.03572268784046173,0.11615571835387228,0.007212591009489047,-0.002911996329203248,-0.03572269529104233,0.030572984367609024 +6200000,4.65285587310791,0.031570982187986374,0.0,0.0,0.0,-0.011985494755208492,0.9999282360076904,0.7452403012295756,0.047960053152610434,0.030163555441893357,0.099247627126282,0.00632686797634311,-0.0024809667374938726,-0.033551789820194244,0.10547895171895368,0.006653293215268474,-0.0024809674359858036,-0.03355179727077484,0.02712843008339405 +6300000,4.72739315032959,0.029892949387431145,0.0,0.0,0.0,-0.010546849109232426,0.9999444484710693,0.7458721598342936,0.04359032805875254,0.0274153006658821,0.08922913329675433,0.005745097975494266,-0.0020983375143259764,-0.03163036331534386,0.09521785738143808,0.006079805143581797,-0.0020983379799872637,-0.031630370765924454,0.023971568793058395 +6400000,4.801994800567627,0.028418084606528282,0.0,0.0,0.0,-0.009243076667189598,0.9999573826789856,0.74644658536517,0.03938207559228064,0.024768601001434364,0.07968244847385479,0.005169323409856362,-0.0017621119040995836,-0.0299367755651474,0.08540788337244748,0.005504873244676495,-0.001762112369760871,-0.029936783015727997,0.021094094961881638 +6500000,4.876654624938965,0.02712707780301571,0.0,0.0,0.0,-0.008068847469985485,0.9999674558639526,0.7469640676655441,0.03535124341221097,0.022233486422774194,0.07063238166793234,0.00460931412230929,-0.0014691485557705164,-0.028450103476643562,0.07607775961752417,0.004939364363243171,-0.0014691483229398727,-0.028450099751353264,0.018486415967345238 +6600000,4.951365947723389,0.026001358404755592,0.0,0.0,0.0,-0.007018376141786575,0.9999753832817078,0.7474260884253563,0.031511029017393256,0.019818257243643556,0.06209759373658196,0.004073363624504124,-0.0012166600208729506,-0.027150245383381844,0.06724985783848081,0.004392520389625596,-0.0012166602537035942,-0.02715025097131729,0.016137875616550446 +6700000,5.026123046875,0.025023169815540314,0.0,0.0,0.0,-0.006085498258471489,0.9999815225601196,0.7478349271845027,0.02787197730974037,0.01752954547782413,0.054091033373039596,0.0035672857191933354,-0.001000850461423397,-0.026017969474196434,0.05894059950078513,0.0038714185173532325,-0.001000850461423397,-0.026017969474196434,0.014036867767572403 +6800000,5.1009202003479,0.02417563460767269,0.0,0.0,0.0,-0.005263758823275566,0.9999862313270569,0.748193484818807,0.024442098976104925,0.01537238929314775,0.04662032592277503,0.003095813669931689,-0.000818414322566241,-0.025034984573721886,0.051160851680633654,0.0033816558442421015,-0.0008184144971892238,-0.025034990161657333,0.012171076610684395 +6900000,5.17575216293335,0.023442799225449562,0.0,0.0,0.0,-0.004546487238258123,0.9999897480010986,0.748505117868107,0.02122700886734615,0.013350320042356065,0.03968821614473704,0.0026617213941825487,-0.0006656446494162083,-0.024183982983231544,0.04391635254377278,0.002927046745423098,-0.000665644824039191,-0.024183988571166992,0.010527568869292736 +7000000,5.250613689422607,0.022809673100709915,0.0,0.0,0.0,-0.003926873207092285,0.9999923706054688,0.7487734873116165,0.01823007961051453,0.011465458874537442,0.03329298462993714,0.0022667339977423488,-0.0005389699945226312,-0.023448670282959938,0.037208131103151285,0.0025100863080707607,-0.000538970110937953,-0.023448675870895386,0.009093008004128933 +7100000,5.325500965118408,0.022262251004576683,0.0,0.0,0.0,-0.003398031694814563,0.9999943375587463,0.7490024428333616,0.015452607149542693,0.009718620848768988,0.02742887724784517,0.001911696713165085,-0.0004345309571363032,-0.022813789546489716,0.031032933094616997,0.0021322278037857727,-0.000434531073551625,-0.022813795134425163,0.007853768765926361 +7200000,5.400409698486328,0.021787531673908234,0.0,0.0,0.0,-0.0029530671890825033,0.9999956488609314,0.7491958982530285,0.012893985436696506,0.008109424802953777,0.022086532172275866,0.0015958597599241699,-0.00034965903614647686,-0.02226514182984829,0.025383647578911265,0.0017935333670421707,-0.00034965903614647686,-0.02226514182984829,0.00679607642814517 +7300000,5.47533655166626,0.02137351967394352,0.0,0.0,0.0,-0.0025851319078356028,0.9999966621398926,0.7493577390457821,0.010551884570416304,0.006636405390198932,0.01725337983184017,0.0013183475757355286,-0.0002810120058711618,-0.02178957127034664,0.020249716714306216,0.001493473015242589,-0.0002810120058711618,-0.02178957127034664,0.005906143691390753 +7400000,5.550278186798096,0.02100921981036663,0.0,0.0,0.0,-0.002287474926561117,0.9999974370002747,0.749491748315973,0.008422431327063076,0.005297126620794387,0.01291405414577522,0.0010772905019115295,-0.00022604092373512685,-0.02137497439980507,0.015617546453188257,0.0012306708152450612,-0.00022604092373512685,-0.02137497439980507,0.005170269403606653 +7500000,5.625232219696045,0.020684625953435898,0.0,0.0,0.0,-0.0020534887444227934,0.9999979734420776,0.7496015462097088,0.0065003884760777625,0.004088294639042618,0.009050773503572076,0.00087062435271301,-0.00018206809181720018,-0.021010272204875946,0.011470898473601951,0.0010033162306004088,-0.00018206813547294587,-0.021010277792811394,0.004574954509735107 +7600000,5.700196266174316,0.020390696823596954,0.0,0.0,0.0,-0.001876750262454152,0.9999983310699463,0.7496905275537232,0.0047793309374624116,0.0030058685141273024,0.005643717415644302,0.0006953374843389873,-0.00014726667723152786,-0.020685404539108276,0.007791272461547433,0.000808840274793974,-0.00014726670633535832,-0.020685410127043724,0.004106981214135885 +7700000,5.775168418884277,0.020119331777095795,0.0,0.0,0.0,-0.0017510545440018177,0.9999984502792358,0.7497618130039546,0.0032518161170237213,0.002045167369197309,0.00267137414046841,0.0005483922474870956,-0.00011972848005825654,-0.02039128914475441,0.004558266509966521,0.0006443780199687283,-0.00011972848005825654,-0.02039128914475441,0.0037535028532147408 +7800000,5.850147247314453,0.01986333541572094,0.0,0.0,0.0,-0.001670445199124515,0.9999986290931702,0.7498182427435613,0.0019095464524112847,0.0012009726115794243,0.00011086945075054436,0.0004270870613045406,-9.795789810596034e-05,-0.020119797438383102,0.0017499175428998428,0.000507167685206067,-9.795791993383318e-05,-0.02011980302631855,0.003502112114802003 +7900000,5.925130844116211,0.0196163859218359,0.0,0.0,0.0,-0.0016292372019961476,0.9999986886978149,0.7498623521805236,0.0007435250426331066,0.0004676258129767967,-0.0020617071948955865,0.0003278841908223139,-8.088538743322715e-05,-0.019863717257976532,-0.0006569672602125302,0.0003939575064069029,-8.088538743322715e-05,-0.019863717257976532,0.0033408920280635357 +8000000,6.000118255615234,0.019372984766960144,0.0,0.0,0.0,-0.0016220381949096918,0.9999986886978149,0.7498963791536435,-0.0002557988833286202,-0.00016087980083561015,-0.003871024860014315,0.0002482809672468794,-6.692403985653073e-05,-0.019616708159446716,-0.002686512906871603,0.00030196874245605796,-6.692403985653073e-05,-0.019616708159446716,0.003258475800976157 +8100000,6.075109004974365,0.01912842132151127,0.0,0.0,0.0,-0.0016437628073617816,0.9999986290931702,0.7499222838685736,-0.0010983940013488825,-0.000690813837326341,-0.005342230203734921,0.00018550518774364464,-5.5407661420758814e-05,-0.019373266026377678,-0.004363489535709416,0.00022842678089457372,-5.5407661420758814e-05,-0.019373266026377678,0.0032440777868032455 +8200000,6.150101661682129,0.018878720700740814,0.0,0.0,0.0,-0.0016896441811695695,0.9999986290931702,0.7499417494035684,-0.0017945209580884967,-0.0011286295333889915,-0.006500702169299895,0.00013679129844521316,-4.611932308762334e-05,-0.019128676503896713,-0.005713051720517751,0.00017056272939279817,-4.6119334001559764e-05,-0.01912868022918701,0.003287528408691287 +8300000,6.225096225738525,0.018620599061250687,0.0,0.0,0.0,-0.0017552408389747143,0.9999985098838806,0.7499561974205653,-0.0023546166313134264,-0.0014808909630902054,-0.007371821142640088,9.979024954118445e-05,-3.786458182730712e-05,-0.018878957256674767,-0.00676049662619934,0.0001258728089648109,-3.7864589103264734e-05,-0.018878960981965065,0.003379290923476219 +8400000,6.300091743469238,0.018351413309574127,0.0,0.0,0.0,-0.001836441340856254,0.9999983906745911,0.7499668052661309,-0.0027891888618112896,-0.0017542068313278552,-0.007980765320737757,7.216276280034893e-05,-3.085800926783122e-05,-0.018620822578668594,-0.0075310462942910165,9.195711610925528e-05,-3.0858016543788835e-05,-0.018620826303958893,0.0035104842390865088 +8500000,6.375088214874268,0.018069110810756683,0.0,0.0,0.0,-0.0019294642843306065,0.9999982118606567,0.7499745352510158,-0.003108721446508239,-0.0019551707210743643,-0.008352325067262396,5.2186051655786614e-05,-2.4340413801837713e-05,-0.01835162565112114,-0.008049650179448633,6.684331891734663e-05,-2.4340419258805923e-05,-0.018351629376411438,0.0036728857085108757 +8600000,6.450085163116455,0.017772182822227478,0.0,0.0,0.0,-0.002030856441706419,0.9999980330467224,0.749980160184864,-0.0033235888663822807,-0.00209030746313351,-0.008510737280694876,3.8097586773757524e-05,-1.8486982298782095e-05,-0.018069317564368248,-0.008340807786307537,4.8691567207717344e-05,-1.8486985936760902e-05,-0.018069321289658546,0.003858931828290224 +8700000,6.525082588195801,0.01745961233973503,0.0,0.0,0.0,-0.002137487754225731,0.9999976754188538,0.7499842811404683,-0.003443980463155688,-0.0021660254485255897,-0.008479540754899276,2.8389860008487353e-05,-1.2977108781342395e-05,-0.017772382125258446,-0.008428411970025593,3.5871871173493976e-05,-1.297711696679471e-05,-0.017772382125258446,0.004061715677380562 +8800000,6.600080490112305,0.017130831256508827,0.0,0.0,0.0,-0.0022465456277132034,0.9999974966049194,0.7499873580648228,-0.0034798340985355627,-0.0021885749047393472,-0.00828145348625036,2.202638901984054e-05,-7.948253369249869e-06,-0.01745980605483055,-0.00833561371059053,2.7129016648243345e-05,-7.948253369249869e-06,-0.01745980605483055,0.004274978768080473 +8900000,6.675078392028809,0.016785666346549988,0.0,0.0,0.0,-0.00235552410595119,0.9999972581863403,0.7499897297198334,-0.003440778375692639,-0.002164011557039395,-0.007938263250281512,1.7869621671383054e-05,-3.5191706047044136e-06,-0.0171310193836689,-0.008084703180247862,2.128208821511799e-05,-3.5191706047044136e-06,-0.0171310193836689,0.004493094980716705 +9000000,6.750076770782471,0.01642429828643799,0.0,0.0,0.0,-0.002462215255945921,0.9999970197677612,0.7499916433124427,-0.003336083238972607,-0.002098165559102268,-0.007470741056479272,1.5458686725357295e-05,6.866320063636522e-07,-0.016785848885774612,-0.007697011262223447,1.7604848845474877e-05,6.866321768939088e-07,-0.01678585261106491,0.004711053799837828 +9100000,6.825075149536133,0.016047218814492226,0.0,0.0,0.0,-0.002564696129411459,0.9999967813491821,0.7499932606728722,-0.0031746181078977477,-0.001996615162199841,-0.006898564998688238,1.3721306413859983e-05,4.110137069801567e-06,-0.016424477100372314,-0.007192825170988174,1.515255096625022e-05,4.110137979296269e-06,-0.016424480825662613,0.004924436565488577 +9200000,6.900073528289795,0.015655186027288437,0.0,0.0,0.0,-0.002661315957084298,0.999996542930603,0.7499946739331659,-0.0029648176624843584,-0.0018646651965310425,-0.006240268271007844,1.2385021512748184e-05,7.163571353885345e-06,-0.016047393903136253,-0.006591324529036589,1.3404969543097685e-05,7.163573172874749e-06,-0.016047397628426552,0.00512939877808094 +9300000,6.975071907043457,0.015249188058078289,0.0,0.0,0.0,-0.0027506817132234573,0.9999963045120239,0.749995933061649,-0.0027146546607451937,-0.0017073299753114423,-0.005513195628911209,1.1197115476883392e-05,9.799151484912727e-06,-0.015655355527997017,-0.00591053031554093,1.2010800188814767e-05,9.799154213396832e-06,-0.015655359253287315,0.005322639364749193 +9400000,7.050070285797119,0.014830401167273521,0.0,0.0,0.0,-0.0028316439129412174,0.9999960660934448,0.7499970600734444,-0.002431619174260839,-0.0015293202353841757,-0.004733476658958025,1.000103121891304e-05,1.198494737764122e-05,-0.015249352902173996,-0.005167269172757021,1.0741713454368535e-05,1.1984950106125325e-05,-0.015249356627464294,0.005501371808350086 +9500000,7.125068664550781,0.014400159008800983,0.0,0.0,0.0,-0.002903280546888709,0.9999958872795105,0.7499980582715793,-0.002122703722861083,-0.0013350337879629458,-0.003916009777398763,8.68025451840866e-06,1.3703711374546401e-05,-0.014830561354756355,-0.0043771501472463295,9.439986624861238e-06,1.3703715012525208e-05,-0.014830565080046654,0.005663296673446894 +9600000,7.200067043304443,0.01395991537719965,0.0,0.0,0.0,-0.0029648812487721443,0.9999955892562866,0.7499989145205216,-0.001794393741813078,-0.0011285495231528795,-0.0030744576295904553,7.058983587180442e-06,1.4951539924368262e-05,-0.014400314539670944,-0.0035545532129826513,7.936480791579001e-06,1.4951539924368262e-05,-0.014400314539670944,0.005806569941341877 +9700000,7.2750654220581055,0.013511217199265957,0.0,0.0,0.0,-0.0030159323941916227,0.9999954700469971,0.7499996180870766,-0.0014526636475868273,-0.0009136249355891997,-0.002221258963239919,5.492206539485975e-06,1.573637200635858e-05,-0.013960067182779312,-0.0027126325391130172,6.3930217795556315e-06,1.5736375644337386e-05,-0.01396007090806961,0.005929773673415184 +9800000,7.350063800811768,0.013055674731731415,0.0,0.0,0.0,-0.0030560989398509264,0.9999953508377075,0.7500001547307179,-0.0011029768951416412,-0.000693696160466441,-0.0013676375735501166,3.607482562274885e-06,1.6076473912107758e-05,-0.013511366210877895,-0.001863323342754837,4.634067928704012e-06,1.6076473912107758e-05,-0.013511366210877895,0.006031873635947704 +9900000,7.42506217956543,0.012594934552907944,0.0,0.0,0.0,-0.0030852104537189007,0.9999952912330627,0.750000509353667,-0.0007502905362966775,-0.0004718808404381619,-0.000523633927008392,1.6823244809627781e-06,1.5998750313883647e-05,-0.01305582094937563,-0.0010173643836586727,2.7701629181693165e-06,1.5998750313883647e-05,-0.01305582094937563,0.006112207658588886 +10000000,7.50006103515625,0.012130659073591232,0.0,0.0,0.0,-0.003103244351223111,0.999995231628418,0.7500006805190981,-0.0003990636963600399,-0.00025098345683021383,0.00030186469562314047,-1.0199690038865283e-07,1.553717265778687e-05,-0.012595078907907009,-0.0001843252884764388,9.565117077420566e-07,1.5537176295765676e-05,-0.012595081701874733,0.006170432083308697 diff --git a/docs/INTERACTIVE_RUNTIME_MVP.md b/docs/INTERACTIVE_RUNTIME_MVP.md new file mode 100644 index 00000000..52881cc3 --- /dev/null +++ b/docs/INTERACTIVE_RUNTIME_MVP.md @@ -0,0 +1,360 @@ +# Interactive Runtime MVP + +This document scopes the first interactive runtime iteration for Alpasim. +The goal is not to replace batch rollout execution. The goal is to add a +camera-first debugging control plane that supports stable single-step +simulation and current-state inspection for a frontend. + +## Goals + +- Support frontend-driven single-step simulation. +- Support viewing current camera outputs from the stepped state. +- Reuse the existing event-based runtime loop and rollout services. +- Keep the existing batch `v0.RuntimeService` unchanged. + +## Non-goals + +- Replacing the existing batch rollout API. +- Reworking sensorsim or NRE service contracts in the first iteration. +- Adding lidar/radar streaming in the first iteration. +- Full replay productization. +- Full online eval streaming. + +## Existing runtime cut points + +The current runtime already contains the key semantic boundary needed for +single-step simulation: + +- `EventBasedRollout` owns the event queue and main loop. +- `StepEvent` is the commit boundary for simulation state. +- `RolloutState` is the authoritative mutable world state. +- `GroupedRenderEvent` already assembles multi-camera renders within a control + step. + +This means the MVP should not introduce a second scheduler. It should expose +controlled execution over the existing event queue until the next committed +`StepEvent`. + +## External API shape + +The frontend-facing API lives in +`src/grpc/alpasim_grpc/v1/interactive_runtime.proto`. + +First iteration RPCs: + +- `CreateSession` +- `StartSession` +- `PauseSession` +- `ResumeSession` +- `StepSession` +- `GetSessionState` +- `ListSensors` +- `GetFrame` +- `StreamSession` + +The key IDs are intentionally split: + +- `interactive_session_id`: frontend-visible session ID +- `rollout_uuid`: underlying runtime rollout/logging identity + +This avoids colliding with existing rollout-scoped service sessions and ASL +logging semantics. + +## Runtime-side components + +### `InteractiveSessionManager` + +Suggested file: +`src/runtime/alpasim_runtime/interactive/session_manager.py` + +Responsibilities: + +- Own the registry of active interactive sessions. +- Create and destroy sessions. +- Route RPCs by `interactive_session_id`. +- Enforce per-session locking so `StepSession` and `ResumeSession` do not race. +- Hold session metadata needed by the servicer: + - scene ID + - status + - latest snapshot + - retained frame refs + - event subscribers + +Suggested interface: + +```python +class InteractiveSessionManager: + async def create_session(...) -> InteractiveSessionHandle: ... + async def get_session(session_id: str) -> InteractiveSessionHandle: ... + async def close_session(session_id: str) -> None: ... +``` + +### `InteractiveSessionRunner` + +Suggested file: +`src/runtime/alpasim_runtime/interactive/session_runner.py` + +Responsibilities: + +- Build the same service objects as a normal rollout. +- Build `UnboundRollout`. +- Initialize service rollout sessions. +- Construct `RolloutState` and `EventQueue`. +- Expose controlled execution: + - `start_background()` + - `pause()` + - `resume()` + - `step(num_steps=1)` +- Stop execution exactly after a committed `StepEvent`. + +Important point: + +This runner should reuse `EventBasedRollout` setup logic instead of duplicating + runtime initialization ad hoc. The cleanest refactor is to split +`EventBasedRollout.run()` into: + +- `initialize()` +- `run_until_step_commit()` +- `run_until_complete()` +- `shutdown()` + +The batch path can continue to call `run_until_complete()`. +The interactive path can repeatedly call `run_until_step_commit()`. + +### `InteractiveSnapshotStore` + +Suggested file: +`src/runtime/alpasim_runtime/interactive/snapshot.py` + +Responsibilities: + +- Build immutable post-commit snapshots from `RolloutState`. +- Assign monotonically increasing `tick_id`. +- Convert runtime state into frontend-safe summary structures. + +First iteration snapshot contents: + +- `interactive_session_id` +- `tick_id` +- `sim_time_us` +- ego pose + dynamics +- actor poses +- frame refs for camera outputs produced for the committed tick + +Source of truth remains `RolloutState`. +The snapshot store is only a projection layer. + +### `InteractiveFrameStore` + +Suggested file: +`src/runtime/alpasim_runtime/interactive/frame_store.py` + +Responsibilities: + +- Retain a bounded in-memory window of rendered camera frames. +- Index by `(tick_id, sensor_id)`. +- Return raw bytes and metadata for `GetFrame`. +- Evict old frames when `max_retained_ticks` is exceeded. + +The snapshot should only carry `FrameRef`. +Raw image bytes should stay out of `SessionState` and `StreamSession`. + +### `InteractiveEventBus` + +Suggested file: +`src/runtime/alpasim_runtime/interactive/event_bus.py` + +Responsibilities: + +- Broadcast state/snapshot updates to `StreamSession` subscribers. +- Decouple runtime execution from gRPC streaming backpressure. + +The MVP can keep this simple: + +- one `asyncio.Queue` per subscriber +- best-effort fanout +- drop/close slow subscribers if necessary + +## Minimal refactor inside `EventBasedRollout` + +Suggested refactor target: +`src/runtime/alpasim_runtime/event_loop.py` + +Current `run()` mixes four phases: + +1. setup +2. service session initialization +3. main event loop +4. teardown/eval/finalization + +To support stepping cleanly, split it into reusable methods: + +```python +async def initialize(self) -> None: ... +async def step_until_commit(self) -> SessionStepResult: ... +async def run_to_completion(self) -> Optional[ScenarioEvalResult]: ... +async def finalize(self) -> Optional[ScenarioEvalResult]: ... +``` + +Recommended behavior of `step_until_commit()`: + +- Pop events sequentially from the existing `EventQueue`. +- Run them normally. +- Stop only after one `StepEvent` has completed and committed state. +- Collect frame metadata generated since the previous commit. +- Build and return a `SessionStepResult`. + +This preserves current event ordering and current multi-clock behavior. + +## How to detect "one step committed" + +Do not infer this indirectly from timestamps. +Make it explicit. + +Suggested approach: + +- Add a lightweight callback hook or result object around `StepEvent.run()`. +- After `StepEvent` commits state, emit a `step_committed` signal with: + - committed simulation timestamp + - committed camera frame refs for this step + +This is more reliable than inspecting queue contents after every event. + +## Frame capture integration + +Suggested integration point: +`src/runtime/alpasim_runtime/events/camera.py` + +`GroupedRenderEvent` already receives all camera images for a control window. +The interactive path should additionally forward those images into the +`InteractiveFrameStore`. + +Recommended approach: + +- keep existing driver submission behavior unchanged +- add an optional frame sink callback +- when images are rendered, send `(camera_logical_id, start_ts, end_ts, bytes)` + to the sink + +This avoids mixing frontend retention concerns into the driver service. + +## Session state model + +Suggested runtime-side model: + +```python +@dataclass +class InteractiveSessionState: + interactive_session_id: str + rollout_uuid: str + scene_id: str + status: SessionStatus + current_tick_id: int + current_sim_time_us: int + latest_snapshot: SessionSnapshot | None + error: str | None +``` + +The gRPC `SessionState` should be derived from this internal state. + +## gRPC servicer integration + +Suggested new file: +`src/runtime/alpasim_runtime/daemon/interactive_servicer.py` + +Responsibilities: + +- Map v1 RPCs to `InteractiveSessionManager`. +- Translate domain errors into gRPC status codes. +- Manage `StreamSession` subscription lifecycle. + +Suggested app integration: + +- keep `RuntimeDaemonServicer` untouched for batch mode +- additionally register `InteractiveRuntimeServiceServicer` in + `src/runtime/alpasim_runtime/daemon/app.py` + +This keeps one daemon process exposing both APIs. + +## Why not use the worker pool for the MVP + +The existing daemon worker model is optimized for stateless rollout jobs: + +- parent expands jobs +- scheduler assigns endpoints +- worker runs one rollout to completion +- worker returns `JobResult` + +Interactive sessions do not fit this model well because they require: + +- long-lived in-memory state +- per-session control operations +- frame retention +- streaming subscriptions + +So the MVP should run interactive sessions in-process in the daemon, not in the +existing worker pool. + +This keeps the batch path unchanged and avoids forcing sessionful behavior into +job IPC designed for fire-and-forget rollouts. + +## Online eval for the MVP + +Do not refactor the evaluation stack in phase 1. + +Recommended MVP behavior: + +- Keep current `RuntimeEvaluator` behavior for batch rollouts unchanged. +- Expose only simple online state derived from committed snapshots: + - current ego speed + - current tick/time + - actor count + +If needed later, add a lightweight online metric publisher that subscribes at +the same commit boundary as snapshots. + +## File-level implementation order + +1. Refactor `EventBasedRollout` for reusable initialization and step execution. +2. Add runtime-side interactive models: + - `interactive/session_manager.py` + - `interactive/session_runner.py` + - `interactive/snapshot.py` + - `interactive/frame_store.py` +3. Add gRPC servicer: + - `daemon/interactive_servicer.py` +4. Register the new servicer in `daemon/app.py`. +5. Compile the new v1 protobufs. +6. Add targeted tests. + +## Tests to add + +Suggested test files: + +- `src/runtime/tests/test_interactive_session_manager.py` +- `src/runtime/tests/test_interactive_step_runner.py` +- `src/runtime/tests/test_interactive_frame_store.py` +- `src/runtime/tests/test_interactive_servicer.py` + +Critical test cases: + +- `StepSession(num_steps=1)` advances exactly one committed step. +- Two consecutive `StepSession` calls return increasing `tick_id`. +- `GetFrame` returns the frame associated with the requested committed tick. +- `PauseSession` prevents background stepping. +- `ResumeSession` continues stepping until paused/completed. +- Interactive sessions do not interfere with batch `simulate`. + +## Immediate next coding task + +The best next implementation step is: + +1. Refactor `EventBasedRollout` so the event loop can stop after a committed + `StepEvent`. +2. Add a small `SessionStepResult` model carrying: + - `tick_id` + - `sim_time_us` + - `frame_refs` +3. Keep everything else mocked or in-memory until that seam is working. + +Once that seam exists, the gRPC layer becomes straightforward. diff --git a/docs/INTERACTIVE_WEB_UI.md b/docs/INTERACTIVE_WEB_UI.md new file mode 100644 index 00000000..abe0a5c0 --- /dev/null +++ b/docs/INTERACTIVE_WEB_UI.md @@ -0,0 +1,75 @@ +# Interactive Web UI + +This is a lightweight browser UI for the interactive runtime work. + +The first version is intentionally split into: + +- a static browser frontend +- a tiny Python HTTP gateway +- an adapter layer that can target either mock data or the interactive runtime + +This keeps the page usable while the gRPC control plane is still under active +development, while still allowing real runtime integration. + +## What it includes + +- ego-centric scene map rendered from artifact vector map data +- camera view panel with sensor switching +- recent speed trace panel +- control panel with session create/start/pause/resume/step actions + +## Current backend status + +The page is wired to HTTP endpoints that mirror the intended interactive +session flow. + +There are now two gateway backends: + +- `MockInteractiveApiAdapter` for local UI work +- `GrpcInteractiveApiAdapter` for a real runtime daemon + +The gRPC adapter calls: + +- `CreateSession` +- `GetSessionState` +- `ListSensors` +- `StepSession` +- `PauseSession` +- `ResumeSession` +- `GetFrame` + +from the interactive runtime. + +The map endpoint is local to the HTTP gateway. It resolves the scene artifact +from `--usdz-glob`, loads the vector map or XODR-backed map through +`alpasim_utils.artifact.Artifact`, and serializes a lightweight line-layer +payload for the browser. + +## Run it + +From the repository root, mock mode: + +```bash +uv run --project src/runtime python -m alpasim_runtime.web_debugger --host 127.0.0.1 --port 8080 +``` + +Against a real runtime daemon: + +```bash +uv run --project src/runtime python -m alpasim_runtime.web_debugger \ + --host 127.0.0.1 \ + --port 8080 \ + --runtime-address 127.0.0.1:50051 \ + --usdz-glob "$PWD/data/nre-artifacts/all-usdzs/**/*.usdz" +``` + +Then open: + +```text +http://127.0.0.1:8080 +``` + +## Runtime behavior + +When a session is in `RUNNING` state, the page polls session state and sensor +frames automatically so pause/resume can be observed without manual refresh. diff --git a/docs/run.md b/docs/run.md new file mode 100644 index 00000000..1e726557 --- /dev/null +++ b/docs/run.md @@ -0,0 +1,98 @@ +# Run Interactive GPU1 Local + +This note records the local interactive simulation commands for the generated +`run_interactive_gpu1_local` run directory. + +Run the three command groups below from three separate terminals. + +## Terminal 1: Microservices + +Starts the driver, controller, physics, and sensorsim services. The generated +compose file pins GPU-backed services to GPU `1`. + +```bash +cd /media/a8001/BigSSD/gjc/alpasim/run_interactive_gpu1_local +docker compose -f docker-compose.yaml up driver-0 controller-0 physics-0 sensorsim-0 +``` + +Expected service ports: + +- driver: `localhost:6000` +- sensorsim: `localhost:6001` +- physics: `localhost:6002` +- controller: `localhost:6003` + +## Terminal 2: Runtime Server + +Starts the interactive runtime gRPC server on `127.0.0.1:50051`. + +```bash +cd /media/a8001/BigSSD/gjc/alpasim +source setup_local_env.sh +cd src/runtime + +env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy \ + NO_PROXY=127.0.0.1,localhost,::1 \ + no_proxy=127.0.0.1,localhost,::1 \ + uv run python -m alpasim_runtime.simulate \ + --serve \ + --listen-address 127.0.0.1:50051 \ + --usdz-glob="/media/a8001/BigSSD/gjc/alpasim/data/nre-artifacts/scenesets/48ef968906b9f603e44089c7d235a66c/**/*.usdz" \ + --user-config=/media/a8001/BigSSD/gjc/alpasim/run_interactive_gpu1_local/generated-user-config-0.yaml \ + --network-config=/media/a8001/BigSSD/gjc/alpasim/run_interactive_gpu1_local/generated-network-config.yaml \ + --log-dir=/media/a8001/BigSSD/gjc/alpasim/run_interactive_gpu1_local \ + --eval-config=/media/a8001/BigSSD/gjc/alpasim/run_interactive_gpu1_local/eval-config.yaml \ + --log-level=INFO +``` + +## Terminal 3: Web Debugger + +Starts the browser UI on `127.0.0.1:8080` and connects it to the runtime server. + +```bash +cd /media/a8001/BigSSD/gjc/alpasim +source setup_local_env.sh +cd src/runtime + +env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy \ + NO_PROXY=127.0.0.1,localhost,::1 \ + no_proxy=127.0.0.1,localhost,::1 \ + uv run python -m alpasim_runtime.web_debugger.server \ + --host 127.0.0.1 \ + --port 8080 \ + --runtime-address 127.0.0.1:50051 \ + --user-config /media/a8001/BigSSD/gjc/alpasim/run_interactive_gpu1_local/generated-user-config-0.yaml \ + --usdz-glob "/media/a8001/BigSSD/gjc/alpasim/data/nre-artifacts/scenesets/48ef968906b9f603e44089c7d235a66c/**/*.usdz" +``` + +Open the UI: + +```text +http://127.0.0.1:8080 +``` + +## VPN And Proxy Notes + +If the UI or runtime reports an error like: + +```text +failed to connect to all addresses; last error: UNAVAILABLE: ipv4:127.0.0.1:7890: Socket closed +``` + +then local gRPC traffic is being routed through a VPN/proxy. The commands above +clear proxy environment variables and set `NO_PROXY` for loopback addresses. + +If the browser still fails, add `127.0.0.1` and `localhost` to the VPN/proxy +extension's bypass/direct list. + +## Stop + +Stop the runtime and web debugger with `Ctrl-C`. + +Stop the microservices from the compose terminal with `Ctrl-C`, then clean up +containers if needed: + +```bash +cd /media/a8001/BigSSD/gjc/alpasim/run_interactive_gpu1_local +docker compose -f docker-compose.yaml down +``` diff --git a/src/controller/alpasim_controller/system.py b/src/controller/alpasim_controller/system.py index d9fdf6b9..5b52f71b 100644 --- a/src/controller/alpasim_controller/system.py +++ b/src/controller/alpasim_controller/system.py @@ -210,10 +210,12 @@ def run_controller_and_vehicle_model( pose_local_to_rig_estimated=pose_grpc, dynamic_state=dyn_state, dynamic_state_estimated=dyn_state, + front_steering_angle_rad=self._vehicle_model.front_steering_angle, ) ) return controller_pb2.RunControllerAndVehicleModelResponse( + front_steering_angle_rad=self._vehicle_model.front_steering_angle, states=reported_states, ) diff --git a/src/controller/tests/test_system.py b/src/controller/tests/test_system.py index ce12847c..2ff80bed 100644 --- a/src/controller/tests/test_system.py +++ b/src/controller/tests/test_system.py @@ -71,6 +71,9 @@ def test_alpasimvdc_one_step(dt_propagation_us) -> None: # sanity check that the integration is approximately working assert len(response.states) > 0 final = response.states[-1] + assert final.front_steering_angle_rad == pytest.approx( + response.front_steering_angle_rad, abs=1e-9 + ) assert final.pose_local_to_rig.vec.x == pytest.approx( X_ORIGINAL + get_vx() * dt_propagation_us / 1e6, abs=TOLERANCE_GT ) diff --git a/src/driver/pyproject.toml b/src/driver/pyproject.toml index 125a909a..dd60fd74 100644 --- a/src/driver/pyproject.toml +++ b/src/driver/pyproject.toml @@ -35,6 +35,7 @@ alpasim_driver_main = "alpasim_driver.main:main" ar1 = "alpasim_driver.models.ar1_model:AR1Model" vam = "alpasim_driver.models.vam_model:VAMModel" manual = "alpasim_driver.models.manual_model:ManualModel" +pdm = "alpasim_driver.models.pdm_model:PDMModel" [tool.uv.sources] vam = { git = "https://github.com/valeoai/VideoActionModel.git" } diff --git a/src/driver/src/alpasim_driver/main.py b/src/driver/src/alpasim_driver/main.py index 857daa36..cae40d4a 100644 --- a/src/driver/src/alpasim_driver/main.py +++ b/src/driver/src/alpasim_driver/main.py @@ -6,7 +6,9 @@ from __future__ import annotations import asyncio +import base64 import functools +import json import logging import os import pickle @@ -139,6 +141,48 @@ def _rig_est_offsets_to_local_positions( # Unique queue marker instructing the worker thread to flush and exit. _SENTINEL_JOB = object() +_DRIVER_CONTROL_PREFIX = b"ALPASIM_DRIVER_CTRL_V1:" + + +def _decode_driver_control( + renderer_data: bytes | None, +) -> tuple[str | None, dict[str, Any] | None, bytes]: + """Decode per-frame driver control envelope from renderer_data.""" + if renderer_data is None: + return None, None, b"" + if not renderer_data.startswith(_DRIVER_CONTROL_PREFIX): + return None, None, renderer_data + + payload = renderer_data[len(_DRIVER_CONTROL_PREFIX) :] + try: + obj = json.loads(payload.decode("utf-8")) + except Exception: + logger.warning("Invalid driver control payload; ignoring override") + return None, None, b"" + + next_model = obj.get("next_model") + if next_model is not None and not isinstance(next_model, str): + logger.warning("driver control next_model must be string; ignoring override") + next_model = None + + planner_context = obj.get("planner_context") + if planner_context is not None and not isinstance(planner_context, dict): + logger.warning("driver control planner_context must be object; ignoring") + planner_context = None + + raw_renderer_data = obj.get("renderer_data_b64") + if raw_renderer_data is None: + return next_model, planner_context, b"" + if not isinstance(raw_renderer_data, str): + logger.warning("driver control renderer_data_b64 must be string") + return next_model, planner_context, b"" + + try: + decoded = base64.b64decode(raw_renderer_data.encode("ascii")) + except Exception: + logger.warning("Failed to decode renderer_data_b64 from driver control payload") + return next_model, planner_context, b"" + return next_model, planner_context, decoded @dataclass @@ -148,9 +192,11 @@ class DriveJob: session_id: str session: "Session" command: DriveCommand + model_type: str + planner_context: dict[str, Any] | None pose: Optional[PoseAtTime] timestamp_us: int - result: asyncio.Future[DriveResponse] + result: asyncio.Future[ModelPrediction] @dataclass @@ -479,7 +525,8 @@ def __init__( cfg.model.device if torch.cuda.is_available() else "cpu" ) - # Create model using factory + # Create default model using factory + self._default_model_type = cfg.model.model_type self._model = _create_model( cfg.model, self._device, @@ -487,6 +534,9 @@ def __init__( context_length=cfg.inference.context_length, output_frequency_hz=cfg.inference.output_frequency_hz, ) + self._models: dict[str, BaseTrajectoryModel] = { + self._default_model_type: self._model + } # Get context length from model or config override self._context_length = ( @@ -497,7 +547,7 @@ def __init__( logger.info( "Initialized %s model with %d cameras, context_length=%d", - cfg.model.model_type, + self._default_model_type, self._model.num_cameras, self._context_length, ) @@ -626,6 +676,62 @@ def _worker_main(self) -> None: continue self._loop.call_soon_threadsafe(leftover.result.cancel) + def _get_model(self, model_type: str) -> BaseTrajectoryModel: + """Return model instance for model_type, creating it lazily.""" + existing = self._models.get(model_type) + if existing is not None: + return existing + + model_cfg = ModelConfig( + model_type=model_type, + checkpoint_path=self._cfg.model.checkpoint_path, + device=self._cfg.model.device, + tokenizer_path=self._cfg.model.tokenizer_path, + ) + candidate = _create_model( + model_cfg, + self._device, + camera_ids=self._cfg.inference.use_cameras, + context_length=self._cfg.inference.context_length, + output_frequency_hz=self._cfg.inference.output_frequency_hz, + ) + + # Switching is only safe when model I/O cadence matches the default runtime setup. + if candidate.camera_ids != self._model.camera_ids: + raise ValueError( + f"Model '{model_type}' camera_ids={candidate.camera_ids} do not match " + f"default model camera_ids={self._model.camera_ids}" + ) + if candidate.context_length != self._model.context_length: + raise ValueError( + f"Model '{model_type}' context_length={candidate.context_length} does " + f"not match default model context_length={self._model.context_length}" + ) + if candidate.output_frequency_hz != self._model.output_frequency_hz: + raise ValueError( + f"Model '{model_type}' output_frequency_hz=" + f"{candidate.output_frequency_hz} does not match default model " + f"output_frequency_hz={self._model.output_frequency_hz}" + ) + + self._models[model_type] = candidate + logger.info("Loaded switchable model '%s'", model_type) + return candidate + + def _resolve_model_type_for_request(self, request: DriveRequest) -> str: + """Resolve per-request model type from optional control payload.""" + override_model, _, _ = _decode_driver_control(request.renderer_data) + if override_model is None or override_model.strip() == "": + return self._default_model_type + return override_model + + def _resolve_planner_context_for_request( + self, request: DriveRequest + ) -> dict[str, Any] | None: + """Resolve per-request planner context from optional control payload.""" + _, planner_context, _ = _decode_driver_control(request.renderer_data) + return planner_context + def _get_speed_and_acceleration(self, session: Session) -> tuple[float, float]: """Extract speed and acceleration from session's dynamic state. @@ -653,7 +759,9 @@ def _get_speed_and_acceleration(self, session: Session) -> tuple[float, float]: return float(speed), float(acceleration) - def _prepare_camera_images(self, session: Session) -> CameraImages: + def _prepare_camera_images( + self, session: Session, model: BaseTrajectoryModel + ) -> CameraImages: """Collect raw images from frame caches for all cameras. Returns dict mapping camera_id to list of CameraFrame tuples. @@ -661,7 +769,7 @@ def _prepare_camera_images(self, session: Session) -> CameraImages: """ camera_images: CameraImages = {} - for cam_id in self._model.camera_ids: + for cam_id in model.camera_ids: frame_cache = session.frame_caches[cam_id] entries = frame_cache.latest_frame_entries(self._context_length) camera_images[cam_id] = [(e.timestamp_us, e.image) for e in entries] @@ -717,19 +825,34 @@ def _run_batch(self, batch: list[DriveJob]) -> list[ModelPrediction]: Builds a PredictionInput per job and delegates to predict_batch(), which models can override for GPU-level batching. """ - inputs = [] - for job in batch: - speed, acceleration = self._get_speed_and_acceleration(job.session) - inputs.append( - PredictionInput( - camera_images=self._prepare_camera_images(job.session), - command=job.command, - speed=speed, - acceleration=acceleration, - ego_pose_history=job.session.poses, + predictions: list[ModelPrediction | None] = [None] * len(batch) + grouped: dict[str, list[tuple[int, DriveJob]]] = {} + for index, job in enumerate(batch): + grouped.setdefault(job.model_type, []).append((index, job)) + + for model_type, indexed_jobs in grouped.items(): + model = self._get_model(model_type) + inputs: list[PredictionInput] = [] + for _, job in indexed_jobs: + speed, acceleration = self._get_speed_and_acceleration(job.session) + inputs.append( + PredictionInput( + camera_images=self._prepare_camera_images(job.session, model), + command=job.command, + speed=speed, + acceleration=acceleration, + ego_pose_history=job.session.poses, + planner_context=job.planner_context, + ) ) - ) - return self._model.predict_batch(inputs) + + model_predictions = model.predict_batch(inputs) + for (index, _), prediction in zip( + indexed_jobs, model_predictions, strict=True + ): + predictions[index] = prediction + + return [cast(ModelPrediction, prediction) for prediction in predictions] @async_log_call async def start_session( @@ -894,11 +1017,18 @@ async def drive( trajectory=empty_traj, ) + selected_model_type = self._resolve_model_type_for_request(request) + planner_context = self._resolve_planner_context_for_request(request) + # Validate model eagerly to fail this request immediately if unavailable. + self._get_model(selected_model_type) + future: asyncio.Future[ModelPrediction] = self._loop.create_future() job = DriveJob( session_id=request.session_uuid, session=session, command=session.current_command, + model_type=selected_model_type, + planner_context=planner_context, pose=pose_snapshot, timestamp_us=request.time_now_us, result=future, @@ -908,12 +1038,18 @@ async def drive( prediction = await future # Convert model prediction to Alpasim trajectory format + selected_model = self._get_model(job.model_type) alpasim_traj: Trajectory = self._convert_prediction_to_alpasim_trajectory( - prediction, job.pose, job.timestamp_us + prediction, + job.pose, + job.timestamp_us, + output_frequency_hz=selected_model.output_frequency_hz, ) reasoning_text: str | None = prediction.reasoning_text debug_data = { + "selected_model_type": job.model_type, + "has_planner_context": job.planner_context is not None, "command": int(session.current_command), "command_name": session.current_command.name, "num_frames": { @@ -925,6 +1061,15 @@ async def drive( "trajectory_points": len(alpasim_traj.poses), "reasoning_text": reasoning_text, } + if prediction.debug_metadata is not None: + debug_data["model_debug"] = prediction.debug_metadata + debug_data.update( + { + key: value + for key, value in prediction.debug_metadata.items() + if key not in debug_data + } + ) debug_info = DriveResponse.DebugInfo( unstructured_debug_info=pickle.dumps(debug_data) ) @@ -938,6 +1083,7 @@ def _convert_prediction_to_alpasim_trajectory( prediction: ModelPrediction, current_pose: PoseAtTime, time_now_us: int, + output_frequency_hz: int, ) -> Trajectory: """Convert model prediction to Alpasim trajectory format. @@ -948,6 +1094,7 @@ def _convert_prediction_to_alpasim_trajectory( prediction: Model prediction with trajectory_xy and optional headings. current_pose: Current vehicle pose in local frame. time_now_us: Current time in microseconds. + output_frequency_hz: Output waypoint frequency of the selected model. Returns: Alpasim Trajectory protobuf message. @@ -961,7 +1108,7 @@ def _convert_prediction_to_alpasim_trajectory( trajectory.poses.append(current_pose) curr_z = current_pose.pose.vec.z - frequency_hz = self._model.output_frequency_hz + frequency_hz = output_frequency_hz time_delta_us = int(1_000_000 / frequency_hz) time_step = 1.0 / frequency_hz diff --git a/src/driver/src/alpasim_driver/models/__init__.py b/src/driver/src/alpasim_driver/models/__init__.py index 5212242a..22595b2c 100644 --- a/src/driver/src/alpasim_driver/models/__init__.py +++ b/src/driver/src/alpasim_driver/models/__init__.py @@ -13,6 +13,7 @@ PredictionInput, ) from .manual_model import ManualModel +from .pdm_model import PDMModel from .vam_model import VAMModel __all__ = [ @@ -23,6 +24,7 @@ "DriveCommand", "ManualModel", "ModelPrediction", + "PDMModel", "PredictionInput", "VAMModel", ] diff --git a/src/driver/src/alpasim_driver/models/base.py b/src/driver/src/alpasim_driver/models/base.py index a03afd02..5a602331 100644 --- a/src/driver/src/alpasim_driver/models/base.py +++ b/src/driver/src/alpasim_driver/models/base.py @@ -53,6 +53,7 @@ class PredictionInput: speed: float # m/s acceleration: float # m/s² ego_pose_history: list[Any] # list[PoseAtTime] + planner_context: dict[str, Any] | None = None @dataclass @@ -64,6 +65,7 @@ class ModelPrediction: reasoning_text: str | None = ( None # optional text output (e.g. chain-of-causation reasoning) ) + debug_metadata: dict[str, Any] | None = None class BaseTrajectoryModel(ABC): diff --git a/src/driver/src/alpasim_driver/models/pdm_bridge.py b/src/driver/src/alpasim_driver/models/pdm_bridge.py new file mode 100644 index 00000000..379c99e0 --- /dev/null +++ b/src/driver/src/alpasim_driver/models/pdm_bridge.py @@ -0,0 +1,438 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +"""Minimal dependency-free PDM-style planning bridge for driver integration. + +This module intentionally does not depend on navsim / nuplan planner APIs. +It consumes the compact planner_context built by runtime PolicyEvent and +produces a closed-loop-scored trajectory in rig frame. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from time import perf_counter +from typing import Any, Sequence + +import numpy as np + +from .base import DriveCommand + + +@dataclass(frozen=True, slots=True) +class PDMEgoState: + speed_mps: float + acceleration_mps2: float + yaw_rad: float + + +@dataclass(frozen=True, slots=True) +class PDMActor: + actor_id: str + position_in_rig: np.ndarray + yaw_rad: float + + +@dataclass(frozen=True, slots=True) +class PDMWaitLine: + points_in_rig: np.ndarray + + +@dataclass(frozen=True, slots=True) +class PDMPlannerInput: + ego: PDMEgoState + route_waypoints_in_rig: np.ndarray + nearby_lane_centerlines_in_rig: list[np.ndarray] + actors: list[PDMActor] + wait_lines_in_rig: list[PDMWaitLine] + crosswalks_in_rig: list[np.ndarray] + fallback_command: DriveCommand + + +@dataclass(frozen=True, slots=True) +class PDMClosedLoopResult: + trajectory_xy: np.ndarray + headings: np.ndarray + debug_metadata: dict[str, Any] + + +@dataclass(frozen=True, slots=True) +class _ProposalCandidate: + proposal_idx: int + source: str + centerline_xy: np.ndarray + trajectory_xy: np.ndarray + headings: np.ndarray + score: float + score_breakdown: dict[str, float] + + +def build_pdm_planner_input( + *, + speed_mps: float, + acceleration_mps2: float, + planner_context: dict[str, Any] | None, + fallback_command: DriveCommand, +) -> PDMPlannerInput: + planner_context = planner_context or {} + + ego_ctx = planner_context.get("ego") + ego_yaw = 0.0 + if isinstance(ego_ctx, dict): + ego_yaw = float(ego_ctx.get("yaw", 0.0)) + + route_waypoints = _points_from_context( + planner_context.get("route_waypoints_in_rig"), + minimum_cols=2, + ) + nearby_lanes_ctx = planner_context.get("nearby_lanes") + nearby_lane_centerlines: list[np.ndarray] = [] + if isinstance(nearby_lanes_ctx, list): + for lane in nearby_lanes_ctx: + if not isinstance(lane, dict): + continue + points = _points_from_context(lane.get("centerline_in_rig"), minimum_cols=2) + if len(points) >= 2: + nearby_lane_centerlines.append(points) + + actors_ctx = planner_context.get("actors") + actors: list[PDMActor] = [] + if isinstance(actors_ctx, list): + for actor in actors_ctx: + if not isinstance(actor, dict): + continue + pos = actor.get("position_in_rig") + if not isinstance(pos, list) or len(pos) < 2: + continue + actors.append( + PDMActor( + actor_id=str(actor.get("id", "")), + position_in_rig=np.asarray(pos[:2], dtype=np.float64), + yaw_rad=float(actor.get("yaw", 0.0)), + ) + ) + + traffic_rules = planner_context.get("traffic_rules") + wait_lines: list[PDMWaitLine] = [] + crosswalks: list[np.ndarray] = [] + if isinstance(traffic_rules, dict): + for wait_line in traffic_rules.get("wait_lines_in_rig", []): + if not isinstance(wait_line, dict): + continue + points = _points_from_context(wait_line.get("points"), minimum_cols=2) + if len(points) >= 2: + wait_lines.append(PDMWaitLine(points_in_rig=points)) + for crosswalk in traffic_rules.get("crosswalks_in_rig", []): + points = _points_from_context(crosswalk, minimum_cols=2) + if len(points) >= 3: + crosswalks.append(points) + + return PDMPlannerInput( + ego=PDMEgoState( + speed_mps=max(0.0, float(speed_mps)), + acceleration_mps2=float(acceleration_mps2), + yaw_rad=ego_yaw, + ), + route_waypoints_in_rig=route_waypoints, + nearby_lane_centerlines_in_rig=nearby_lane_centerlines, + actors=actors, + wait_lines_in_rig=wait_lines, + crosswalks_in_rig=crosswalks, + fallback_command=fallback_command, + ) + + +class PDMClosedLoopPlanner: + """Dependency-free closed-loop planner with proposal generation and scoring.""" + + def __init__( + self, + *, + horizon_s: float, + output_frequency_hz: int, + min_turn_radius_m: float, + max_accel_mps2: float, + max_speed_mps: float, + ) -> None: + self._horizon_s = horizon_s + self._output_frequency_hz = output_frequency_hz + self._num_waypoints = max(1, int(round(horizon_s * output_frequency_hz))) + self._dt = 1.0 / output_frequency_hz + self._min_turn_radius_m = min_turn_radius_m + self._max_accel_mps2 = max_accel_mps2 + self._max_speed_mps = max_speed_mps + + def plan(self, planner_input: PDMPlannerInput) -> PDMClosedLoopResult: + start = perf_counter() + route_available = len(planner_input.route_waypoints_in_rig) >= 2 + nearby_lane_count = len(planner_input.nearby_lane_centerlines_in_rig) + + proposals = self._build_proposals(planner_input) + fallback_reason: str | None = None + if not proposals: + proposals = [ + self._fallback_centerline( + planner_input.fallback_command, + planner_input.ego.speed_mps, + ) + ] + fallback_reason = "heuristic_centerline" + elif not route_available: + fallback_reason = "missing_route" + elif nearby_lane_count == 0: + fallback_reason = "missing_nearby_lanes" + + scored_candidates = [ + self._simulate_and_score(planner_input, proposal_idx=i, source=source, centerline_xy=proposal) + for i, (source, proposal) in enumerate(proposals) + ] + best = min(scored_candidates, key=lambda candidate: candidate.score) + runtime_ms = (perf_counter() - start) * 1000.0 + + return PDMClosedLoopResult( + trajectory_xy=best.trajectory_xy.astype(np.float32), + headings=best.headings.astype(np.float32), + debug_metadata={ + "proposal_count": len(scored_candidates), + "selected_proposal_idx": best.proposal_idx, + "selected_proposal_source": best.source, + "route_available": route_available, + "nearby_lane_count": nearby_lane_count, + "actor_count": len(planner_input.actors), + "wait_line_count": len(planner_input.wait_lines_in_rig), + "crosswalk_count": len(planner_input.crosswalks_in_rig), + "fallback_reason": fallback_reason, + "planner_runtime_ms": runtime_ms, + "selected_score": best.score, + "selected_score_breakdown": best.score_breakdown, + }, + ) + + def _build_proposals( + self, planner_input: PDMPlannerInput + ) -> list[tuple[str, np.ndarray]]: + proposals: list[tuple[str, np.ndarray]] = [] + if len(planner_input.route_waypoints_in_rig) >= 2: + proposals.append(("route_centerline", planner_input.route_waypoints_in_rig)) + for idx, lane in enumerate(planner_input.nearby_lane_centerlines_in_rig[:6]): + proposals.append((f"nearby_lane_{idx}", lane)) + return proposals + + def _fallback_centerline( + self, command: DriveCommand, speed_mps: float + ) -> tuple[str, np.ndarray]: + horizon_distance = max(speed_mps * self._horizon_s, 12.0) + samples = np.linspace(0.0, horizon_distance, num=max(self._num_waypoints, 6)) + if command == DriveCommand.LEFT: + radius = self._min_turn_radius_m + theta = samples / radius + centerline = np.column_stack( + (radius * np.sin(theta), radius * (1.0 - np.cos(theta))) + ) + elif command == DriveCommand.RIGHT: + radius = self._min_turn_radius_m + theta = samples / radius + centerline = np.column_stack( + (radius * np.sin(theta), -radius * (1.0 - np.cos(theta))) + ) + else: + centerline = np.column_stack((samples, np.zeros_like(samples))) + return ("heuristic_fallback", centerline.astype(np.float64)) + + def _simulate_and_score( + self, + planner_input: PDMPlannerInput, + *, + proposal_idx: int, + source: str, + centerline_xy: np.ndarray, + ) -> _ProposalCandidate: + trajectory_xy = _sample_along_polyline( + centerline_xy, + _motion_profile_distances( + speed_mps=planner_input.ego.speed_mps, + acceleration_mps2=planner_input.ego.acceleration_mps2, + dt=self._dt, + num_waypoints=self._num_waypoints, + max_accel_mps2=self._max_accel_mps2, + max_speed_mps=self._max_speed_mps, + ), + ) + headings = _compute_headings(trajectory_xy) + score_breakdown = self._score_trajectory( + planner_input=planner_input, + centerline_xy=centerline_xy, + trajectory_xy=trajectory_xy, + ) + score = float(sum(score_breakdown.values())) + return _ProposalCandidate( + proposal_idx=proposal_idx, + source=source, + centerline_xy=centerline_xy, + trajectory_xy=trajectory_xy, + headings=headings, + score=score, + score_breakdown=score_breakdown, + ) + + def _score_trajectory( + self, + *, + planner_input: PDMPlannerInput, + centerline_xy: np.ndarray, + trajectory_xy: np.ndarray, + ) -> dict[str, float]: + score_breakdown: dict[str, float] = { + "progress_reward": -float(np.max(trajectory_xy[:, 0])), + "curvature_penalty": _curvature_penalty(trajectory_xy), + "actor_penalty": _actor_penalty(trajectory_xy, planner_input.actors), + "lane_penalty": _lane_penalty(trajectory_xy, centerline_xy), + "wait_line_penalty": _wait_line_penalty( + trajectory_xy, planner_input.wait_lines_in_rig + ), + "crosswalk_penalty": _crosswalk_penalty( + trajectory_xy, planner_input.crosswalks_in_rig + ), + } + return score_breakdown + + +def _points_from_context( + raw_points: Any, + *, + minimum_cols: int, +) -> np.ndarray: + if not isinstance(raw_points, list) or len(raw_points) == 0: + return np.zeros((0, minimum_cols), dtype=np.float64) + try: + points = np.asarray(raw_points, dtype=np.float64) + except (TypeError, ValueError): + return np.zeros((0, minimum_cols), dtype=np.float64) + if points.ndim != 2 or points.shape[1] < minimum_cols: + return np.zeros((0, minimum_cols), dtype=np.float64) + return points[:, :minimum_cols] + + +def _motion_profile_distances( + *, + speed_mps: float, + acceleration_mps2: float, + dt: float, + num_waypoints: int, + max_accel_mps2: float, + max_speed_mps: float, +) -> np.ndarray: + t = np.arange(1, num_waypoints + 1, dtype=np.float64) * dt + accel = float(np.clip(acceleration_mps2, -max_accel_mps2, max_accel_mps2)) + speed = float(np.clip(speed_mps, 0.0, max_speed_mps)) + return np.maximum(speed * t + 0.5 * accel * t**2, 0.0) + + +def _sample_along_polyline(polyline_xy: np.ndarray, distances: np.ndarray) -> np.ndarray: + if len(polyline_xy) == 0: + return np.column_stack((distances, np.zeros_like(distances))) + if len(polyline_xy) == 1: + return np.repeat(polyline_xy.astype(np.float64), len(distances), axis=0) + + deltas = np.diff(polyline_xy, axis=0) + segment_lengths = np.linalg.norm(deltas, axis=1) + cumulative = np.concatenate(([0.0], np.cumsum(segment_lengths))) + total_length = cumulative[-1] + if total_length <= 1e-6: + return np.repeat(polyline_xy[:1].astype(np.float64), len(distances), axis=0) + + clipped = np.clip(distances, 0.0, total_length) + out = np.zeros((len(clipped), 2), dtype=np.float64) + for i, distance in enumerate(clipped): + seg_idx = min(np.searchsorted(cumulative, distance, side="right") - 1, len(deltas) - 1) + seg_len = max(segment_lengths[seg_idx], 1e-6) + alpha = (distance - cumulative[seg_idx]) / seg_len + out[i] = polyline_xy[seg_idx] + alpha * deltas[seg_idx] + return out + + +def _compute_headings(trajectory_xy: np.ndarray) -> np.ndarray: + prev = np.zeros_like(trajectory_xy) + prev[1:, :] = trajectory_xy[:-1, :] + deltas = trajectory_xy - prev + return np.arctan2(deltas[:, 1], deltas[:, 0]) + + +def _curvature_penalty(trajectory_xy: np.ndarray) -> float: + if len(trajectory_xy) < 3: + return 0.0 + headings = _compute_headings(trajectory_xy) + heading_deltas = np.diff(headings) + return float(np.sum(np.abs(np.unwrap(heading_deltas))) * 0.4) + + +def _lane_penalty(trajectory_xy: np.ndarray, centerline_xy: np.ndarray) -> float: + if len(centerline_xy) < 2: + return 0.0 + sampled_centerline = _sample_along_polyline( + centerline_xy, + _trajectory_arc_lengths(trajectory_xy), + ) + deviation = np.linalg.norm(trajectory_xy - sampled_centerline, axis=1) + return float(np.mean(deviation) * 0.5) + + +def _actor_penalty(trajectory_xy: np.ndarray, actors: Sequence[PDMActor]) -> float: + if not actors: + return 0.0 + penalty = 0.0 + for actor in actors: + deltas = trajectory_xy - actor.position_in_rig[np.newaxis, :] + dists = np.linalg.norm(deltas, axis=1) + min_dist = float(np.min(dists)) + if min_dist < 1.5: + penalty += 1_000.0 + elif min_dist < 4.0: + penalty += (4.0 - min_dist) * 15.0 + return penalty + + +def _wait_line_penalty( + trajectory_xy: np.ndarray, + wait_lines: Sequence[PDMWaitLine], +) -> float: + if not wait_lines or len(trajectory_xy) == 0: + return 0.0 + penalty = 0.0 + for wait_line in wait_lines: + mean_x = float(np.mean(wait_line.points_in_rig[:, 0])) + if mean_x <= 0.0: + continue + crossed = np.any(trajectory_xy[:, 0] >= mean_x) + if crossed and mean_x < 8.0: + penalty += 120.0 + elif crossed: + penalty += 15.0 + return penalty + + +def _crosswalk_penalty( + trajectory_xy: np.ndarray, + crosswalks: Sequence[np.ndarray], +) -> float: + if not crosswalks or len(trajectory_xy) == 0: + return 0.0 + penalty = 0.0 + for polygon in crosswalks: + min_xy = np.min(polygon[:, :2], axis=0) + max_xy = np.max(polygon[:, :2], axis=0) + inside = np.logical_and( + np.logical_and(trajectory_xy[:, 0] >= min_xy[0], trajectory_xy[:, 0] <= max_xy[0]), + np.logical_and(trajectory_xy[:, 1] >= min_xy[1], trajectory_xy[:, 1] <= max_xy[1]), + ) + if np.any(inside): + penalty += 20.0 + return penalty + + +def _trajectory_arc_lengths(trajectory_xy: np.ndarray) -> np.ndarray: + if len(trajectory_xy) == 0: + return np.zeros(0, dtype=np.float64) + deltas = np.diff(trajectory_xy, axis=0) + segment_lengths = np.linalg.norm(deltas, axis=1) + return np.concatenate(([0.0], np.cumsum(segment_lengths))) diff --git a/src/driver/src/alpasim_driver/models/pdm_model.py b/src/driver/src/alpasim_driver/models/pdm_model.py new file mode 100644 index 00000000..27d8f6e0 --- /dev/null +++ b/src/driver/src/alpasim_driver/models/pdm_model.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +"""Dependency-free PDMClosed-style planner model adapter.""" + +from __future__ import annotations + +import logging + +import numpy as np +import torch + +from ..schema import ModelConfig +from .base import BaseTrajectoryModel, DriveCommand, ModelPrediction, PredictionInput +from .pdm_bridge import PDMClosedLoopPlanner, build_pdm_planner_input + +logger = logging.getLogger(__name__) + + +class PDMModel(BaseTrajectoryModel): + """PDMClosed-style planner backend for driver model switching. + + This implementation uses the compact runtime planner_context as its formal + input source and does not depend on camera frames for inference. Camera + validation is intentionally kept to preserve compatibility with the driver + model interface and model-switch safety checks. + """ + + TRAJECTORY_HORIZON_S = 4.0 + DEFAULT_CONTEXT_LENGTH = 1 + DEFAULT_CRUISE_SPEED_MPS = 5.0 + MIN_SPEED_MPS = 0.0 + MAX_SPEED_MPS = 20.0 + MIN_TURN_RADIUS_M = 12.0 + MAX_ACCEL_MPS2 = 4.0 + + @classmethod + def from_config( + cls, + model_cfg: ModelConfig, + device: torch.device, + camera_ids: list[str], + context_length: int | None, + output_frequency_hz: int, + ) -> "PDMModel": + del model_cfg, device + return cls( + camera_ids=camera_ids, + context_length=context_length or cls.DEFAULT_CONTEXT_LENGTH, + output_frequency_hz=output_frequency_hz, + ) + + def __init__( + self, + camera_ids: list[str], + context_length: int, + output_frequency_hz: int, + ) -> None: + self._camera_ids = camera_ids + self._context_length = context_length + self._output_frequency_hz = output_frequency_hz + self._planner = PDMClosedLoopPlanner( + horizon_s=self.TRAJECTORY_HORIZON_S, + output_frequency_hz=output_frequency_hz, + min_turn_radius_m=self.MIN_TURN_RADIUS_M, + max_accel_mps2=self.MAX_ACCEL_MPS2, + max_speed_mps=self.MAX_SPEED_MPS, + ) + logger.info( + "Initialized PDMModel with %d camera(s), context_length=%d, output_frequency=%dHz", + len(camera_ids), + context_length, + output_frequency_hz, + ) + + @property + def camera_ids(self) -> list[str]: + return self._camera_ids + + @property + def context_length(self) -> int: + return self._context_length + + @property + def output_frequency_hz(self) -> int: + return self._output_frequency_hz + + def _encode_command(self, command: DriveCommand) -> int: + return int(command) + + def _normalize_speed(self, speed_mps: float) -> float: + if speed_mps < self.MIN_SPEED_MPS: + return self.DEFAULT_CRUISE_SPEED_MPS + return float(np.clip(speed_mps, self.MIN_SPEED_MPS, self.MAX_SPEED_MPS)) + + def predict(self, prediction_input: PredictionInput) -> ModelPrediction: + self._validate_cameras(prediction_input.camera_images) + + planner_input = build_pdm_planner_input( + speed_mps=self._normalize_speed(prediction_input.speed), + acceleration_mps2=prediction_input.acceleration, + planner_context=prediction_input.planner_context, + fallback_command=prediction_input.command, + ) + result = self._planner.plan(planner_input) + debug_metadata = {"planner_backend": "pdm_closed", **result.debug_metadata} + return ModelPrediction( + trajectory_xy=result.trajectory_xy, + headings=result.headings, + debug_metadata=debug_metadata, + ) diff --git a/src/driver/src/alpasim_driver/models/vam_model.py b/src/driver/src/alpasim_driver/models/vam_model.py index ae185f84..b93a9241 100644 --- a/src/driver/src/alpasim_driver/models/vam_model.py +++ b/src/driver/src/alpasim_driver/models/vam_model.py @@ -78,9 +78,10 @@ def _format_trajs(trajs: torch.Tensor) -> np.ndarray: class VAMModel(BaseTrajectoryModel): """VAM wrapper implementing the common interface.""" - # VAM uses float16 for inference; float32 on ARM (torch.amp.autocast on aarch64 - # doesn't auto-cast Float32 weights in F.linear — fails across PyTorch versions) - DTYPE = torch.float32 if platform.machine() == "aarch64" else torch.float16 + # Keep VAM inference in float32. Some VAM checkpoints keep the action expert + # weights in fp32; passing fp16 action tensors into that path fails in + # F.linear with a Half-vs-Float dtype mismatch. + DTYPE = torch.float32 # VAM only supports single camera NUM_CAMERAS = 1 # NeuroNCAPTransform expects 900x1600 input @@ -137,7 +138,7 @@ def __init__( self._camera_ids = camera_ids self._context_length = context_length self._preproc_pipeline = NeuroNCAPTransform() - self._use_autocast = device.type == "cuda" and platform.machine() != "aarch64" + self._use_autocast = device.type == "cuda" and self.DTYPE != torch.float32 @property def camera_ids(self) -> list[str]: diff --git a/src/driver/src/alpasim_driver/tests/test_pdm_model.py b/src/driver/src/alpasim_driver/tests/test_pdm_model.py new file mode 100644 index 00000000..4b011dcd --- /dev/null +++ b/src/driver/src/alpasim_driver/tests/test_pdm_model.py @@ -0,0 +1,149 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +from __future__ import annotations + +import numpy as np +import pytest +import torch + +from alpasim_driver.models.base import ( + CameraFrame, + DriveCommand, + PredictionInput, +) +from alpasim_driver.models.pdm_model import PDMModel +from alpasim_driver.schema import ModelConfig + + +CAMERA_ID = "camera_front_wide_120fov" + + +def _make_model() -> PDMModel: + return PDMModel.from_config( + ModelConfig(model_type="pdm", checkpoint_path="unused", device="cpu"), + device=torch.device("cpu"), + camera_ids=[CAMERA_ID], + context_length=1, + output_frequency_hz=10, + ) + + +def _make_prediction_input( + *, + planner_context: dict | None, + command: DriveCommand = DriveCommand.STRAIGHT, +) -> PredictionInput: + image = np.zeros((32, 32, 3), dtype=np.uint8) + return PredictionInput( + camera_images={CAMERA_ID: [CameraFrame(timestamp_us=1_000_000, image=image)]}, + command=command, + speed=6.0, + acceleration=0.0, + ego_pose_history=[], + planner_context=planner_context, + ) + + +def _full_planner_context() -> dict: + return { + "ego": { + "position": [0.0, 0.0, 0.0], + "yaw": 0.0, + }, + "route_waypoints_in_rig": [ + [0.0, 0.0, 0.0], + [10.0, 0.0, 0.0], + [20.0, 0.0, 0.0], + [30.0, 1.0, 0.0], + ], + "nearby_lanes": [ + { + "id": "lane_main", + "centerline_in_rig": [[0.0, 0.0], [10.0, 0.0], [25.0, 1.0]], + }, + { + "id": "lane_left", + "centerline_in_rig": [[0.0, 2.0], [10.0, 2.0], [25.0, 3.0]], + }, + ], + "actors": [ + { + "id": "lead", + "position_in_rig": [14.0, 0.2], + "yaw": 0.0, + } + ], + "traffic_rules": { + "wait_lines_in_rig": [ + { + "type": "Stop", + "points": [[6.0, -2.0], [6.0, 2.0]], + } + ], + "crosswalks_in_rig": [ + [[18.0, -2.0], [20.0, -2.0], [20.0, 2.0], [18.0, 2.0]] + ], + }, + } + + +def test_pdm_predict_with_full_context_returns_non_empty_trajectory() -> None: + model = _make_model() + prediction = model.predict(_make_prediction_input(planner_context=_full_planner_context())) + + assert prediction.trajectory_xy.shape == (40, 2) + assert prediction.headings.shape == (40,) + assert prediction.debug_metadata is not None + assert prediction.debug_metadata["planner_backend"] == "pdm_closed" + assert prediction.debug_metadata["proposal_count"] >= 1 + assert prediction.debug_metadata["route_available"] is True + assert prediction.debug_metadata["actor_count"] == 1 + assert prediction.debug_metadata["wait_line_count"] == 1 + assert prediction.debug_metadata["crosswalk_count"] == 1 + + +@pytest.mark.parametrize( + ("planner_context", "expected_fallback"), + [ + ({}, "heuristic_centerline"), + ( + { + "ego": {"position": [0.0, 0.0, 0.0], "yaw": 0.0}, + "actors": [], + "traffic_rules": {}, + }, + "heuristic_centerline", + ), + ( + { + "ego": {"position": [0.0, 0.0, 0.0], "yaw": 0.0}, + "route_waypoints_in_rig": [[0.0, 0.0, 0.0], [20.0, 0.0, 0.0]], + "actors": [], + "traffic_rules": {}, + }, + "missing_nearby_lanes", + ), + ], +) +def test_pdm_predict_degrades_gracefully_when_context_missing( + planner_context: dict, + expected_fallback: str, +) -> None: + model = _make_model() + prediction = model.predict(_make_prediction_input(planner_context=planner_context)) + + assert prediction.trajectory_xy.shape == (40, 2) + assert prediction.headings.shape == (40,) + assert prediction.debug_metadata is not None + assert prediction.debug_metadata["fallback_reason"] == expected_fallback + + +def test_pdm_predict_respects_rig_frame_heading_for_turns() -> None: + model = _make_model() + prediction = model.predict( + _make_prediction_input(planner_context=None, command=DriveCommand.LEFT) + ) + + assert prediction.trajectory_xy[-1, 1] > 0.0 + assert np.max(prediction.headings) > 0.1 diff --git a/src/driver/src/alpasim_driver/tests/test_pdm_service_flow.py b/src/driver/src/alpasim_driver/tests/test_pdm_service_flow.py new file mode 100644 index 00000000..8b8ec552 --- /dev/null +++ b/src/driver/src/alpasim_driver/tests/test_pdm_service_flow.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +from __future__ import annotations + +import asyncio +import base64 +import json +import pickle +from io import BytesIO +from pathlib import Path + +import grpc.aio +import numpy as np +import pytest +from alpasim_grpc.v0.common_pb2 import DynamicState, Pose, PoseAtTime, Quat, Vec3 +from alpasim_grpc.v0.common_pb2 import Trajectory as TrajectoryMsg +from alpasim_grpc.v0.egodriver_pb2 import ( + DriveRequest, + DriveResponse, + DriveSessionRequest, + RolloutCameraImage, + RolloutEgoTrajectory, +) +from omegaconf import OmegaConf +from PIL import Image + +from ..main import EgoDriverService, _DRIVER_CONTROL_PREFIX +from ..schema import DriverConfig + + +def _get_repo_root() -> Path: + return Path(__file__).resolve().parents[5] + + +def _make_png_bytes() -> bytes: + image = np.zeros((32, 32, 3), dtype=np.uint8) + buffer = BytesIO() + Image.fromarray(image).save(buffer, format="PNG") + return buffer.getvalue() + + +def _encode_planner_context(planner_context: dict) -> bytes: + payload = { + "planner_context": planner_context, + "renderer_data_b64": base64.b64encode(b"").decode("ascii"), + } + return _DRIVER_CONTROL_PREFIX + json.dumps(payload, separators=(",", ":")).encode( + "utf-8" + ) + + +@pytest.mark.asyncio +async def test_pdm_drive_response_contains_debug_metadata(tmp_path: Path) -> None: + repo_root = _get_repo_root() + cfg_path = repo_root / "src" / "wizard" / "configs" / "driver" / "pdm.yaml" + raw_cfg = OmegaConf.load(cfg_path) + if "defaults" in raw_cfg: + del raw_cfg["defaults"] + raw_cfg.output_dir = str(tmp_path) + raw_cfg.port = 0 + + schema = OmegaConf.structured(DriverConfig) + cfg = OmegaConf.merge(schema, raw_cfg) + + loop = asyncio.get_running_loop() + server = grpc.aio.server() + service = EgoDriverService(cfg=cfg, loop=loop, grpc_server=server) + + session_uuid = "pdm-session" + camera_id = cfg.inference.use_cameras[0] + rollout_spec = DriveSessionRequest.RolloutSpec() + camera_def = rollout_spec.vehicle.available_cameras.add() + camera_def.logical_id = camera_id + camera_def.intrinsics.resolution_h = 32 + camera_def.intrinsics.resolution_w = 32 + ftheta = camera_def.intrinsics.ftheta_param + ftheta.principal_point_x = 16.0 + ftheta.principal_point_y = 16.0 + ftheta.angle_to_pixeldist_poly.extend([0.0, 1.0]) + ftheta.pixeldist_to_angle_poly.extend([0.0, 1.0]) + + start_request = DriveSessionRequest( + session_uuid=session_uuid, + random_seed=0, + rollout_spec=rollout_spec, + ) + + try: + await service.start_session(start_request, None) + + pose_ts = 1_000_000 + traj_msg = TrajectoryMsg() + traj_msg.poses.append( + PoseAtTime( + pose=Pose( + vec=Vec3(x=0.0, y=0.0, z=0.0), + quat=Quat(w=1.0, x=0.0, y=0.0, z=0.0), + ), + timestamp_us=pose_ts, + ) + ) + dynamic_state = DynamicState( + linear_velocity=Vec3(x=6.0, y=0.0, z=0.0), + angular_velocity=Vec3(x=0.0, y=0.0, z=0.0), + linear_acceleration=Vec3(x=0.0, y=0.0, z=0.0), + angular_acceleration=Vec3(x=0.0, y=0.0, z=0.0), + ) + await service.submit_egomotion_observation( + RolloutEgoTrajectory( + session_uuid=session_uuid, + trajectory=traj_msg, + dynamic_states=[dynamic_state], + ), + None, + ) + await service.submit_image_observation( + RolloutCameraImage( + session_uuid=session_uuid, + camera_image=RolloutCameraImage.CameraImage( + logical_id=camera_id, + frame_start_us=pose_ts, + frame_end_us=pose_ts + 50_000, + image_bytes=_make_png_bytes(), + ), + ), + None, + ) + + planner_context = { + "ego": {"position": [0.0, 0.0, 0.0], "yaw": 0.0}, + "route_waypoints_in_rig": [[0.0, 0.0, 0.0], [12.0, 0.0, 0.0], [24.0, 0.5, 0.0]], + "nearby_lanes": [ + { + "id": "lane_main", + "centerline_in_rig": [[0.0, 0.0], [12.0, 0.0], [24.0, 0.5]], + } + ], + "actors": [{"id": "lead", "position_in_rig": [18.0, 0.1], "yaw": 0.0}], + "traffic_rules": { + "wait_lines_in_rig": [{"type": "Stop", "points": [[8.0, -1.0], [8.0, 1.0]]}], + "crosswalks_in_rig": [], + }, + } + response: DriveResponse = await service.drive( + DriveRequest( + session_uuid=session_uuid, + time_now_us=pose_ts, + time_query_us=pose_ts + 100_000, + renderer_data=_encode_planner_context(planner_context), + ), + None, + ) + + assert len(response.trajectory.poses) > 1 + debug_info = pickle.loads(response.debug_info.unstructured_debug_info) + assert debug_info["selected_model_type"] == "pdm" + assert debug_info["planner_backend"] == "pdm_closed" + assert debug_info["proposal_count"] >= 1 + assert debug_info["route_available"] is True + assert debug_info["actor_count"] == 1 + assert debug_info["wait_line_count"] == 1 + finally: + await service.stop_worker() + if session_uuid in service._sessions: + del service._sessions[session_uuid] diff --git a/src/driver/src/alpasim_driver/tests/test_schema.py b/src/driver/src/alpasim_driver/tests/test_schema.py index a46d8238..6ebe20a4 100644 --- a/src/driver/src/alpasim_driver/tests/test_schema.py +++ b/src/driver/src/alpasim_driver/tests/test_schema.py @@ -9,7 +9,7 @@ # The entry-point names that the driver package registers. # transfuser is provided by the optional alpasim_transfuser plugin, not the core driver. -EXPECTED_MODELS = ["ar1", "manual", "vam"] +EXPECTED_MODELS = ["ar1", "manual", "pdm", "vam"] def test_all_expected_models_registered() -> None: diff --git a/src/grpc/alpasim_grpc/v0/controller.proto b/src/grpc/alpasim_grpc/v0/controller.proto index c2c20d99..4d14581a 100644 --- a/src/grpc/alpasim_grpc/v0/controller.proto +++ b/src/grpc/alpasim_grpc/v0/controller.proto @@ -67,6 +67,7 @@ message RunControllerAndVehicleModelResponse { common.PoseAtTime pose_local_to_rig_estimated = 2 [deprecated = true]; common.DynamicState dynamic_state = 3 [deprecated = true]; common.DynamicState dynamic_state_estimated = 4 [deprecated = true]; + double front_steering_angle_rad = 5 [deprecated = true]; // Vehicle state at a point in time, produced by stepping the vehicle model. message PropagatedState { @@ -75,10 +76,11 @@ message RunControllerAndVehicleModelResponse { common.Pose pose_local_to_rig_estimated = 3; common.DynamicState dynamic_state = 4; common.DynamicState dynamic_state_estimated = 5; + double front_steering_angle_rad = 6; } // All propagated states in chronological order. // Contains intermediate states at pose_reporting_interval_us boundaries // followed by the final state at future_time_us. - repeated PropagatedState states = 6; + repeated PropagatedState states = 7; } diff --git a/src/grpc/alpasim_grpc/v1/__init__.py b/src/grpc/alpasim_grpc/v1/__init__.py new file mode 100644 index 00000000..7c504cd4 --- /dev/null +++ b/src/grpc/alpasim_grpc/v1/__init__.py @@ -0,0 +1 @@ +"""Experimental v1 gRPC APIs for interactive runtime control.""" diff --git a/src/grpc/alpasim_grpc/v1/interactive_runtime.proto b/src/grpc/alpasim_grpc/v1/interactive_runtime.proto new file mode 100644 index 00000000..08cfede9 --- /dev/null +++ b/src/grpc/alpasim_grpc/v1/interactive_runtime.proto @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (c) 2026 NVIDIA Corporation + +syntax = "proto3"; + +import "alpasim_grpc/v0/common.proto"; + +// Frontend-facing interactive control plane for runtime. +// +// This API intentionally coexists with v0 RuntimeService: +// - v0 RuntimeService: batch rollout submission +// - v1 InteractiveRuntimeService: session-oriented debugging and stepping +// +// The first iteration is camera-first and step-oriented. It exposes stable +// post-commit snapshots and separate frame retrieval by tick. + +service InteractiveRuntimeService { + rpc CreateSession(CreateSessionRequest) returns (CreateSessionResponse); + rpc ListSessions(ListSessionsRequest) returns (ListSessionsResponse); + rpc StartSession(StartSessionRequest) returns (SessionState); + rpc PauseSession(PauseSessionRequest) returns (SessionState); + rpc ResumeSession(ResumeSessionRequest) returns (SessionState); + rpc StepSession(StepSessionRequest) returns (StepSessionResponse); + rpc GetSessionState(GetSessionStateRequest) returns (SessionState); + rpc SetActiveBackends(SetActiveBackendsRequest) returns (SessionState); + rpc ListCandidates(ListCandidatesRequest) returns (ListCandidatesResponse); + rpc SelectCandidate(SelectCandidateRequest) returns (SessionState); + rpc RecomputeCandidate(RecomputeCandidateRequest) returns (SessionState); + rpc ListCheckpoints(ListCheckpointsRequest) returns (ListCheckpointsResponse); + rpc RestoreCheckpoint(RestoreCheckpointRequest) returns (SessionState); + rpc ListSensors(ListSensorsRequest) returns (ListSensorsResponse); + rpc GetFrame(GetFrameRequest) returns (GetFrameResponse); + rpc StreamSession(StreamSessionRequest) returns (stream SessionEvent); +} + +enum SessionStatus { + SESSION_STATUS_UNSPECIFIED = 0; + SESSION_STATUS_CREATED = 1; + SESSION_STATUS_RUNNING = 2; + SESSION_STATUS_PAUSED = 3; + SESSION_STATUS_COMPLETED = 4; + SESSION_STATUS_FAILED = 5; +} + +enum SensorType { + SENSOR_TYPE_UNSPECIFIED = 0; + SENSOR_TYPE_CAMERA = 1; +} + +enum FrameEncoding { + FRAME_ENCODING_UNSPECIFIED = 0; + FRAME_ENCODING_JPEG = 1; + FRAME_ENCODING_PNG = 2; +} + +message CreateSessionRequest { + string scene_id = 1; + repeated string sensor_ids = 2; + bool start_paused = 3; + uint32 max_retained_ticks = 4; +} + +message CreateSessionResponse { + string interactive_session_id = 1; + SessionState initial_state = 2; +} + +message ListSessionsRequest {} + +message StartSessionRequest { + string interactive_session_id = 1; +} + +message PauseSessionRequest { + string interactive_session_id = 1; +} + +message ResumeSessionRequest { + string interactive_session_id = 1; +} + +message StepSessionRequest { + string interactive_session_id = 1; + uint32 num_steps = 2; +} + +message GetSessionStateRequest { + string interactive_session_id = 1; +} + +message ListCandidatesRequest { + string interactive_session_id = 1; +} + +message SetActiveBackendsRequest { + string interactive_session_id = 1; + repeated string backend_ids = 2; +} + +message RecomputeCandidateRequest { + string interactive_session_id = 1; + string backend_id = 2; +} + +message SelectCandidateRequest { + string interactive_session_id = 1; + string candidate_id = 2; +} + +message ListCheckpointsRequest { + string interactive_session_id = 1; +} + +message RestoreCheckpointRequest { + string interactive_session_id = 1; + string checkpoint_id = 2; +} + +message ListSensorsRequest { + string interactive_session_id = 1; +} + +message GetFrameRequest { + string interactive_session_id = 1; + string sensor_id = 2; + uint64 tick_id = 3; +} + +message StreamSessionRequest { + string interactive_session_id = 1; + bool include_snapshots = 2; +} + +message SensorDescriptor { + string sensor_id = 1; + string logical_id = 2; + SensorType sensor_type = 3; + uint32 nominal_width = 4; + uint32 nominal_height = 5; + uint64 nominal_frame_interval_us = 6; + common.Pose rig_to_sensor = 7; + FrameEncoding frame_encoding = 8; +} + +message FrameRef { + string sensor_id = 1; + uint64 tick_id = 2; + uint64 frame_start_us = 3; + uint64 frame_end_us = 4; + FrameEncoding frame_encoding = 5; +} + +message EgoState { + common.Pose pose = 1; + common.DynamicState dynamics = 2; + double front_steering_angle_rad = 3; +} + +message ActorState { + string actor_id = 1; + common.Pose pose = 2; +} + +message PolylinePoint { + double x = 1; + double y = 2; +} + +message CandidatePlan { + string candidate_id = 1; + string backend_id = 2; + bool selected = 3; + repeated PolylinePoint points = 4; +} + +message CandidateSummary { + string candidate_id = 1; + string backend_id = 2; + string status = 3; + bool selected = 4; + string error = 5; + string diagnostics_json = 6; +} + +message DecisionSummary { + uint64 step_id = 1; + string input_snapshot_id = 2; + string selected_candidate_id = 3; + repeated CandidateSummary candidates = 4; + string arbitration_reason = 5; +} + +message CheckpointSummary { + string checkpoint_id = 1; + uint64 tick_id = 2; + uint64 sim_time_us = 3; + SessionStatus status = 4; + bool restore_supported = 5; + repeated string unsupported_backend_ids = 6; +} + +message SessionSnapshot { + string interactive_session_id = 1; + uint64 tick_id = 2; + uint64 sim_time_us = 3; + EgoState ego = 4; + repeated ActorState actors = 5; + repeated FrameRef frame_refs = 6; + DecisionSummary latest_decision = 7; + repeated PolylinePoint ego_history = 8; + repeated PolylinePoint selected_plan = 9; + repeated CandidatePlan candidate_plans = 10; + string context_diagnostics_json = 11; +} + +message SessionState { + string interactive_session_id = 1; + string rollout_uuid = 2; + string scene_id = 3; + SessionStatus status = 4; + uint64 current_tick_id = 5; + uint64 current_sim_time_us = 6; + string error = 7; + SessionSnapshot latest_snapshot = 8; + DecisionSummary latest_decision = 9; + repeated string active_backend_ids = 10; + repeated string available_backend_ids = 11; +} + +message StepSessionResponse { + SessionState state = 1; + repeated SessionSnapshot committed_snapshots = 2; +} + +message ListSensorsResponse { + repeated SensorDescriptor sensors = 1; +} + +message ListSessionsResponse { + repeated SessionState sessions = 1; +} + +message ListCandidatesResponse { + repeated CandidateSummary candidates = 1; +} + +message ListCheckpointsResponse { + repeated CheckpointSummary checkpoints = 1; +} + +message GetFrameResponse { + FrameRef frame_ref = 1; + bytes image_bytes = 2; +} + +message SessionEvent { + oneof event { + SessionState state = 1; + SessionSnapshot snapshot = 2; + } +} diff --git a/src/mtgs_sensorsim/README.md b/src/mtgs_sensorsim/README.md new file mode 100644 index 00000000..ab416f4d --- /dev/null +++ b/src/mtgs_sensorsim/README.md @@ -0,0 +1,16 @@ +# AlpaSim MTGS Sensorsim + +This package is a deployment wrapper for using MTGS as an AlpaSim +`SensorsimService`. + +It intentionally does not vendor the full MTGS repository or checkpoint into +AlpaSim. The container image must provide the MTGS Python package and its CUDA / +Nerfstudio dependencies, while AlpaSim provides the gRPC API and wizard service +configuration. + +Typical container paths: + +- `/repo/src/mtgs_sensorsim`: this wrapper +- `/repo/src/grpc`: AlpaSim gRPC Python package +- `/mnt/mtgs`: MTGS repository, experiments, exported AlpaSim artifact directory + diff --git a/src/mtgs_sensorsim/alpasim_mtgs_sensorsim/__init__.py b/src/mtgs_sensorsim/alpasim_mtgs_sensorsim/__init__.py new file mode 100644 index 00000000..e4044a89 --- /dev/null +++ b/src/mtgs_sensorsim/alpasim_mtgs_sensorsim/__init__.py @@ -0,0 +1,2 @@ +"""MTGS-backed implementation of the AlpaSim SensorsimService.""" + diff --git a/src/mtgs_sensorsim/alpasim_mtgs_sensorsim/__main__.py b/src/mtgs_sensorsim/alpasim_mtgs_sensorsim/__main__.py new file mode 100644 index 00000000..48601b6d --- /dev/null +++ b/src/mtgs_sensorsim/alpasim_mtgs_sensorsim/__main__.py @@ -0,0 +1,6 @@ +from alpasim_mtgs_sensorsim.server import main + + +if __name__ == "__main__": + main() + diff --git a/src/mtgs_sensorsim/alpasim_mtgs_sensorsim/server.py b/src/mtgs_sensorsim/alpasim_mtgs_sensorsim/server.py new file mode 100644 index 00000000..c39e1704 --- /dev/null +++ b/src/mtgs_sensorsim/alpasim_mtgs_sensorsim/server.py @@ -0,0 +1,42 @@ +"""Launch the MTGS-backed AlpaSim SensorsimService. + +This module lives in the AlpaSim source tree so the wizard can deploy it as a +normal sensorsim service. It delegates the renderer implementation to the MTGS +Python package, which must still be present in the runtime image or mounted into +the container because the trained model, Nerfstudio config, and CUDA kernels are +MTGS-owned assets. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + + +def _prepend_path(path: str | Path | None) -> None: + if path is None: + return + path = str(path) + if path and path not in sys.path: + sys.path.insert(0, path) + + +def configure_import_paths() -> None: + """Make AlpaSim grpc and MTGS imports available inside the sensorsim image.""" + repo_root = Path(__file__).resolve().parents[3] + _prepend_path(repo_root / "src" / "grpc") + + # Container default used by local_mtgs_sensorsim.yaml. + _prepend_path(os.environ.get("MTGS_REPO", "/mnt/mtgs")) + + +def main() -> None: + configure_import_paths() + from mtgs_sensorsim_server import main as mtgs_server_main + + mtgs_server_main() + + +if __name__ == "__main__": + main() diff --git a/src/mtgs_sensorsim/mtgs_sensorsim_server.py b/src/mtgs_sensorsim/mtgs_sensorsim_server.py new file mode 100644 index 00000000..2486ba17 --- /dev/null +++ b/src/mtgs_sensorsim/mtgs_sensorsim_server.py @@ -0,0 +1,485 @@ +#!/usr/bin/env python3 +"""Serve MTGS rendering behind the AlpaSim sensorsim gRPC API. + +This is a compatibility shim: AlpaSim still talks to a SensorsimService, but the +implementation renders RGB images from an MTGS/Nerfstudio checkpoint instead of +NuRec. The first version focuses on camera RGB rendering; dynamic object pose +overrides are intentionally not applied yet. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import sys +import time +from pathlib import Path +from typing import Optional + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +import cv2 +import grpc +import numpy as np +import torch +from scipy.spatial.transform import Rotation + +from nerfstudio.cameras.cameras import Cameras, CameraType +from nerfstudio.utils import colormaps + +DEFAULT_ALPASIM_GRPC = Path("/home/one/src/alpasim/src/grpc") +if DEFAULT_ALPASIM_GRPC.exists() and str(DEFAULT_ALPASIM_GRPC) not in sys.path: + sys.path.append(str(DEFAULT_ALPASIM_GRPC)) + +from alpasim_grpc import API_VERSION_MESSAGE # noqa: E402 +from alpasim_grpc.v0 import common_pb2, sensorsim_pb2, sensorsim_pb2_grpc # noqa: E402 +from mtgs.tools.render import eval_setup # noqa: E402 + + +LOGGER = logging.getLogger("mtgs_sensorsim") +OPENCV_TO_NERFSTUDIO = np.diag([1.0, -1.0, -1.0, 1.0]).astype(np.float32) + + +def pose_proto_from_se3(matrix: np.ndarray) -> common_pb2.Pose: + quat_xyzw = Rotation.from_matrix(matrix[:3, :3]).as_quat() + return common_pb2.Pose( + vec=common_pb2.Vec3( + x=float(matrix[0, 3]), + y=float(matrix[1, 3]), + z=float(matrix[2, 3]), + ), + quat=common_pb2.Quat( + w=float(quat_xyzw[3]), + x=float(quat_xyzw[0]), + y=float(quat_xyzw[1]), + z=float(quat_xyzw[2]), + ), + ) + + +def se3_from_pose_proto(pose: common_pb2.Pose) -> np.ndarray: + matrix = np.eye(4, dtype=np.float32) + matrix[:3, 3] = [pose.vec.x, pose.vec.y, pose.vec.z] + matrix[:3, :3] = Rotation.from_quat( + [pose.quat.x, pose.quat.y, pose.quat.z, pose.quat.w] + ).as_matrix() + return matrix + + +def opencv_distortion_to_proto(distortion: list[float]) -> tuple[list[float], list[float]]: + """Map OpenCV k1,k2,p1,p2,k3 into sensorsim's radial/tangential layout.""" + + if len(distortion) >= 5: + radial = [distortion[0], distortion[1], distortion[4], 0.0, 0.0, 0.0] + tangential = [distortion[2], distortion[3]] + elif len(distortion) >= 4: + radial = [distortion[0], distortion[1], 0.0, 0.0, 0.0, 0.0] + tangential = [distortion[2], distortion[3]] + else: + radial = [0.0] * 6 + tangential = [0.0] * 2 + return radial, tangential + + +def rgb_tensor_to_encoded_bytes( + rgb: torch.Tensor, + image_format: int, + quality: float, +) -> bytes: + rgb_np = ( + torch.clamp(rgb.detach(), 0.0, 1.0).mul(255.0).byte().cpu().numpy() + ) + if rgb_np.ndim != 3 or rgb_np.shape[-1] < 3: + raise ValueError(f"Expected RGB image [H,W,3], got {rgb_np.shape}") + rgb_np = rgb_np[..., :3] + + if image_format == sensorsim_pb2.ImageFormat.PNG: + ok, encoded = cv2.imencode(".png", cv2.cvtColor(rgb_np, cv2.COLOR_RGB2BGR)) + else: + jpeg_quality = int(np.clip(quality if quality else 95, 1, 100)) + ok, encoded = cv2.imencode( + ".jpg", + cv2.cvtColor(rgb_np, cv2.COLOR_RGB2BGR), + [int(cv2.IMWRITE_JPEG_QUALITY), jpeg_quality], + ) + if not ok: + raise RuntimeError("OpenCV image encoding failed") + return encoded.tobytes() + + +def configure_cuda_extension_arch() -> None: + """Avoid compiling CUDA extensions for every visible architecture on first render.""" + + if os.environ.get("TORCH_CUDA_ARCH_LIST") or not torch.cuda.is_available(): + return + major, minor = torch.cuda.get_device_capability() + arch = f"{major}.{minor}" + os.environ["TORCH_CUDA_ARCH_LIST"] = arch + LOGGER.info("Set TORCH_CUDA_ARCH_LIST=%s for first-render CUDA extension builds", arch) + + +class MTGSSensorsimServicer(sensorsim_pb2_grpc.SensorsimServiceServicer): + def __init__( + self, + config_path: Path, + artifact_dir: Path, + scene_id: Optional[str], + travel_id: int, + native_height: int, + native_width: int, + eval_num_rays_per_chunk: Optional[int], + background_rgb: tuple[float, float, float], + ): + self.config_path = config_path + self.artifact_dir = artifact_dir + self.rig_json = json.loads((artifact_dir / "rig_trajectories.json").read_text()) + self.scene_id = scene_id or self.rig_json["rig_trajectories"][0]["sequence_id"] + self.travel_id = travel_id + self.native_height = native_height + self.native_width = native_width + + rig = self.rig_json["rig_trajectories"][0] + self.min_timestamp_us = int(min(rig["T_rig_world_timestamps_us"])) + self.max_timestamp_us = int(max(rig["T_rig_world_timestamps_us"])) + + configure_cuda_extension_arch() + LOGGER.info("Loading MTGS pipeline from %s", config_path) + _config, pipeline, checkpoint_path, step = eval_setup( + config_path, + eval_num_rays_per_chunk=eval_num_rays_per_chunk, + test_mode="test", + ) + self.pipeline = pipeline + self.pipeline.eval() + self.device = pipeline.device + self.render_lock = asyncio.Lock() + self.server: Optional[grpc.aio.Server] = None + LOGGER.info("Loaded checkpoint %s at step %s on %s", checkpoint_path, step, self.device) + + model = self.pipeline.model + if hasattr(model, "set_background"): + model.set_background(torch.tensor(background_rgb, dtype=torch.float32, device=self.device)) + + def _normalised_time(self, timestamp_us: int) -> float: + denom = max(1, self.max_timestamp_us - self.min_timestamp_us) + return float(np.clip((int(timestamp_us) - self.min_timestamp_us) / denom, 0.0, 1.0)) + + def _available_camera_from_calibration( + self, + unique_id: str, + calibration: dict, + ) -> sensorsim_pb2.AvailableCamerasReturn.AvailableCamera: + params = calibration["camera_model"]["parameters"] + intr = np.asarray(params["intrinsics"], dtype=np.float64) + distortion = [float(v) for v in params.get("distortion", [])] + radial, tangential = opencv_distortion_to_proto(distortion) + + spec = sensorsim_pb2.CameraSpec( + logical_id=calibration["logical_sensor_name"], + resolution_h=self.native_height, + resolution_w=self.native_width, + shutter_type=sensorsim_pb2.ShutterType.GLOBAL, + ) + pinhole = spec.opencv_pinhole_param + pinhole.principal_point_x = float(intr[0, 2]) + pinhole.principal_point_y = float(intr[1, 2]) + pinhole.focal_length_x = float(intr[0, 0]) + pinhole.focal_length_y = float(intr[1, 1]) + pinhole.radial_coeffs.extend(radial) + pinhole.tangential_coeffs.extend(tangential) + pinhole.thin_prism_coeffs.extend([0.0, 0.0, 0.0, 0.0]) + + pose = pose_proto_from_se3(np.asarray(calibration["T_sensor_rig"], dtype=np.float32)) + return sensorsim_pb2.AvailableCamerasReturn.AvailableCamera( + intrinsics=spec, + rig_to_camera=pose, + logical_id=calibration["logical_sensor_name"], + ) + + def _intrinsics_from_request(self, request: sensorsim_pb2.RGBRenderRequest) -> tuple[float, float, float, float]: + spec = request.camera_intrinsics + model = spec.WhichOneof("camera_param") + if model != "opencv_pinhole_param": + raise ValueError( + f"Only opencv_pinhole_param is supported by the first MTGS sensorsim shim, got {model}" + ) + + pinhole = spec.opencv_pinhole_param + native_w = float(spec.resolution_w or request.resolution_w or self.native_width) + native_h = float(spec.resolution_h or request.resolution_h or self.native_height) + scale_x = float(request.resolution_w) / native_w + scale_y = float(request.resolution_h) / native_h + return ( + float(pinhole.focal_length_x) * scale_x, + float(pinhole.focal_length_y) * scale_y, + float(pinhole.principal_point_x) * scale_x, + float(pinhole.principal_point_y) * scale_y, + ) + + def _camera_from_request(self, request: sensorsim_pb2.RGBRenderRequest) -> Cameras: + height = int(request.resolution_h) + width = int(request.resolution_w) + if height <= 0 or width <= 0: + height = int(request.camera_intrinsics.resolution_h or self.native_height) + width = int(request.camera_intrinsics.resolution_w or self.native_width) + + camera_to_world_opencv = se3_from_pose_proto(request.sensor_pose.start_pose) + camera_to_world_ns = camera_to_world_opencv @ OPENCV_TO_NERFSTUDIO + fx, fy, cx, cy = self._intrinsics_from_request(request) + + timestamp_us = int(request.frame_start_us) + camera = Cameras( + fx=torch.tensor([fx], dtype=torch.float32), + fy=torch.tensor([fy], dtype=torch.float32), + cx=torch.tensor([cx], dtype=torch.float32), + cy=torch.tensor([cy], dtype=torch.float32), + height=torch.tensor([height], dtype=torch.int32), + width=torch.tensor([width], dtype=torch.int32), + camera_to_worlds=torch.from_numpy(camera_to_world_ns[:3, :4])[None, ...].float(), + camera_type=CameraType.PERSPECTIVE, + times=torch.tensor([self._normalised_time(timestamp_us)], dtype=torch.float32), + metadata={ + "travel_id": self.travel_id, + "multicolor_travel_id": self.travel_id, + "linear_velocity": np.zeros(3, dtype=np.float32), + "angular_velocity": np.zeros(3, dtype=np.float32), + }, + ) + return camera.to(self.device) + + def _warmup_request(self, height: int, width: int) -> sensorsim_pb2.RGBRenderRequest: + rig = self.rig_json["rig_trajectories"][0] + camera_calibration = next(iter(self.rig_json["camera_calibrations"].values())) + camera = self._available_camera_from_calibration("warmup", camera_calibration) + + rig_to_world = np.asarray(rig["T_rig_worlds"][0], dtype=np.float32) + rig_to_camera = np.asarray(camera_calibration["T_sensor_rig"], dtype=np.float32) + camera_to_world = rig_to_world @ rig_to_camera + pose = pose_proto_from_se3(camera_to_world) + + intrinsics = sensorsim_pb2.CameraSpec() + intrinsics.CopyFrom(camera.intrinsics) + return sensorsim_pb2.RGBRenderRequest( + scene_id=self.scene_id, + resolution_h=height, + resolution_w=width, + camera_intrinsics=intrinsics, + frame_start_us=int(rig["T_rig_world_timestamps_us"][0]), + frame_end_us=int(rig["T_rig_world_timestamps_us"][0]), + sensor_pose=sensorsim_pb2.PosePair(start_pose=pose, end_pose=pose), + image_format=sensorsim_pb2.ImageFormat.JPEG, + image_quality=85, + ) + + def warmup(self, num_renders: int, height: int, width: int) -> None: + if num_renders <= 0: + return + request = self._warmup_request(height=height, width=width) + LOGGER.info("Warming MTGS renderer with %d render(s) at %dx%d", num_renders, width, height) + for idx in range(num_renders): + started = time.perf_counter() + camera = self._camera_from_request(request) + with torch.no_grad(): + outputs = self.pipeline.model.get_outputs_for_camera(camera) + rgb = outputs.get("rgb") + if rgb is None: + keys = ", ".join(outputs.keys()) + raise RuntimeError(f"MTGS model output has no 'rgb'. Available outputs: {keys}") + encoded = rgb_tensor_to_encoded_bytes( + colormaps.apply_colormap(rgb), + request.image_format, + request.image_quality, + ) + torch.cuda.synchronize(self.device) if self.device.type == "cuda" else None + LOGGER.info( + "Warmup render %d/%d completed in %.3fs, image_bytes=%d", + idx + 1, + num_renders, + time.perf_counter() - started, + len(encoded), + ) + + async def get_version(self, request, context): + LOGGER.info("RPC get_version peer=%s", context.peer()) + return common_pb2.VersionId( + version_id="mtgs-sensorsim-local", + git_hash="unknown", + grpc_api_version=API_VERSION_MESSAGE, + ) + + async def get_available_scenes(self, request, context): + LOGGER.info("RPC get_available_scenes peer=%s", context.peer()) + return common_pb2.AvailableScenesReturn(scene_ids=[self.scene_id]) + + async def get_available_cameras(self, request, context): + LOGGER.info("RPC get_available_cameras scene_id=%s peer=%s", request.scene_id, context.peer()) + if request.scene_id and request.scene_id != self.scene_id: + LOGGER.warning("Requested scene_id=%s, serving %s", request.scene_id, self.scene_id) + response = sensorsim_pb2.AvailableCamerasReturn() + for unique_id, calibration in self.rig_json["camera_calibrations"].items(): + response.available_cameras.append( + self._available_camera_from_calibration(unique_id, calibration) + ) + return response + + async def get_available_ego_masks(self, request, context): + LOGGER.info("RPC get_available_ego_masks peer=%s", context.peer()) + return sensorsim_pb2.AvailableEgoMasksReturn() + + async def get_available_trajectories(self, request, context): + LOGGER.info("RPC get_available_trajectories peer=%s", context.peer()) + return sensorsim_pb2.AvailableTrajectoriesReturn() + + async def render_lidar(self, request, context): + LOGGER.info("RPC render_lidar scene_id=%s peer=%s", request.scene_id, context.peer()) + return sensorsim_pb2.LidarRenderReturn(num_points=0) + + async def render_rgb(self, request, context): + request_started = time.perf_counter() + LOGGER.info( + "RPC render_rgb received scene_id=%s size=%dx%d frame_start_us=%s peer=%s", + request.scene_id, + int(request.resolution_w), + int(request.resolution_h), + int(request.frame_start_us), + context.peer(), + ) + if request.dynamic_objects: + LOGGER.debug( + "Ignoring %d dynamic object overrides in first MTGS sensorsim shim", + len(request.dynamic_objects), + ) + + camera_started = time.perf_counter() + camera = self._camera_from_request(request) + camera_elapsed = time.perf_counter() - camera_started + async with self.render_lock: + render_started = time.perf_counter() + with torch.no_grad(): + outputs = self.pipeline.model.get_outputs_for_camera(camera) + if self.device.type == "cuda": + torch.cuda.synchronize(self.device) + render_elapsed = time.perf_counter() - render_started + rgb = outputs.get("rgb") + if rgb is None: + keys = ", ".join(outputs.keys()) + raise RuntimeError(f"MTGS model output has no 'rgb'. Available outputs: {keys}") + + encode_started = time.perf_counter() + encoded = rgb_tensor_to_encoded_bytes( + colormaps.apply_colormap(rgb), + request.image_format, + request.image_quality, + ) + encode_elapsed = time.perf_counter() - encode_started + LOGGER.info( + "render_rgb scene_id=%s camera=%s size=%dx%d frame_start_us=%s timings camera=%.3fs render=%.3fs encode=%.3fs total=%.3fs bytes=%d", + request.scene_id, + request.camera_intrinsics.logical_id, + int(request.resolution_w), + int(request.resolution_h), + int(request.frame_start_us), + camera_elapsed, + render_elapsed, + encode_elapsed, + time.perf_counter() - request_started, + len(encoded), + ) + return sensorsim_pb2.RGBRenderReturn(image_bytes=encoded) + + async def render_aggregated(self, request, context): + LOGGER.info("RPC render_aggregated rgb_requests=%d peer=%s", len(request.rgb_requests), context.peer()) + response = sensorsim_pb2.AggregatedRenderReturn() + for rgb_request in request.rgb_requests: + response.rgb_returns.append(await self.render_rgb(rgb_request, context)) + return response + + async def shut_down(self, request, context): + LOGGER.info("Received shut_down request") + if self.server is not None: + asyncio.create_task(self.server.stop(grace=1.0)) + return common_pb2.Empty() + + +async def serve(args: argparse.Namespace) -> None: + logging.basicConfig( + level=getattr(logging, args.log_level.upper()), + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + servicer = MTGSSensorsimServicer( + config_path=args.config, + artifact_dir=args.artifact_dir, + scene_id=args.scene_id, + travel_id=args.travel_id, + native_height=args.native_height, + native_width=args.native_width, + eval_num_rays_per_chunk=args.eval_num_rays_per_chunk, + background_rgb=tuple(args.background_rgb), + ) + servicer.warmup( + num_renders=args.warmup_renders, + height=args.warmup_height, + width=args.warmup_width, + ) + server = grpc.aio.server( + options=[ + ("grpc.max_send_message_length", args.max_message_bytes), + ("grpc.max_receive_message_length", args.max_message_bytes), + ] + ) + sensorsim_pb2_grpc.add_SensorsimServiceServicer_to_server(servicer, server) + servicer.server = server + listen_addr = f"{args.host}:{args.port}" + server.add_insecure_port(listen_addr) + await server.start() + LOGGER.info("MTGS sensorsim server listening on %s for scene_id=%s", listen_addr, servicer.scene_id) + await server.wait_for_termination() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run an MTGS-backed AlpaSim sensorsim gRPC service.") + parser.add_argument( + "--config", + type=Path, + default=Path("experiments/main_mt/MTGS/road_block-331220_4690660_331190_4690710/config.yml"), + help="MTGS/Nerfstudio config.yml that points to the trained checkpoint directory.", + ) + parser.add_argument( + "--artifact-dir", + type=Path, + default=Path("exports/alpasim/road_block-331220_4690660_331190_4690710"), + help="AlpaSim artifact directory containing rig_trajectories.json.", + ) + parser.add_argument("--scene-id", default=None, help="Override scene_id reported to AlpaSim.") + parser.add_argument("--travel-id", type=int, default=7, help="MTGS traversal id used for time/color conditioning.") + parser.add_argument("--host", default="0.0.0.0") + parser.add_argument("--port", type=int, default=50053) + parser.add_argument("--native-height", type=int, default=1080) + parser.add_argument("--native-width", type=int, default=1920) + parser.add_argument("--eval-num-rays-per-chunk", type=int, default=None) + parser.add_argument("--max-message-bytes", type=int, default=64 * 1024 * 1024) + parser.add_argument("--background-rgb", type=float, nargs=3, default=(0.0, 0.0, 0.0)) + parser.add_argument( + "--warmup-renders", + type=int, + default=0, + help="Run this many local renders before accepting gRPC requests. This pays first-render CUDA JIT cost at startup.", + ) + parser.add_argument("--warmup-height", type=int, default=180) + parser.add_argument("--warmup-width", type=int, default=320) + parser.add_argument("--log-level", default="INFO") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + asyncio.run(serve(args)) + + +if __name__ == "__main__": + main() diff --git a/src/mtgs_sensorsim/pyproject.toml b/src/mtgs_sensorsim/pyproject.toml new file mode 100644 index 00000000..ca913d5b --- /dev/null +++ b/src/mtgs_sensorsim/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "alpasim-mtgs-sensorsim" +version = "0.1.0" +description = "MTGS-backed implementation of the AlpaSim SensorsimService" +requires-python = ">=3.11,<3.13" +dependencies = [ + "alpasim-grpc", + "grpcio", + "numpy", + "opencv-python-headless", + "scipy", + "torch", +] + +[project.scripts] +alpasim_mtgs_sensorsim = "alpasim_mtgs_sensorsim.server:main" + +[tool.uv.sources] +alpasim-grpc = { workspace = true } + +[tool.setuptools] +py-modules = ["mtgs_sensorsim_server"] + +[tool.setuptools.packages.find] +include = ["alpasim_mtgs_sensorsim*"] diff --git a/src/mtgs_sensorsim/render.py b/src/mtgs_sensorsim/render.py new file mode 100644 index 00000000..6dfbd38d --- /dev/null +++ b/src/mtgs_sensorsim/render.py @@ -0,0 +1,770 @@ +#-------------------------------------------------------------------------------# +# MTGS: Multi-Traversal Gaussian Splatting (https://arxiv.org/abs/2503.12552) # +# Source code: https://github.com/OpenDriveLab/MTGS # +# Copyright (c) OpenDriveLab. All rights reserved. # +#-------------------------------------------------------------------------------# + +#!/usr/bin/env python +from __future__ import annotations + +import gzip +import json +import os +import sys +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Literal, Optional, Union, Tuple, Callable + +import mediapy as media +import numpy as np +import torch +import tyro +from jaxtyping import Float +from rich import box, style +from rich.panel import Panel +from rich.progress import BarColumn, Progress, TaskProgressColumn, TextColumn, TimeElapsedColumn, TimeRemainingColumn +from rich.table import Table +from torch import Tensor +from typing_extensions import Annotated +import yaml + +from nerfstudio.cameras.camera_paths import get_interpolated_poses_many, get_path_from_json +from nerfstudio.cameras.cameras import Cameras, CameraType +from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager, VanillaDataManagerConfig +from nerfstudio.data.datamanagers.full_images_datamanager import FullImageDatamanagerConfig +from nerfstudio.data.datasets.base_dataset import Dataset +from nerfstudio.data.scene_box import OrientedBox +from nerfstudio.data.utils.dataloaders import FixedIndicesEvalDataloader +from nerfstudio.engine.trainer import TrainerConfig +from nerfstudio.model_components import renderers +from nerfstudio.pipelines.base_pipeline import Pipeline +from nerfstudio.utils import colormaps, install_checks +from nerfstudio.utils.rich_utils import CONSOLE, ItersPerSecColumn + + +def _render_trajectory_video( + pipeline: Pipeline, + cameras: Cameras, + output_filename: Path, + rendered_output_names: List[str], + crop_data: Optional[CropData] = None, + rendered_resolution_scaling_factor: float = 1.0, + seconds: float = 5.0, + output_format: Literal["images", "video"] = "video", + image_format: Literal["jpeg", "png"] = "jpeg", + jpeg_quality: int = 100, + depth_near_plane: Optional[float] = None, + depth_far_plane: Optional[float] = None, + colormap_options: colormaps.ColormapOptions = colormaps.ColormapOptions(), + filenames: Optional[List[str]] = None, +) -> None: + """Helper function to create a video of the spiral trajectory. + + Args: + pipeline: Pipeline to evaluate with. + cameras: Cameras to render. + output_filename: Name of the output file. + rendered_output_names: List of outputs to visualise. + crop_data: Crop data to apply to the rendered images. + rendered_resolution_scaling_factor: Scaling factor to apply to the camera image resolution. + seconds: Length of output video. + output_format: How to save output data. + depth_near_plane: Closest depth to consider when using the colormap for depth. If None, use min value. + depth_far_plane: Furthest depth to consider when using the colormap for depth. If None, use max value. + colormap_options: Options for colormap. + """ + CONSOLE.print("[bold green]Creating trajectory " + output_format) + cameras.rescale_output_resolution(rendered_resolution_scaling_factor) + cameras = cameras.to(pipeline.device) + fps = len(cameras) / seconds + + progress = Progress( + TextColumn(":movie_camera: Rendering :movie_camera:"), + BarColumn(), + TaskProgressColumn( + text_format="[progress.percentage]{task.completed}/{task.total:>.0f}({task.percentage:>3.1f}%)", + show_speed=True, + ), + ItersPerSecColumn(suffix="fps"), + TimeRemainingColumn(elapsed_when_finished=False, compact=False), + TimeElapsedColumn(), + ) + output_image_dir = output_filename.parent / output_filename.stem + if "images" in output_format: + output_image_dir.mkdir(parents=True, exist_ok=True) + if "video" in output_format: + # make the folder if it doesn't exist + output_filename.parent.mkdir(parents=True, exist_ok=True) + # NOTE: + # we could use ffmpeg_args "-movflags faststart" for progressive download, + # which would force moov atom into known position before mdat, + # but then we would have to move all of mdat to insert metadata atom + # (unless we reserve enough space to overwrite with our uuid tag, + # but we don't know how big the video file will be, so it's not certain!) + if filenames is not None: + assert len(filenames) == len(cameras), "filenames must have the same length as cameras" + + with ExitStack() as stack: + writer = None + render_images = [] + with progress: + for camera_idx in progress.track(range(cameras.size), description=""): + obb_box = None + if crop_data is not None: + obb_box = crop_data.obb + + camera = cameras[camera_idx : camera_idx + 1] + if crop_data is not None: + with renderers.background_color_override_context( + crop_data.background_color.to(pipeline.device) + ), torch.no_grad(): + outputs = pipeline.model.get_outputs_for_camera( + camera, obb_box=obb_box + ) + else: + with torch.no_grad(): + outputs = pipeline.model.get_outputs_for_camera( + camera, obb_box=obb_box + ) + + render_image = [] + for rendered_output_name in rendered_output_names: + if rendered_output_name not in outputs: + CONSOLE.rule("Error", style="red") + CONSOLE.print(f"Could not find {rendered_output_name} in the model outputs", justify="center") + CONSOLE.print( + f"Please set --rendered_output_name to one of: {outputs.keys()}", justify="center" + ) + sys.exit(1) + output_image = outputs[rendered_output_name] + is_depth = rendered_output_name.find("depth") != -1 + if is_depth: + output_image = ( + colormaps.apply_depth_colormap( + output_image, + accumulation=outputs["accumulation"], + near_plane=depth_near_plane, + far_plane=depth_far_plane, + colormap_options=colormap_options, + ) + .cpu() + .numpy() + ) + else: + output_image = ( + colormaps.apply_colormap( + image=output_image, + colormap_options=colormap_options, + ) + .cpu() + .numpy() + ) + render_image.append(output_image) + + render_image = np.concatenate(render_image, axis=1) + render_images.append(render_image) + if "images" in output_format: + img_filename = filenames[camera_idx] if filenames is not None else f"{camera_idx:05d}" + if image_format == "png": + media.write_image(output_image_dir / f"{img_filename}.png", render_image, fmt="png") + if image_format == "jpeg": + media.write_image( + output_image_dir / f"{img_filename}.jpg", render_image, fmt="jpeg", quality=jpeg_quality + ) + if "video" in output_format: + if writer is None: + render_width = int(render_image.shape[1]) + render_height = int(render_image.shape[0]) + writer = stack.enter_context( + media.VideoWriter( + path=output_filename, + shape=(render_height, render_width), + fps=fps, + ) + ) + writer.add_image(render_image) + + table = Table( + title=None, + show_header=False, + box=box.MINIMAL, + title_style=style.Style(bold=True), + ) + if output_format == "video": + table.add_row("Video", str(output_filename)) + else: + table.add_row("Images", str(output_image_dir)) + CONSOLE.print(Panel(table, title="[bold][green]:tada: Render Complete :tada:[/bold]", expand=False)) + + return render_images + +def find_ckpt_path(config: TrainerConfig) -> Tuple[Path, int]: + assert config.load_dir is not None + if config.load_step is None: + CONSOLE.print("Loading latest checkpoint from load_dir") + # NOTE: this is specific to the checkpoint name format + if not os.path.exists(config.load_dir): + CONSOLE.rule("Error", style="red") + CONSOLE.print(f"No checkpoint directory found at {config.load_dir}, ", justify="center") + CONSOLE.print( + "Please make sure the checkpoint exists, they should be generated periodically during training", + justify="center", + ) + sys.exit(1) + load_step = sorted(int(x[x.find("-") + 1 : x.find(".")]) for x in os.listdir(config.load_dir))[-1] + else: + load_step = config.load_step + load_path = config.load_dir / f"step-{load_step:09d}.ckpt" + assert load_path.exists(), f"Checkpoint {load_path} does not exist" + return load_path + +def eval_setup( + config_path: Path, + eval_num_rays_per_chunk: Optional[int] = None, + test_mode: Literal["test", "val", "inference"] = "test", + update_config_callback: Optional[Callable[[TrainerConfig], TrainerConfig]] = None, +) -> Tuple[TrainerConfig, Pipeline, Path, int]: + # load save config + config = yaml.load(config_path.read_text(), Loader=yaml.Loader) + assert isinstance(config, TrainerConfig) + + # config.pipeline.datamanager._target = all_methods[config.method_name].pipeline.datamanager._target + if eval_num_rays_per_chunk: + config.pipeline.model.eval_num_rays_per_chunk = eval_num_rays_per_chunk + + # load checkpoints from wherever they were saved + config.load_dir = config.get_checkpoint_dir() + checkpoint_path = find_ckpt_path(config) + + if 'camera_optimizer' in config.pipeline.model.__dict__ \ + and config.pipeline.model.camera_optimizer.mode == "SO3xR3": + config.pipeline.datamanager.dataparser.load_cam_optim_from = checkpoint_path + config.pipeline.datamanager.dataparser.eval_2hz = False + + if update_config_callback is not None: + config = update_config_callback(config) + + + # setup pipeline (which includes the DataManager) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + pipeline = config.pipeline.setup(device=device, test_mode=test_mode) + assert isinstance(pipeline, Pipeline) + pipeline.eval() + + # load checkpointed information + loaded_state = torch.load(checkpoint_path, map_location="cpu") + pipeline.load_pipeline(loaded_state["pipeline"], loaded_state["step"]) + + return config, pipeline, checkpoint_path, loaded_state["step"] + +def _get_interpolated_camera_path(cameras: Cameras, steps: int, order_poses: bool) -> Cameras: + """Generate a camera path between two cameras. Uses the camera type of the first camera + + Args: + cameras: Cameras object containing intrinsics of all cameras. + steps: The number of steps to interpolate between the two cameras. + + Returns: + A new set of cameras along a path. + """ + Ks = cameras.get_intrinsics_matrices() + poses = cameras.camera_to_worlds + poses, Ks = get_interpolated_poses_many(poses, Ks, steps_per_transition=steps+1, order_poses=order_poses) + camera_timestamps = cameras.times.squeeze() + + timestamps_new = [] + for i in range(len(camera_timestamps) - 1): + timestamps_new.append( + torch.linspace(camera_timestamps[i], camera_timestamps[i+1], steps+1, dtype=torch.float)[:-1]) + timestamps_new.append(camera_timestamps[-1][None]) + timestamps_new = torch.cat(timestamps_new, dim=0) + + mask = torch.cat([torch.ones(steps, dtype=torch.bool), torch.zeros(1, dtype=torch.bool)]) + mask = mask.repeat(len(cameras) - 1) + mask[-1] = True + + poses = poses[mask] + Ks = Ks[mask] + + cameras = Cameras( + fx=Ks[:, 0, 0], + fy=Ks[:, 1, 1], + cx=Ks[0, 0, 2], + cy=Ks[0, 1, 2], + camera_type=cameras.camera_type[0], + camera_to_worlds=poses, + times=timestamps_new, + width=cameras.width[0], + height=cameras.height[0], + ) + return cameras + +@dataclass +class CropData: + """Data for cropping an image.""" + + background_color: Float[Tensor, "3"] = torch.Tensor([0.0, 0.0, 0.0]) + """background color""" + obb: OrientedBox = field(default_factory=lambda: OrientedBox(R=torch.eye(3), T=torch.zeros(3), S=torch.ones(3) * 2)) + """Oriented box representing the crop region""" + + # properties for backwards-compatibility interface + @property + def center(self): + return self.obb.T + + @property + def scale(self): + return self.obb.S + + +def get_crop_from_json(camera_json: Dict[str, Any]) -> Optional[CropData]: + """Load crop data from a camera path JSON + + args: + camera_json: camera path data + returns: + Crop data + """ + if "crop" not in camera_json or camera_json["crop"] is None: + return None + bg_color = camera_json["crop"]["crop_bg_color"] + center = camera_json["crop"]["crop_center"] + scale = camera_json["crop"]["crop_scale"] + rot = (0.0, 0.0, 0.0) if "crop_rot" not in camera_json["crop"] else tuple(camera_json["crop"]["crop_rot"]) + assert len(center) == 3 + assert len(scale) == 3 + assert len(rot) == 3 + return CropData( + background_color=torch.Tensor([bg_color["r"] / 255.0, bg_color["g"] / 255.0, bg_color["b"] / 255.0]), + obb=OrientedBox.from_params(center, rot, scale), + ) + + +@dataclass +class BaseRender: + """Base class for rendering.""" + + load_config: Path + """Path to config YAML file.""" + output_path: Path = Path("renders/output.mp4") + """Path to output video file.""" + image_format: Literal["jpeg", "png"] = "jpeg" + """Image format""" + jpeg_quality: int = 100 + """JPEG quality""" + downscale_factor: float = 2.0 + """Scaling factor to apply to the camera image resolution.""" + eval_num_rays_per_chunk: Optional[int] = None + """Specifies number of rays per chunk during eval. If None, use the value in the config file.""" + rendered_output_names: List[str] = field(default_factory=lambda: ["rgb"]) + """Name of the renderer outputs to use. rgb, depth, etc. concatenates them along y axis""" + depth_near_plane: Optional[float] = None + """Closest depth to consider when using the colormap for depth. If None, use min value.""" + depth_far_plane: Optional[float] = None + """Furthest depth to consider when using the colormap for depth. If None, use max value.""" + colormap_options: colormaps.ColormapOptions = colormaps.ColormapOptions() + """Colormap options.""" + multi_view_camera: Tuple[Literal['CAM_F0', 'CAM_L0', 'CAM_R0', 'CAM_L1', 'CAM_R1', 'CAM_L2', 'CAM_R2', 'CAM_B0'], ...] = \ + ('CAM_F0', 'CAM_L0', 'CAM_R0', 'CAM_L1', 'CAM_R1', 'CAM_L2', 'CAM_R2', 'CAM_B0') + + +@dataclass +class RenderCameraPath(BaseRender): + """Render a camera path generated by the viewer or blender add-on.""" + + camera_path_filename: Path = Path("camera_path.json") + """Filename of the camera path to render.""" + output_format: Literal["images", "video"] = "video" + """How to save output data.""" + + def main(self) -> None: + """Main function.""" + _, pipeline, _, _ = eval_setup( + self.load_config, + eval_num_rays_per_chunk=self.eval_num_rays_per_chunk, + test_mode="inference", + ) + + install_checks.check_ffmpeg_installed() + + with open(self.camera_path_filename, "r", encoding="utf-8") as f: + camera_path = json.load(f) + seconds = camera_path["seconds"] + crop_data = get_crop_from_json(camera_path) + camera_path = get_path_from_json(camera_path) + + if ( + camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value + or camera_path.camera_type[0] == CameraType.VR180_L.value + ): + # temp folder for writing left and right view renders + temp_folder_path = self.output_path.parent / (self.output_path.stem + "_temp") + + Path(temp_folder_path).mkdir(parents=True, exist_ok=True) + left_eye_path = temp_folder_path / "render_left.mp4" + + self.output_path = left_eye_path + + if camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value: + CONSOLE.print("[bold green]:goggles: Omni-directional Stereo VR :goggles:") + else: + CONSOLE.print("[bold green]:goggles: VR180 :goggles:") + + CONSOLE.print("Rendering left eye view") + + # add mp4 suffix to video output if none is specified + if self.output_format == "video" and str(self.output_path.suffix) == "": + self.output_path = self.output_path.with_suffix(".mp4") + + _render_trajectory_video( + pipeline, + camera_path, + output_filename=self.output_path, + rendered_output_names=self.rendered_output_names, + rendered_resolution_scaling_factor=1.0 / self.downscale_factor, + crop_data=crop_data, + seconds=seconds, + output_format=self.output_format, + image_format=self.image_format, + jpeg_quality=self.jpeg_quality, + depth_near_plane=self.depth_near_plane, + depth_far_plane=self.depth_far_plane, + colormap_options=self.colormap_options + ) + + +@dataclass +class RenderInterpolated(BaseRender): + """Render a trajectory that interpolates between training or eval dataset images.""" + + pose_source: Literal["eval", "train"] = "eval" + """Pose source to render.""" + output_path: Optional[Path] = None + """Path to output video for multi-traversal""" + interpolation_steps: int = 6 + """Number of interpolation steps between eval dataset cameras.""" + order_poses: bool = False + """Whether to order camera poses by proximity.""" + frame_rate: int = 60 + """Frame rate of the output video.""" + output_format: Literal["images", "video"] = "video" + """How to save output data.""" + + def main(self) -> None: + """Main function.""" + def update_config(config: TrainerConfig) -> TrainerConfig: + config.pipeline.datamanager.dataparser.train_split_fraction = 1.0 + config.pipeline.datamanager.dataparser.cameras = self.multi_view_camera + CONSOLE.print(f'Using cams {self.multi_view_camera}.', style="bold green") + return config + + config, pipeline, _, _ = eval_setup( + self.load_config, + eval_num_rays_per_chunk=self.eval_num_rays_per_chunk, + test_mode="inference", + update_config_callback = update_config + ) + + install_checks.check_ffmpeg_installed() + + if self.pose_source == "eval": + assert pipeline.datamanager.eval_dataset is not None + cameras = pipeline.datamanager.eval_dataset.cameras + dataparser_outputs = pipeline.datamanager.eval_dataset._dataparser_outputs + else: + assert pipeline.datamanager.train_dataset is not None + cameras = pipeline.datamanager.train_dataset.cameras + dataparser_outputs = pipeline.datamanager.train_dataset._dataparser_outputs + + num_cameras = len(self.multi_view_camera) + + def render_travel(output_path, cameras, num_cameras, travel_id=None): + cam_split = [] + for i in range(num_cameras): + cam_split.append(cameras[i::num_cameras]) + + rendered = {} + for cam_id, cameras in enumerate(cam_split): + cam_name = self.multi_view_camera[cam_id] + seconds = (self.interpolation_steps * (len(cameras) - 1) + 1) / self.frame_rate + camera_path = _get_interpolated_camera_path( + cameras=cameras, + steps=self.interpolation_steps, + order_poses=self.order_poses + ) + # the metadata will be kept after slicing the camera path + camera_path.metadata = {'travel_id': travel_id} + assert self.output_format == "video" + output_filename = output_path / f"{cam_name}.mp4" + + images = _render_trajectory_video( + pipeline, + camera_path, + output_filename=Path(output_filename), + rendered_output_names=self.rendered_output_names, + rendered_resolution_scaling_factor=1.0 / self.downscale_factor, + seconds=seconds, + output_format=self.output_format, + image_format=self.image_format, + depth_near_plane=self.depth_near_plane, + depth_far_plane=self.depth_far_plane, + colormap_options=self.colormap_options + ) + rendered[cam_name] = images + + # concat and save videos + videos = [] + for cam in ('CAM_L0', 'CAM_F0', 'CAM_R0'): + video = rendered[cam] + videos.append(video) + videos = np.concatenate(videos, axis=-2) + media.write_video(output_path / "concat_front.mp4", videos, fps=self.frame_rate) + + videos = [] + for cam in ('CAM_R2', 'CAM_B0', 'CAM_L2'): + video = rendered[cam] + videos.append(video) + videos = np.concatenate(videos, axis=-2) + media.write_video(output_path / "concat_back.mp4", videos, fps=self.frame_rate) + + base_output_path = Path() + if self.output_path is not None: + base_output_path = self.output_path + else: + if hasattr(config, "base_dir"): + base_output_path = Path(f"renders/scene_videos/{os.path.basename(config.base_dir)}") + else: + base_output_path = Path(f"renders/{config.experiment_name}") + + if hasattr(dataparser_outputs, "travel_ids") and dataparser_outputs.travel_ids is not None: + travel_ids = dataparser_outputs.travel_ids + travel_id_set = list(set(travel_ids)) + cameras_travels = {k:[] for k in travel_id_set} + for idx, travel_id in enumerate(travel_ids): + cameras_travels[travel_id].append(idx) + for travel_id in travel_id_set: + output_path = base_output_path / f"travel_{travel_id}" + travel_indices = torch.tensor(cameras_travels[travel_id], dtype=torch.int64) + render_travel(output_path, cameras[travel_indices], num_cameras, travel_id) + else: + render_travel(base_output_path, cameras, num_cameras) + +@contextmanager +def _disable_datamanager_setup(cls): + """ + Disables setup_train or setup_eval for faster initialization. + """ + old_setup_train = getattr(cls, "setup_train") + old_setup_eval = getattr(cls, "setup_eval") + setattr(cls, "setup_train", lambda *args, **kwargs: None) + setattr(cls, "setup_eval", lambda *args, **kwargs: None) + yield cls + setattr(cls, "setup_train", old_setup_train) + setattr(cls, "setup_eval", old_setup_eval) + + +@dataclass +class DatasetRender(BaseRender): + """Render all images in the dataset.""" + + output_path: Path = Path("renders") + """Path to output video file.""" + data: Optional[Path] = None + """Override path to the dataset.""" + downscale_factor: Optional[float] = None + """Scaling factor to apply to the camera image resolution.""" + split: Literal["train", "val", "test", "train+test"] = "test" + """Split to render.""" + rendered_output_names: Optional[List[str]] = field(default_factory=lambda: None) + """Name of the renderer outputs to use. rgb, depth, raw-depth, gt-rgb etc. By default all outputs are rendered.""" + + def main(self): + config: TrainerConfig + + def update_config(config: TrainerConfig) -> TrainerConfig: + data_manager_config = config.pipeline.datamanager + assert isinstance(data_manager_config, (VanillaDataManagerConfig, FullImageDatamanagerConfig)) + data_manager_config.eval_num_images_to_sample_from = -1 + data_manager_config.eval_num_times_to_repeat_images = -1 + if isinstance(data_manager_config, VanillaDataManagerConfig): + data_manager_config.train_num_images_to_sample_from = -1 + data_manager_config.train_num_times_to_repeat_images = -1 + if self.data is not None: + data_manager_config.data = self.data + if self.downscale_factor is not None: + assert hasattr(data_manager_config.dataparser, "downscale_factor") + setattr(data_manager_config.dataparser, "downscale_factor", self.downscale_factor) + return config + + config, pipeline, _, _ = eval_setup( + self.load_config, + eval_num_rays_per_chunk=self.eval_num_rays_per_chunk, + test_mode="inference", + update_config_callback=update_config, + ) + data_manager_config = config.pipeline.datamanager + assert isinstance(data_manager_config, (VanillaDataManagerConfig, FullImageDatamanagerConfig)) + + for split in self.split.split("+"): + datamanager: VanillaDataManager + dataset: Dataset + if split == "train": + with _disable_datamanager_setup(data_manager_config._target): # pylint: disable=protected-access + datamanager = data_manager_config.setup(test_mode="test", device=pipeline.device) + + dataset = datamanager.train_dataset + dataparser_outputs = getattr(dataset, "_dataparser_outputs", datamanager.train_dataparser_outputs) + else: + with _disable_datamanager_setup(data_manager_config._target): # pylint: disable=protected-access + datamanager = data_manager_config.setup(test_mode=split, device=pipeline.device) + + dataset = datamanager.eval_dataset + dataparser_outputs = getattr(dataset, "_dataparser_outputs", None) + if dataparser_outputs is None: + dataparser_outputs = datamanager.dataparser.get_dataparser_outputs(split=datamanager.test_split) + dataloader = FixedIndicesEvalDataloader( + input_dataset=dataset, + device=datamanager.device, + num_workers=datamanager.world_size * 4, + ) + images_root = Path(os.path.commonpath(dataparser_outputs.image_filenames)) + with Progress( + TextColumn(f":movie_camera: Rendering split {split} :movie_camera:"), + BarColumn(), + TaskProgressColumn( + text_format="[progress.percentage]{task.completed}/{task.total:>.0f}({task.percentage:>3.1f}%)", + show_speed=True, + ), + ItersPerSecColumn(suffix="fps"), + TimeRemainingColumn(elapsed_when_finished=False, compact=False), + TimeElapsedColumn(), + ) as progress: + for camera_idx, (camera, batch) in enumerate(progress.track(dataloader, total=len(dataset))): + with torch.no_grad(): + outputs = pipeline.model.get_outputs_for_camera(camera) + + gt_batch = batch.copy() + gt_batch["rgb"] = gt_batch.pop("image") + all_outputs = ( + list(outputs.keys()) + + [f"raw-{x}" for x in outputs.keys()] + + [f"gt-{x}" for x in gt_batch.keys()] + + [f"raw-gt-{x}" for x in gt_batch.keys()] + ) + rendered_output_names = self.rendered_output_names + if rendered_output_names is None: + rendered_output_names = ["gt-rgb"] + list(outputs.keys()) + for rendered_output_name in rendered_output_names: + if rendered_output_name not in all_outputs: + CONSOLE.rule("Error", style="red") + CONSOLE.print( + f"Could not find {rendered_output_name} in the model outputs", justify="center" + ) + CONSOLE.print( + f"Please set --rendered-output-name to one of: {all_outputs}", justify="center" + ) + sys.exit(1) + + is_raw = False + is_depth = rendered_output_name.find("depth") != -1 + image_name = f"{camera_idx:05d}" + + # Try to get the original filename + image_name = dataparser_outputs.image_filenames[camera_idx].relative_to(images_root) + + output_path = self.output_path / split / rendered_output_name / image_name + output_path.parent.mkdir(exist_ok=True, parents=True) + + output_name = rendered_output_name + if output_name.startswith("raw-"): + output_name = output_name[4:] + is_raw = True + if output_name.startswith("gt-"): + output_name = output_name[3:] + output_image = gt_batch[output_name] + else: + output_image = outputs[output_name] + if is_depth: + # Divide by the dataparser scale factor + output_image.div_(dataparser_outputs.dataparser_scale) + else: + if output_name.startswith("gt-"): + output_name = output_name[3:] + output_image = gt_batch[output_name] + else: + output_image = outputs[output_name] + del output_name + + # Map to color spaces / numpy + if is_raw: + output_image = output_image.cpu().numpy() + elif is_depth: + output_image = ( + colormaps.apply_depth_colormap( + output_image, + accumulation=outputs["accumulation"], + near_plane=self.depth_near_plane, + far_plane=self.depth_far_plane, + colormap_options=self.colormap_options, + ) + .cpu() + .numpy() + ) + else: + output_image = ( + colormaps.apply_colormap( + image=output_image, + colormap_options=self.colormap_options, + ) + .cpu() + .numpy() + ) + + # Save to file + if is_raw: + with gzip.open(output_path.with_suffix(".npy.gz"), "wb") as f: + np.save(f, output_image) + elif self.image_format == "png": + media.write_image(output_path.with_suffix(".png"), output_image, fmt="png") + elif self.image_format == "jpeg": + media.write_image( + output_path.with_suffix(".jpg"), output_image, fmt="jpeg", quality=self.jpeg_quality + ) + else: + raise ValueError(f"Unknown image format {self.image_format}") + + table = Table( + title=None, + show_header=False, + box=box.MINIMAL, + title_style=style.Style(bold=True), + ) + for split in self.split.split("+"): + table.add_row(f"Outputs {split}", str(self.output_path / split)) + CONSOLE.print(Panel(table, title="[bold][green]:tada: Render on split {} Complete :tada:[/bold]", expand=False)) + + +Commands = tyro.conf.FlagConversionOff[ + Union[ + Annotated[RenderCameraPath, tyro.conf.subcommand(name="camera-path")], + Annotated[RenderInterpolated, tyro.conf.subcommand(name="interpolate")], + Annotated[DatasetRender, tyro.conf.subcommand(name="dataset")], + ] +] + + +def entrypoint(): + """Entrypoint for use with pyproject scripts.""" + tyro.extras.set_accent_color("bright_yellow") + tyro.cli(Commands).main() + + +if __name__ == "__main__": + entrypoint() + + +def get_parser_fn(): + """Get the parser function for the sphinx docs.""" + return tyro.extras.get_parser(Commands) # noqa diff --git a/src/mtgs_sensorsim/run_conda_mtgs_sensorsim.sh b/src/mtgs_sensorsim/run_conda_mtgs_sensorsim.sh new file mode 100755 index 00000000..a7e521bd --- /dev/null +++ b/src/mtgs_sensorsim/run_conda_mtgs_sensorsim.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +CONDA_ENV_PREFIX="${CONDA_ENV_PREFIX:-/home/a8001/anaconda3/envs/mtgs}" +MTGS_REPO="${MTGS_REPO:-${REPO_ROOT}/MTGS}" +ARTIFACT_DIR="${ARTIFACT_DIR:-${REPO_ROOT}/data/nre-artifacts/road_block-331220_4690660_331190_4690710}" +CONFIG_PATH="${CONFIG_PATH:-${MTGS_REPO}/experiments/main_mt/MTGS/road_block-331220_4690660_331190_4690710/config.yml}" + +export PYTHONPATH="${REPO_ROOT}/src/mtgs_sensorsim:${REPO_ROOT}/src/grpc:${MTGS_REPO}:${PYTHONPATH:-}" +export MTGS_REPO +export CUDA_HOME="${CUDA_HOME:-${CONDA_ENV_PREFIX}}" +export PATH="${CONDA_ENV_PREFIX}/bin:${CUDA_HOME}/bin:${PATH}" +export LD_LIBRARY_PATH="${CONDA_ENV_PREFIX}/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH:-}" +export MPLCONFIGDIR="${MPLCONFIGDIR:-/tmp/mplconfig}" +export TORCH_EXTENSIONS_DIR="${TORCH_EXTENSIONS_DIR:-/tmp/torch_extensions_mtgs_${USER:-user}}" +# Keep gsplat JIT from failing when CUDA devices are not visible during compile. +# Override this for a known GPU architecture if needed, e.g. TORCH_CUDA_ARCH_LIST=8.6. +export TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-8.0;8.6;8.9}" +mkdir -p "${MPLCONFIGDIR}" "${TORCH_EXTENSIONS_DIR}" + +cd "${MTGS_REPO}" + +if [[ "$#" -gt 0 ]]; then + exec "${CONDA_ENV_PREFIX}/bin/python" -m alpasim_mtgs_sensorsim "$@" +fi + +exec "${CONDA_ENV_PREFIX}/bin/python" -m alpasim_mtgs_sensorsim \ + --host "${HOST:-127.0.0.1}" \ + --port "${PORT:-50053}" \ + --config "${CONFIG_PATH}" \ + --artifact-dir "${ARTIFACT_DIR}" \ + --scene-id "${SCENE_ID:-mtgs-road_block-331220_4690660_331190_4690710}" \ + --travel-id "${TRAVEL_ID:-7}" \ + --native-height "${NATIVE_HEIGHT:-1080}" \ + --native-width "${NATIVE_WIDTH:-1920}" \ + --warmup-renders "${WARMUP_RENDERS:-1}" \ + --warmup-height "${WARMUP_HEIGHT:-180}" \ + --warmup-width "${WARMUP_WIDTH:-320}" \ + --log-level "${LOG_LEVEL:-INFO}" diff --git a/src/runtime/alpasim_runtime/config.py b/src/runtime/alpasim_runtime/config.py index e6f883c9..05768b5b 100644 --- a/src/runtime/alpasim_runtime/config.py +++ b/src/runtime/alpasim_runtime/config.py @@ -70,6 +70,18 @@ class RuntimeCameraConfig: first_frame_offset_us: int = 0 +@dataclass +class DriverBackendConfig: + """Optional runtime-side backend registry entry for multi-model orchestration.""" + + backend_id: str = MISSING + model_type: str = MISSING + priority: int = 0 + backend_type: str = "driver_model" + supports_parallel: bool = False + supports_hot_switch: bool = True + + @dataclass class PoseConfig: translation_m: tuple[float, float, float] @@ -202,6 +214,11 @@ class SimulationConfig: # Whether to send optional messages to the driver send_recording_ground_truth: bool = False + # Optional runtime-side multi-backend registry. Empty keeps legacy single-driver behavior. + driver_backends: list[DriverBackendConfig] = field(default_factory=list) + observation_cache_size: int = 32 + observation_window_summary_size: int = 4 + # Actors that appear for less than this amount of time will be dropped # before the simulation starts. Set to 0 to disable filtering. min_traffic_duration_us: int = 3_000_000 # 3 s diff --git a/src/runtime/alpasim_runtime/daemon/app.py b/src/runtime/alpasim_runtime/daemon/app.py index 8978dc54..e5edcc1c 100644 --- a/src/runtime/alpasim_runtime/daemon/app.py +++ b/src/runtime/alpasim_runtime/daemon/app.py @@ -8,7 +8,9 @@ from contextlib import suppress from alpasim_grpc.v0 import runtime_pb2_grpc +from alpasim_grpc.v1 import interactive_runtime_pb2_grpc from alpasim_runtime.daemon.engine import DaemonEngine +from alpasim_runtime.daemon.interactive_servicer import InteractiveRuntimeServicer from alpasim_runtime.daemon.servicer import RuntimeDaemonServicer import grpc @@ -79,6 +81,12 @@ async def run(self) -> None: ), server, ) + interactive_runtime_pb2_grpc.add_InteractiveRuntimeServiceServicer_to_server( + InteractiveRuntimeServicer( + manager=self._engine.interactive_session_manager, + ), + server, + ) server.add_insecure_port(self._listen_address) await server.start() diff --git a/src/runtime/alpasim_runtime/daemon/engine.py b/src/runtime/alpasim_runtime/daemon/engine.py index 7985dc6d..17e94876 100644 --- a/src/runtime/alpasim_runtime/daemon/engine.py +++ b/src/runtime/alpasim_runtime/daemon/engine.py @@ -10,9 +10,11 @@ from alpasim_grpc.v0 import logging_pb2, runtime_pb2 from alpasim_runtime.address_pool import AddressPool from alpasim_runtime.daemon.scheduler import DaemonScheduler, DaemonUnavailableError +from alpasim_runtime.interactive.session_manager import InteractiveSessionManager from alpasim_runtime.runtime_context import ( build_runtime_context, compute_num_consumers_per_worker, + create_address_pools, ) from alpasim_runtime.worker.ipc import JobResult, PendingRolloutJob from alpasim_runtime.worker.runtime import WorkerRuntime, start_worker_runtime @@ -181,6 +183,7 @@ def __init__( self._scene_id_to_artifact_path: dict[str, str] = {} self._scheduler: DaemonScheduler | None = None self._worker_runtime: WorkerRuntime | None = None + self._interactive_session_manager: InteractiveSessionManager | None = None self._started = False @property @@ -189,6 +192,12 @@ def version_ids(self) -> logging_pb2.RolloutMetadata.VersionIds: raise RuntimeError("daemon is not started") return self._version_ids + @property + def interactive_session_manager(self) -> InteractiveSessionManager: + if self._interactive_session_manager is None: + raise RuntimeError("daemon is not started") + return self._interactive_session_manager + async def startup(self) -> None: """Initialize the runtime context, start workers, and begin scheduling. @@ -231,6 +240,14 @@ async def startup(self) -> None: self._scene_id_to_artifact_path = runtime_context.scene_id_to_artifact_path self._worker_runtime = worker_runtime self._scheduler = scheduler + self._interactive_session_manager = InteractiveSessionManager( + user_config=runtime_context.config.user, + eval_config=runtime_context.eval_config, + version_ids=runtime_context.version_ids, + scene_id_to_artifact_path=runtime_context.scene_id_to_artifact_path, + pools=create_address_pools(runtime_context.config), + rollouts_dir=f"{self._log_dir}/rollouts", + ) self._started = True async def simulate( @@ -291,9 +308,13 @@ async def shutdown(self) -> None: assert self._scheduler is not None assert self._worker_runtime is not None + if self._interactive_session_manager is not None: + await self._interactive_session_manager.close_all() + await self._scheduler.shutdown(reason="daemon shutting down") await self._worker_runtime.stop() self._scheduler = None self._worker_runtime = None + self._interactive_session_manager = None self._started = False diff --git a/src/runtime/alpasim_runtime/daemon/interactive_servicer.py b/src/runtime/alpasim_runtime/daemon/interactive_servicer.py new file mode 100644 index 00000000..b5727bbb --- /dev/null +++ b/src/runtime/alpasim_runtime/daemon/interactive_servicer.py @@ -0,0 +1,415 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +from __future__ import annotations + +import asyncio +import json +import logging + +from alpasim_grpc.v1 import interactive_runtime_pb2, interactive_runtime_pb2_grpc +from alpasim_runtime.interactive.models import ( + ActorStateModel, + CandidatePlanModel, + CandidateSummaryModel, + CheckpointSummaryModel, + DecisionSummaryModel, + EgoStateModel, + FrameRefModel, + PolylinePointModel, + SensorDescriptorModel, + SessionSnapshotModel, + SessionStateModel, + SessionUpdateModel, +) +from alpasim_runtime.interactive.session_manager import InteractiveSessionManager + +import grpc + +logger = logging.getLogger(__name__) + + +def _status_to_proto(status: str) -> interactive_runtime_pb2.SessionStatus.ValueType: + mapping = { + "CREATED": interactive_runtime_pb2.SESSION_STATUS_CREATED, + "RUNNING": interactive_runtime_pb2.SESSION_STATUS_RUNNING, + "PAUSED": interactive_runtime_pb2.SESSION_STATUS_PAUSED, + "COMPLETED": interactive_runtime_pb2.SESSION_STATUS_COMPLETED, + "FAILED": interactive_runtime_pb2.SESSION_STATUS_FAILED, + } + return mapping.get(status, interactive_runtime_pb2.SESSION_STATUS_UNSPECIFIED) + + +def _frame_encoding_to_proto(encoding: str) -> interactive_runtime_pb2.FrameEncoding.ValueType: + if encoding == "PNG": + return interactive_runtime_pb2.FRAME_ENCODING_PNG + if encoding == "JPEG": + return interactive_runtime_pb2.FRAME_ENCODING_JPEG + return interactive_runtime_pb2.FRAME_ENCODING_UNSPECIFIED + + +def _sensor_to_proto(sensor: SensorDescriptorModel) -> interactive_runtime_pb2.SensorDescriptor: + return interactive_runtime_pb2.SensorDescriptor( + sensor_id=sensor.sensor_id, + logical_id=sensor.logical_id, + sensor_type=interactive_runtime_pb2.SENSOR_TYPE_CAMERA, + nominal_width=sensor.nominal_width, + nominal_height=sensor.nominal_height, + nominal_frame_interval_us=sensor.nominal_frame_interval_us, + rig_to_sensor=sensor.rig_to_sensor, + frame_encoding=_frame_encoding_to_proto(sensor.frame_encoding), + ) + + +def _frame_ref_to_proto(frame_ref: FrameRefModel) -> interactive_runtime_pb2.FrameRef: + return interactive_runtime_pb2.FrameRef( + sensor_id=frame_ref.sensor_id, + tick_id=frame_ref.tick_id, + frame_start_us=frame_ref.frame_start_us, + frame_end_us=frame_ref.frame_end_us, + frame_encoding=_frame_encoding_to_proto(frame_ref.frame_encoding), + ) + + +def _ego_to_proto(ego: EgoStateModel) -> interactive_runtime_pb2.EgoState: + return interactive_runtime_pb2.EgoState( + pose=ego.pose, + dynamics=ego.dynamics, + front_steering_angle_rad=ego.front_steering_angle_rad, + ) + + +def _actor_to_proto(actor: ActorStateModel) -> interactive_runtime_pb2.ActorState: + return interactive_runtime_pb2.ActorState(actor_id=actor.actor_id, pose=actor.pose) + + +def _polyline_point_to_proto( + point: PolylinePointModel, +) -> interactive_runtime_pb2.PolylinePoint: + return interactive_runtime_pb2.PolylinePoint(x=point.x, y=point.y) + + +def _candidate_plan_to_proto( + candidate_plan: CandidatePlanModel, +) -> interactive_runtime_pb2.CandidatePlan: + return interactive_runtime_pb2.CandidatePlan( + candidate_id=candidate_plan.candidate_id, + backend_id=candidate_plan.backend_id, + selected=candidate_plan.selected, + points=[_polyline_point_to_proto(point) for point in candidate_plan.points], + ) + + +def _snapshot_to_proto(snapshot: SessionSnapshotModel) -> interactive_runtime_pb2.SessionSnapshot: + message = interactive_runtime_pb2.SessionSnapshot( + interactive_session_id=snapshot.interactive_session_id, + tick_id=snapshot.tick_id, + sim_time_us=snapshot.sim_time_us, + ego=_ego_to_proto(snapshot.ego), + actors=[_actor_to_proto(actor) for actor in snapshot.actors], + frame_refs=[_frame_ref_to_proto(frame_ref) for frame_ref in snapshot.frame_refs], + ego_history=[_polyline_point_to_proto(point) for point in snapshot.ego_history], + selected_plan=[_polyline_point_to_proto(point) for point in snapshot.selected_plan], + candidate_plans=[ + _candidate_plan_to_proto(candidate_plan) + for candidate_plan in snapshot.candidate_plans + ], + context_diagnostics_json=json.dumps( + snapshot.context_diagnostics, + sort_keys=True, + ), + ) + if snapshot.latest_decision is not None and hasattr(message, "latest_decision"): + message.latest_decision.CopyFrom(_decision_to_proto(snapshot.latest_decision)) + return message + + +def _state_to_proto(state: SessionStateModel) -> interactive_runtime_pb2.SessionState: + message = interactive_runtime_pb2.SessionState( + interactive_session_id=state.interactive_session_id, + rollout_uuid=state.rollout_uuid, + scene_id=state.scene_id, + status=_status_to_proto(state.status), + current_tick_id=state.current_tick_id, + current_sim_time_us=state.current_sim_time_us, + error=state.error, + active_backend_ids=state.active_backend_ids, + available_backend_ids=state.available_backend_ids, + ) + if state.latest_snapshot is not None: + message.latest_snapshot.CopyFrom(_snapshot_to_proto(state.latest_snapshot)) + if state.latest_decision is not None and hasattr(message, "latest_decision"): + message.latest_decision.CopyFrom(_decision_to_proto(state.latest_decision)) + return message + + +def _candidate_to_proto( + candidate: CandidateSummaryModel, +) -> interactive_runtime_pb2.CandidateSummary: + return interactive_runtime_pb2.CandidateSummary( + candidate_id=candidate.candidate_id, + backend_id=candidate.backend_id, + status=candidate.status, + selected=candidate.selected, + error=candidate.error, + diagnostics_json=json.dumps(candidate.diagnostics, sort_keys=True), + ) + + +def _decision_to_proto( + decision: DecisionSummaryModel, +) -> interactive_runtime_pb2.DecisionSummary: + return interactive_runtime_pb2.DecisionSummary( + step_id=decision.step_id, + input_snapshot_id=decision.input_snapshot_id, + selected_candidate_id=decision.selected_candidate_id or "", + candidates=[_candidate_to_proto(candidate) for candidate in decision.candidates], + arbitration_reason=decision.arbitration_reason, + ) + + +def _checkpoint_to_proto( + checkpoint: CheckpointSummaryModel, +) -> interactive_runtime_pb2.CheckpointSummary: + return interactive_runtime_pb2.CheckpointSummary( + checkpoint_id=checkpoint.checkpoint_id, + tick_id=checkpoint.tick_id, + sim_time_us=checkpoint.sim_time_us, + status=_status_to_proto(checkpoint.status), + restore_supported=checkpoint.restore_supported, + unsupported_backend_ids=checkpoint.unsupported_backend_ids, + ) + + +def _update_to_proto( + update: SessionUpdateModel, +) -> interactive_runtime_pb2.StepSessionResponse: + return interactive_runtime_pb2.StepSessionResponse( + state=_state_to_proto(update.state), + committed_snapshots=[ + _snapshot_to_proto(snapshot) for snapshot in update.committed_snapshots + ], + ) + + +class InteractiveRuntimeServicer( + interactive_runtime_pb2_grpc.InteractiveRuntimeServiceServicer +): + def __init__(self, manager: InteractiveSessionManager): + self._manager = manager + + async def CreateSession(self, request, context): + try: + state = await self._manager.create_session( + scene_id=request.scene_id, + start_paused=request.start_paused, + max_retained_ticks=request.max_retained_ticks or 64, + ) + return interactive_runtime_pb2.CreateSessionResponse( + interactive_session_id=state.interactive_session_id, + initial_state=_state_to_proto(state), + ) + except KeyError as exc: + await context.abort(grpc.StatusCode.NOT_FOUND, str(exc)) + except RuntimeError as exc: + if "No service capacity available" in str(exc): + await context.abort(grpc.StatusCode.RESOURCE_EXHAUSTED, str(exc)) + logger.exception("interactive CreateSession failed") + await context.abort(grpc.StatusCode.INTERNAL, str(exc)) + except Exception as exc: # noqa: BLE001 + logger.exception("interactive CreateSession failed") + await context.abort(grpc.StatusCode.INTERNAL, str(exc)) + raise RuntimeError("context.abort did not terminate request") + + async def ListSessions(self, request, context): + del request + try: + sessions = await self._manager.list_sessions() + return interactive_runtime_pb2.ListSessionsResponse( + sessions=[_state_to_proto(state) for state in sessions] + ) + except Exception as exc: # noqa: BLE001 + logger.exception("interactive ListSessions failed") + await context.abort(grpc.StatusCode.INTERNAL, str(exc)) + raise RuntimeError("context.abort did not terminate request") + + async def StartSession(self, request, context): + return await self._with_state( + context, self._manager.start_session(request.interactive_session_id) + ) + + async def PauseSession(self, request, context): + return await self._with_state( + context, self._manager.pause_session(request.interactive_session_id) + ) + + async def ResumeSession(self, request, context): + return await self._with_state( + context, self._manager.resume_session(request.interactive_session_id) + ) + + async def StepSession(self, request, context): + try: + update = await self._manager.step_session( + request.interactive_session_id, + max(int(request.num_steps), 1), + ) + return _update_to_proto(update) + except KeyError as exc: + await context.abort(grpc.StatusCode.NOT_FOUND, str(exc)) + raise RuntimeError("context.abort did not terminate request") + + async def GetSessionState(self, request, context): + return await self._with_state( + context, self._manager.get_state(request.interactive_session_id) + ) + + async def SetActiveBackends(self, request, context): + try: + state = await self._manager.set_active_backends( + request.interactive_session_id, + list(request.backend_ids), + ) + return _state_to_proto(state) + except KeyError as exc: + await context.abort(grpc.StatusCode.NOT_FOUND, str(exc)) + raise RuntimeError("context.abort did not terminate request") + + async def ListCandidates(self, request, context): + try: + candidates = await self._manager.list_candidates( + request.interactive_session_id + ) + return interactive_runtime_pb2.ListCandidatesResponse( + candidates=[_candidate_to_proto(candidate) for candidate in candidates] + ) + except KeyError as exc: + await context.abort(grpc.StatusCode.NOT_FOUND, str(exc)) + raise RuntimeError("context.abort did not terminate request") + + async def RecomputeCandidate(self, request, context): + try: + state = await self._manager.recompute_candidate( + request.interactive_session_id, + request.backend_id, + ) + return _state_to_proto(state) + except KeyError as exc: + await context.abort(grpc.StatusCode.NOT_FOUND, str(exc)) + except RuntimeError as exc: + await context.abort(grpc.StatusCode.FAILED_PRECONDITION, str(exc)) + raise RuntimeError("context.abort did not terminate request") + + async def SelectCandidate(self, request, context): + try: + state = await self._manager.select_candidate( + request.interactive_session_id, + request.candidate_id, + ) + return _state_to_proto(state) + except KeyError as exc: + await context.abort(grpc.StatusCode.NOT_FOUND, str(exc)) + except RuntimeError as exc: + await context.abort(grpc.StatusCode.FAILED_PRECONDITION, str(exc)) + raise RuntimeError("context.abort did not terminate request") + + async def ListCheckpoints(self, request, context): + try: + checkpoints = await self._manager.list_checkpoints( + request.interactive_session_id + ) + return interactive_runtime_pb2.ListCheckpointsResponse( + checkpoints=[ + _checkpoint_to_proto(checkpoint) for checkpoint in checkpoints + ] + ) + except KeyError as exc: + await context.abort(grpc.StatusCode.NOT_FOUND, str(exc)) + raise RuntimeError("context.abort did not terminate request") + + async def RestoreCheckpoint(self, request, context): + try: + state = await self._manager.restore_checkpoint( + request.interactive_session_id, + request.checkpoint_id, + ) + return _state_to_proto(state) + except KeyError as exc: + await context.abort(grpc.StatusCode.NOT_FOUND, str(exc)) + except RuntimeError as exc: + await context.abort(grpc.StatusCode.FAILED_PRECONDITION, str(exc)) + raise RuntimeError("context.abort did not terminate request") + + async def ListSensors(self, request, context): + try: + sensors = await self._manager.list_sensors(request.interactive_session_id) + return interactive_runtime_pb2.ListSensorsResponse( + sensors=[_sensor_to_proto(sensor) for sensor in sensors] + ) + except KeyError as exc: + await context.abort(grpc.StatusCode.NOT_FOUND, str(exc)) + raise RuntimeError("context.abort did not terminate request") + + async def GetFrame(self, request, context): + try: + frame = await self._manager.get_frame( + request.interactive_session_id, + request.sensor_id, + int(request.tick_id), + ) + return interactive_runtime_pb2.GetFrameResponse( + frame_ref=_frame_ref_to_proto( + FrameRefModel( + sensor_id=frame.sensor_id, + tick_id=int(request.tick_id), + frame_start_us=frame.frame_start_us, + frame_end_us=frame.frame_end_us, + frame_encoding=frame.frame_encoding, + ) + ), + image_bytes=frame.image_bytes, + ) + except KeyError as exc: + await context.abort(grpc.StatusCode.NOT_FOUND, str(exc)) + raise RuntimeError("context.abort did not terminate request") + + async def StreamSession(self, request, context): + queue: asyncio.Queue | None = None + try: + state = await self._manager.get_state(request.interactive_session_id) + yield interactive_runtime_pb2.SessionEvent(state=_state_to_proto(state)) + queue = await self._manager.subscribe(request.interactive_session_id) + while True: + event = await queue.get() + if event.snapshot is not None: + if request.include_snapshots: + yield interactive_runtime_pb2.SessionEvent( + snapshot=_snapshot_to_proto(event.snapshot) + ) + continue + if event.state is not None: + yield interactive_runtime_pb2.SessionEvent( + state=_state_to_proto(event.state) + ) + except KeyError as exc: + await context.abort(grpc.StatusCode.NOT_FOUND, str(exc)) + except asyncio.CancelledError: + return + finally: + if queue is not None: + try: + await self._manager.unsubscribe( + request.interactive_session_id, queue + ) + except KeyError: + pass + return + + @staticmethod + async def _with_state(context, awaitable): + try: + state = await awaitable + return _state_to_proto(state) + except KeyError as exc: + await context.abort(grpc.StatusCode.NOT_FOUND, str(exc)) + raise RuntimeError("context.abort did not terminate request") diff --git a/src/runtime/alpasim_runtime/decision.py b/src/runtime/alpasim_runtime/decision.py new file mode 100644 index 00000000..470c8572 --- /dev/null +++ b/src/runtime/alpasim_runtime/decision.py @@ -0,0 +1,574 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +"""Decision orchestration primitives for runtime policy evaluation.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +from dataclasses import dataclass, field, replace +from enum import Enum +from typing import Any, Callable, Protocol, Sequence + +from alpasim_runtime.services.driver_service import DriverService +from alpasim_utils import geometry + + +class CandidateStatus(str, Enum): + """Lifecycle state for a candidate decision.""" + + PENDING = "PENDING" + READY = "READY" + SELECTED = "SELECTED" + REJECTED = "REJECTED" + STALE = "STALE" + NEEDS_RECOMPUTE = "NEEDS_RECOMPUTE" + FAILED = "FAILED" + + +@dataclass(frozen=True, slots=True) +class BackendMetadata: + """Declared capabilities for a driver backend.""" + + backend_id: str + backend_type: str + supports_parallel: bool = False + supports_hot_switch: bool = False + supports_restore: bool = False + priority: int = 0 + + +@dataclass(frozen=True, slots=True) +class DecisionSnapshot: + """A stable per-step input snapshot shared by candidate generation.""" + + step_id: int + input_snapshot_id: str + time_now_us: int + time_query_us: int + ego_pose_history_timestamps_us: list[int] + traffic_actor_ids: list[str] + route_waypoints_in_rig: list[list[float]] + planner_context: dict[str, Any] | None + renderer_data: bytes | None + camera_frame_timestamps_us: dict[str, int] + + +@dataclass(frozen=True, slots=True) +class CandidateDecision: + """One backend's candidate output for a given input snapshot.""" + + candidate_id: str + step_id: int + input_snapshot_id: str + backend_id: str + status: CandidateStatus + trajectory: geometry.Trajectory | None + diagnostics: dict[str, Any] = field(default_factory=dict) + generated_at_us: int = 0 + recompute_count: int = 0 + error: str | None = None + + +@dataclass(frozen=True, slots=True) +class DecisionBundle: + """Candidate set and selection state for one policy step.""" + + snapshot: DecisionSnapshot + candidates: list[CandidateDecision] + selected_candidate_id: str | None = None + arbitration_reason: str | None = None + + +def _normalize_for_hash(value: Any) -> Any: + """Recursively normalize arbitrary values into a hashable JSON shape.""" + if isinstance(value, dict): + return { + str(key): _normalize_for_hash(item) + for key, item in sorted(value.items(), key=lambda kv: str(kv[0])) + } + if isinstance(value, (list, tuple)): + return [_normalize_for_hash(item) for item in value] + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if hasattr(value, "tolist"): + try: + return _normalize_for_hash(value.tolist()) + except Exception: + pass + return repr(value) + + +def build_input_snapshot_id( + *, + step_id: int, + time_now_us: int, + time_query_us: int, + planner_context: dict[str, Any] | None, + route_waypoints_in_rig: Sequence[Sequence[float]], + traffic_actor_ids: Sequence[str], + ego_pose_history_timestamps_us: Sequence[int], + camera_frame_timestamps_us: dict[str, int], + renderer_data: bytes | None, +) -> str: + """Build a deterministic identity for a per-step policy input snapshot.""" + payload = { + "camera_frame_timestamps_us": dict(sorted(camera_frame_timestamps_us.items())), + "ego_pose_history_timestamps_us": list(ego_pose_history_timestamps_us), + "planner_context": _normalize_for_hash(planner_context), + "renderer_data_sha256": hashlib.sha256(renderer_data or b"").hexdigest(), + "route_waypoints_in_rig": [ + list(waypoint) for waypoint in route_waypoints_in_rig + ], + "step_id": step_id, + "time_now_us": time_now_us, + "time_query_us": time_query_us, + "traffic_actor_ids": list(traffic_actor_ids), + } + payload_json = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload_json.encode("utf-8")).hexdigest() + + +class DriverBackendAdapter(Protocol): + """Interface used by the runtime orchestrator to query backends.""" + + metadata: BackendMetadata + + async def infer(self, snapshot: DecisionSnapshot) -> CandidateDecision: ... + + def capture_backend_state(self) -> Any | None: ... + + async def restore_backend_state(self, state: Any) -> None: ... + + +class DriverOrchestrator(Protocol): + """Runtime-facing orchestrator interface for candidate generation.""" + + async def generate_candidates( + self, + snapshot: DecisionSnapshot, + backend_ids: Sequence[str] | None = None, + ) -> DecisionBundle: ... + + async def select_candidate(self, bundle: DecisionBundle) -> CandidateDecision: ... + + async def recompute_candidate( + self, + bundle: DecisionBundle, + backend_id: str, + ) -> DecisionBundle: ... + + def capture_backend_checkpoint(self) -> tuple[dict[str, Any], list[str]]: ... + + async def restore_backend_checkpoint(self, checkpoint: dict[str, Any]) -> None: ... + + +class DriverBackendRegistry: + """Static registry of available driver backends for one rollout session.""" + + def __init__(self, backends: Sequence[DriverBackendAdapter]): + if not backends: + raise ValueError("DriverBackendRegistry requires at least one backend") + self._backends = list(backends) + + @property + def backends(self) -> list[DriverBackendAdapter]: + return list(self._backends) + + @property + def backend_ids(self) -> list[str]: + return [backend.metadata.backend_id for backend in self._backends] + + def get_backend(self, backend_id: str) -> DriverBackendAdapter: + for backend in self._backends: + if backend.metadata.backend_id == backend_id: + return backend + raise KeyError(f"Unknown backend_id: {backend_id}") + + +def _next_recompute_index(bundle: DecisionBundle, backend_id: str) -> int: + return ( + max( + ( + candidate.recompute_count + for candidate in bundle.candidates + if candidate.backend_id == backend_id + ), + default=-1, + ) + + 1 + ) + + +def _merge_recomputed_candidate( + bundle: DecisionBundle, + candidate: CandidateDecision, +) -> DecisionBundle: + updated_candidates: list[CandidateDecision] = [] + for existing in bundle.candidates: + if ( + existing.backend_id == candidate.backend_id + and existing.candidate_id != bundle.selected_candidate_id + and existing.status in {CandidateStatus.READY, CandidateStatus.FAILED} + ): + updated_candidates.append(replace(existing, status=CandidateStatus.STALE)) + continue + updated_candidates.append(existing) + updated_candidates.append(candidate) + return DecisionBundle( + snapshot=bundle.snapshot, + candidates=updated_candidates, + selected_candidate_id=bundle.selected_candidate_id, + arbitration_reason=( + f"{bundle.arbitration_reason};recomputed:{candidate.backend_id}" + if bundle.arbitration_reason + else f"recomputed:{candidate.backend_id}" + ), + ) + + +def select_candidate_in_bundle( + bundle: DecisionBundle, + candidate_id: str, +) -> DecisionBundle: + """Return a new bundle with an explicitly selected candidate.""" + selected_found = False + updated_candidates: list[CandidateDecision] = [] + for candidate in bundle.candidates: + if candidate.candidate_id == candidate_id: + selected_found = True + updated_candidates.append(replace(candidate, status=CandidateStatus.SELECTED)) + continue + if candidate.candidate_id == bundle.selected_candidate_id: + updated_candidates.append(replace(candidate, status=CandidateStatus.REJECTED)) + continue + updated_candidates.append(candidate) + + if not selected_found: + raise KeyError(f"Unknown candidate_id: {candidate_id}") + + return DecisionBundle( + snapshot=bundle.snapshot, + candidates=updated_candidates, + selected_candidate_id=candidate_id, + arbitration_reason="manual_selection", + ) + + +class DriverServiceBackendAdapter: + """Adapter that treats the existing DriverService as a single backend.""" + + def __init__( + self, + driver: DriverService, + *, + backend_id: str = "default_driver", + backend_type: str = "grpc_driver_service", + model_type_override: str | None = None, + supports_parallel: bool = False, + supports_hot_switch: bool = True, + priority: int = 0, + observation_window_summary_getter: ( + Callable[[str, int], dict[str, Any]] | None + ) = None, + observation_window_summary_size: int = 0, + ) -> None: + self._driver = driver + self._model_type_override = model_type_override + self._observation_window_summary_getter = observation_window_summary_getter + self._observation_window_summary_size = max(0, observation_window_summary_size) + self.metadata = BackendMetadata( + backend_id=backend_id, + backend_type=backend_type, + supports_parallel=supports_parallel, + supports_hot_switch=supports_hot_switch, + priority=priority, + ) + + async def infer(self, snapshot: DecisionSnapshot) -> CandidateDecision: + candidate_id = f"{snapshot.input_snapshot_id}:{self.metadata.backend_id}:0" + planner_context = dict(snapshot.planner_context or {}) + planner_context["decision_metadata"] = { + "backend_id": self.metadata.backend_id, + "candidate_id": candidate_id, + "input_snapshot_id": snapshot.input_snapshot_id, + "step_id": snapshot.step_id, + } + if ( + self._observation_window_summary_getter is not None + and self._observation_window_summary_size > 0 + ): + planner_context["decision_metadata"]["observation_window"] = ( + self._observation_window_summary_getter( + snapshot.input_snapshot_id, + self._observation_window_summary_size, + ) + ) + if self._model_type_override is not None: + self._driver.set_next_model_for_next_drive(self._model_type_override) + self._driver.set_planner_context_for_next_drive(planner_context) + trajectory = await self._driver.drive( + time_now_us=snapshot.time_now_us, + time_query_us=snapshot.time_query_us, + renderer_data=snapshot.renderer_data, + ) + driver_debug_getter = getattr(self._driver, "consume_last_drive_debug_info", None) + driver_debug = driver_debug_getter() if callable(driver_debug_getter) else {} + return CandidateDecision( + candidate_id=candidate_id, + step_id=snapshot.step_id, + input_snapshot_id=snapshot.input_snapshot_id, + backend_id=self.metadata.backend_id, + status=CandidateStatus.READY, + trajectory=trajectory, + diagnostics={ + "backend_type": self.metadata.backend_type, + "input_snapshot_id": snapshot.input_snapshot_id, + "model_type_override": self._model_type_override, + "driver_debug": driver_debug, + "planner_context_quality": (snapshot.planner_context or {}).get("quality", {}), + "planner_context_timing": (snapshot.planner_context or {}).get("timing", {}), + }, + generated_at_us=snapshot.time_now_us, + ) + + def capture_backend_state(self) -> Any | None: + return None + + async def restore_backend_state(self, state: Any) -> None: + del state + raise RuntimeError( + f"Backend {self.metadata.backend_id} does not support restore" + ) + + +class SingleBackendDriverOrchestrator: + """Compatibility orchestrator preserving existing single-backend behavior.""" + + def __init__(self, backend: DriverBackendAdapter) -> None: + self._backend = backend + + @property + def backend_ids(self) -> list[str]: + return [self._backend.metadata.backend_id] + + async def generate_candidates( + self, + snapshot: DecisionSnapshot, + backend_ids: Sequence[str] | None = None, + ) -> DecisionBundle: + if backend_ids and self._backend.metadata.backend_id not in set(backend_ids): + raise RuntimeError( + f"Requested backends {list(backend_ids)} exclude the only available backend " + f"{self._backend.metadata.backend_id}" + ) + candidate = await self._backend.infer(snapshot) + return DecisionBundle(snapshot=snapshot, candidates=[candidate]) + + async def select_candidate(self, bundle: DecisionBundle) -> CandidateDecision: + if len(bundle.candidates) != 1: + raise ValueError("Single-backend orchestrator expects exactly one candidate") + candidate = bundle.candidates[0] + if candidate.trajectory is None: + raise RuntimeError("Single-backend candidate is missing a trajectory") + return CandidateDecision( + candidate_id=candidate.candidate_id, + step_id=candidate.step_id, + input_snapshot_id=candidate.input_snapshot_id, + backend_id=candidate.backend_id, + status=CandidateStatus.SELECTED, + trajectory=candidate.trajectory, + diagnostics=candidate.diagnostics, + generated_at_us=candidate.generated_at_us, + recompute_count=candidate.recompute_count, + error=candidate.error, + ) + + async def recompute_candidate( + self, + bundle: DecisionBundle, + backend_id: str, + ) -> DecisionBundle: + if backend_id != self._backend.metadata.backend_id: + raise KeyError(f"Unknown backend_id: {backend_id}") + recompute_count = _next_recompute_index(bundle, backend_id) + candidate = await self._backend.infer(bundle.snapshot) + candidate = replace( + candidate, + candidate_id=( + f"{bundle.snapshot.input_snapshot_id}:{self._backend.metadata.backend_id}:" + f"{recompute_count}" + ), + recompute_count=recompute_count, + ) + return _merge_recomputed_candidate(bundle, candidate) + + def capture_backend_checkpoint(self) -> tuple[dict[str, Any], list[str]]: + if not self._backend.metadata.supports_restore: + return ({}, [self._backend.metadata.backend_id]) + return ( + {self._backend.metadata.backend_id: self._backend.capture_backend_state()}, + [], + ) + + async def restore_backend_checkpoint(self, checkpoint: dict[str, Any]) -> None: + if not self._backend.metadata.supports_restore: + return + backend_id = self._backend.metadata.backend_id + if backend_id not in checkpoint: + raise RuntimeError(f"Missing backend checkpoint for {backend_id}") + await self._backend.restore_backend_state(checkpoint[backend_id]) + + +class MultiBackendDriverOrchestrator: + """Multi-backend orchestrator with default priority-based arbitration.""" + + def __init__(self, registry: DriverBackendRegistry) -> None: + self._registry = registry + + @property + def backend_ids(self) -> list[str]: + return self._registry.backend_ids + + async def generate_candidates( + self, + snapshot: DecisionSnapshot, + backend_ids: Sequence[str] | None = None, + ) -> DecisionBundle: + selected_backend_ids = set(backend_ids or self._registry.backend_ids) + candidate_backends = [ + backend + for backend in self._registry.backends + if backend.metadata.backend_id in selected_backend_ids + ] + if not candidate_backends: + raise RuntimeError( + f"No registered backends match requested backend_ids={sorted(selected_backend_ids)}" + ) + + candidates: list[CandidateDecision] = [] + parallel_backends = [ + backend + for backend in candidate_backends + if backend.metadata.supports_parallel + ] + serial_backends = [ + backend + for backend in candidate_backends + if not backend.metadata.supports_parallel + ] + + if parallel_backends: + results = await asyncio.gather( + *[backend.infer(snapshot) for backend in parallel_backends], + return_exceptions=True, + ) + for backend, result in zip(parallel_backends, results, strict=True): + if isinstance(result, Exception): + candidates.append( + CandidateDecision( + candidate_id=f"{snapshot.input_snapshot_id}:{backend.metadata.backend_id}:0", + step_id=snapshot.step_id, + input_snapshot_id=snapshot.input_snapshot_id, + backend_id=backend.metadata.backend_id, + status=CandidateStatus.FAILED, + trajectory=None, + diagnostics={"backend_type": backend.metadata.backend_type}, + generated_at_us=snapshot.time_now_us, + error=str(result), + ) + ) + continue + candidates.append(result) + + for backend in serial_backends: + try: + result = await backend.infer(snapshot) + except Exception as exc: # noqa: BLE001 + result = exc + if isinstance(result, Exception): + candidates.append( + CandidateDecision( + candidate_id=f"{snapshot.input_snapshot_id}:{backend.metadata.backend_id}:0", + step_id=snapshot.step_id, + input_snapshot_id=snapshot.input_snapshot_id, + backend_id=backend.metadata.backend_id, + status=CandidateStatus.FAILED, + trajectory=None, + diagnostics={"backend_type": backend.metadata.backend_type}, + generated_at_us=snapshot.time_now_us, + error=str(result), + ) + ) + continue + candidates.append(result) + return DecisionBundle(snapshot=snapshot, candidates=candidates) + + async def select_candidate(self, bundle: DecisionBundle) -> CandidateDecision: + ready_candidates = [ + candidate + for candidate in bundle.candidates + if candidate.status in {CandidateStatus.READY, CandidateStatus.SELECTED} + and candidate.trajectory is not None + ] + if not ready_candidates: + raise RuntimeError( + f"No usable candidates for input_snapshot_id={bundle.snapshot.input_snapshot_id}" + ) + + priority_by_backend = { + backend.metadata.backend_id: backend.metadata.priority + for backend in self._registry.backends + } + selected = min( + ready_candidates, + key=lambda candidate: priority_by_backend.get(candidate.backend_id, 1_000_000), + ) + return CandidateDecision( + candidate_id=selected.candidate_id, + step_id=selected.step_id, + input_snapshot_id=selected.input_snapshot_id, + backend_id=selected.backend_id, + status=CandidateStatus.SELECTED, + trajectory=selected.trajectory, + diagnostics=selected.diagnostics, + generated_at_us=selected.generated_at_us, + recompute_count=selected.recompute_count, + error=selected.error, + ) + + async def recompute_candidate( + self, + bundle: DecisionBundle, + backend_id: str, + ) -> DecisionBundle: + backend = self._registry.get_backend(backend_id) + recompute_count = _next_recompute_index(bundle, backend_id) + candidate = await backend.infer(bundle.snapshot) + candidate = replace( + candidate, + candidate_id=f"{bundle.snapshot.input_snapshot_id}:{backend_id}:{recompute_count}", + recompute_count=recompute_count, + ) + return _merge_recomputed_candidate(bundle, candidate) + + def capture_backend_checkpoint(self) -> tuple[dict[str, Any], list[str]]: + checkpoint: dict[str, Any] = {} + unsupported_backend_ids: list[str] = [] + for backend in self._registry.backends: + if not backend.metadata.supports_restore: + unsupported_backend_ids.append(backend.metadata.backend_id) + continue + checkpoint[backend.metadata.backend_id] = backend.capture_backend_state() + return (checkpoint, unsupported_backend_ids) + + async def restore_backend_checkpoint(self, checkpoint: dict[str, Any]) -> None: + for backend in self._registry.backends: + if not backend.metadata.supports_restore: + continue + backend_id = backend.metadata.backend_id + if backend_id not in checkpoint: + raise RuntimeError(f"Missing backend checkpoint for {backend_id}") + await backend.restore_backend_state(checkpoint[backend_id]) diff --git a/src/runtime/alpasim_runtime/event_loop.py b/src/runtime/alpasim_runtime/event_loop.py index fc7149d1..b773754c 100644 --- a/src/runtime/alpasim_runtime/event_loop.py +++ b/src/runtime/alpasim_runtime/event_loop.py @@ -9,15 +9,25 @@ from __future__ import annotations +import copy import contextlib import logging import os import time +from dataclasses import replace from dataclasses import dataclass, field -from typing import Optional +from typing import Any, Optional import numpy as np from alpasim_grpc.v0.logging_pb2 import LogEntry, RolloutMetadata +from alpasim_runtime.decision import ( + CandidateDecision, + DecisionBundle, + DecisionSnapshot, + DriverBackendRegistry, + DriverServiceBackendAdapter, + MultiBackendDriverOrchestrator, +) from alpasim_runtime.autoresume import mark_rollout_complete from alpasim_runtime.broadcaster import MessageBroadcaster from alpasim_runtime.camera_catalog import CameraCatalog @@ -35,6 +45,7 @@ from alpasim_runtime.events.state import RolloutState, ServiceBundle from alpasim_runtime.events.step import StepEvent from alpasim_runtime.events.traffic import TrafficEvent +from alpasim_runtime.observation_cache import ObservationCache from alpasim_runtime.route_generator import RouteGenerator from alpasim_runtime.services.controller_service import ControllerService from alpasim_runtime.services.driver_service import DriverService @@ -51,6 +62,7 @@ from alpasim_utils import geometry from alpasim_utils.logs import LogWriter from alpasim_utils.scenario import TrafficObjects +from alpasim_utils.types import ImageWithMetadata from eval.runtime_evaluator import RuntimeEvaluator from eval.scenario_evaluator import ScenarioEvalResult @@ -59,6 +71,133 @@ logger = logging.getLogger(__name__) +def _clone_trajectory( + trajectory: geometry.Trajectory | None, +) -> geometry.Trajectory | None: + if trajectory is None: + return None + time_range = trajectory.time_range_us + return trajectory.clip(time_range.start, time_range.stop) + + +def _clone_dynamic_trajectory( + trajectory: geometry.DynamicTrajectory | None, +) -> geometry.DynamicTrajectory | None: + if trajectory is None: + return None + return trajectory.clone() + + +def _clone_image_with_metadata(image: ImageWithMetadata) -> ImageWithMetadata: + return ImageWithMetadata( + start_timestamp_us=image.start_timestamp_us, + end_timestamp_us=image.end_timestamp_us, + image_bytes=image.image_bytes, + camera_logical_id=image.camera_logical_id, + ) + + +def _clone_decision_snapshot(snapshot: DecisionSnapshot) -> DecisionSnapshot: + return DecisionSnapshot( + step_id=snapshot.step_id, + input_snapshot_id=snapshot.input_snapshot_id, + time_now_us=snapshot.time_now_us, + time_query_us=snapshot.time_query_us, + ego_pose_history_timestamps_us=list(snapshot.ego_pose_history_timestamps_us), + traffic_actor_ids=list(snapshot.traffic_actor_ids), + route_waypoints_in_rig=[list(waypoint) for waypoint in snapshot.route_waypoints_in_rig], + planner_context=copy.deepcopy(snapshot.planner_context), + renderer_data=snapshot.renderer_data, + camera_frame_timestamps_us=dict(snapshot.camera_frame_timestamps_us), + ) + + +def _clone_candidate_decision(candidate: CandidateDecision) -> CandidateDecision: + return CandidateDecision( + candidate_id=candidate.candidate_id, + step_id=candidate.step_id, + input_snapshot_id=candidate.input_snapshot_id, + backend_id=candidate.backend_id, + status=candidate.status, + trajectory=_clone_trajectory(candidate.trajectory), + diagnostics=copy.deepcopy(candidate.diagnostics), + generated_at_us=candidate.generated_at_us, + recompute_count=candidate.recompute_count, + error=candidate.error, + ) + + +def _clone_decision_bundle(bundle: DecisionBundle | None) -> DecisionBundle | None: + if bundle is None: + return None + return DecisionBundle( + snapshot=_clone_decision_snapshot(bundle.snapshot), + candidates=[_clone_candidate_decision(candidate) for candidate in bundle.candidates], + selected_candidate_id=bundle.selected_candidate_id, + arbitration_reason=bundle.arbitration_reason, + ) + + +def _clone_rollout_state(state: RolloutState) -> RolloutState: + cloned_step_context = None + if state.step_context is not None: + if state.step_context.outstanding_tasks: + raise RuntimeError("Cannot checkpoint with outstanding step tasks") + cloned_step_context = replace( + state.step_context, + decision_bundle=_clone_decision_bundle(state.step_context.decision_bundle), + driver_trajectory=_clone_trajectory(state.step_context.driver_trajectory), + ego_true=_clone_dynamic_trajectory(state.step_context.ego_true), + ego_estimated=_clone_dynamic_trajectory(state.step_context.ego_estimated), + corrected_ego_trajectory=_clone_trajectory( + state.step_context.corrected_ego_trajectory + ), + traffic_response=copy.deepcopy(state.step_context.traffic_response), + traffic_trajectories={ + object_id: _clone_trajectory(trajectory) + for object_id, trajectory in state.step_context.traffic_trajectories.items() + }, + outstanding_tasks=[], + ) + + observation_cache = None + if state.observation_cache is not None: + observation_cache = ObservationCache(max_frames=state.observation_cache._max_frames) + observation_cache.restore(state.observation_cache.checkpoint()) + + return RolloutState( + unbound=state.unbound, + ego_trajectory=state.ego_trajectory.clone(), + ego_trajectory_estimate=state.ego_trajectory_estimate.clone(), + traffic_objs=state.traffic_objs.clip_trajectories( + state.unbound.control_timestamps_us[0], + state.unbound.control_timestamps_us[-1] + 1, + ), + last_egopose_update_us=state.last_egopose_update_us, + current_front_steering_angle_rad=state.current_front_steering_angle_rad, + last_camera_frame_us=dict(state.last_camera_frame_us), + last_decision_step_id=state.last_decision_step_id, + last_committed_decision_bundle=_clone_decision_bundle( + state.last_committed_decision_bundle + ), + available_driver_backend_ids=list(state.available_driver_backend_ids), + active_driver_backend_ids=( + list(state.active_driver_backend_ids) + if state.active_driver_backend_ids is not None + else None + ), + observation_cache=observation_cache, + data_sensorsim_to_driver=state.data_sensorsim_to_driver, + last_rendered_images={ + camera_id: _clone_image_with_metadata(image) + for camera_id, image in state.last_rendered_images.items() + }, + rendered_images_handler=state.rendered_images_handler, + step_wall_start=state.step_wall_start, + step_context=cloned_step_context, + ) + + def _build_traffic_session_trajectory(unbound: UnboundRollout) -> geometry.Trajectory: """Build the ego AABB trajectory used for traffic session initialization.""" return unbound.gt_ego_trajectory.clip( @@ -75,6 +214,30 @@ def _simulated_duration_us(unbound: UnboundRollout) -> int: return unbound.control_timestamps_us[-1] - unbound.control_timestamps_us[0] +@dataclass(frozen=True) +class EventLoopAdvanceResult: + """Result of advancing the event loop. + + `step_committed` becomes true after a `StepEvent` has finished mutating + state. `simulation_finished` becomes true once the loop reaches the end + condition, either via `SimulationEndEvent` or an empty queue. + """ + + step_committed: bool = False + simulation_finished: bool = False + committed_step_timestamp_us: int | None = None + + +@dataclass(frozen=True) +class RuntimeCheckpoint: + """Deep-copied runtime state captured at a committed step boundary.""" + + state: RolloutState + event_queue: EventQueue + backend_checkpoint: dict[str, Any] = field(default_factory=dict) + unsupported_backend_ids: list[str] = field(default_factory=list) + + @dataclass class EventBasedRollout: """Event-based simulation loop implementation. @@ -103,6 +266,14 @@ class EventBasedRollout: runtime_cameras: list[RuntimeCamera] = field(init=False, default_factory=list) _runtime_evaluator: RuntimeEvaluator = field(init=False) + _async_stack: contextlib.AsyncExitStack | None = field(init=False, default=None) + _state: RolloutState | None = field(init=False, default=None) + _event_queue: EventQueue | None = field(init=False, default=None) + _rollout_start_time: float = field(init=False, default=0.0) + _loop_start_time: float = field(init=False, default=0.0) + _initialized: bool = field(init=False, default=False) + _closed: bool = field(init=False, default=False) + _default_driver_orchestrator: Any = field(init=False, default=None) def __post_init__(self) -> None: """Initialize mutable state.""" @@ -270,11 +441,21 @@ async def _apply_physics_to_trajectory( def _create_rollout_state(self) -> RolloutState: """Create the RolloutState from the current rollout.""" + available_backend_ids = ( + list(self._default_driver_orchestrator.backend_ids) + if self._build_default_driver_orchestrator() is not None + else ["default_driver"] + ) return RolloutState( unbound=self.unbound, ego_trajectory=self.ego_trajectory, ego_trajectory_estimate=self.ego_trajectory_estimate, traffic_objs=self.traffic_objs, + available_driver_backend_ids=available_backend_ids, + active_driver_backend_ids=list(available_backend_ids), + observation_cache=ObservationCache( + max_frames=self.unbound.observation_cache_size + ), ) def _create_service_bundle(self) -> ServiceBundle: @@ -286,7 +467,55 @@ def _create_service_bundle(self) -> ServiceBundle: trafficsim=self.trafficsim, broadcaster=self.broadcaster, planner_delay_buffer=self.planner_delay_buffer, + driver_orchestrator=self._build_default_driver_orchestrator(), + ) + + def _build_default_driver_orchestrator(self): + """Build the default per-rollout driver orchestrator from runtime config.""" + cached_orchestrator = getattr(self, "_default_driver_orchestrator", None) + if cached_orchestrator is not None: + return cached_orchestrator + + backend_cfgs = getattr(self.unbound, "driver_backends", []) or [] + if not backend_cfgs: + self._default_driver_orchestrator = None + return None + + def _observation_window_summary( + input_snapshot_id: str, + window_size: int, + ) -> dict[str, Any]: + if self._state is None or self._state.observation_cache is None: + return { + "anchor_snapshot_id": input_snapshot_id, + "window_size": window_size, + "available_frames": 0, + "frames": [], + } + from alpasim_runtime.observation_cache import ObservationCacheReader + + return ObservationCacheReader( + self._state.observation_cache + ).build_window_summary(input_snapshot_id, window_size) + + adapters = [ + DriverServiceBackendAdapter( + self.driver, + backend_id=backend.backend_id, + backend_type=backend.backend_type, + model_type_override=backend.model_type, + supports_parallel=backend.supports_parallel, + supports_hot_switch=backend.supports_hot_switch, + priority=backend.priority, + observation_window_summary_getter=_observation_window_summary, + observation_window_summary_size=self.unbound.observation_window_summary_size, + ) + for backend in backend_cfgs + ] + self._default_driver_orchestrator = MultiBackendDriverOrchestrator( + DriverBackendRegistry(adapters) ) + return self._default_driver_orchestrator def _create_initial_events(self) -> EventQueue: """Create and schedule the initial set of events.""" @@ -369,33 +598,88 @@ def _create_initial_events(self) -> EventQueue: return queue - async def run(self) -> Optional[ScenarioEvalResult]: - """Run the event-based simulation loop. + def _require_initialized(self) -> tuple[RolloutState, EventQueue]: + """Return the active rollout state and queue after `initialize()`.""" + if not self._initialized or self._state is None or self._event_queue is None: + raise RuntimeError("EventBasedRollout must be initialized before stepping") + return (self._state, self._event_queue) + + @property + def current_state(self) -> RolloutState: + """Expose the active rollout state after initialization.""" + state, _ = self._require_initialized() + return state + + def capture_runtime_checkpoint(self) -> RuntimeCheckpoint: + """Capture a runtime-only checkpoint at a committed step boundary.""" + state, event_queue = self._require_initialized() + cloned_state = _clone_rollout_state(state) + cloned_events = [copy.copy(event) for event in event_queue.queue] + cloned_queue = EventQueue.init_from_sequence(cloned_events) + orchestrator = self._build_default_driver_orchestrator() + backend_checkpoint: dict[str, Any] = {} + unsupported_backend_ids: list[str] = [] + if orchestrator is not None and hasattr(orchestrator, "capture_backend_checkpoint"): + backend_checkpoint, unsupported_backend_ids = ( + orchestrator.capture_backend_checkpoint() + ) + return RuntimeCheckpoint( + state=cloned_state, + event_queue=cloned_queue, + backend_checkpoint=backend_checkpoint, + unsupported_backend_ids=unsupported_backend_ids, + ) - Returns: - ScenarioEvalResult if in-runtime evaluation is enabled, None otherwise. + async def restore_runtime_checkpoint(self, checkpoint: RuntimeCheckpoint) -> None: + """Restore a previously captured runtime checkpoint.""" + self._require_initialized() + restored_state = _clone_rollout_state(checkpoint.state) + restored_state.rendered_images_handler = self.current_state.rendered_images_handler + restored_queue = EventQueue.init_from_sequence( + [copy.copy(event) for event in checkpoint.event_queue.queue] + ) + orchestrator = self._build_default_driver_orchestrator() + if ( + orchestrator is not None + and checkpoint.backend_checkpoint + and hasattr(orchestrator, "restore_backend_checkpoint") + ): + await orchestrator.restore_backend_checkpoint(checkpoint.backend_checkpoint) + self._state = restored_state + self._event_queue = restored_queue + + async def initialize(self) -> None: + """Initialize services and create the event loop state. + + This splits setup from execution so callers can either run the rollout + to completion or stop after a committed `StepEvent`. """ - async with contextlib.AsyncExitStack() as async_stack: - rollout_start_time = time.perf_counter() + if self._initialized: + return + if self._closed: + raise RuntimeError("EventBasedRollout cannot be re-initialized after close") - # Enter broadcaster context - await async_stack.enter_async_context(self.broadcaster) + stack = contextlib.AsyncExitStack() + await stack.__aenter__() + self._async_stack = stack + self._rollout_start_time = time.perf_counter() + + try: + await stack.enter_async_context(self.broadcaster) await self._log_metadata( session_metadata=self.unbound.get_log_metadata(), version_ids=self.unbound.version_ids, ) - # Initialize service sessions for service in [self.sensorsim, self.physics, self.controller]: - await async_stack.enter_async_context( + await stack.enter_async_context( service.rollout_session( uuid=str(self.unbound.rollout_uuid), broadcaster=self.broadcaster, ) ) - # Get available cameras and merge sensorsim_cameras = await self.sensorsim.get_available_cameras( self.unbound.scene_id ) @@ -403,7 +687,6 @@ async def run(self) -> Optional[ScenarioEvalResult]: self.unbound.scene_id, sensorsim_cameras ) - # Build runtime cameras self.runtime_cameras = [] rig_start_us = self.unbound.gt_ego_trajectory.time_range_us.start for camera_cfg in self.unbound.camera_configs: @@ -416,7 +699,6 @@ async def run(self) -> Optional[ScenarioEvalResult]: ) ) - # Send cameras to driver available_camera_protos = [ self.camera_catalog.get_camera_definition( self.unbound.scene_id, camera_cfg.logical_id @@ -424,7 +706,7 @@ async def run(self) -> Optional[ScenarioEvalResult]: for camera_cfg in self.unbound.camera_configs ] - await async_stack.enter_async_context( + await stack.enter_async_context( self.driver.rollout_session( uuid=str(self.unbound.rollout_uuid), broadcaster=self.broadcaster, @@ -435,10 +717,9 @@ async def run(self) -> Optional[ScenarioEvalResult]: ) ) - # Create traffic session gt_ego_aabb_trajectory = _build_traffic_session_trajectory(self.unbound) - await async_stack.enter_async_context( + await stack.enter_async_context( self.trafficsim.rollout_session( uuid=str(self.unbound.rollout_uuid), broadcaster=self.broadcaster, @@ -452,7 +733,6 @@ async def run(self) -> Optional[ScenarioEvalResult]: ) ) - # Apply physics to initial trajectory (corrects poses, keeps dynamics) if self.unbound.physics_update_mode != PhysicsUpdateMode.NONE: corrected_traj = await self._apply_physics_to_trajectory( self.ego_trajectory.trajectory() @@ -470,70 +750,144 @@ async def run(self) -> Optional[ScenarioEvalResult]: self.unbound.n_sim_steps, ) - # Warmup sensorsim if self.runtime_cameras and not self.sensorsim.skip: await self._warmup_sensorsim() - # Start timing the main loop - loop_start_time = time.perf_counter() + self._state = self._create_rollout_state() + self._event_queue = self._create_initial_events() + self._loop_start_time = time.perf_counter() + self._initialized = True logger.info("Event-based simulation loop timer started") + except BaseException: + await stack.aclose() + self._async_stack = None + self._state = None + self._event_queue = None + self._rollout_start_time = 0.0 + self._loop_start_time = 0.0 + raise + + async def _advance_one_event(self) -> EventLoopAdvanceResult: + """Execute the next queued event and summarize the outcome.""" + state, event_queue = self._require_initialized() + + if not event_queue: + return EventLoopAdvanceResult(simulation_finished=True) + + event = event_queue.pop() + event_timestamp_us = event.timestamp_us + logger.info(f"sim_time {event_timestamp_us:_}us: {event.description()}") + + try: + await event.handle(state, event_queue) + except EndSimulationException: + logger.info("Simulation ended via SimulationEndEvent") + return EventLoopAdvanceResult(simulation_finished=True) + except Exception: + logger.exception( + "Error during event handling. Pending events in queue (%d):", + len(event_queue), + ) + for event_desc in event_queue.pending_events_summary(): + logger.error(" - %s", event_desc) + raise + + step_committed = isinstance(event, StepEvent) + return EventLoopAdvanceResult( + step_committed=step_committed, + simulation_finished=not bool(event_queue), + committed_step_timestamp_us=( + event_timestamp_us if step_committed else None + ), + ) - # Create state and initial events - state = self._create_rollout_state() - event_queue = self._create_initial_events() - - # Main event loop - try: - while event_queue: - event = event_queue.pop() - logger.info( - f"sim_time {event.timestamp_us:_}us: {event.description()}" - ) - await event.handle(state, event_queue) - except EndSimulationException: - logger.info("Simulation ended via SimulationEndEvent") - except Exception: - logger.exception( - "Error during event handling. Pending events in queue (%d):", - len(event_queue), + async def run_until_step_commit(self) -> EventLoopAdvanceResult: + """Advance the loop until one `StepEvent` has committed state.""" + while True: + result = await self._advance_one_event() + if result.step_committed or result.simulation_finished: + return result + + async def run_until_complete(self) -> None: + """Run the event loop until completion.""" + while True: + result = await self._advance_one_event() + if result.simulation_finished: + return + + async def aclose( + self, + *, + mark_complete: bool = False, + run_evaluation: bool = False, + ) -> Optional[ScenarioEvalResult]: + """Drain outstanding work, optionally evaluate, and close sessions.""" + if self._closed: + return None + + eval_result: Optional[ScenarioEvalResult] = None + try: + if self._state is not None and self._state.step_context is not None: + await self._state.step_context.drain_outstanding_tasks() + + loop_duration = 0.0 + if self._loop_start_time > 0.0: + loop_duration = time.perf_counter() - self._loop_start_time + logger.info("Event-based simulation loop timer stopped") + + rollout_duration = 0.0 + if self._rollout_start_time > 0.0: + rollout_duration = time.perf_counter() - self._rollout_start_time + ctx = try_get_context() + if ctx is not None: + ctx.rollout_duration.observe(rollout_duration) + + if run_evaluation: + eval_result = self._runtime_evaluator.run_evaluation() + + if mark_complete: + mark_rollout_complete( + self.unbound.save_path_root, self.unbound.rollout_uuid ) - for event_desc in event_queue.pending_events_summary(): - logger.error(" - %s", event_desc) - raise - if state.step_context is not None: - await state.step_context.drain_outstanding_tasks() - - # Record timing - loop_duration = time.perf_counter() - loop_start_time - logger.info("Event-based simulation loop timer stopped") - - rollout_duration = time.perf_counter() - rollout_start_time - ctx = try_get_context() - if ctx is not None: - ctx.rollout_duration.observe(rollout_duration) - - eval_result = self._runtime_evaluator.run_evaluation() - - mark_rollout_complete( - self.unbound.save_path_root, self.unbound.rollout_uuid - ) + simulated_duration_us = _simulated_duration_us(self.unbound) + simulated_duration_s = simulated_duration_us / 1e6 + realtime_ratio = ( + simulated_duration_s / loop_duration if loop_duration > 0 else 0.0 + ) - # Calculate realtime ratio - simulated_duration_us = _simulated_duration_us(self.unbound) - simulated_duration_s = simulated_duration_us / 1e6 - realtime_ratio = simulated_duration_s / loop_duration + logger.info( + "Session COMPLETED: uuid=%s scene=%s " + "simulated %.2f sim seconds in %.2f wall clock seconds for %.2fx real time " + "(total rollout %.2fs incl. setup/warmup)", + self.unbound.rollout_uuid, + self.unbound.scene_id, + simulated_duration_s, + loop_duration, + realtime_ratio, + rollout_duration, + ) + finally: + if self._async_stack is not None: + await self._async_stack.aclose() + self._async_stack = None + self._state = None + self._event_queue = None + self._initialized = False + self._closed = True + self._rollout_start_time = 0.0 + self._loop_start_time = 0.0 - logger.info( - "Session COMPLETED: uuid=%s scene=%s " - "simulated %.2f sim seconds in %.2f wall clock seconds for %.2fx real time " - "(total rollout %.2fs incl. setup/warmup)", - self.unbound.rollout_uuid, - self.unbound.scene_id, - simulated_duration_s, - loop_duration, - realtime_ratio, - rollout_duration, - ) + async def run(self) -> Optional[ScenarioEvalResult]: + """Run the event-based simulation loop. - return eval_result + Returns: + ScenarioEvalResult if in-runtime evaluation is enabled, None otherwise. + """ + await self.initialize() + try: + await self.run_until_complete() + except BaseException: + await self.aclose() + raise + return await self.aclose(mark_complete=True, run_evaluation=True) diff --git a/src/runtime/alpasim_runtime/events/camera.py b/src/runtime/alpasim_runtime/events/camera.py index d6e654aa..b9ce543e 100644 --- a/src/runtime/alpasim_runtime/events/camera.py +++ b/src/runtime/alpasim_runtime/events/camera.py @@ -25,6 +25,14 @@ logger = logging.getLogger(__name__) +def _remember_rendered_images( + state: RolloutState, + images: list, +) -> None: + for image in images: + state.last_rendered_images[image.camera_logical_id] = image + + class GroupedRenderEvent(RecurringEvent): """Render all cameras whose shutters closed within a control step window. @@ -119,6 +127,11 @@ async def _render_aggregated( ego_mask_rig_config_id=state.unbound.ego_mask_rig_config_id, ) + if state.rendered_images_handler is not None: + await state.rendered_images_handler(images_with_metadata) + + _remember_rendered_images(state, images_with_metadata) + for image in images_with_metadata: state.step_context.track_task(self.driver.submit_image(image)) @@ -149,5 +162,10 @@ async def _render_parallel( ] ) + if state.rendered_images_handler is not None: + await state.rendered_images_handler(images) + + _remember_rendered_images(state, images) + for image in images: state.step_context.track_task(self.driver.submit_image(image)) diff --git a/src/runtime/alpasim_runtime/events/controller.py b/src/runtime/alpasim_runtime/events/controller.py index 51ef7017..ad845fda 100644 --- a/src/runtime/alpasim_runtime/events/controller.py +++ b/src/runtime/alpasim_runtime/events/controller.py @@ -123,6 +123,10 @@ async def _run_controller( pose_reporting_interval_us=state.unbound.pose_reporting_interval_us or state.unbound.control_timestep_us, ) + if propagated_states: + state.current_front_steering_angle_rad = float( + propagated_states[-1].front_steering_angle_rad + ) # Convert list[PropagatedPosesAtTime] → two DynamicTrajectory instances. timestamps = np.array( diff --git a/src/runtime/alpasim_runtime/events/policy.py b/src/runtime/alpasim_runtime/events/policy.py index 3d2c9c06..4b3ebfed 100644 --- a/src/runtime/alpasim_runtime/events/policy.py +++ b/src/runtime/alpasim_runtime/events/policy.py @@ -11,15 +11,363 @@ from __future__ import annotations import logging -from typing import Optional +from dataclasses import replace +from typing import Any, Optional import numpy as np +from alpasim_runtime.decision import ( + DecisionBundle, + DecisionSnapshot, + DriverServiceBackendAdapter, + SingleBackendDriverOrchestrator, + build_input_snapshot_id, +) from alpasim_runtime.events.base import EventPriority, EventQueue, RecurringEvent from alpasim_runtime.events.state import RolloutState, ServiceBundle +from alpasim_runtime.observation_cache import ObservationFrame from alpasim_runtime.route_generator import RouteGenerator from alpasim_utils import geometry logger = logging.getLogger(__name__) +try: + from trajdata.maps.vec_map_elements import MapElementType +except Exception: # pragma: no cover - optional dependency boundary + MapElementType = None + + +def _pose_to_position_and_yaw(pose: geometry.Pose) -> tuple[list[float], float]: + """Extract position xyz and yaw from a Pose.""" + position, quat_wxyz = pose.to_proto() + w, x, y, z = quat_wxyz + yaw = float( + np.arctan2( + 2.0 * (w * z + x * y), + 1.0 - 2.0 * (y * y + z * z), + ) + ) + return [float(position[0]), float(position[1]), float(position[2])], yaw + + +def _local_to_rig_xy( + point_xy: np.ndarray, + ego_xy: np.ndarray, + ego_yaw: float, +) -> np.ndarray: + """Transform a 2D local-frame point into the current rig frame.""" + dxdy = np.asarray(point_xy[:2], dtype=np.float64) - ego_xy + cos_yaw = float(np.cos(ego_yaw)) + sin_yaw = float(np.sin(ego_yaw)) + return np.array( + [ + cos_yaw * dxdy[0] + sin_yaw * dxdy[1], + -sin_yaw * dxdy[0] + cos_yaw * dxdy[1], + ], + dtype=np.float64, + ) + + +def _polyline_points_xy(obj: Any) -> np.ndarray | None: + """Extract [N,2] points from map polyline/polygon-like objects.""" + points = getattr(obj, "points", None) + if points is None: + return None + points_np = np.asarray(points) + if points_np.ndim != 2 or points_np.shape[0] == 0 or points_np.shape[1] < 2: + return None + return points_np[:, :2].astype(np.float64) + + +def _build_planner_context( + state: RolloutState, + timestamp_us: int, + route_in_rig: Optional[geometry.Polyline], + max_actors: int = 64, +) -> dict[str, Any]: + """Build a compact per-frame planning context for the driver.""" + freshness_thresholds_ms = { + "ego": 150.0, + "actor": 300.0, + "route": 1000.0, + "map": 1000.0, + "camera": 300.0, + } + ego_pose = state.ego_trajectory_estimate.last_pose + ego_position, ego_yaw = _pose_to_position_and_yaw(ego_pose) + ego_xy = np.array(ego_position[:2], dtype=np.float64) + ego_pose_latest_us = int(state.ego_trajectory_estimate.timestamps_us[-1]) + ego_age_ms = max(0.0, (timestamp_us - ego_pose_latest_us) / 1000.0) + + actor_rows: list[tuple[float, dict[str, Any]]] = [] + sample_ts = np.array([timestamp_us], dtype=np.uint64) + for actor_id, actor in state.traffic_objs.items(): + if actor.is_static: + continue + if timestamp_us not in actor.trajectory.time_range_us: + continue + actor_pose = actor.trajectory.interpolate_poses_list(sample_ts)[0] + actor_pos, actor_yaw = _pose_to_position_and_yaw(actor_pose) + actor_xy = np.array(actor_pos[:2], dtype=np.float64) + actor_rig_xy = _local_to_rig_xy(actor_xy, ego_xy, ego_yaw) + distance = float(np.linalg.norm(actor_xy - ego_xy)) + actor_rows.append( + ( + distance, + { + "id": actor_id, + "label": actor.label_class, + "position_in_local": actor_pos, + "position_in_rig": [float(actor_rig_xy[0]), float(actor_rig_xy[1])], + "yaw": actor_yaw, + "distance": distance, + }, + ) + ) + + actor_rows.sort(key=lambda item: item[0]) + actors = [row for _, row in actor_rows[:max_actors]] + + route_waypoints: list[list[float]] = [] + if route_in_rig is not None: + waypoints_np = np.asarray(route_in_rig.waypoints) + if waypoints_np.size > 0: + route_waypoints = waypoints_np[:, :3].astype(float).tolist() + + vector_map = state.unbound.vector_map + map_summary: dict[str, Any] = {"available": vector_map is not None} + nearby_lanes: list[dict[str, Any]] = [] + traffic_rules: dict[str, Any] = { + "traffic_sign_count": 0, + "wait_lines_in_rig": [], + "crosswalks_in_rig": [], + } + if vector_map is not None: + map_summary["map_id"] = getattr(vector_map, "map_id", None) + lanes = getattr(vector_map, "lanes", None) + road_edges = getattr(vector_map, "road_edges", None) + map_summary["lane_count"] = len(lanes) if lanes is not None else None + map_summary["road_edge_count"] = ( + len(road_edges) if road_edges is not None else None + ) + + if lanes is not None: + lane_candidates: list[tuple[float, dict[str, Any]]] = [] + for lane in lanes: + lane_id = getattr(lane, "id", None) + center = getattr(lane, "center", None) + center_xy = _polyline_points_xy(center) if center is not None else None + if center_xy is None: + continue + + lane_center_xy = center_xy.mean(axis=0) + lane_dist = float(np.linalg.norm(lane_center_xy - ego_xy)) + center_in_rig = np.array( + [_local_to_rig_xy(p, ego_xy, ego_yaw) for p in center_xy], + dtype=np.float64, + ) + lane_candidates.append( + ( + lane_dist, + { + "id": lane_id, + "centerline_in_rig": center_in_rig.tolist(), + }, + ) + ) + + lane_candidates.sort(key=lambda item: item[0]) + nearby_lanes = [lane_info for _, lane_info in lane_candidates[:20]] + + if MapElementType is not None: + elements = getattr(vector_map, "elements", {}) or {} + traffic_signs = elements.get(MapElementType.TRAFFIC_SIGN, {}) + wait_lines = elements.get(MapElementType.WAIT_LINE, {}) + crosswalks = elements.get(MapElementType.PED_CROSSWALK, {}) + traffic_rules["traffic_sign_count"] = len(traffic_signs) + + for wait_line in wait_lines.values(): + wl_type = str(getattr(wait_line, "wait_line_type", "Stop")) + polyline = getattr(wait_line, "polyline", None) + points_xy = _polyline_points_xy(polyline) if polyline is not None else None + if points_xy is None: + continue + points_in_rig = np.array( + [_local_to_rig_xy(p, ego_xy, ego_yaw) for p in points_xy], + dtype=np.float64, + ) + traffic_rules["wait_lines_in_rig"].append( + { + "type": wl_type, + "points": points_in_rig.tolist(), + } + ) + + for crosswalk in crosswalks.values(): + polygon = getattr(crosswalk, "polygon", None) + points_xy = _polyline_points_xy(polygon) if polygon is not None else None + if points_xy is None: + continue + points_in_rig = np.array( + [_local_to_rig_xy(p, ego_xy, ego_yaw) for p in points_xy], + dtype=np.float64, + ) + traffic_rules["crosswalks_in_rig"].append(points_in_rig.tolist()) + + route_timestamp_us = int(timestamp_us) if route_in_rig is not None else 0 + route_age_ms = 0.0 if route_in_rig is not None else None + actor_snapshot_us = int(timestamp_us) + actor_age_ms = 0.0 + map_context_timestamp_us = int(timestamp_us) if vector_map is not None else 0 + map_age_ms = 0.0 if vector_map is not None else None + camera_latest_us = max(state.last_camera_frame_us.values(), default=0) + camera_age_ms = ( + max(0.0, (timestamp_us - int(camera_latest_us)) / 1000.0) + if camera_latest_us + else None + ) + + quality = { + "route_available": bool(route_waypoints), + "nearby_lane_count": len(nearby_lanes), + "actor_count": len(actors), + "wait_line_count": len(traffic_rules["wait_lines_in_rig"]), + "crosswalk_count": len(traffic_rules["crosswalks_in_rig"]), + } + stale_flags = { + "ego": ego_age_ms > freshness_thresholds_ms["ego"], + "actor": actor_age_ms > freshness_thresholds_ms["actor"], + "route": route_age_ms is None or route_age_ms > freshness_thresholds_ms["route"], + "map": map_age_ms is None or map_age_ms > freshness_thresholds_ms["map"], + "camera": camera_age_ms is None or camera_age_ms > freshness_thresholds_ms["camera"], + } + + return { + "timestamp_us": int(timestamp_us), + "ego": { + "position": ego_position, + "yaw": ego_yaw, + }, + "route_waypoints_in_rig": route_waypoints, + "actors": actors, + "nearby_lanes": nearby_lanes, + "traffic_rules": traffic_rules, + "map_summary": map_summary, + "timing": { + "policy_tick_us": int(timestamp_us), + "ego_pose_latest_us": ego_pose_latest_us, + "route_timestamp_us": route_timestamp_us, + "actor_snapshot_us": actor_snapshot_us, + "map_context_timestamp_us": map_context_timestamp_us, + "camera_latest_us": int(camera_latest_us), + "ego_age_ms": ego_age_ms, + "route_age_ms": route_age_ms, + "actor_age_ms": actor_age_ms, + "map_age_ms": map_age_ms, + "camera_age_ms": camera_age_ms, + "freshness_thresholds_ms": freshness_thresholds_ms, + }, + "quality": { + **quality, + "stale_flags": stale_flags, + }, + } + + +def _compute_policy_step_id(state: RolloutState, timestamp_us: int, interval_us: int) -> int: + """Compute a deterministic per-step id for policy evaluation.""" + scene_start_us = int(state.unbound.control_timestamps_us[0]) + if interval_us <= 0: + raise ValueError(f"policy interval must be positive, got {interval_us}") + return int((timestamp_us - scene_start_us) // interval_us) + + +def _build_decision_snapshot( + state: RolloutState, + *, + step_id: int, + time_now_us: int, + time_query_us: int, + planner_context: dict[str, Any] | None, + route_in_rig: Optional[geometry.Polyline], + renderer_data: bytes | None, +) -> DecisionSnapshot: + """Build a stable snapshot identity for the current policy input.""" + route_waypoints_in_rig: list[list[float]] = [] + if route_in_rig is not None: + route_waypoints_np = np.asarray(route_in_rig.waypoints) + if route_waypoints_np.size > 0: + route_waypoints_in_rig = route_waypoints_np[:, :3].astype(float).tolist() + + camera_frame_timestamps_us = dict(sorted(state.last_camera_frame_us.items())) + ego_pose_history_timestamps_us = [ + int(timestamp) for timestamp in state.ego_trajectory_estimate.timestamps_us.tolist() + ] + traffic_actor_ids = sorted(state.traffic_objs.keys()) + input_snapshot_id = build_input_snapshot_id( + step_id=step_id, + time_now_us=time_now_us, + time_query_us=time_query_us, + planner_context=planner_context, + route_waypoints_in_rig=route_waypoints_in_rig, + traffic_actor_ids=traffic_actor_ids, + ego_pose_history_timestamps_us=ego_pose_history_timestamps_us, + camera_frame_timestamps_us=camera_frame_timestamps_us, + renderer_data=renderer_data, + ) + return DecisionSnapshot( + step_id=step_id, + input_snapshot_id=input_snapshot_id, + time_now_us=time_now_us, + time_query_us=time_query_us, + ego_pose_history_timestamps_us=ego_pose_history_timestamps_us, + traffic_actor_ids=traffic_actor_ids, + route_waypoints_in_rig=route_waypoints_in_rig, + planner_context=planner_context, + renderer_data=renderer_data, + camera_frame_timestamps_us=camera_frame_timestamps_us, + ) + + +def _append_observation_frame( + state: RolloutState, + *, + decision_snapshot: DecisionSnapshot, + planner_context: dict[str, Any] | None, + route_in_rig: geometry.Polyline | None, + renderer_data: bytes | None, +) -> None: + if state.observation_cache is None: + return + state.observation_cache.append( + ObservationFrame( + step_id=decision_snapshot.step_id, + input_snapshot_id=decision_snapshot.input_snapshot_id, + time_now_us=decision_snapshot.time_now_us, + time_query_us=decision_snapshot.time_query_us, + camera_frame_timestamps_us=dict(decision_snapshot.camera_frame_timestamps_us), + rendered_images=dict(state.last_rendered_images), + renderer_data=renderer_data, + ego_trajectory=state.ego_trajectory.clone(), + ego_trajectory_estimate=state.ego_trajectory_estimate.clone(), + traffic_objs=state.traffic_objs.clip_trajectories( + int(state.unbound.control_timestamps_us[0]), + int(state.unbound.control_timestamps_us[-1]) + 1, + ), + ego_pose_history_timestamps_us=list( + decision_snapshot.ego_pose_history_timestamps_us + ), + route_waypoints_in_rig=( + [] + if route_in_rig is None + else [list(point) for point in route_in_rig.points.tolist()] + ), + planner_context=planner_context, + active_backend_ids=( + list(state.active_driver_backend_ids) + if state.active_driver_backend_ids is not None + else list(state.available_driver_backend_ids) + ), + ) + ) class PolicyEvent(RecurringEvent): @@ -93,13 +441,14 @@ async def run(self, state: RolloutState, queue: EventQueue) -> None: svc.driver.submit_trajectory(ego_trajectory, dynamic_states_in_rig) ) + route_for_policy: Optional[geometry.Polyline] = None if self.route_generator is not None: pose_local_to_rig = state.ego_trajectory.last_pose route = self.route_generator.generate_route( step_start_us, pose_local_to_rig ) - route = RouteGenerator.prepare_for_policy(route) - ctx.track_task(svc.driver.submit_route(step_start_us, route)) + route_for_policy = RouteGenerator.prepare_for_policy(route) + ctx.track_task(svc.driver.submit_route(step_start_us, route_for_policy)) if self.send_recording_ground_truth: gt_traj = state.unbound.gt_ego_trajectory @@ -114,19 +463,68 @@ async def run(self, state: RolloutState, queue: EventQueue) -> None: await ctx.drain_outstanding_tasks() state.last_egopose_update_us = step_start_us - # --- Driver query --- - drive_trajectory_noisy = await svc.driver.drive( + planner_context = _build_planner_context( + state=state, + timestamp_us=step_start_us, + route_in_rig=route_for_policy, + ) + step_id = _compute_policy_step_id( + state, + timestamp_us=step_start_us, + interval_us=self.interval_us, + ) + decision_snapshot = _build_decision_snapshot( + state, + step_id=step_id, time_now_us=step_start_us, time_query_us=target_time_us, + planner_context=planner_context, + route_in_rig=route_for_policy, renderer_data=state.data_sensorsim_to_driver, ) + _append_observation_frame( + state, + decision_snapshot=decision_snapshot, + planner_context=planner_context, + route_in_rig=route_for_policy, + renderer_data=state.data_sensorsim_to_driver, + ) + orchestrator = svc.driver_orchestrator or SingleBackendDriverOrchestrator( + DriverServiceBackendAdapter(svc.driver) + ) + decision_bundle = await orchestrator.generate_candidates( + decision_snapshot, + backend_ids=state.active_driver_backend_ids, + ) + selected_candidate = await orchestrator.select_candidate(decision_bundle) state.data_sensorsim_to_driver = None # Consumed - # --- Transform from noisy to true local frame --- - drive_trajectory = transform_trajectory_from_noisy_to_true_local_frame( - state, drive_trajectory_noisy + transformed_candidates = [] + for candidate in decision_bundle.candidates: + transformed_trajectory = candidate.trajectory + if transformed_trajectory is not None: + transformed_trajectory = transform_trajectory_from_noisy_to_true_local_frame( + state, + transformed_trajectory, + ) + transformed_candidates.append( + replace(candidate, trajectory=transformed_trajectory) + ) + + transformed_bundle = DecisionBundle( + snapshot=decision_bundle.snapshot, + candidates=transformed_candidates, + selected_candidate_id=selected_candidate.candidate_id, + arbitration_reason=decision_bundle.arbitration_reason or "single_backend_default", + ) + selected_candidate = next( + candidate + for candidate in transformed_bundle.candidates + if candidate.candidate_id == selected_candidate.candidate_id ) - state.step_context.driver_trajectory = drive_trajectory + state.last_decision_step_id = step_id + state.step_context.decision_bundle = transformed_bundle + state.step_context.driver_trajectory = selected_candidate.trajectory # --------------------------------------------------------------------------- diff --git a/src/runtime/alpasim_runtime/events/state.py b/src/runtime/alpasim_runtime/events/state.py index f160cb6f..a2eb6422 100644 --- a/src/runtime/alpasim_runtime/events/state.py +++ b/src/runtime/alpasim_runtime/events/state.py @@ -11,11 +11,13 @@ import asyncio import logging from dataclasses import dataclass, field -from typing import Any, Coroutine, Optional +from typing import Any, Awaitable, Callable, Coroutine, Optional from alpasim_grpc.v0.traffic_pb2 import TrafficReturn from alpasim_runtime.broadcaster import MessageBroadcaster +from alpasim_runtime.decision import DecisionBundle, DriverOrchestrator from alpasim_runtime.delay_buffer import DelayBuffer +from alpasim_runtime.observation_cache import ObservationCache from alpasim_runtime.services.controller_service import ControllerService from alpasim_runtime.services.driver_service import DriverService from alpasim_runtime.services.physics_service import PhysicsService @@ -23,6 +25,7 @@ from alpasim_runtime.unbound_rollout import UnboundRollout from alpasim_utils import geometry from alpasim_utils.scenario import TrafficObjects +from alpasim_utils.types import ImageWithMetadata logger = logging.getLogger(__name__) @@ -41,6 +44,7 @@ class ServiceBundle: trafficsim: TrafficService broadcaster: MessageBroadcaster planner_delay_buffer: DelayBuffer + driver_orchestrator: DriverOrchestrator | None = None @dataclass(slots=True) @@ -60,6 +64,8 @@ class StepContext: force_gt: bool = False # PolicyEvent → ControllerEvent + # Candidate set and selected output for the current policy step. + decision_bundle: DecisionBundle | None = None # Driver output transformed to the true local frame. driver_trajectory: Optional[geometry.Trajectory] = None @@ -116,10 +122,20 @@ class RolloutState: # === Assertion tracking (for assert_zero_decision_delay) === last_egopose_update_us: int = 0 + current_front_steering_angle_rad: float = 0.0 last_camera_frame_us: dict[str, int] = field(default_factory=dict) + last_decision_step_id: int = 0 + last_committed_decision_bundle: DecisionBundle | None = None + available_driver_backend_ids: list[str] = field(default_factory=list) + active_driver_backend_ids: list[str] | None = None + observation_cache: ObservationCache | None = None # === Inter-event data === data_sensorsim_to_driver: Optional[bytes] = None + last_rendered_images: dict[str, ImageWithMetadata] = field(default_factory=dict) + rendered_images_handler: ( + Callable[[list[ImageWithMetadata]], Awaitable[None]] | None + ) = None # === Step timing (for step_duration telemetry) === step_wall_start: float = 0.0 diff --git a/src/runtime/alpasim_runtime/events/step.py b/src/runtime/alpasim_runtime/events/step.py index 19101e24..6ca5be77 100644 --- a/src/runtime/alpasim_runtime/events/step.py +++ b/src/runtime/alpasim_runtime/events/step.py @@ -48,6 +48,7 @@ async def run(self, state: RolloutState, queue: EventQueue) -> None: ), f"StepEvent timestamp mismatch: {ctx.step_start_us} != {self.timestamp_us}" assert ctx.ego_estimated is not None assert ctx.corrected_ego_trajectory is not None + state.last_committed_decision_bundle = ctx.decision_bundle # Commit ego trajectories (bulk concat) corrected_ego = geometry.DynamicTrajectory.from_trajectory_and_dynamics( diff --git a/src/runtime/alpasim_runtime/interactive/__init__.py b/src/runtime/alpasim_runtime/interactive/__init__.py new file mode 100644 index 00000000..50834793 --- /dev/null +++ b/src/runtime/alpasim_runtime/interactive/__init__.py @@ -0,0 +1 @@ +"""Interactive runtime session management.""" diff --git a/src/runtime/alpasim_runtime/interactive/frame_store.py b/src/runtime/alpasim_runtime/interactive/frame_store.py new file mode 100644 index 00000000..e5ce36a8 --- /dev/null +++ b/src/runtime/alpasim_runtime/interactive/frame_store.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +from __future__ import annotations + +import copy +from collections import OrderedDict + +from .models import FrameDataModel, FrameRefModel + + +class FrameStore: + """Bounded in-memory store for rendered frames keyed by tick and sensor.""" + + def __init__(self, max_retained_ticks: int) -> None: + self._max_retained_ticks = max(1, max_retained_ticks) + self._frames_by_tick: OrderedDict[int, dict[str, FrameDataModel]] = OrderedDict() + + def store_tick_frames( + self, + tick_id: int, + frames: list[FrameDataModel], + ) -> list[FrameRefModel]: + bucket = {frame.sensor_id: frame for frame in frames} + self._frames_by_tick[tick_id] = bucket + self._frames_by_tick.move_to_end(tick_id) + while len(self._frames_by_tick) > self._max_retained_ticks: + self._frames_by_tick.popitem(last=False) + + return [ + FrameRefModel( + sensor_id=frame.sensor_id, + tick_id=tick_id, + frame_start_us=frame.frame_start_us, + frame_end_us=frame.frame_end_us, + frame_encoding=frame.frame_encoding, + ) + for frame in bucket.values() + ] + + def get_frame(self, sensor_id: str, tick_id: int) -> FrameDataModel: + """Return the newest retained frame for `sensor_id` at or before `tick_id`.""" + for stored_tick in reversed(self._frames_by_tick): + if stored_tick > tick_id: + continue + bucket = self._frames_by_tick[stored_tick] + if sensor_id in bucket: + return bucket[sensor_id] + raise KeyError(f"No retained frame for sensor_id={sensor_id} tick_id<={tick_id}") + + def snapshot(self) -> OrderedDict[int, dict[str, FrameDataModel]]: + """Return a deep-copied snapshot of retained frames.""" + return copy.deepcopy(self._frames_by_tick) + + def restore(self, snapshot: OrderedDict[int, dict[str, FrameDataModel]]) -> None: + """Restore retained frames from a previous snapshot.""" + self._frames_by_tick = copy.deepcopy(snapshot) diff --git a/src/runtime/alpasim_runtime/interactive/models.py b/src/runtime/alpasim_runtime/interactive/models.py new file mode 100644 index 00000000..d1dd4118 --- /dev/null +++ b/src/runtime/alpasim_runtime/interactive/models.py @@ -0,0 +1,135 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class SensorDescriptorModel: + sensor_id: str + logical_id: str + nominal_width: int + nominal_height: int + nominal_frame_interval_us: int + rig_to_sensor: object + frame_encoding: str + + +@dataclass(frozen=True) +class FrameDataModel: + sensor_id: str + frame_start_us: int + frame_end_us: int + frame_encoding: str + content_type: str + image_bytes: bytes + + +@dataclass(frozen=True) +class FrameRefModel: + sensor_id: str + tick_id: int + frame_start_us: int + frame_end_us: int + frame_encoding: str + + +@dataclass(frozen=True) +class EgoStateModel: + pose: object + dynamics: object + front_steering_angle_rad: float = 0.0 + + +@dataclass(frozen=True) +class ActorStateModel: + actor_id: str + pose: object + + +@dataclass(frozen=True) +class PolylinePointModel: + x: float + y: float + + +@dataclass(frozen=True) +class CandidatePlanModel: + candidate_id: str + backend_id: str + selected: bool + points: list[PolylinePointModel] + + +@dataclass(frozen=True) +class CandidateSummaryModel: + candidate_id: str + backend_id: str + status: str + selected: bool + error: str = "" + diagnostics: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class DecisionSummaryModel: + step_id: int + input_snapshot_id: str + selected_candidate_id: str | None + candidates: list[CandidateSummaryModel] + arbitration_reason: str = "" + + +@dataclass(frozen=True) +class CheckpointSummaryModel: + checkpoint_id: str + tick_id: int + sim_time_us: int + status: str = "PAUSED" + restore_supported: bool = True + unsupported_backend_ids: list[str] = field(default_factory=list) + + +@dataclass(frozen=True) +class SessionSnapshotModel: + interactive_session_id: str + tick_id: int + sim_time_us: int + ego: EgoStateModel + actors: list[ActorStateModel] + frame_refs: list[FrameRefModel] + latest_decision: DecisionSummaryModel | None = None + ego_history: list[PolylinePointModel] = field(default_factory=list) + selected_plan: list[PolylinePointModel] = field(default_factory=list) + candidate_plans: list[CandidatePlanModel] = field(default_factory=list) + context_diagnostics: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class SessionStateModel: + interactive_session_id: str + rollout_uuid: str + scene_id: str + status: str + current_tick_id: int + current_sim_time_us: int + latest_snapshot: SessionSnapshotModel | None + latest_decision: DecisionSummaryModel | None = None + active_backend_ids: list[str] = field(default_factory=list) + available_backend_ids: list[str] = field(default_factory=list) + error: str = "" + + +@dataclass +class SessionEventModel: + state: SessionStateModel | None = None + snapshot: SessionSnapshotModel | None = None + + +@dataclass +class SessionUpdateModel: + state: SessionStateModel + committed_snapshots: list[SessionSnapshotModel] = field(default_factory=list) diff --git a/src/runtime/alpasim_runtime/interactive/session_manager.py b/src/runtime/alpasim_runtime/interactive/session_manager.py new file mode 100644 index 00000000..16850db0 --- /dev/null +++ b/src/runtime/alpasim_runtime/interactive/session_manager.py @@ -0,0 +1,191 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +from __future__ import annotations + +import asyncio +import uuid +from dataclasses import dataclass + +from alpasim_runtime.address_pool import AddressPool, release_all, try_acquire_all +from alpasim_runtime.config import UserSimulatorConfig +from alpasim_runtime.worker.ipc import ServiceEndpoints +from alpasim_grpc.v0.logging_pb2 import RolloutMetadata + +from eval.schema import EvalConfig + +from .models import ( + CandidateSummaryModel, + CheckpointSummaryModel, + FrameDataModel, + SensorDescriptorModel, + SessionStateModel, + SessionUpdateModel, +) +from .session_runner import InteractiveSessionRunner + + +@dataclass +class _SessionEntry: + runner: InteractiveSessionRunner + + +class InteractiveSessionManager: + def __init__( + self, + *, + user_config: UserSimulatorConfig, + eval_config: EvalConfig, + version_ids: RolloutMetadata.VersionIds, + scene_id_to_artifact_path: dict[str, str], + pools: dict[str, AddressPool], + rollouts_dir: str, + ) -> None: + self._user_config = user_config + self._eval_config = eval_config + self._version_ids = version_ids + self._scene_id_to_artifact_path = scene_id_to_artifact_path + self._pools = pools + self._rollouts_dir = rollouts_dir + self._sessions: dict[str, _SessionEntry] = {} + self._lock = asyncio.Lock() + + async def create_session( + self, + *, + scene_id: str, + start_paused: bool, + max_retained_ticks: int, + ) -> SessionStateModel: + session_id: str | None = None + runner: InteractiveSessionRunner | None = None + async with self._lock: + if scene_id not in self._scene_id_to_artifact_path: + raise KeyError(f"Unknown scene_id: {scene_id}") + acquired = try_acquire_all(self._pools) + if acquired is None: + raise RuntimeError("No service capacity available for interactive session") + + async def _release() -> None: + release_all(self._pools, acquired) + + session_id = uuid.uuid4().hex + runner = InteractiveSessionRunner( + interactive_session_id=session_id, + scene_id=scene_id, + artifact_path=self._scene_id_to_artifact_path[scene_id], + endpoints=ServiceEndpoints( + driver=acquired["driver"], + sensorsim=acquired["sensorsim"], + physics=acquired["physics"], + trafficsim=acquired["trafficsim"], + controller=acquired["controller"], + ), + user_config=self._user_config, + eval_config=self._eval_config, + version_ids=self._version_ids, + rollouts_dir=self._rollouts_dir, + max_retained_ticks=max_retained_ticks, + on_released=_release, + ) + self._sessions[session_id] = _SessionEntry(runner=runner) + + assert runner is not None + assert session_id is not None + try: + state = await runner.initialize() + if not start_paused: + state = await runner.start() + return state + except Exception: + async with self._lock: + self._sessions.pop(session_id, None) + await runner.close() + raise + + async def get_state(self, session_id: str) -> SessionStateModel: + runner = await self._get_runner(session_id) + return await runner.get_state() + + async def list_sessions(self) -> list[SessionStateModel]: + async with self._lock: + runners = [entry.runner for entry in self._sessions.values()] + if not runners: + return [] + return [await runner.get_state() for runner in runners] + + async def list_sensors(self, session_id: str) -> list[SensorDescriptorModel]: + runner = await self._get_runner(session_id) + return await runner.list_sensors() + + async def list_candidates(self, session_id: str) -> list[CandidateSummaryModel]: + runner = await self._get_runner(session_id) + return await runner.list_candidates() + + async def recompute_candidate( + self, session_id: str, backend_id: str + ) -> SessionStateModel: + runner = await self._get_runner(session_id) + return await runner.recompute_candidate(backend_id) + + async def select_candidate( + self, session_id: str, candidate_id: str + ) -> SessionStateModel: + runner = await self._get_runner(session_id) + return await runner.select_candidate(candidate_id) + + async def set_active_backends( + self, session_id: str, backend_ids: list[str] + ) -> SessionStateModel: + runner = await self._get_runner(session_id) + return await runner.set_active_backends(backend_ids) + + async def list_checkpoints(self, session_id: str) -> list[CheckpointSummaryModel]: + runner = await self._get_runner(session_id) + return await runner.list_checkpoints() + + async def restore_checkpoint( + self, session_id: str, checkpoint_id: str + ) -> SessionStateModel: + runner = await self._get_runner(session_id) + return await runner.restore_checkpoint(checkpoint_id) + + async def start_session(self, session_id: str) -> SessionStateModel: + runner = await self._get_runner(session_id) + return await runner.start() + + async def pause_session(self, session_id: str) -> SessionStateModel: + runner = await self._get_runner(session_id) + return await runner.pause() + + async def resume_session(self, session_id: str) -> SessionStateModel: + runner = await self._get_runner(session_id) + return await runner.resume() + + async def step_session(self, session_id: str, num_steps: int) -> SessionUpdateModel: + runner = await self._get_runner(session_id) + return await runner.step(num_steps=num_steps) + + async def get_frame(self, session_id: str, sensor_id: str, tick_id: int) -> FrameDataModel: + runner = await self._get_runner(session_id) + return await runner.get_frame(sensor_id=sensor_id, tick_id=tick_id) + + async def subscribe(self, session_id: str) -> asyncio.Queue: + return (await self._get_runner(session_id)).subscribe() + + async def unsubscribe(self, session_id: str, queue: asyncio.Queue) -> None: + runner = await self._get_runner(session_id) + runner.unsubscribe(queue) + + async def close_all(self) -> None: + async with self._lock: + entries = list(self._sessions.items()) + self._sessions.clear() + for _, entry in entries: + await entry.runner.close() + + async def _get_runner(self, session_id: str) -> InteractiveSessionRunner: + async with self._lock: + if session_id not in self._sessions: + raise KeyError(f"Unknown interactive_session_id: {session_id}") + return self._sessions[session_id].runner diff --git a/src/runtime/alpasim_runtime/interactive/session_runner.py b/src/runtime/alpasim_runtime/interactive/session_runner.py new file mode 100644 index 00000000..0514589f --- /dev/null +++ b/src/runtime/alpasim_runtime/interactive/session_runner.py @@ -0,0 +1,736 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +from __future__ import annotations + +import asyncio +import copy +import logging +from collections import OrderedDict +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, replace + +import numpy as np +from alpasim_grpc.v0 import sensorsim_pb2 +from alpasim_runtime.decision import ( + CandidateDecision, + DecisionBundle, + DriverServiceBackendAdapter, + SingleBackendDriverOrchestrator, + select_candidate_in_bundle, +) +from alpasim_runtime.camera_catalog import CameraCatalog +from alpasim_runtime.config import UserSimulatorConfig +from alpasim_runtime.event_loop import EventBasedRollout, RuntimeCheckpoint +from alpasim_runtime.services.controller_service import ControllerService +from alpasim_runtime.services.driver_service import DriverService +from alpasim_runtime.services.physics_service import PhysicsService +from alpasim_runtime.services.sensorsim_service import SensorsimService +from alpasim_runtime.services.traffic_service import TrafficService +from alpasim_runtime.unbound_rollout import UnboundRollout +from alpasim_runtime.worker.artifact_cache import make_artifact_loader +from alpasim_runtime.worker.ipc import ServiceEndpoints +from alpasim_utils import geometry +from alpasim_utils.geometry import pose_to_grpc +from alpasim_utils.types import ImageWithMetadata +from alpasim_grpc.v0.common_pb2 import DynamicState, Vec3 +from alpasim_grpc.v0.logging_pb2 import RolloutMetadata + +from eval.schema import EvalConfig + +from .frame_store import FrameStore +from .models import ( + ActorStateModel, + CandidatePlanModel, + CandidateSummaryModel, + CheckpointSummaryModel, + DecisionSummaryModel, + EgoStateModel, + FrameDataModel, + PolylinePointModel, + SensorDescriptorModel, + SessionEventModel, + SessionSnapshotModel, + SessionStateModel, + SessionUpdateModel, +) + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class _InteractiveCheckpoint: + checkpoint_id: str + tick_id: int + sim_time_us: int + runtime_checkpoint: RuntimeCheckpoint + frame_store_snapshot: OrderedDict[int, dict[str, FrameDataModel]] + latest_snapshot: SessionSnapshotModel | None + pending_frames: list[FrameDataModel] + status: str + + +def _frame_encoding_name(image_format_enum: int) -> tuple[str, str]: + if sensorsim_pb2.ImageFormat.Name(image_format_enum) == "PNG": + return ("PNG", "image/png") + return ("JPEG", "image/jpeg") + + +def _dynamic_state_from_row(row: np.ndarray) -> DynamicState: + return DynamicState( + linear_velocity=Vec3(x=float(row[0]), y=float(row[1]), z=float(row[2])), + angular_velocity=Vec3(x=float(row[3]), y=float(row[4]), z=float(row[5])), + linear_acceleration=Vec3(x=float(row[6]), y=float(row[7]), z=float(row[8])), + angular_acceleration=Vec3( + x=float(row[9]), + y=float(row[10]), + z=float(row[11]), + ), + ) + + +class InteractiveSessionRunner: + """Owns one live interactive simulation session.""" + + def __init__( + self, + *, + interactive_session_id: str, + scene_id: str, + artifact_path: str, + endpoints: ServiceEndpoints, + user_config: UserSimulatorConfig, + eval_config: EvalConfig, + version_ids: RolloutMetadata.VersionIds, + rollouts_dir: str, + max_retained_ticks: int, + on_released: Callable[[], Awaitable[None]], + ) -> None: + self._interactive_session_id = interactive_session_id + self._scene_id = scene_id + self._artifact_path = artifact_path + self._endpoints = endpoints + self._user_config = user_config + self._eval_config = eval_config + self._version_ids = version_ids + self._rollouts_dir = rollouts_dir + self._on_released = on_released + + self._lock = asyncio.Lock() + self._subscribers: set[asyncio.Queue[SessionEventModel]] = set() + self._run_continuously = False + self._background_task: asyncio.Task[None] | None = None + self._released = False + self._closed = False + self._tick_id = -1 + self._latest_snapshot: SessionSnapshotModel | None = None + self._status = "CREATED" + self._error = "" + self._camera_catalog = CameraCatalog(user_config.extra_cameras) + self._frame_store = FrameStore(max_retained_ticks=max_retained_ticks) + self._max_retained_ticks = max(1, max_retained_ticks) + self._pending_frames: list[FrameDataModel] = [] + self._artifact_loader = make_artifact_loader( + smooth_trajectories=user_config.smooth_trajectories, + max_cache_size=user_config.artifact_cache_size, + ) + self._rollout: EventBasedRollout | None = None + self._sensors: list[SensorDescriptorModel] = [] + self._checkpoints: OrderedDict[str, _InteractiveCheckpoint] = OrderedDict() + + async def initialize(self) -> SessionStateModel: + async with self._lock: + if self._rollout is None: + artifact = self._artifact_loader(self._scene_id, self._artifact_path) + rollout = EventBasedRollout( + unbound=UnboundRollout.create( + simulation_config=self._user_config.simulation_config, + scene_id=self._scene_id, + version_ids=self._version_ids, + available_artifacts={self._scene_id: artifact}, + rollouts_dir=self._rollouts_dir, + ), + driver=DriverService( + self._endpoints.driver.address, + skip=self._endpoints.driver.skip, + ), + sensorsim=SensorsimService( + self._endpoints.sensorsim.address, + skip=self._endpoints.sensorsim.skip, + camera_catalog=self._camera_catalog, + ), + physics=PhysicsService( + self._endpoints.physics.address, + skip=self._endpoints.physics.skip, + ), + trafficsim=TrafficService( + self._endpoints.trafficsim.address, + skip=self._endpoints.trafficsim.skip, + ), + controller=ControllerService( + self._endpoints.controller.address, + skip=self._endpoints.controller.skip, + ), + camera_catalog=self._camera_catalog, + eval_config=self._eval_config, + ) + await rollout.initialize() + rollout.current_state.rendered_images_handler = self._capture_images + self._rollout = rollout + self._sensors = self._build_sensors() + + if self._latest_snapshot is None: + await self._advance_once_locked() + + self._status = "PAUSED" + state = self._build_state() + await self._publish(SessionEventModel(state=state)) + return state + + async def start(self) -> SessionStateModel: + return await self.resume() + + async def resume(self) -> SessionStateModel: + async with self._lock: + if self._closed: + return self._build_state() + if self._rollout is None: + raise RuntimeError("session not initialized") + if self._status == "COMPLETED": + return self._build_state() + self._run_continuously = True + self._status = "RUNNING" + if self._background_task is None or self._background_task.done(): + self._background_task = asyncio.create_task(self._run_forever()) + state = self._build_state() + await self._publish(SessionEventModel(state=state)) + return state + + async def pause(self) -> SessionStateModel: + task: asyncio.Task[None] | None + async with self._lock: + self._run_continuously = False + task = self._background_task + if task is not None: + await task + async with self._lock: + if self._status not in {"COMPLETED", "FAILED"}: + self._status = "PAUSED" + state = self._build_state() + await self._publish(SessionEventModel(state=state)) + return state + + async def step(self, num_steps: int) -> SessionUpdateModel: + committed: list[SessionSnapshotModel] = [] + await self.pause() + async with self._lock: + for _ in range(max(1, num_steps)): + if self._status in {"COMPLETED", "FAILED"}: + break + snapshot = await self._advance_once_locked() + if snapshot is not None: + committed.append(snapshot) + if self._status == "COMPLETED": + break + state = self._build_state() + await self._publish(SessionEventModel(state=state)) + return SessionUpdateModel(state=state, committed_snapshots=committed) + + async def close(self) -> None: + task: asyncio.Task[None] | None + async with self._lock: + if self._closed: + return + self._run_continuously = False + task = self._background_task + if task is not None: + await task + async with self._lock: + completed = self._status == "COMPLETED" + await self._finalize_locked( + mark_complete=completed, + run_evaluation=completed, + ) + self._closed = True + + async def get_state(self) -> SessionStateModel: + async with self._lock: + return self._build_state() + + async def set_active_backends(self, backend_ids: list[str]) -> SessionStateModel: + async with self._lock: + if self._rollout is None: + raise RuntimeError("session not initialized") + available = set(self._rollout.current_state.available_driver_backend_ids) + selected = list(backend_ids) + if selected: + unknown = [backend_id for backend_id in selected if backend_id not in available] + if unknown: + raise KeyError(f"Unknown backend ids: {unknown}") + self._rollout.current_state.active_driver_backend_ids = selected + else: + self._rollout.current_state.active_driver_backend_ids = list(available) + state = self._build_state() + await self._publish(SessionEventModel(state=state)) + return state + + async def list_candidates(self) -> list[CandidateSummaryModel]: + async with self._lock: + decision = self._latest_decision_summary() + if decision is None: + return [] + return list(decision.candidates) + + async def list_checkpoints(self) -> list[CheckpointSummaryModel]: + async with self._lock: + return [ + CheckpointSummaryModel( + checkpoint_id=checkpoint.checkpoint_id, + tick_id=checkpoint.tick_id, + sim_time_us=checkpoint.sim_time_us, + status=checkpoint.status, + restore_supported=not bool( + checkpoint.runtime_checkpoint.unsupported_backend_ids + ), + unsupported_backend_ids=list( + checkpoint.runtime_checkpoint.unsupported_backend_ids + ), + ) + for checkpoint in self._checkpoints.values() + ] + + async def recompute_candidate(self, backend_id: str) -> SessionStateModel: + await self.pause() + async with self._lock: + if self._rollout is None: + raise RuntimeError("session not initialized") + bundle = self._rollout.current_state.last_committed_decision_bundle + if bundle is None: + raise RuntimeError("No committed decision bundle available for recompute") + available = set(self._rollout.current_state.available_driver_backend_ids) + if backend_id not in available: + raise KeyError(f"Unknown backend_id: {backend_id}") + orchestrator = self._driver_orchestrator() + updated_bundle = await orchestrator.recompute_candidate(bundle, backend_id) + self._rollout.current_state.last_committed_decision_bundle = updated_bundle + if self._latest_snapshot is not None: + self._latest_snapshot = replace( + self._latest_snapshot, + latest_decision=_decision_summary_from_bundle(updated_bundle), + selected_plan=_selected_plan_from_bundle(updated_bundle), + candidate_plans=_candidate_plans_from_bundle(updated_bundle), + context_diagnostics=_context_diagnostics_from_bundle(updated_bundle), + ) + if self._latest_snapshot is not None: + self._record_checkpoint_locked(self._latest_snapshot) + state = self._build_state() + await self._publish(SessionEventModel(state=state)) + return state + + async def select_candidate(self, candidate_id: str) -> SessionStateModel: + await self.pause() + async with self._lock: + if self._rollout is None: + raise RuntimeError("session not initialized") + bundle = self._rollout.current_state.last_committed_decision_bundle + if bundle is None: + raise RuntimeError("No committed decision bundle available for selection") + updated_bundle = select_candidate_in_bundle(bundle, candidate_id) + self._rollout.current_state.last_committed_decision_bundle = updated_bundle + if self._latest_snapshot is not None: + self._latest_snapshot = replace( + self._latest_snapshot, + latest_decision=_decision_summary_from_bundle(updated_bundle), + selected_plan=_selected_plan_from_bundle(updated_bundle), + candidate_plans=_candidate_plans_from_bundle(updated_bundle), + context_diagnostics=_context_diagnostics_from_bundle(updated_bundle), + ) + self._record_checkpoint_locked(self._latest_snapshot) + state = self._build_state() + await self._publish(SessionEventModel(state=state)) + return state + + async def restore_checkpoint(self, checkpoint_id: str) -> SessionStateModel: + await self.pause() + async with self._lock: + if self._rollout is None: + raise RuntimeError("session not initialized") + checkpoint = self._checkpoints.get(checkpoint_id) + if checkpoint is None: + raise KeyError(f"Unknown checkpoint_id: {checkpoint_id}") + if checkpoint.runtime_checkpoint.unsupported_backend_ids: + raise RuntimeError( + "Checkpoint restore is not supported for backends: " + + ", ".join(checkpoint.runtime_checkpoint.unsupported_backend_ids) + ) + await self._rollout.restore_runtime_checkpoint(checkpoint.runtime_checkpoint) + self._rollout.current_state.rendered_images_handler = self._capture_images + self._frame_store.restore(checkpoint.frame_store_snapshot) + self._tick_id = checkpoint.tick_id + self._latest_snapshot = copy.deepcopy(checkpoint.latest_snapshot) + self._pending_frames = copy.deepcopy(checkpoint.pending_frames) + self._status = "PAUSED" + self._error = "" + state = self._build_state() + await self._publish(SessionEventModel(state=state)) + return state + + async def list_sensors(self) -> list[SensorDescriptorModel]: + async with self._lock: + return list(self._sensors) + + async def get_frame(self, sensor_id: str, tick_id: int) -> FrameDataModel: + async with self._lock: + return self._frame_store.get_frame(sensor_id=sensor_id, tick_id=tick_id) + + def subscribe(self) -> asyncio.Queue[SessionEventModel]: + queue: asyncio.Queue[SessionEventModel] = asyncio.Queue() + self._subscribers.add(queue) + return queue + + def unsubscribe(self, queue: asyncio.Queue[SessionEventModel]) -> None: + self._subscribers.discard(queue) + + async def _run_forever(self) -> None: + try: + while True: + async with self._lock: + if not self._run_continuously or self._status in { + "COMPLETED", + "FAILED", + }: + break + await self._advance_once_locked() + state = self._build_state() + await self._publish(SessionEventModel(state=state)) + except Exception as exc: # noqa: BLE001 + logger.exception("interactive session failed: %s", self._interactive_session_id) + async with self._lock: + self._status = "FAILED" + self._error = str(exc) + await self._finalize_locked(mark_complete=False, run_evaluation=False) + state = self._build_state() + await self._publish(SessionEventModel(state=state)) + finally: + async with self._lock: + if self._status not in {"COMPLETED", "FAILED"}: + self._status = "PAUSED" + self._background_task = None + + async def _advance_once_locked(self) -> SessionSnapshotModel | None: + assert self._rollout is not None + result = await self._rollout.run_until_step_commit() + snapshot = None + if result.step_committed: + self._tick_id += 1 + frame_refs = self._frame_store.store_tick_frames( + tick_id=self._tick_id, + frames=self._pending_frames, + ) + self._pending_frames = [] + snapshot = self._build_snapshot(frame_refs) + self._latest_snapshot = snapshot + self._record_checkpoint_locked(snapshot) + await self._publish(SessionEventModel(snapshot=snapshot)) + if result.simulation_finished: + self._status = "COMPLETED" + return snapshot + + async def _capture_images(self, images: list[ImageWithMetadata]) -> None: + if self._rollout is None: + return + frame_encoding, content_type = _frame_encoding_name( + self._rollout.unbound.image_format + ) + for image in images: + self._pending_frames.append( + FrameDataModel( + sensor_id=image.camera_logical_id, + frame_start_us=image.start_timestamp_us, + frame_end_us=image.end_timestamp_us, + frame_encoding=frame_encoding, + content_type=content_type, + image_bytes=image.image_bytes, + ) + ) + + def _build_sensors(self) -> list[SensorDescriptorModel]: + assert self._rollout is not None + sensors: list[SensorDescriptorModel] = [] + frame_encoding, _ = _frame_encoding_name(self._rollout.unbound.image_format) + for camera in self._rollout.runtime_cameras: + definition = self._camera_catalog.get_camera_definition( + self._scene_id, camera.logical_id + ) + sensors.append( + SensorDescriptorModel( + sensor_id=camera.logical_id, + logical_id=camera.logical_id, + nominal_width=camera.render_resolution_hw[1], + nominal_height=camera.render_resolution_hw[0], + nominal_frame_interval_us=camera.clock.interval_us, + rig_to_sensor=pose_to_grpc(definition.rig_to_camera), + frame_encoding=frame_encoding, + ) + ) + return sensors + + def _build_snapshot( + self, + frame_refs: list[object], + ) -> SessionSnapshotModel: + assert self._rollout is not None + state = self._rollout.current_state + sim_time_us = int(state.ego_trajectory.timestamps_us[-1]) + ego_pose = pose_to_grpc(state.ego_trajectory.get_pose(-1)) + ego_dynamics = _dynamic_state_from_row(state.ego_trajectory.dynamics[-1]) + + actors: list[ActorStateModel] = [] + for actor_id, traffic_obj in state.traffic_objs.items(): + if sim_time_us not in traffic_obj.trajectory.time_range_us: + continue + actors.append( + ActorStateModel( + actor_id=actor_id, + pose=geometry.pose_to_grpc( + traffic_obj.trajectory.interpolate_pose(sim_time_us) + ), + ) + ) + + ego_history = _polyline_from_positions(state.ego_trajectory.positions) + candidate_plans = _candidate_plans_from_bundle(state.last_committed_decision_bundle) + selected_plan = next( + (candidate.points for candidate in candidate_plans if candidate.selected), + [], + ) + + return SessionSnapshotModel( + interactive_session_id=self._interactive_session_id, + tick_id=self._tick_id, + sim_time_us=sim_time_us, + ego=EgoStateModel( + pose=ego_pose, + dynamics=ego_dynamics, + front_steering_angle_rad=state.current_front_steering_angle_rad, + ), + actors=actors, + frame_refs=list(frame_refs), + latest_decision=self._latest_decision_summary(), + ego_history=ego_history, + selected_plan=selected_plan, + candidate_plans=candidate_plans, + context_diagnostics=_context_diagnostics_from_bundle( + state.last_committed_decision_bundle + ), + ) + + def _build_state(self) -> SessionStateModel: + sim_time_us = self._latest_snapshot.sim_time_us if self._latest_snapshot else 0 + return SessionStateModel( + interactive_session_id=self._interactive_session_id, + rollout_uuid=( + self._rollout.unbound.rollout_uuid if self._rollout is not None else "" + ), + scene_id=self._scene_id, + status=self._status, + current_tick_id=max(self._tick_id, 0) if self._latest_snapshot else 0, + current_sim_time_us=sim_time_us, + latest_snapshot=self._latest_snapshot, + latest_decision=self._latest_decision_summary(), + active_backend_ids=self._active_backend_ids(), + available_backend_ids=self._available_backend_ids(), + error=self._error, + ) + + def _latest_decision_summary(self) -> DecisionSummaryModel | None: + if self._rollout is None: + return self._latest_snapshot.latest_decision if self._latest_snapshot else None + bundle = self._rollout.current_state.last_committed_decision_bundle + if bundle is None: + return self._latest_snapshot.latest_decision if self._latest_snapshot else None + return _decision_summary_from_bundle(bundle) + + def _active_backend_ids(self) -> list[str]: + if self._rollout is None: + return [] + active = self._rollout.current_state.active_driver_backend_ids + return list(active) if active is not None else [] + + def _available_backend_ids(self) -> list[str]: + if self._rollout is None: + return [] + return list(self._rollout.current_state.available_driver_backend_ids) + + def _driver_orchestrator(self): + assert self._rollout is not None + orchestrator = self._rollout._build_default_driver_orchestrator() + if orchestrator is not None: + return orchestrator + return SingleBackendDriverOrchestrator( + DriverServiceBackendAdapter(self._rollout.driver) + ) + + def _record_checkpoint_locked(self, snapshot: SessionSnapshotModel) -> None: + assert self._rollout is not None + checkpoint_id = f"tick-{self._tick_id}" + checkpoint = _InteractiveCheckpoint( + checkpoint_id=checkpoint_id, + tick_id=self._tick_id, + sim_time_us=snapshot.sim_time_us, + runtime_checkpoint=self._rollout.capture_runtime_checkpoint(), + frame_store_snapshot=self._frame_store.snapshot(), + latest_snapshot=copy.deepcopy(snapshot), + pending_frames=copy.deepcopy(self._pending_frames), + status="PAUSED", + ) + self._checkpoints[checkpoint_id] = checkpoint + self._checkpoints.move_to_end(checkpoint_id) + while len(self._checkpoints) > self._max_retained_ticks: + self._checkpoints.popitem(last=False) + + async def _publish(self, event: SessionEventModel) -> None: + dead: list[asyncio.Queue[SessionEventModel]] = [] + for queue in list(self._subscribers): + try: + queue.put_nowait(event) + except asyncio.QueueFull: + dead.append(queue) + for queue in dead: + self._subscribers.discard(queue) + + async def _finalize_locked( + self, + *, + mark_complete: bool, + run_evaluation: bool, + ) -> None: + if self._rollout is not None: + await self._rollout.aclose( + mark_complete=mark_complete, + run_evaluation=run_evaluation, + ) + self._rollout = None + if not self._released: + await self._on_released() + self._released = True + + +def _candidate_summary_from_decision( + candidate: CandidateDecision, + *, + selected_candidate_id: str | None, +) -> CandidateSummaryModel: + return CandidateSummaryModel( + candidate_id=candidate.candidate_id, + backend_id=candidate.backend_id, + status=candidate.status.value, + selected=candidate.candidate_id == selected_candidate_id, + error=candidate.error or "", + diagnostics=_normalize_debug_payload(candidate.diagnostics), + ) + + +def _decision_summary_from_bundle(bundle: DecisionBundle) -> DecisionSummaryModel: + return DecisionSummaryModel( + step_id=bundle.snapshot.step_id, + input_snapshot_id=bundle.snapshot.input_snapshot_id, + selected_candidate_id=bundle.selected_candidate_id, + candidates=[ + _candidate_summary_from_decision( + candidate, + selected_candidate_id=bundle.selected_candidate_id, + ) + for candidate in bundle.candidates + ], + arbitration_reason=bundle.arbitration_reason or "", + ) + + +def _polyline_from_positions(positions: np.ndarray | None) -> list[PolylinePointModel]: + if positions is None or len(positions) == 0: + return [] + return [ + PolylinePointModel(x=float(position[0]), y=float(position[1])) + for position in positions + ] + + +def _selected_plan_from_bundle( + bundle: DecisionBundle | None, +) -> list[PolylinePointModel]: + if bundle is None: + return [] + + selected_candidate: CandidateDecision | None = None + if bundle.selected_candidate_id is not None: + selected_candidate = next( + ( + candidate + for candidate in bundle.candidates + if candidate.candidate_id == bundle.selected_candidate_id + ), + None, + ) + if selected_candidate is None: + selected_candidate = next( + (candidate for candidate in bundle.candidates if candidate.status.value == "SELECTED"), + None, + ) + if selected_candidate is None or selected_candidate.trajectory is None: + return [] + return _polyline_from_positions(selected_candidate.trajectory.positions) + + +def _candidate_plans_from_bundle( + bundle: DecisionBundle | None, +) -> list[CandidatePlanModel]: + if bundle is None: + return [] + plans: list[CandidatePlanModel] = [] + for candidate in bundle.candidates: + if candidate.trajectory is None: + continue + plans.append( + CandidatePlanModel( + candidate_id=candidate.candidate_id, + backend_id=candidate.backend_id, + selected=candidate.candidate_id == bundle.selected_candidate_id, + points=_polyline_from_positions(candidate.trajectory.positions), + ) + ) + return plans + + +def _context_diagnostics_from_bundle( + bundle: DecisionBundle | None, +) -> dict[str, object]: + if bundle is None: + return {} + planner_context = bundle.snapshot.planner_context or {} + diagnostics = { + "timing": planner_context.get("timing", {}), + "quality": planner_context.get("quality", {}), + "map_summary": planner_context.get("map_summary", {}), + "route_waypoint_count": len(planner_context.get("route_waypoints_in_rig", []) or []), + "candidate_count": len(bundle.candidates), + "selected_candidate_id": bundle.selected_candidate_id, + "arbitration_reason": bundle.arbitration_reason or "", + } + return _normalize_debug_payload(diagnostics) + + +def _normalize_debug_payload(value): + if isinstance(value, dict): + return {str(key): _normalize_debug_payload(item) for key, item in value.items()} + if isinstance(value, list): + return [_normalize_debug_payload(item) for item in value] + if isinstance(value, tuple): + return [_normalize_debug_payload(item) for item in value] + if isinstance(value, np.generic): + return value.item() + if isinstance(value, np.ndarray): + return value.tolist() + if isinstance(value, (str, int, float, bool)) or value is None: + return value + return repr(value) diff --git a/src/runtime/alpasim_runtime/observation_cache.py b/src/runtime/alpasim_runtime/observation_cache.py new file mode 100644 index 00000000..65ed86b8 --- /dev/null +++ b/src/runtime/alpasim_runtime/observation_cache.py @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +"""Shared recent-frame observation cache for runtime-side driver orchestration.""" + +from __future__ import annotations + +from collections import OrderedDict +from dataclasses import dataclass +from typing import Any + +from alpasim_utils import geometry +from alpasim_utils.scenario import TrafficObject, TrafficObjects +from alpasim_utils.types import ImageWithMetadata + + +def _clone_image(image: ImageWithMetadata) -> ImageWithMetadata: + return ImageWithMetadata( + start_timestamp_us=image.start_timestamp_us, + end_timestamp_us=image.end_timestamp_us, + image_bytes=image.image_bytes, + camera_logical_id=image.camera_logical_id, + ) + + +def _clone_traffic_objects(traffic_objs: TrafficObjects) -> TrafficObjects: + cloned = {} + for object_id, traffic_obj in traffic_objs.items(): + time_range = traffic_obj.trajectory.time_range_us + cloned[object_id] = TrafficObject( + track_id=traffic_obj.track_id, + aabb=traffic_obj.aabb, + trajectory=traffic_obj.trajectory.clip(time_range.start, time_range.stop), + is_static=traffic_obj.is_static, + label_class=traffic_obj.label_class, + ) + return TrafficObjects(cloned) + + +def _normalize_cache_value(value: Any) -> Any: + if isinstance(value, dict): + return { + str(key): _normalize_cache_value(item) + for key, item in sorted(value.items(), key=lambda kv: str(kv[0])) + } + if isinstance(value, (list, tuple)): + return [_normalize_cache_value(item) for item in value] + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if hasattr(value, "tolist"): + try: + return _normalize_cache_value(value.tolist()) + except Exception: + pass + return repr(value) + + +@dataclass(frozen=True, slots=True) +class ObservationFrame: + step_id: int + input_snapshot_id: str + time_now_us: int + time_query_us: int + camera_frame_timestamps_us: dict[str, int] + rendered_images: dict[str, ImageWithMetadata] + renderer_data: bytes | None + ego_trajectory: geometry.DynamicTrajectory + ego_trajectory_estimate: geometry.DynamicTrajectory | None + traffic_objs: TrafficObjects + ego_pose_history_timestamps_us: list[int] + route_waypoints_in_rig: list[list[float]] + planner_context: dict[str, Any] | None + active_backend_ids: list[str] + + def clone(self) -> ObservationFrame: + return ObservationFrame( + step_id=self.step_id, + input_snapshot_id=self.input_snapshot_id, + time_now_us=self.time_now_us, + time_query_us=self.time_query_us, + camera_frame_timestamps_us=dict(self.camera_frame_timestamps_us), + rendered_images={ + camera_id: _clone_image(image) + for camera_id, image in self.rendered_images.items() + }, + renderer_data=self.renderer_data, + ego_trajectory=self.ego_trajectory.clone(), + ego_trajectory_estimate=( + self.ego_trajectory_estimate.clone() + if self.ego_trajectory_estimate is not None + else None + ), + traffic_objs=_clone_traffic_objects(self.traffic_objs), + ego_pose_history_timestamps_us=list(self.ego_pose_history_timestamps_us), + route_waypoints_in_rig=[list(waypoint) for waypoint in self.route_waypoints_in_rig], + planner_context=_normalize_cache_value(self.planner_context), + active_backend_ids=list(self.active_backend_ids), + ) + + +@dataclass(frozen=True, slots=True) +class ObservationWindow: + anchor_snapshot_id: str + frames: list[ObservationFrame] + + +@dataclass(frozen=True, slots=True) +class ObservationCacheCheckpoint: + latest_snapshot_id: str | None + frames_by_snapshot_id: dict[str, ObservationFrame] + snapshot_order: list[str] + + +class ObservationCache: + """Bounded in-memory cache of recent shared observation frames.""" + + def __init__(self, max_frames: int = 32) -> None: + self._max_frames = max(1, max_frames) + self._frames_by_snapshot_id: OrderedDict[str, ObservationFrame] = OrderedDict() + self._latest_snapshot_id: str | None = None + + def append(self, frame: ObservationFrame) -> None: + self._frames_by_snapshot_id[frame.input_snapshot_id] = frame.clone() + self._frames_by_snapshot_id.move_to_end(frame.input_snapshot_id) + self._latest_snapshot_id = frame.input_snapshot_id + while len(self._frames_by_snapshot_id) > self._max_frames: + self._frames_by_snapshot_id.popitem(last=False) + if self._latest_snapshot_id not in self._frames_by_snapshot_id: + self._latest_snapshot_id = ( + next(reversed(self._frames_by_snapshot_id)) + if self._frames_by_snapshot_id + else None + ) + + def get(self, input_snapshot_id: str) -> ObservationFrame: + return self._frames_by_snapshot_id[input_snapshot_id].clone() + + def latest(self) -> ObservationFrame | None: + if self._latest_snapshot_id is None: + return None + return self.get(self._latest_snapshot_id) + + def get_window( + self, + input_snapshot_id: str, + window_size: int, + ) -> ObservationWindow: + snapshot_ids = list(self._frames_by_snapshot_id.keys()) + try: + end_idx = snapshot_ids.index(input_snapshot_id) + 1 + except ValueError as exc: + raise KeyError(f"Unknown input_snapshot_id: {input_snapshot_id}") from exc + start_idx = max(0, end_idx - max(1, window_size)) + selected_ids = snapshot_ids[start_idx:end_idx] + return ObservationWindow( + anchor_snapshot_id=input_snapshot_id, + frames=[self._frames_by_snapshot_id[snapshot_id].clone() for snapshot_id in selected_ids], + ) + + def list_snapshot_ids(self) -> list[str]: + return list(self._frames_by_snapshot_id.keys()) + + def checkpoint(self) -> ObservationCacheCheckpoint: + return ObservationCacheCheckpoint( + latest_snapshot_id=self._latest_snapshot_id, + frames_by_snapshot_id={ + snapshot_id: frame.clone() + for snapshot_id, frame in self._frames_by_snapshot_id.items() + }, + snapshot_order=list(self._frames_by_snapshot_id.keys()), + ) + + def restore(self, checkpoint: ObservationCacheCheckpoint) -> None: + self._frames_by_snapshot_id = OrderedDict( + (snapshot_id, checkpoint.frames_by_snapshot_id[snapshot_id].clone()) + for snapshot_id in checkpoint.snapshot_order + ) + self._latest_snapshot_id = checkpoint.latest_snapshot_id + + +class ObservationCacheReader: + """Read-only helper for backend adapters to access shared observation windows.""" + + def __init__(self, cache: ObservationCache) -> None: + self._cache = cache + + def get_frame(self, input_snapshot_id: str) -> ObservationFrame: + return self._cache.get(input_snapshot_id) + + def latest(self) -> ObservationFrame | None: + return self._cache.latest() + + def get_window( + self, + input_snapshot_id: str, + window_size: int, + ) -> ObservationWindow: + return self._cache.get_window(input_snapshot_id, window_size) + + def build_window_summary( + self, + input_snapshot_id: str, + window_size: int, + ) -> dict[str, Any]: + window = self.get_window(input_snapshot_id, window_size) + return { + "anchor_snapshot_id": window.anchor_snapshot_id, + "window_size": window_size, + "available_frames": len(window.frames), + "frames": [ + { + "input_snapshot_id": frame.input_snapshot_id, + "step_id": frame.step_id, + "time_now_us": frame.time_now_us, + "time_query_us": frame.time_query_us, + "camera_ids": sorted(frame.rendered_images.keys()), + "camera_frame_timestamps_us": dict(frame.camera_frame_timestamps_us), + "active_backend_ids": list(frame.active_backend_ids), + } + for frame in window.frames + ], + } diff --git a/src/runtime/alpasim_runtime/runtime_context.py b/src/runtime/alpasim_runtime/runtime_context.py index 2cd7175a..81ae71c0 100644 --- a/src/runtime/alpasim_runtime/runtime_context.py +++ b/src/runtime/alpasim_runtime/runtime_context.py @@ -4,7 +4,7 @@ from __future__ import annotations import math -from dataclasses import dataclass, replace +from dataclasses import dataclass, is_dataclass, replace from alpasim_grpc.v0.logging_pb2 import RolloutMetadata from alpasim_runtime.address_pool import AddressPool @@ -19,6 +19,7 @@ validate_scenarios, ) from alpasim_utils.artifact import Artifact +from omegaconf import OmegaConf from eval.schema import EvalConfig @@ -176,8 +177,12 @@ async def build_runtime_context( ) config_for_validation = config if not validate_config_scenes: + if is_dataclass(config.user): + user_for_validation = replace(config.user, scenes=[]) + else: + user_for_validation = OmegaConf.merge(config.user, {"scenes": []}) config_for_validation = SimulatorConfig( - user=replace(config.user, scenes=[]), + user=user_for_validation, network=config.network, ) await validate_scenarios(config_for_validation) diff --git a/src/runtime/alpasim_runtime/services/controller_service.py b/src/runtime/alpasim_runtime/services/controller_service.py index 21fa12cd..3290f7f5 100644 --- a/src/runtime/alpasim_runtime/services/controller_service.py +++ b/src/runtime/alpasim_runtime/services/controller_service.py @@ -40,6 +40,7 @@ class PropagatedPosesAtTime: pose_local_to_rig_estimate: Pose # The "software" estimated pose in local frame dynamic_state: DynamicState # The true dynamic state (velocities, accelerations) dynamic_state_estimated: DynamicState # The estimated dynamic state + front_steering_angle_rad: float = 0.0 # True front wheel steering angle class ControllerService(ServiceBase[VDCServiceStub]): @@ -228,6 +229,7 @@ async def run_controller_and_vehicle( pose_local_to_rig_estimate=fallback_pose_local_to_rig, dynamic_state=DynamicState(), dynamic_state_estimated=DynamicState(), + front_steering_angle_rad=0.0, ) ] return self._ensure_intermediates( @@ -274,6 +276,7 @@ async def run_controller_and_vehicle( pose_local_to_rig_estimate=fallback_pose_local_to_rig, dynamic_state=DynamicState(), dynamic_state_estimated=DynamicState(), + front_steering_angle_rad=0.0, ) ] elif response.states: @@ -287,6 +290,7 @@ async def run_controller_and_vehicle( ), dynamic_state=s.dynamic_state, dynamic_state_estimated=s.dynamic_state_estimated, + front_steering_angle_rad=float(s.front_steering_angle_rad), ) for s in response.states ] @@ -300,6 +304,9 @@ async def run_controller_and_vehicle( ), dynamic_state=response.dynamic_state, dynamic_state_estimated=response.dynamic_state_estimated, + front_steering_angle_rad=float( + getattr(response, "front_steering_angle_rad", 0.0) + ), ) ] diff --git a/src/runtime/alpasim_runtime/services/driver_service.py b/src/runtime/alpasim_runtime/services/driver_service.py index 8d8edaca..046a60eb 100644 --- a/src/runtime/alpasim_runtime/services/driver_service.py +++ b/src/runtime/alpasim_runtime/services/driver_service.py @@ -5,7 +5,10 @@ from __future__ import annotations +import base64 +import json import logging +import pickle import random from typing import Optional, Type @@ -38,6 +41,28 @@ from alpasim_utils.types import ImageWithMetadata logger = logging.getLogger(__name__) +_DRIVER_CONTROL_PREFIX = b"ALPASIM_DRIVER_CTRL_V1:" + + +def _encode_driver_control( + renderer_data: Optional[bytes], + next_model: Optional[str], + planner_context: Optional[dict[str, object]], +) -> bytes: + """Encode optional model-switch control in renderer_data envelope.""" + if next_model is None and planner_context is None: + return renderer_data or b"" + + payload: dict[str, object] = {} + if next_model is not None: + payload["next_model"] = next_model + if planner_context is not None: + payload["planner_context"] = planner_context + if renderer_data: + payload["renderer_data_b64"] = base64.b64encode(renderer_data).decode("ascii") + + encoded = json.dumps(payload, separators=(",", ":")).encode("utf-8") + return _DRIVER_CONTROL_PREFIX + encoded class DriverService(ServiceBase[EgodriverServiceStub]): @@ -48,6 +73,28 @@ class DriverService(ServiceBase[EgodriverServiceStub]): submitting sensor observations and receiving driving decisions. """ + def __init__(self, address: str, skip: bool = False): + super().__init__(address=address, skip=skip) + self._next_model_override: str | None = None + self._planner_context_override: dict[str, object] | None = None + self._last_drive_debug_info: dict[str, object] = {} + + def set_next_model_for_next_drive(self, model_type: str | None) -> None: + """Request a model switch for the next drive() call only.""" + self._next_model_override = model_type + + def set_planner_context_for_next_drive( + self, planner_context: dict[str, object] | None + ) -> None: + """Set per-frame planner context payload for the next drive() call only.""" + self._planner_context_override = planner_context + + def consume_last_drive_debug_info(self) -> dict[str, object]: + """Return and clear the latest parsed driver debug payload.""" + payload = dict(self._last_drive_debug_info) + self._last_drive_debug_info = {} + return payload + @property def stub_class(self) -> Type[EgodriverServiceStub]: return EgodriverServiceStub @@ -213,11 +260,19 @@ async def drive( """ session_info = self._require_session_info() # Create request with both old and new fields for backward compatibility + packed_renderer_data = _encode_driver_control( + renderer_data=renderer_data, + next_model=self._next_model_override, + planner_context=self._planner_context_override, + ) + self._next_model_override = None + self._planner_context_override = None + request = DriveRequest( session_uuid=session_info.uuid, time_now_us=time_now_us, time_query_us=time_query_us, - renderer_data=renderer_data or b"", + renderer_data=packed_renderer_data, ) await session_info.broadcaster.broadcast(LogEntry(driver_request=request)) @@ -257,7 +312,21 @@ async def drive( response = await profiled_rpc_call( "drive", "driver", self.stub.drive, request ) + self._last_drive_debug_info = _decode_drive_debug_info(response) await session_info.broadcaster.broadcast(LogEntry(driver_return=response)) return trajectory_from_grpc(response.trajectory) + + +def _decode_drive_debug_info(response: DriveResponse) -> dict[str, object]: + debug_info = getattr(response, "debug_info", None) + raw = getattr(debug_info, "unstructured_debug_info", b"") + if not raw: + return {} + try: + payload = pickle.loads(raw) + except Exception: + logger.warning("Failed to decode driver debug info payload") + return {} + return payload if isinstance(payload, dict) else {} diff --git a/src/runtime/alpasim_runtime/simulate/__main__.py b/src/runtime/alpasim_runtime/simulate/__main__.py index ac8649b5..3c902f37 100644 --- a/src/runtime/alpasim_runtime/simulate/__main__.py +++ b/src/runtime/alpasim_runtime/simulate/__main__.py @@ -43,9 +43,13 @@ def get_run_name(log_dir: str) -> str: run_metadata_path = os.path.join(log_dir, "run_metadata.yaml") + if not os.path.exists(run_metadata_path): + return Path(log_dir).name or "alpasim_run" with open(run_metadata_path, "r") as f: run_metadata = yaml.safe_load(f) - return run_metadata.get("run_name") + if not isinstance(run_metadata, dict): + return Path(log_dir).name or "alpasim_run" + return run_metadata.get("run_name") or Path(log_dir).name or "alpasim_run" def create_arg_parser() -> argparse.ArgumentParser: diff --git a/src/runtime/alpasim_runtime/unbound_rollout.py b/src/runtime/alpasim_runtime/unbound_rollout.py index 8166f7a9..8adf114e 100644 --- a/src/runtime/alpasim_runtime/unbound_rollout.py +++ b/src/runtime/alpasim_runtime/unbound_rollout.py @@ -14,6 +14,7 @@ import numpy as np from alpasim_grpc.v0.logging_pb2 import RolloutMetadata from alpasim_runtime.config import ( + DriverBackendConfig, PhysicsUpdateMode, RouteGeneratorType, RuntimeCameraConfig, @@ -89,6 +90,9 @@ class UnboundRollout: planner_delay_us: int route_generator_type: RouteGeneratorType send_recording_ground_truth: bool + driver_backends: list[DriverBackendConfig] + observation_cache_size: int + observation_window_summary_size: int nre_runid: str nre_version: str nre_uuid: str @@ -235,6 +239,9 @@ def create( pose_reporting_interval_us=simulation_config.pose_reporting_interval_us, route_generator_type=simulation_config.route_generator_type, send_recording_ground_truth=simulation_config.send_recording_ground_truth, + driver_backends=list(simulation_config.driver_backends), + observation_cache_size=simulation_config.observation_cache_size, + observation_window_summary_size=simulation_config.observation_window_summary_size, vehicle_config=vehicle, vector_map=artifact.map, hidden_traffic_objs=hidden_traffic_objs, diff --git a/src/runtime/alpasim_runtime/web_debugger/__init__.py b/src/runtime/alpasim_runtime/web_debugger/__init__.py new file mode 100644 index 00000000..71c50a33 --- /dev/null +++ b/src/runtime/alpasim_runtime/web_debugger/__init__.py @@ -0,0 +1 @@ +"""Simple web debugger for interactive runtime development.""" diff --git a/src/runtime/alpasim_runtime/web_debugger/__main__.py b/src/runtime/alpasim_runtime/web_debugger/__main__.py new file mode 100644 index 00000000..6bbb55d3 --- /dev/null +++ b/src/runtime/alpasim_runtime/web_debugger/__main__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 + +from .server import main + + +if __name__ == "__main__": + main() diff --git a/src/runtime/alpasim_runtime/web_debugger/map_provider.py b/src/runtime/alpasim_runtime/web_debugger/map_provider.py new file mode 100644 index 00000000..a7335d8d --- /dev/null +++ b/src/runtime/alpasim_runtime/web_debugger/map_provider.py @@ -0,0 +1,568 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +from __future__ import annotations + +import math +import threading +import xml.etree.ElementTree as ET +import zipfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + +from alpasim_utils.artifact import Artifact + +try: + from trajdata import maps +except ImportError: # pragma: no cover - handled at runtime based on env + maps = None + + +@dataclass(frozen=True) +class SceneMapPayload: + scene_id: str + bounds: dict[str, float] + layers: dict[str, list[list[list[float]]]] + source: str = "empty" + + def as_dict(self) -> dict[str, Any]: + return { + "scene_id": self.scene_id, + "bounds": self.bounds, + "layers": self.layers, + "source": self.source, + } + + +def default_usdz_glob() -> str: + repo_root = Path(__file__).resolve().parents[4] + return str(repo_root / "data/nre-artifacts/all-usdzs/**/*.usdz") + + +class SceneMapProvider: + def __init__(self, usdz_glob: str): + self._usdz_glob = usdz_glob + self._lock = threading.Lock() + self._artifacts: dict[str, Artifact] | None = None + self._payloads: dict[str, SceneMapPayload] = {} + + def list_scene_ids(self) -> list[str]: + with self._lock: + if self._artifacts is None: + self._artifacts = Artifact.discover_from_glob(self._usdz_glob) + return sorted(self._artifacts.keys()) + + def get_scene_map(self, scene_id: str) -> SceneMapPayload: + with self._lock: + if scene_id in self._payloads: + return self._payloads[scene_id] + + if self._artifacts is None: + self._artifacts = Artifact.discover_from_glob(self._usdz_glob) + + if scene_id not in self._artifacts: + raise KeyError(f"Unknown scene_id: {scene_id}") + + payload = self._build_payload(self._artifacts[scene_id]) + self._payloads[scene_id] = payload + return payload + + def _build_payload(self, artifact: Artifact) -> SceneMapPayload: + if maps is not None and artifact.map is not None: + return _build_payload_from_vec_map(artifact) + + xodr_payload = self._build_payload_from_xodr(artifact) + if xodr_payload is not None: + return xodr_payload + + return SceneMapPayload( + scene_id=artifact.scene_id, + bounds={"min_x": -50.0, "max_x": 50.0, "min_y": -50.0, "max_y": 50.0}, + layers={}, + source="empty", + ) + + def _build_payload_from_xodr(self, artifact: Artifact) -> SceneMapPayload | None: + try: + with zipfile.ZipFile(artifact.source, "r") as zip_file: + if "map.xodr" not in zip_file.namelist(): + return None + + xodr_xml = zip_file.read("map.xodr").decode("utf-8") + root = ET.fromstring(xodr_xml) + transform = self._load_xodr_transform(artifact, zip_file, xodr_xml) + except (zipfile.BadZipFile, ET.ParseError): + return None + + layers: dict[str, list[list[list[float]]]] = { + "road_lane_center": [], + "road_lane_left_edge": [], + "road_lane_right_edge": [], + "road_edge": [], + "stop_line": [], + "other_line": [], + } + bounds = { + "min_x": float("inf"), + "max_x": float("-inf"), + "min_y": float("inf"), + "max_y": float("-inf"), + } + + def _append(layer_name: str, points: list[list[float]]) -> None: + normalized = _normalize_points(points) + if len(normalized) < 2: + return + layers[layer_name].append(normalized) + for x, y in normalized: + bounds["min_x"] = min(bounds["min_x"], x) + bounds["max_x"] = max(bounds["max_x"], x) + bounds["min_y"] = min(bounds["min_y"], y) + bounds["max_y"] = max(bounds["max_y"], y) + + for road in root.findall("road"): + road_layers = _build_xodr_road_layers(road, transform) + for layer_name, polylines in road_layers.items(): + for polyline in polylines: + _append(layer_name, polyline) + + if not np.isfinite(list(bounds.values())).all(): + return None + + pad = 8.0 + return SceneMapPayload( + scene_id=artifact.scene_id, + bounds={ + "min_x": bounds["min_x"] - pad, + "max_x": bounds["max_x"] + pad, + "min_y": bounds["min_y"] - pad, + "max_y": bounds["max_y"] + pad, + }, + layers={k: v for k, v in layers.items() if v}, + source="xodr", + ) + + @staticmethod + def _load_xodr_transform( + artifact: Artifact, + zip_file: zipfile.ZipFile, + xodr_xml: str, + ) -> np.ndarray | None: + try: + return artifact._get_xodr_transform(zip_file, xodr_xml) # noqa: SLF001 + except Exception: + return None + + +def _build_payload_from_vec_map(artifact: Artifact) -> SceneMapPayload: + assert maps is not None + assert artifact.map is not None + + vec_map = artifact.map + layers: dict[str, list[list[list[float]]]] = { + "road_lane_center": [], + "road_lane_left_edge": [], + "road_lane_right_edge": [], + "road_edge": [], + "stop_line": [], + "other_line": [], + } + bounds = { + "min_x": float("inf"), + "max_x": float("-inf"), + "min_y": float("inf"), + "max_y": float("-inf"), + } + + def _append(name: str, xy: Any) -> None: + points = _normalize_points(xy) + if len(points) < 2: + return + layers[name].append(points) + for x, y in points: + bounds["min_x"] = min(bounds["min_x"], x) + bounds["max_x"] = max(bounds["max_x"], x) + bounds["min_y"] = min(bounds["min_y"], y) + bounds["max_y"] = max(bounds["max_y"], y) + + road_lanes = vec_map.elements[maps.vec_map_elements.MapElementType.ROAD_LANE] + road_edges = vec_map.elements[maps.vec_map_elements.MapElementType.ROAD_EDGE] + wait_lines = vec_map.elements[maps.vec_map_elements.MapElementType.WAIT_LINE] + + for element in road_lanes.values(): + _append("road_lane_center", element.center.xy) + if element.left_edge is not None: + _append("road_lane_left_edge", element.left_edge.xy) + if element.right_edge is not None: + _append("road_lane_right_edge", element.right_edge.xy) + + for element in road_edges.values(): + _append("road_edge", element.polyline.xy) + + for element in wait_lines.values(): + layer_name = "stop_line" if element.wait_line_type == "STOP" else "other_line" + _append(layer_name, element.polyline.xy) + + if not np.isfinite(list(bounds.values())).all(): + bounds = {"min_x": -50.0, "max_x": 50.0, "min_y": -50.0, "max_y": 50.0} + else: + pad = 8.0 + bounds = { + "min_x": bounds["min_x"] - pad, + "max_x": bounds["max_x"] + pad, + "min_y": bounds["min_y"] - pad, + "max_y": bounds["max_y"] + pad, + } + + return SceneMapPayload( + scene_id=artifact.scene_id, + bounds=bounds, + layers={k: v for k, v in layers.items() if v}, + source="xodr-vector-map", + ) + + +@dataclass(frozen=True) +class _PlanGeometry: + s: float + x: float + y: float + hdg: float + length: float + kind: str + curvature: float = 0.0 + + def sample(self, s_abs: float) -> tuple[float, float, float]: + ds = min(max(s_abs - self.s, 0.0), self.length) + if self.kind == "line": + x = self.x + math.cos(self.hdg) * ds + y = self.y + math.sin(self.hdg) * ds + return (x, y, self.hdg) + + if self.kind == "arc": + if abs(self.curvature) < 1e-9: + x = self.x + math.cos(self.hdg) * ds + y = self.y + math.sin(self.hdg) * ds + return (x, y, self.hdg) + + radius = 1.0 / self.curvature + center_x = self.x - math.sin(self.hdg) * radius + center_y = self.y + math.cos(self.hdg) * radius + start_angle = math.atan2(self.y - center_y, self.x - center_x) + angle = start_angle + ds * self.curvature + x = center_x + math.cos(angle) * radius + y = center_y + math.sin(angle) * radius + return (x, y, self.hdg + ds * self.curvature) + + raise ValueError(f"Unsupported geometry kind: {self.kind}") + + +@dataclass(frozen=True) +class _Poly3: + s: float + a: float + b: float + c: float + d: float + + def value_at(self, s_abs: float) -> float: + ds = max(s_abs - self.s, 0.0) + return self.a + self.b * ds + self.c * ds * ds + self.d * ds * ds * ds + + +@dataclass(frozen=True) +class _LaneDef: + lane_id: int + lane_type: str + widths: list[_Poly3] + + def width_at(self, s_section: float, section_s: float) -> float: + if not self.widths: + return 0.0 + active = self.widths[0] + s_local = max(s_section - section_s, 0.0) + for width in self.widths: + if width.s <= s_local: + active = width + else: + break + return max(active.value_at(s_local), 0.0) + + +def _build_xodr_road_layers( + road: ET.Element, + transform: np.ndarray | None, +) -> dict[str, list[list[list[float]]]]: + plan_view = road.find("planView") + lanes_elem = road.find("lanes") + if plan_view is None or lanes_elem is None: + return {} + + geometries = _parse_plan_geometries(plan_view) + if not geometries: + return {} + + lane_offsets = _parse_poly3_nodes(lanes_elem.findall("laneOffset")) + lane_sections = lanes_elem.findall("laneSection") + if not lane_sections: + return {} + + road_length = float(road.attrib.get("length", "0") or 0.0) + layers: dict[str, list[list[list[float]]]] = { + "road_lane_center": [], + "road_lane_left_edge": [], + "road_lane_right_edge": [], + "road_edge": [], + } + + for index, lane_section in enumerate(lane_sections): + section_s = float(lane_section.attrib.get("s", "0") or 0.0) + next_s = ( + float(lane_sections[index + 1].attrib.get("s", "0") or 0.0) + if index + 1 < len(lane_sections) + else road_length + ) + if next_s <= section_s: + continue + + sample_s = _sample_range(section_s, next_s) + ref_samples = [_sample_reference(geometries, s_abs) for s_abs in sample_s] + lane_offset_values = [_eval_poly3(lane_offsets, s_abs) for s_abs in sample_s] + + left_lanes = _parse_lane_group(lane_section.find("left"), reverse=False) + right_lanes = _parse_lane_group(lane_section.find("right"), reverse=True) + + road_left, road_right = [], [] + outer_left_offsets = lane_offset_values.copy() + outer_right_offsets = lane_offset_values.copy() + + for lane in left_lanes: + center, left_edge, right_edge = _build_lane_polylines( + lane=lane, + section_s=section_s, + sample_s=sample_s, + ref_samples=ref_samples, + inner_offsets=outer_left_offsets, + side=1.0, + transform=transform, + ) + if center: + layers["road_lane_center"].append(center) + layers["road_lane_left_edge"].append(left_edge) + layers["road_lane_right_edge"].append(right_edge) + outer_left_offsets = [ + inner + lane.width_at(s_abs, section_s) + for inner, s_abs in zip(outer_left_offsets, sample_s, strict=True) + ] + road_left = [ + _offset_point(sample, offset, transform) + for sample, offset in zip(ref_samples, outer_left_offsets, strict=True) + ] + + for lane in right_lanes: + center, left_edge, right_edge = _build_lane_polylines( + lane=lane, + section_s=section_s, + sample_s=sample_s, + ref_samples=ref_samples, + inner_offsets=outer_right_offsets, + side=-1.0, + transform=transform, + ) + if center: + layers["road_lane_center"].append(center) + layers["road_lane_left_edge"].append(left_edge) + layers["road_lane_right_edge"].append(right_edge) + outer_right_offsets = [ + inner - lane.width_at(s_abs, section_s) + for inner, s_abs in zip(outer_right_offsets, sample_s, strict=True) + ] + road_right = [ + _offset_point(sample, offset, transform) + for sample, offset in zip(ref_samples, outer_right_offsets, strict=True) + ] + + if len(road_left) >= 2: + layers["road_edge"].append(road_left) + if len(road_right) >= 2: + layers["road_edge"].append(road_right) + + return layers + + +def _parse_plan_geometries(plan_view: ET.Element) -> list[_PlanGeometry]: + geometries: list[_PlanGeometry] = [] + for geometry in plan_view.findall("geometry"): + children = list(geometry) + if not children: + continue + child = children[0] + if child.tag not in {"line", "arc"}: + continue + geometries.append( + _PlanGeometry( + s=float(geometry.attrib.get("s", "0") or 0.0), + x=float(geometry.attrib.get("x", "0") or 0.0), + y=float(geometry.attrib.get("y", "0") or 0.0), + hdg=float(geometry.attrib.get("hdg", "0") or 0.0), + length=float(geometry.attrib.get("length", "0") or 0.0), + kind=child.tag, + curvature=float(child.attrib.get("curvature", "0") or 0.0), + ) + ) + geometries.sort(key=lambda item: item.s) + return geometries + + +def _parse_poly3_nodes(nodes: list[ET.Element]) -> list[_Poly3]: + entries = [ + _Poly3( + s=float(node.attrib.get("s", "0") or 0.0), + a=float(node.attrib.get("a", "0") or 0.0), + b=float(node.attrib.get("b", "0") or 0.0), + c=float(node.attrib.get("c", "0") or 0.0), + d=float(node.attrib.get("d", "0") or 0.0), + ) + for node in nodes + ] + entries.sort(key=lambda item: item.s) + return entries + + +def _parse_lane_group(group: ET.Element | None, reverse: bool) -> list[_LaneDef]: + if group is None: + return [] + lanes = [] + for lane in group.findall("lane"): + widths = [ + _Poly3( + s=float(width.attrib.get("sOffset", "0") or 0.0), + a=float(width.attrib.get("a", "0") or 0.0), + b=float(width.attrib.get("b", "0") or 0.0), + c=float(width.attrib.get("c", "0") or 0.0), + d=float(width.attrib.get("d", "0") or 0.0), + ) + for width in lane.findall("width") + ] + widths.sort(key=lambda item: item.s) + lanes.append( + _LaneDef( + lane_id=int(lane.attrib.get("id", "0") or 0), + lane_type=lane.attrib.get("type", "none"), + widths=widths, + ) + ) + + lanes.sort(key=lambda item: item.lane_id, reverse=reverse) + return [lane for lane in lanes if lane.lane_id != 0] + + +def _sample_range(start_s: float, end_s: float) -> list[float]: + length = max(end_s - start_s, 0.0) + if length <= 0.0: + return [start_s] + target_step = 2.0 + num_points = max(12, min(96, int(math.ceil(length / target_step)) + 1)) + return np.linspace(start_s, end_s, num_points).tolist() + + +def _sample_reference( + geometries: list[_PlanGeometry], + s_abs: float, +) -> tuple[float, float, float]: + active = geometries[0] + for geometry in geometries: + if geometry.s <= s_abs: + active = geometry + else: + break + return active.sample(s_abs) + + +def _eval_poly3(entries: list[_Poly3], s_abs: float) -> float: + if not entries: + return 0.0 + active = entries[0] + for entry in entries: + if entry.s <= s_abs: + active = entry + else: + break + return active.value_at(s_abs) + + +def _build_lane_polylines( + lane: _LaneDef, + section_s: float, + sample_s: list[float], + ref_samples: list[tuple[float, float, float]], + inner_offsets: list[float], + side: float, + transform: np.ndarray | None, +) -> tuple[list[list[float]], list[list[float]], list[list[float]]]: + centers: list[list[float]] = [] + left_edge: list[list[float]] = [] + right_edge: list[list[float]] = [] + + for s_abs, ref_sample, inner_offset in zip( + sample_s, ref_samples, inner_offsets, strict=True + ): + width = lane.width_at(s_abs, section_s) + if width <= 1e-3: + continue + if side > 0: + right_offset = inner_offset + left_offset = inner_offset + width + else: + left_offset = inner_offset + right_offset = inner_offset - width + center_offset = 0.5 * (left_offset + right_offset) + centers.append(_offset_point(ref_sample, center_offset, transform)) + left_edge.append(_offset_point(ref_sample, left_offset, transform)) + right_edge.append(_offset_point(ref_sample, right_offset, transform)) + + return centers, left_edge, right_edge + + +def _offset_point( + ref_sample: tuple[float, float, float], + lateral_offset: float, + transform: np.ndarray | None, +) -> list[float]: + x, y, hdg = ref_sample + point = np.array( + [ + x - math.sin(hdg) * lateral_offset, + y + math.cos(hdg) * lateral_offset, + 0.0, + 1.0, + ], + dtype=np.float64, + ) + if transform is not None: + point = transform @ point + return [float(point[0]), float(point[1])] + + +def _normalize_points(xy: Any) -> list[list[float]]: + arr = np.asarray(xy, dtype=np.float64) + if arr.ndim != 2: + return [] + if arr.shape[0] == 2 and arr.shape[1] != 2: + arr = arr.T + if arr.shape[1] > 2: + arr = arr[:, :2] + if arr.shape[1] != 2: + return [] + + original = arr.copy() + if len(arr) > 160: + stride = max(1, len(arr) // 160) + arr = arr[::stride] + if not np.array_equal(arr[-1], original[-1]): + arr = np.vstack([arr, original[-1]]) + + return [[round(float(point[0]), 3), round(float(point[1]), 3)] for point in arr] diff --git a/src/runtime/alpasim_runtime/web_debugger/server.py b/src/runtime/alpasim_runtime/web_debugger/server.py new file mode 100644 index 00000000..e00cf1e9 --- /dev/null +++ b/src/runtime/alpasim_runtime/web_debugger/server.py @@ -0,0 +1,1418 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +from __future__ import annotations + +import argparse +import json +import math +import threading +import uuid +from collections.abc import Iterable +from dataclasses import dataclass, field +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +import grpc +import yaml +from alpasim_grpc.v1 import interactive_runtime_pb2, interactive_runtime_pb2_grpc + +from .map_provider import SceneMapProvider, default_usdz_glob + + +@dataclass(frozen=True) +class SensorView: + sensor_id: str + logical_id: str + nominal_width: int + nominal_height: int + frame_encoding: str = "svg" + + +@dataclass +class ActorView: + actor_id: str + x: float + y: float + heading_deg: float = 0.0 + speed_mps: float = 0.0 + yaw_rate_rps: float = 0.0 + front_steering_angle_rad: float = 0.0 + + +@dataclass(frozen=True) +class CandidateView: + candidate_id: str + backend_id: str + status: str + selected: bool + error: str = "" + diagnostics: dict[str, object] = field(default_factory=dict) + + +@dataclass(frozen=True) +class DecisionView: + step_id: int + input_snapshot_id: str + selected_candidate_id: str | None + candidates: list[CandidateView] + arbitration_reason: str = "" + + +@dataclass(frozen=True) +class CandidatePlanView: + candidate_id: str + backend_id: str + selected: bool + points: list[dict[str, float]] + + +@dataclass(frozen=True) +class CheckpointView: + checkpoint_id: str + tick_id: int + sim_time_us: int + status: str = "PAUSED" + restore_supported: bool = True + unsupported_backend_ids: list[str] = field(default_factory=list) + + +@dataclass +class SessionSnapshotView: + interactive_session_id: str + tick_id: int + sim_time_us: int + ego: ActorView + actors: list[ActorView] + frame_refs: list[dict[str, object]] + latest_decision: DecisionView | None = None + ego_history: list[dict[str, float]] = field(default_factory=list) + selected_plan: list[dict[str, float]] = field(default_factory=list) + candidate_plans: list[CandidatePlanView] = field(default_factory=list) + context_diagnostics: dict[str, object] = field(default_factory=dict) + + +@dataclass +class SessionStateView: + interactive_session_id: str + rollout_uuid: str + scene_id: str + status: str + current_tick_id: int + current_sim_time_us: int + latest_snapshot: SessionSnapshotView | None + latest_decision: DecisionView | None = None + active_backend_ids: list[str] = field(default_factory=list) + available_backend_ids: list[str] = field(default_factory=list) + error: str = "" + + +class InteractiveApiAdapter: + """HTTP gateway abstraction over the future interactive runtime API.""" + + def create_session(self, scene_id: str) -> SessionStateView: + raise NotImplementedError + + def list_sessions(self) -> list[SessionStateView]: + raise NotImplementedError + + def get_state(self, session_id: str) -> SessionStateView: + raise NotImplementedError + + def list_sensors(self, session_id: str) -> list[SensorView]: + raise NotImplementedError + + def list_candidates(self, session_id: str) -> list[CandidateView]: + raise NotImplementedError + + def list_checkpoints(self, session_id: str) -> list[CheckpointView]: + raise NotImplementedError + + def list_all_sessions(self) -> list[str]: + raise NotImplementedError + + def step_session(self, session_id: str, num_steps: int) -> SessionStateView: + raise NotImplementedError + + def start_session(self, session_id: str) -> SessionStateView: + raise NotImplementedError + + def pause_session(self, session_id: str) -> SessionStateView: + raise NotImplementedError + + def resume_session(self, session_id: str) -> SessionStateView: + raise NotImplementedError + + def set_active_backends(self, session_id: str, backend_ids: list[str]) -> SessionStateView: + raise NotImplementedError + + def recompute_candidate(self, session_id: str, backend_id: str) -> SessionStateView: + raise NotImplementedError + + def select_candidate(self, session_id: str, candidate_id: str) -> SessionStateView: + raise NotImplementedError + + def restore_checkpoint(self, session_id: str, checkpoint_id: str) -> SessionStateView: + raise NotImplementedError + + def get_frame_payload( + self, + session_id: str, + sensor_id: str, + tick_id: int, + ) -> tuple[bytes, str]: + raise NotImplementedError + + def close(self) -> None: + return None + + +def _yaw_deg_from_pose(pose) -> float: + quat = pose.quat + norm_sq = ( + float(quat.w) ** 2 + + float(quat.x) ** 2 + + float(quat.y) ** 2 + + float(quat.z) ** 2 + ) + if not math.isfinite(norm_sq) or norm_sq < 1.0e-12: + return 0.0 + siny_cosp = 2.0 * (quat.w * quat.z + quat.x * quat.y) + cosy_cosp = 1.0 - 2.0 * (quat.y * quat.y + quat.z * quat.z) + return math.degrees(math.atan2(siny_cosp, cosy_cosp)) + + +def _speed_mps_from_dynamics(dynamics) -> float: + velocity = dynamics.linear_velocity + return math.sqrt( + float(velocity.x) ** 2 + float(velocity.y) ** 2 + float(velocity.z) ** 2 + ) + + +def _decode_json_field(payload: str) -> dict[str, object]: + if not payload: + return {} + try: + decoded = json.loads(payload) + except Exception: + return {} + return decoded if isinstance(decoded, dict) else {} + + +@dataclass +class _MockSession: + scene_id: str + interactive_session_id: str + rollout_uuid: str + status: str = "CREATED" + current_tick_id: int = 0 + current_sim_time_us: int = 0 + sensors: list[SensorView] = field(default_factory=list) + ego_x: float = 0.0 + ego_y: float = 0.0 + latest_snapshot: SessionSnapshotView | None = None + active_backend_ids: list[str] = field(default_factory=lambda: ["vla_default"]) + available_backend_ids: list[str] = field( + default_factory=lambda: ["vla_default", "vla_shadow"] + ) + checkpoints: list[CheckpointView] = field(default_factory=list) + + +class MockInteractiveApiAdapter(InteractiveApiAdapter): + """In-memory adapter used until the real interactive runtime is wired in.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._sessions: dict[str, _MockSession] = {} + + def create_session(self, scene_id: str) -> SessionStateView: + with self._lock: + session_id = uuid.uuid4().hex[:12] + sensors = [ + SensorView( + sensor_id="camera_front_wide_120fov", + logical_id="camera_front_wide_120fov", + nominal_width=1280, + nominal_height=720, + ), + SensorView( + sensor_id="camera_front_left_120fov", + logical_id="camera_front_left_120fov", + nominal_width=1280, + nominal_height=720, + ), + SensorView( + sensor_id="camera_front_right_120fov", + logical_id="camera_front_right_120fov", + nominal_width=1280, + nominal_height=720, + ), + ] + session = _MockSession( + scene_id=scene_id, + interactive_session_id=session_id, + rollout_uuid=f"mock-{session_id}", + sensors=sensors, + ) + session.latest_snapshot = self._build_snapshot(session) + self._record_checkpoint(session) + self._sessions[session_id] = session + return self._build_state(session) + + def list_sessions(self) -> list[SessionStateView]: + with self._lock: + return [self._build_state(session) for session in self._sessions.values()] + + def get_state(self, session_id: str) -> SessionStateView: + with self._lock: + return self._build_state(self._require_session(session_id)) + + def list_sensors(self, session_id: str) -> list[SensorView]: + with self._lock: + return list(self._require_session(session_id).sensors) + + def list_candidates(self, session_id: str) -> list[CandidateView]: + with self._lock: + session = self._require_session(session_id) + decision = self._current_decision(session) + return list(decision.candidates if decision is not None else []) + + def list_checkpoints(self, session_id: str) -> list[CheckpointView]: + with self._lock: + session = self._require_session(session_id) + return list(session.checkpoints) + + def list_all_sessions(self) -> list[str]: + with self._lock: + return list(self._sessions.keys()) + + def step_session(self, session_id: str, num_steps: int) -> SessionStateView: + with self._lock: + session = self._require_session(session_id) + steps = max(num_steps, 1) + session.status = "PAUSED" + for _ in range(steps): + session.current_tick_id += 1 + session.current_sim_time_us += 100_000 + session.ego_x += 2.5 + session.ego_y += 0.45 + session.latest_snapshot = self._build_snapshot(session) + self._record_checkpoint(session) + return self._build_state(session) + + def start_session(self, session_id: str) -> SessionStateView: + with self._lock: + session = self._require_session(session_id) + session.status = "RUNNING" + return self._build_state(session) + + def pause_session(self, session_id: str) -> SessionStateView: + with self._lock: + session = self._require_session(session_id) + session.status = "PAUSED" + return self._build_state(session) + + def resume_session(self, session_id: str) -> SessionStateView: + with self._lock: + session = self._require_session(session_id) + session.status = "RUNNING" + session.current_tick_id += 1 + session.current_sim_time_us += 100_000 + session.ego_x += 2.0 + session.ego_y += 0.25 + session.latest_snapshot = self._build_snapshot(session) + self._record_checkpoint(session) + return self._build_state(session) + + def set_active_backends(self, session_id: str, backend_ids: list[str]) -> SessionStateView: + with self._lock: + session = self._require_session(session_id) + selected = list(backend_ids) if backend_ids else list(session.available_backend_ids) + unknown = [backend_id for backend_id in selected if backend_id not in session.available_backend_ids] + if unknown: + raise KeyError(f"Unknown backend ids: {unknown}") + session.active_backend_ids = selected + session.latest_snapshot = self._build_snapshot(session) + self._record_checkpoint(session) + return self._build_state(session) + + def recompute_candidate(self, session_id: str, backend_id: str) -> SessionStateView: + with self._lock: + session = self._require_session(session_id) + if backend_id not in session.available_backend_ids: + raise KeyError(f"Unknown backend_id: {backend_id}") + session.status = "PAUSED" + decision = self._current_decision(session) + if decision is None: + raise RuntimeError("No committed decision bundle available for recompute") + updated = [] + recompute_suffix = 1 + for candidate in decision.candidates: + if candidate.backend_id == backend_id and not candidate.selected: + updated.append(CandidateView( + candidate_id=candidate.candidate_id, + backend_id=candidate.backend_id, + status="STALE", + selected=False, + error=candidate.error, + )) + recompute_suffix += 1 + else: + updated.append(candidate) + updated.append( + CandidateView( + candidate_id=f"{backend_id}:step{decision.step_id}:r{recompute_suffix}", + backend_id=backend_id, + status="READY", + selected=False, + ) + ) + new_decision = DecisionView( + step_id=decision.step_id, + input_snapshot_id=decision.input_snapshot_id, + selected_candidate_id=decision.selected_candidate_id, + candidates=updated, + arbitration_reason="recomputed", + ) + session.latest_snapshot = self._replace_snapshot_decision(session, new_decision) + self._record_checkpoint(session) + return self._build_state(session) + + def select_candidate(self, session_id: str, candidate_id: str) -> SessionStateView: + with self._lock: + session = self._require_session(session_id) + session.status = "PAUSED" + decision = self._current_decision(session) + if decision is None: + raise RuntimeError("No committed decision bundle available for selection") + found = False + updated = [] + for candidate in decision.candidates: + if candidate.candidate_id == candidate_id: + found = True + updated.append(CandidateView( + candidate_id=candidate.candidate_id, + backend_id=candidate.backend_id, + status="SELECTED", + selected=True, + error=candidate.error, + )) + elif candidate.selected: + updated.append(CandidateView( + candidate_id=candidate.candidate_id, + backend_id=candidate.backend_id, + status="REJECTED", + selected=False, + error=candidate.error, + )) + else: + updated.append(candidate) + if not found: + raise KeyError(f"Unknown candidate_id: {candidate_id}") + new_decision = DecisionView( + step_id=decision.step_id, + input_snapshot_id=decision.input_snapshot_id, + selected_candidate_id=candidate_id, + candidates=updated, + arbitration_reason="manual_selection", + ) + session.latest_snapshot = self._replace_snapshot_decision(session, new_decision) + self._record_checkpoint(session) + return self._build_state(session) + + def restore_checkpoint(self, session_id: str, checkpoint_id: str) -> SessionStateView: + with self._lock: + session = self._require_session(session_id) + checkpoint = next( + (item for item in session.checkpoints if item.checkpoint_id == checkpoint_id), + None, + ) + if checkpoint is None: + raise KeyError(f"Unknown checkpoint_id: {checkpoint_id}") + session.status = "PAUSED" + session.current_tick_id = checkpoint.tick_id + session.current_sim_time_us = checkpoint.sim_time_us + session.ego_x = checkpoint.tick_id * 2.5 + session.ego_y = checkpoint.tick_id * 0.45 + session.latest_snapshot = self._build_snapshot(session) + return self._build_state(session) + + def get_frame_payload( + self, + session_id: str, + sensor_id: str, + tick_id: int, + ) -> tuple[bytes, str]: + with self._lock: + session = self._require_session(session_id) + self._require_sensor(session.sensors, sensor_id) + status = session.status + sim_seconds = tick_id * 0.1 + color = { + "RUNNING": "#2d8f5b", + "PAUSED": "#b17b17", + "CREATED": "#4d6aa5", + "FAILED": "#9f2f2f", + }.get(status, "#4d6aa5") + svg = f""" + + + + + + + + + + + + {sensor_id} + Tick {tick_id} + Session {session_id} + Scene {session.scene_id} + Status {status} + Sim time {sim_seconds:.1f}s +""" + return (svg.encode("utf-8"), "image/svg+xml; charset=utf-8") + + def _build_snapshot(self, session: _MockSession) -> SessionSnapshotView: + ego = ActorView( + actor_id="EGO", + x=session.ego_x, + y=session.ego_y, + heading_deg=10.0, + speed_mps=5.6, + yaw_rate_rps=0.08, + front_steering_angle_rad=0.12, + ) + actors = [ + ActorView( + actor_id="veh_a", + x=session.ego_x + 16.0, + y=session.ego_y + 2.5, + heading_deg=0.0, + speed_mps=4.2, + ), + ActorView( + actor_id="veh_b", + x=session.ego_x - 12.5, + y=session.ego_y - 3.0, + heading_deg=180.0, + speed_mps=1.5, + ), + ActorView( + actor_id="ped_a", + x=session.ego_x + 5.0, + y=session.ego_y + 9.0, + heading_deg=-90.0, + speed_mps=0.9, + ), + ] + frame_refs = [ + { + "sensor_id": sensor.sensor_id, + "tick_id": session.current_tick_id, + "frame_start_us": max(session.current_sim_time_us - 33_000, 0), + "frame_end_us": session.current_sim_time_us, + "frame_encoding": sensor.frame_encoding, + } + for sensor in session.sensors + ] + return SessionSnapshotView( + interactive_session_id=session.interactive_session_id, + tick_id=session.current_tick_id, + sim_time_us=session.current_sim_time_us, + ego=ego, + actors=actors, + frame_refs=frame_refs, + latest_decision=self._build_decision(session), + ego_history=[ + {"x": session.ego_x - 12.0, "y": session.ego_y}, + {"x": session.ego_x - 8.0, "y": session.ego_y + 0.3}, + {"x": session.ego_x - 4.0, "y": session.ego_y + 0.4}, + {"x": session.ego_x, "y": session.ego_y}, + ], + selected_plan=[ + {"x": session.ego_x, "y": session.ego_y}, + {"x": session.ego_x + 8.0, "y": session.ego_y + 0.8}, + {"x": session.ego_x + 16.0, "y": session.ego_y + 1.2}, + {"x": session.ego_x + 24.0, "y": session.ego_y + 1.4}, + ], + candidate_plans=[ + CandidatePlanView( + candidate_id=f"vla_default:step{session.current_tick_id}:r0", + backend_id="vla_default", + selected=True, + points=[ + {"x": session.ego_x, "y": session.ego_y}, + {"x": session.ego_x + 8.0, "y": session.ego_y + 0.8}, + {"x": session.ego_x + 16.0, "y": session.ego_y + 1.2}, + ], + ) + ], + context_diagnostics={ + "timing": { + "policy_tick_us": session.current_sim_time_us, + "ego_age_ms": 0.0, + "actor_age_ms": 0.0, + "route_age_ms": 0.0, + "map_age_ms": 0.0, + "camera_age_ms": 0.0, + }, + "quality": { + "route_available": True, + "nearby_lane_count": 3, + "actor_count": len(actors), + "wait_line_count": 1, + "crosswalk_count": 1, + "stale_flags": {}, + }, + }, + ) + + def _build_state(self, session: _MockSession) -> SessionStateView: + latest_decision = self._current_decision(session) + return SessionStateView( + interactive_session_id=session.interactive_session_id, + rollout_uuid=session.rollout_uuid, + scene_id=session.scene_id, + status=session.status, + current_tick_id=session.current_tick_id, + current_sim_time_us=session.current_sim_time_us, + latest_snapshot=session.latest_snapshot, + latest_decision=latest_decision, + active_backend_ids=list(session.active_backend_ids), + available_backend_ids=list(session.available_backend_ids), + ) + + def _build_decision(self, session: _MockSession) -> DecisionView: + selected_backend_id = session.active_backend_ids[0] if session.active_backend_ids else session.available_backend_ids[0] + candidates = [] + for index, backend_id in enumerate(session.active_backend_ids or session.available_backend_ids): + is_selected = backend_id == selected_backend_id + status = "SELECTED" if is_selected else "READY" + candidates.append( + CandidateView( + candidate_id=f"{backend_id}:step{session.current_tick_id}:r0", + backend_id=backend_id, + status=status, + selected=is_selected, + diagnostics={ + "driver_debug": { + "selected_model_type": backend_id, + "fallback_reason": None if is_selected else "shadow_candidate", + "proposal_count": 2, + "route_available": True, + "nearby_lane_count": 3, + "actor_count": 4, + } + }, + ) + ) + if not candidates: + candidates = [ + CandidateView( + candidate_id=f"vla_default:step{session.current_tick_id}:r0", + backend_id="vla_default", + status="SELECTED", + selected=True, + diagnostics={"driver_debug": {"selected_model_type": "vla_default"}}, + ) + ] + selected_backend_id = "vla_default" + return DecisionView( + step_id=session.current_tick_id, + input_snapshot_id=f"mock-step-{session.current_tick_id}", + selected_candidate_id=f"{selected_backend_id}:step{session.current_tick_id}:r0", + candidates=candidates, + arbitration_reason="priority_default", + ) + + def _current_decision(self, session: _MockSession) -> DecisionView | None: + return session.latest_snapshot.latest_decision if session.latest_snapshot is not None else None + + def _replace_snapshot_decision(self, session: _MockSession, decision: DecisionView) -> SessionSnapshotView: + snapshot = session.latest_snapshot or self._build_snapshot(session) + return SessionSnapshotView( + interactive_session_id=snapshot.interactive_session_id, + tick_id=snapshot.tick_id, + sim_time_us=snapshot.sim_time_us, + ego=snapshot.ego, + actors=snapshot.actors, + frame_refs=snapshot.frame_refs, + latest_decision=decision, + ego_history=snapshot.ego_history, + selected_plan=snapshot.selected_plan, + candidate_plans=snapshot.candidate_plans, + context_diagnostics=snapshot.context_diagnostics, + ) + + def _record_checkpoint(self, session: _MockSession) -> None: + checkpoint = CheckpointView( + checkpoint_id=f"ckpt-{session.current_tick_id:04d}-{len(session.checkpoints)}", + tick_id=session.current_tick_id, + sim_time_us=session.current_sim_time_us, + status="PAUSED" if session.status != "FAILED" else "FAILED", + restore_supported=True, + unsupported_backend_ids=[], + ) + session.checkpoints = [item for item in session.checkpoints if item.tick_id != checkpoint.tick_id] + session.checkpoints.append(checkpoint) + session.checkpoints = session.checkpoints[-16:] + + def _require_session(self, session_id: str) -> _MockSession: + if session_id not in self._sessions: + raise KeyError(f"Unknown session_id: {session_id}") + return self._sessions[session_id] + + @staticmethod + def _require_sensor(sensors: Iterable[SensorView], sensor_id: str) -> SensorView: + for sensor in sensors: + if sensor.sensor_id == sensor_id: + return sensor + raise KeyError(f"Unknown sensor_id: {sensor_id}") + + +class DebuggerRequestHandler(BaseHTTPRequestHandler): + server: "InteractiveDebuggerServer" + + def do_GET(self) -> None: # noqa: N802 + parsed = urlparse(self.path) + if parsed.path == "/": + return self._serve_static("index.html", "text/html; charset=utf-8") + if parsed.path == "/decision": + return self._serve_static("decision.html", "text/html; charset=utf-8") + if parsed.path == "/assets/styles.css": + return self._serve_static("styles.css", "text/css; charset=utf-8") + if parsed.path == "/assets/app.js": + return self._serve_static("app.js", "application/javascript; charset=utf-8") + if parsed.path == "/assets/decision.js": + return self._serve_static("decision.js", "application/javascript; charset=utf-8") + if parsed.path == "/api/scenes": + return self._handle_list_scenes() + if parsed.path == "/api/sessions": + return self._handle_list_sessions() + if parsed.path == "/api/session/state": + return self._handle_get_state(parsed) + if parsed.path == "/api/sensors": + return self._handle_list_sensors(parsed) + if parsed.path == "/api/candidates": + return self._handle_list_candidates(parsed) + if parsed.path == "/api/checkpoints": + return self._handle_list_checkpoints(parsed) + if parsed.path == "/api/sessions": + return self._handle_list_all_sessions() + if parsed.path == "/api/map": + return self._handle_get_map(parsed) + if parsed.path == "/api/frame": + return self._handle_get_frame(parsed) + self.send_error(HTTPStatus.NOT_FOUND) + + def do_POST(self) -> None: # noqa: N802 + parsed = urlparse(self.path) + if parsed.path == "/api/session/create": + return self._handle_create_session() + if parsed.path == "/api/session/start": + return self._handle_mutation(parsed.path) + if parsed.path == "/api/session/pause": + return self._handle_mutation(parsed.path) + if parsed.path == "/api/session/resume": + return self._handle_mutation(parsed.path) + if parsed.path == "/api/session/step": + return self._handle_mutation(parsed.path) + if parsed.path == "/api/backends/active": + return self._handle_mutation(parsed.path) + if parsed.path == "/api/candidates/recompute": + return self._handle_mutation(parsed.path) + if parsed.path == "/api/candidates/select": + return self._handle_mutation(parsed.path) + if parsed.path == "/api/checkpoints/restore": + return self._handle_mutation(parsed.path) + self.send_error(HTTPStatus.NOT_FOUND) + + def log_message(self, format: str, *args: object) -> None: # noqa: A003 + del format, args + + def _handle_create_session(self) -> None: + try: + payload = self._read_json_body() + scene_id = str(payload.get("scene_id") or "clipgt-demo") + state = self.server.adapter.create_session(scene_id=scene_id) + self._write_json(self._state_to_dict(state), status=HTTPStatus.CREATED) + except Exception as exc: # noqa: BLE001 + self._write_exception(exc) + + def _handle_get_state(self, parsed) -> None: + try: + params = parse_qs(parsed.query) + session_id = self._require_query_param(params, "session_id") + state = self.server.adapter.get_state(session_id) + self._write_json(self._state_to_dict(state)) + except Exception as exc: # noqa: BLE001 + self._write_exception(exc) + + def _handle_list_sessions(self) -> None: + try: + sessions = self.server.adapter.list_sessions() + self._write_json( + {"sessions": [self._session_summary_to_dict(item) for item in sessions]} + ) + except Exception as exc: # noqa: BLE001 + self._write_exception(exc) + + def _handle_list_scenes(self) -> None: + try: + scene_ids = list(self.server.scene_ids) + self._write_json( + { + "scenes": [ + { + "scene_id": scene_id, + "label": scene_id, + } + for scene_id in scene_ids + ] + } + ) + except Exception as exc: # noqa: BLE001 + self._write_exception(exc) + + def _handle_list_sensors(self, parsed) -> None: + try: + params = parse_qs(parsed.query) + session_id = self._require_query_param(params, "session_id") + sensors = self.server.adapter.list_sensors(session_id) + self._write_json({"sensors": [sensor.__dict__ for sensor in sensors]}) + except Exception as exc: # noqa: BLE001 + self._write_exception(exc) + + def _handle_list_candidates(self, parsed) -> None: + try: + params = parse_qs(parsed.query) + session_id = self._require_query_param(params, "session_id") + candidates = self.server.adapter.list_candidates(session_id) + self._write_json({"candidates": [self._candidate_to_dict(item) for item in candidates]}) + except Exception as exc: # noqa: BLE001 + self._write_exception(exc) + + def _handle_list_checkpoints(self, parsed) -> None: + try: + params = parse_qs(parsed.query) + session_id = self._require_query_param(params, "session_id") + checkpoints = self.server.adapter.list_checkpoints(session_id) + self._write_json({"checkpoints": [self._checkpoint_to_dict(item) for item in checkpoints]}) + except Exception as exc: # noqa: BLE001 + self._write_exception(exc) + + def _handle_list_all_sessions(self) -> None: + try: + session_ids = self.server.adapter.list_all_sessions() + self._write_json({"session_ids": session_ids}) + except Exception as exc: # noqa: BLE001 + self._write_exception(exc) + + def _handle_get_map(self, parsed) -> None: + try: + params = parse_qs(parsed.query) + scene_id = self._require_query_param(params, "scene_id") + self._write_json(self.server.map_provider.get_scene_map(scene_id).as_dict()) + except Exception as exc: # noqa: BLE001 + self._write_exception(exc) + + def _handle_get_frame(self, parsed) -> None: + try: + params = parse_qs(parsed.query) + session_id = self._require_query_param(params, "session_id") + sensor_id = self._require_query_param(params, "sensor_id") + tick_id = int(self._require_query_param(params, "tick_id")) + body, content_type = self.server.adapter.get_frame_payload( + session_id, sensor_id, tick_id + ) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store, max-age=0") + self.send_header("Pragma", "no-cache") + self.send_header("Expires", "0") + self.end_headers() + self.wfile.write(body) + except Exception as exc: # noqa: BLE001 + self._write_exception(exc) + + def _handle_mutation(self, path: str) -> None: + try: + payload = self._read_json_body() + session_id = str(payload["session_id"]) + if path.endswith("/start"): + state = self.server.adapter.start_session(session_id) + elif path.endswith("/pause"): + state = self.server.adapter.pause_session(session_id) + elif path.endswith("/resume"): + state = self.server.adapter.resume_session(session_id) + elif path.endswith("/step"): + state = self.server.adapter.step_session( + session_id=session_id, + num_steps=int(payload.get("num_steps", 1)), + ) + elif path.endswith("/active"): + backend_ids = payload.get("backend_ids") or [] + state = self.server.adapter.set_active_backends( + session_id=session_id, + backend_ids=[str(item) for item in backend_ids], + ) + elif path.endswith("/recompute"): + state = self.server.adapter.recompute_candidate( + session_id=session_id, + backend_id=str(payload["backend_id"]), + ) + elif path.endswith("/select"): + state = self.server.adapter.select_candidate( + session_id=session_id, + candidate_id=str(payload["candidate_id"]), + ) + elif path.endswith("/restore"): + state = self.server.adapter.restore_checkpoint( + session_id=session_id, + checkpoint_id=str(payload["checkpoint_id"]), + ) + else: + raise AssertionError(f"Unhandled mutation path: {path}") + self._write_json(self._state_to_dict(state)) + except Exception as exc: # noqa: BLE001 + self._write_exception(exc) + + def _serve_static(self, filename: str, content_type: str) -> None: + static_root = Path(__file__).with_name("static") + path = static_root / filename + body = path.read_bytes() + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store, max-age=0") + self.send_header("Pragma", "no-cache") + self.send_header("Expires", "0") + self.end_headers() + self.wfile.write(body) + + def _read_json_body(self) -> dict[str, object]: + length = int(self.headers.get("Content-Length", "0")) + if length == 0: + return {} + raw = self.rfile.read(length) + return json.loads(raw.decode("utf-8")) + + def _write_json(self, payload: dict[str, object], status: HTTPStatus = HTTPStatus.OK) -> None: + body = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store, max-age=0") + self.send_header("Pragma", "no-cache") + self.send_header("Expires", "0") + self.end_headers() + self.wfile.write(body) + + def _write_exception(self, exc: Exception) -> None: + status = HTTPStatus.INTERNAL_SERVER_ERROR + payload: dict[str, object] = {"error": str(exc)} + if isinstance(exc, grpc.RpcError): + payload["grpc_status"] = exc.code().name if exc.code() is not None else None + payload["grpc_details"] = exc.details() + if exc.code() == grpc.StatusCode.NOT_FOUND: + status = HTTPStatus.NOT_FOUND + elif exc.code() == grpc.StatusCode.RESOURCE_EXHAUSTED: + status = HTTPStatus.TOO_MANY_REQUESTS + elif exc.code() == grpc.StatusCode.INVALID_ARGUMENT: + status = HTTPStatus.BAD_REQUEST + elif exc.code() == grpc.StatusCode.FAILED_PRECONDITION: + status = HTTPStatus.PRECONDITION_FAILED + elif isinstance(exc, KeyError): + status = HTTPStatus.BAD_REQUEST + elif isinstance(exc, RuntimeError): + status = HTTPStatus.PRECONDITION_FAILED + self._write_json(payload, status=status) + + @staticmethod + def _require_query_param(params: dict[str, list[str]], key: str) -> str: + values = params.get(key) + if not values: + raise KeyError(f"Missing query parameter: {key}") + return values[0] + + def _state_to_dict(self, state: SessionStateView) -> dict[str, object]: + latest_snapshot = self._snapshot_to_dict(state.latest_snapshot) + return { + "interactive_session_id": state.interactive_session_id, + "rollout_uuid": state.rollout_uuid, + "scene_id": state.scene_id, + "status": state.status, + "current_tick_id": state.current_tick_id, + "current_sim_time_us": state.current_sim_time_us, + "latest_snapshot": latest_snapshot, + "latest_decision": self._decision_to_dict(state.latest_decision), + "active_backend_ids": state.active_backend_ids, + "available_backend_ids": state.available_backend_ids, + "error": state.error, + } + + def _snapshot_to_dict(self, snapshot: SessionSnapshotView | None) -> dict[str, object] | None: + if snapshot is None: + return None + return { + "interactive_session_id": snapshot.interactive_session_id, + "tick_id": snapshot.tick_id, + "sim_time_us": snapshot.sim_time_us, + "ego": snapshot.ego.__dict__, + "actors": [actor.__dict__ for actor in snapshot.actors], + "frame_refs": snapshot.frame_refs, + "latest_decision": self._decision_to_dict(snapshot.latest_decision), + "ego_history": snapshot.ego_history, + "selected_plan": snapshot.selected_plan, + "candidate_plans": [ + { + "candidate_id": item.candidate_id, + "backend_id": item.backend_id, + "selected": item.selected, + "points": item.points, + } + for item in snapshot.candidate_plans + ], + "context_diagnostics": snapshot.context_diagnostics, + } + + @staticmethod + def _candidate_to_dict(candidate: CandidateView) -> dict[str, object]: + return { + "candidate_id": candidate.candidate_id, + "backend_id": candidate.backend_id, + "status": candidate.status, + "selected": candidate.selected, + "error": candidate.error, + "diagnostics": candidate.diagnostics, + } + + def _decision_to_dict(self, decision: DecisionView | None) -> dict[str, object] | None: + if decision is None: + return None + return { + "step_id": decision.step_id, + "input_snapshot_id": decision.input_snapshot_id, + "selected_candidate_id": decision.selected_candidate_id, + "arbitration_reason": decision.arbitration_reason, + "candidates": [self._candidate_to_dict(item) for item in decision.candidates], + } + + @staticmethod + def _checkpoint_to_dict(checkpoint: CheckpointView) -> dict[str, object]: + return { + "checkpoint_id": checkpoint.checkpoint_id, + "tick_id": checkpoint.tick_id, + "sim_time_us": checkpoint.sim_time_us, + "status": checkpoint.status, + "restore_supported": checkpoint.restore_supported, + "unsupported_backend_ids": checkpoint.unsupported_backend_ids, + } + + @staticmethod + def _session_summary_to_dict(state: SessionStateView) -> dict[str, object]: + return { + "interactive_session_id": state.interactive_session_id, + "rollout_uuid": state.rollout_uuid, + "scene_id": state.scene_id, + "status": state.status, + "current_tick_id": state.current_tick_id, + "current_sim_time_us": state.current_sim_time_us, + } + + +class InteractiveDebuggerServer(ThreadingHTTPServer): + def __init__( + self, + server_address: tuple[str, int], + adapter: InteractiveApiAdapter, + map_provider: SceneMapProvider, + scene_ids: list[str] | None = None, + ): + super().__init__(server_address, DebuggerRequestHandler) + self.adapter = adapter + self.map_provider = map_provider + self.scene_ids = scene_ids or [] + + +class GrpcInteractiveApiAdapter(InteractiveApiAdapter): + """HTTP gateway adapter backed by the runtime interactive gRPC service.""" + + def __init__(self, runtime_address: str) -> None: + self._channel = grpc.insecure_channel(runtime_address) + self._stub = interactive_runtime_pb2_grpc.InteractiveRuntimeServiceStub( + self._channel + ) + + def create_session(self, scene_id: str) -> SessionStateView: + response = self._stub.CreateSession( + interactive_runtime_pb2.CreateSessionRequest( + scene_id=scene_id, + start_paused=True, + max_retained_ticks=64, + ) + ) + return self._state_from_proto(response.initial_state) + + def list_sessions(self) -> list[SessionStateView]: + response = self._stub.ListSessions(interactive_runtime_pb2.ListSessionsRequest()) + return [self._state_from_proto(item) for item in response.sessions] + + def get_state(self, session_id: str) -> SessionStateView: + response = self._stub.GetSessionState( + interactive_runtime_pb2.GetSessionStateRequest( + interactive_session_id=session_id + ) + ) + return self._state_from_proto(response) + + def list_sensors(self, session_id: str) -> list[SensorView]: + response = self._stub.ListSensors( + interactive_runtime_pb2.ListSensorsRequest( + interactive_session_id=session_id + ) + ) + return [self._sensor_from_proto(sensor) for sensor in response.sensors] + + def list_candidates(self, session_id: str) -> list[CandidateView]: + response = self._stub.ListCandidates( + interactive_runtime_pb2.ListCandidatesRequest( + interactive_session_id=session_id + ) + ) + return [self._candidate_from_proto(item) for item in response.candidates] + + def list_checkpoints(self, session_id: str) -> list[CheckpointView]: + response = self._stub.ListCheckpoints( + interactive_runtime_pb2.ListCheckpointsRequest( + interactive_session_id=session_id + ) + ) + return [self._checkpoint_from_proto(item) for item in response.checkpoints] + + def list_all_sessions(self) -> list[str]: + # gRPC backend may not support listing all sessions yet + return [] + + def step_session(self, session_id: str, num_steps: int) -> SessionStateView: + response = self._stub.StepSession( + interactive_runtime_pb2.StepSessionRequest( + interactive_session_id=session_id, + num_steps=max(int(num_steps), 1), + ) + ) + return self._state_from_proto(response.state) + + def start_session(self, session_id: str) -> SessionStateView: + response = self._stub.StartSession( + interactive_runtime_pb2.StartSessionRequest( + interactive_session_id=session_id + ) + ) + return self._state_from_proto(response) + + def pause_session(self, session_id: str) -> SessionStateView: + response = self._stub.PauseSession( + interactive_runtime_pb2.PauseSessionRequest( + interactive_session_id=session_id + ) + ) + return self._state_from_proto(response) + + def resume_session(self, session_id: str) -> SessionStateView: + response = self._stub.ResumeSession( + interactive_runtime_pb2.ResumeSessionRequest( + interactive_session_id=session_id + ) + ) + return self._state_from_proto(response) + + def set_active_backends(self, session_id: str, backend_ids: list[str]) -> SessionStateView: + response = self._stub.SetActiveBackends( + interactive_runtime_pb2.SetActiveBackendsRequest( + interactive_session_id=session_id, + backend_ids=backend_ids, + ) + ) + return self._state_from_proto(response) + + def recompute_candidate(self, session_id: str, backend_id: str) -> SessionStateView: + response = self._stub.RecomputeCandidate( + interactive_runtime_pb2.RecomputeCandidateRequest( + interactive_session_id=session_id, + backend_id=backend_id, + ) + ) + return self._state_from_proto(response) + + def select_candidate(self, session_id: str, candidate_id: str) -> SessionStateView: + response = self._stub.SelectCandidate( + interactive_runtime_pb2.SelectCandidateRequest( + interactive_session_id=session_id, + candidate_id=candidate_id, + ) + ) + return self._state_from_proto(response) + + def restore_checkpoint(self, session_id: str, checkpoint_id: str) -> SessionStateView: + response = self._stub.RestoreCheckpoint( + interactive_runtime_pb2.RestoreCheckpointRequest( + interactive_session_id=session_id, + checkpoint_id=checkpoint_id, + ) + ) + return self._state_from_proto(response) + + def get_frame_payload( + self, + session_id: str, + sensor_id: str, + tick_id: int, + ) -> tuple[bytes, str]: + response = self._stub.GetFrame( + interactive_runtime_pb2.GetFrameRequest( + interactive_session_id=session_id, + sensor_id=sensor_id, + tick_id=tick_id, + ) + ) + content_type = "image/jpeg" + if ( + response.frame_ref.frame_encoding + == interactive_runtime_pb2.FRAME_ENCODING_PNG + ): + content_type = "image/png" + return (bytes(response.image_bytes), content_type) + + def close(self) -> None: + self._channel.close() + + @staticmethod + def _sensor_from_proto(sensor) -> SensorView: + return SensorView( + sensor_id=sensor.sensor_id, + logical_id=sensor.logical_id, + nominal_width=sensor.nominal_width, + nominal_height=sensor.nominal_height, + frame_encoding=interactive_runtime_pb2.FrameEncoding.Name( + sensor.frame_encoding + ).removeprefix("FRAME_ENCODING_").lower(), + ) + + @staticmethod + def _candidate_from_proto(candidate) -> CandidateView: + return CandidateView( + candidate_id=candidate.candidate_id, + backend_id=candidate.backend_id, + status=candidate.status, + selected=bool(candidate.selected), + error=candidate.error, + diagnostics=_decode_json_field(getattr(candidate, "diagnostics_json", "")), + ) + + @classmethod + def _decision_from_proto(cls, decision) -> DecisionView: + return DecisionView( + step_id=int(decision.step_id), + input_snapshot_id=decision.input_snapshot_id, + selected_candidate_id=decision.selected_candidate_id or None, + candidates=[cls._candidate_from_proto(item) for item in decision.candidates], + arbitration_reason=decision.arbitration_reason, + ) + + @staticmethod + def _checkpoint_from_proto(checkpoint) -> CheckpointView: + return CheckpointView( + checkpoint_id=checkpoint.checkpoint_id, + tick_id=int(checkpoint.tick_id), + sim_time_us=int(checkpoint.sim_time_us), + status=interactive_runtime_pb2.SessionStatus.Name(checkpoint.status).removeprefix( + "SESSION_STATUS_" + ), + restore_supported=bool(checkpoint.restore_supported), + unsupported_backend_ids=list(checkpoint.unsupported_backend_ids), + ) + + @classmethod + def _state_from_proto(cls, state) -> SessionStateView: + latest_snapshot = ( + cls._snapshot_from_proto(state.latest_snapshot) + if state.HasField("latest_snapshot") + else None + ) + latest_decision = ( + cls._decision_from_proto(state.latest_decision) + if hasattr(state, "HasField") and state.HasField("latest_decision") + else None + ) + return SessionStateView( + interactive_session_id=state.interactive_session_id, + rollout_uuid=state.rollout_uuid, + scene_id=state.scene_id, + status=interactive_runtime_pb2.SessionStatus.Name(state.status).removeprefix( + "SESSION_STATUS_" + ), + current_tick_id=int(state.current_tick_id), + current_sim_time_us=int(state.current_sim_time_us), + latest_snapshot=latest_snapshot, + latest_decision=latest_decision, + active_backend_ids=list(getattr(state, "active_backend_ids", [])), + available_backend_ids=list(getattr(state, "available_backend_ids", [])), + error=state.error, + ) + + @classmethod + def _snapshot_from_proto(cls, snapshot) -> SessionSnapshotView: + latest_decision = ( + cls._decision_from_proto(snapshot.latest_decision) + if hasattr(snapshot, "HasField") and snapshot.HasField("latest_decision") + else None + ) + return SessionSnapshotView( + interactive_session_id=snapshot.interactive_session_id, + tick_id=int(snapshot.tick_id), + sim_time_us=int(snapshot.sim_time_us), + ego=ActorView( + actor_id="EGO", + x=float(snapshot.ego.pose.vec.x), + y=float(snapshot.ego.pose.vec.y), + heading_deg=_yaw_deg_from_pose(snapshot.ego.pose), + speed_mps=_speed_mps_from_dynamics(snapshot.ego.dynamics), + yaw_rate_rps=float(snapshot.ego.dynamics.angular_velocity.z), + front_steering_angle_rad=float( + getattr(snapshot.ego, "front_steering_angle_rad", 0.0) + ), + ), + actors=[ + ActorView( + actor_id=actor.actor_id, + x=float(actor.pose.vec.x), + y=float(actor.pose.vec.y), + heading_deg=_yaw_deg_from_pose(actor.pose), + ) + for actor in snapshot.actors + ], + frame_refs=[ + { + "sensor_id": frame_ref.sensor_id, + "tick_id": int(frame_ref.tick_id), + "frame_start_us": int(frame_ref.frame_start_us), + "frame_end_us": int(frame_ref.frame_end_us), + "frame_encoding": interactive_runtime_pb2.FrameEncoding.Name( + frame_ref.frame_encoding + ).removeprefix("FRAME_ENCODING_"), + } + for frame_ref in snapshot.frame_refs + ], + latest_decision=latest_decision, + ego_history=[ + {"x": float(point.x), "y": float(point.y)} + for point in getattr(snapshot, "ego_history", []) + ], + selected_plan=[ + {"x": float(point.x), "y": float(point.y)} + for point in getattr(snapshot, "selected_plan", []) + ], + candidate_plans=[ + CandidatePlanView( + candidate_id=item.candidate_id, + backend_id=item.backend_id, + selected=bool(item.selected), + points=[ + {"x": float(point.x), "y": float(point.y)} + for point in item.points + ], + ) + for item in getattr(snapshot, "candidate_plans", []) + ], + context_diagnostics=_decode_json_field( + getattr(snapshot, "context_diagnostics_json", "") + ), + ) + + +def create_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Run the interactive web debugger") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8080) + parser.add_argument( + "--runtime-address", + default=None, + help="Interactive runtime gRPC address, for example 127.0.0.1:50051", + ) + parser.add_argument( + "--user-config", + default=None, + help="Optional runtime user config YAML used to supplement discovered scenes", + ) + parser.add_argument( + "--usdz-glob", + default=default_usdz_glob(), + help="Glob used to resolve scene artifacts for map rendering", + ) + return parser + + +def _load_scene_ids_from_user_config(user_config_path: str | None) -> list[str]: + if not user_config_path: + return [] + with open(user_config_path, "r", encoding="utf-8") as handle: + payload = yaml.safe_load(handle) or {} + scenes = payload.get("scenes") or [] + scene_ids: list[str] = [] + for item in scenes: + if not isinstance(item, dict): + continue + scene_id = item.get("scene_id") + if scene_id: + scene_ids.append(str(scene_id)) + return sorted(set(scene_ids)) + + +def _load_available_scene_ids( + map_provider: SceneMapProvider, + user_config_path: str | None, +) -> list[str]: + scene_ids: set[str] = set() + try: + scene_ids.update(map_provider.list_scene_ids()) + except Exception: + # Keep the debugger usable even if artifact discovery is temporarily unavailable. + pass + scene_ids.update(_load_scene_ids_from_user_config(user_config_path)) + return sorted(scene_ids) + + +def main() -> None: + args = create_arg_parser().parse_args() + adapter: InteractiveApiAdapter + if args.runtime_address: + adapter = GrpcInteractiveApiAdapter(args.runtime_address) + else: + adapter = MockInteractiveApiAdapter() + map_provider = SceneMapProvider(args.usdz_glob) + scene_ids = _load_available_scene_ids(map_provider, args.user_config) + server = InteractiveDebuggerServer( + (args.host, args.port), + adapter, + map_provider, + scene_ids=scene_ids, + ) + print(f"Interactive debugger listening on http://{args.host}:{args.port}") + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() + adapter.close() + + +if __name__ == "__main__": + main() diff --git a/src/runtime/alpasim_runtime/web_debugger/static/app.js b/src/runtime/alpasim_runtime/web_debugger/static/app.js new file mode 100644 index 00000000..d037cdf2 --- /dev/null +++ b/src/runtime/alpasim_runtime/web_debugger/static/app.js @@ -0,0 +1,1192 @@ +/** + * Alpasim Standard Console - Core Logic + */ + +const STORAGE_KEYS = { + sessionId: "alpasim.web_debugger.session_id", +}; + +const state = { + sessionId: null, + sessionState: null, + availableScenes: [], + availableSessions: [], + sensors: [], + selectedSensorId: null, + sceneMap: null, + snapshotHistory: [], + viewedTickId: null, + refreshInFlight: false, + mapView: { + zoom: 1, + panX: 0, + panY: 0, + rotation: 0, + hasCustomView: false, + isDragging: false, + dragMode: null, + lastPointerX: 0, + lastPointerY: 0, + }, + sensorOverlay: { + offsetX: 0, + offsetY: 0, + dragging: false, + lastPointerX: 0, + lastPointerY: 0, + }, +}; + +const MAP_STYLE = { + background: "#f5f7fb", + roadEdgeOutline: "#ffffff", + roadEdge: "#b8c4d4", + laneBoundaryOutline: "#ffffff", + laneBoundary: "#d5deea", + laneCenterOutline: "#ffffff", + laneCenter: "#8ea2b8", + laneCenterDash: "#6f879f", + stopLineOutline: "#fff7ed", + stopLine: "#f97316", + otherLineOutline: "#f5f3ff", + otherLine: "#a855f7", + egoFill: "#0ea5e9", + egoStroke: "#0369a1", + actorFill: "#f59e0b", + actorStroke: "#b45309", + egoHistory: "#2563eb", + egoHistoryGlow: "rgba(37, 99, 235, 0.18)", + selectedPlan: "#10b981", + selectedPlanGlow: "rgba(16, 185, 129, 0.18)", + backendPlans: { + vam: { stroke: "#10b981", glow: "rgba(16, 185, 129, 0.18)" }, + pdm: { stroke: "#ef4444", glow: "rgba(239, 68, 68, 0.18)" }, + default: { stroke: "#a855f7", glow: "rgba(168, 85, 247, 0.18)" }, + }, + grid: "#e9eef5", +}; + +const CHART_STYLE = { + axis: "#94a3b8", + grid: "#e2e8f0", + speed: "#0ea5e9", + speedFill: "rgba(14, 165, 233, 0.12)", + steering: "#f97316", + steeringFill: "rgba(249, 115, 22, 0.12)", + text: "#475569", + mutedText: "#94a3b8", +}; + +const els = { + errorBanner: document.getElementById("error-banner"), + sceneId: document.getElementById("scene-id"), + createSession: document.getElementById("create-session"), + reconnectSession: document.getElementById("reconnect-session"), + refreshState: document.getElementById("refresh-state"), + playSession: document.getElementById("play-session"), + pauseSession: document.getElementById("pause-session"), + prevFrame: document.getElementById("prev-frame"), + nextFrame: document.getElementById("next-frame"), + liveJump: document.getElementById("live-jump"), + timelineSlider: document.getElementById("timeline-slider"), + playbackMode: document.getElementById("playback-mode"), + timelineCurrent: document.getElementById("timeline-current"), + timelineEnd: document.getElementById("timeline-end"), + sessionStatus: document.getElementById("session-status"), + statusDot: document.getElementById("status-dot"), + decisionPageLink: document.getElementById("decision-page-link"), + sessionTick: document.getElementById("session-tick"), + sessionTime: document.getElementById("session-time"), + egoSpeed: document.getElementById("ego-speed"), + sceneLabel: document.getElementById("scene-label"), + sensorTabs: document.getElementById("sensor-tabs"), + primarySensorImage: document.getElementById("primary-sensor-image"), + sensorOverlay: document.getElementById("sensor-overlay"), + sensorOverlayHeader: document.getElementById("sensor-overlay-header"), + mapCanvas: document.getElementById("map-canvas"), + chartCanvas: document.getElementById("chart-canvas"), + steeringChartCanvas: document.getElementById("steering-chart-canvas"), + speedCurrent: document.getElementById("speed-current"), + steeringCurrent: document.getElementById("steering-current"), + backendSelector: document.getElementById("backend-selector"), + applyBackends: document.getElementById("apply-backends"), + decisionReason: document.getElementById("decision-reason"), + candidateList: document.getElementById("candidate-list"), + checkpointList: document.getElementById("checkpoint-list"), + sessionList: document.getElementById("session-list"), + resetMapView: document.getElementById("reset-map-view"), + mapCompassNeedle: document.getElementById("map-compass-needle"), +}; + +async function readResponse(response) { + if (response.ok) { + return response.json(); + } + const payload = await response.json().catch(() => ({})); + throw new Error(payload.grpc_details || payload.error || `${response.status} ${response.statusText}`); +} + +async function api(path, body = null) { + const response = await fetch( + path, + body + ? { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + } + : undefined, + ); + return readResponse(response); +} + +function showError(message) { + if (!els.errorBanner) return; + els.errorBanner.textContent = message; + els.errorBanner.classList.remove("hidden"); +} + +function clearError() { + if (!els.errorBanner) return; + els.errorBanner.textContent = ""; + els.errorBanner.classList.add("hidden"); +} + +function saveSessionId(sessionId) { + if (!sessionId) return; + window.localStorage.setItem(STORAGE_KEYS.sessionId, sessionId); +} + +function loadSavedSessionId() { + return window.localStorage.getItem(STORAGE_KEYS.sessionId) || ""; +} + +function clearSavedSessionId() { + window.localStorage.removeItem(STORAGE_KEYS.sessionId); +} + +function activeSnapshot() { + if (state.viewedTickId === null) return state.snapshotHistory.at(-1); + return ( + state.snapshotHistory.find((snapshot) => snapshot.tick_id === state.viewedTickId) || + state.snapshotHistory.at(-1) + ); +} + +function formatSimTime(us) { + return `${(us / 1e6).toFixed(2)}s`; +} + +function formatSpeed(mps) { + return `${(mps * 3.6).toFixed(1)} km/h`; +} + +function formatSteering(rad) { + return `${((rad || 0) * 180 / Math.PI).toFixed(2)} deg`; +} + +function renderSceneOptions() { + const currentValue = els.sceneId.value; + els.sceneId.innerHTML = ""; + if (!state.availableScenes.length) { + const option = document.createElement("option"); + option.value = ""; + option.textContent = "无可用场景"; + els.sceneId.appendChild(option); + return; + } + state.availableScenes.forEach((scene) => { + const option = document.createElement("option"); + option.value = scene.scene_id; + option.textContent = scene.label || scene.scene_id; + els.sceneId.appendChild(option); + }); + const preferredSceneId = new URLSearchParams(window.location.search).get("scene_id"); + els.sceneId.value = currentValue || preferredSceneId || state.availableScenes[0].scene_id; +} + +function renderMap(snapshot) { + const canvas = els.mapCanvas; + const ctx = canvas.getContext("2d"); + canvas.width = canvas.offsetWidth; + canvas.height = canvas.offsetHeight; + ctx.clearRect(0, 0, canvas.width, canvas.height); + ctx.fillStyle = MAP_STYLE.background; + ctx.fillRect(0, 0, canvas.width, canvas.height); + if (!snapshot) return; + + const projection = createProjection(canvas, snapshot, state.sceneMap); + drawMapGrid(ctx, canvas, projection); + drawMapLayers(ctx, projection, state.sceneMap?.layers || {}); + drawCandidatePlans(ctx, projection, snapshot.candidate_plans || []); + drawTrajectory( + ctx, + projection, + buildHistoryPath(snapshot), + { + strokeStyle: MAP_STYLE.egoHistory, + glowStyle: MAP_STYLE.egoHistoryGlow, + lineWidth: 3.2, + glowWidth: 8.0, + }, + ); + drawActor(ctx, projection, snapshot.ego, { + fillStyle: MAP_STYLE.egoFill, + strokeStyle: MAP_STYLE.egoStroke, + lengthM: 4.8, + widthM: 2.0, + }); + (snapshot.actors || []).forEach((actor) => + drawActor(ctx, projection, actor, { + fillStyle: MAP_STYLE.actorFill, + strokeStyle: MAP_STYLE.actorStroke, + lengthM: 4.5, + widthM: 1.8, + }), + ); + positionSensorOverlay(); +} + +function createProjection(canvas, snapshot, sceneMap) { + const bounds = sceneMap?.bounds; + const mapView = state.mapView; + if (bounds) { + const width = Math.max(bounds.max_x - bounds.min_x, 1); + const height = Math.max(bounds.max_y - bounds.min_y, 1); + const padding = 24; + const baseScale = Math.min( + (canvas.width - padding * 2) / width, + (canvas.height - padding * 2) / height, + ); + const scale = baseScale * mapView.zoom; + const worldCenterX = (bounds.min_x + bounds.max_x) / 2; + const worldCenterY = (bounds.min_y + bounds.max_y) / 2; + return { + scale, + projectPoint(x, y) { + const rotated = rotateWorldPoint( + x - worldCenterX, + y - worldCenterY, + mapView.rotation, + ); + return { + x: canvas.width / 2 + rotated.x * scale + mapView.panX, + y: canvas.height / 2 - rotated.y * scale + mapView.panY, + }; + }, + }; + } + + const centerX = snapshot.ego?.x ?? 0; + const centerY = snapshot.ego?.y ?? 0; + const scale = 8 * mapView.zoom; + return { + scale, + projectPoint(x, y) { + const rotated = rotateWorldPoint( + x - centerX, + y - centerY, + mapView.rotation, + ); + return { + x: canvas.width / 2 + rotated.x * scale + mapView.panX, + y: canvas.height / 2 - rotated.y * scale + mapView.panY, + }; + }, + }; +} + +function rotateWorldPoint(x, y, rotation) { + const cos = Math.cos(rotation); + const sin = Math.sin(rotation); + return { + x: x * cos - y * sin, + y: x * sin + y * cos, + }; +} + +function drawMapGrid(ctx, canvas, projection) { + const gridSpacingPx = projection.scale * 10; + if (!Number.isFinite(gridSpacingPx) || gridSpacingPx < 24) return; + const spacing = Math.min(gridSpacingPx, 160); + const offsetX = ((state.mapView.panX % spacing) + spacing) % spacing; + const offsetY = ((state.mapView.panY % spacing) + spacing) % spacing; + + ctx.save(); + ctx.strokeStyle = MAP_STYLE.grid; + ctx.lineWidth = 1; + ctx.beginPath(); + for (let x = offsetX; x <= canvas.width; x += spacing) { + ctx.moveTo(x, 0); + ctx.lineTo(x, canvas.height); + } + for (let y = offsetY; y <= canvas.height; y += spacing) { + ctx.moveTo(0, y); + ctx.lineTo(canvas.width, y); + } + ctx.stroke(); + ctx.restore(); +} + +function drawMapLayers(ctx, projection, layers) { + const drawPolylines = ( + polylines, + primaryStrokeStyle, + primaryLineWidth, + dashed = false, + outlineStrokeStyle = null, + outlineLineWidth = null, + ) => { + if (!polylines?.length) return; + polylines.forEach((polyline) => { + if (!polyline?.length) return; + const screenPoints = polyline.map(([x, y]) => projection.projectPoint(x, y)); + const strokePath = () => { + ctx.beginPath(); + screenPoints.forEach((point, index) => { + if (index === 0) { + ctx.moveTo(point.x, point.y); + } else { + ctx.lineTo(point.x, point.y); + } + }); + ctx.stroke(); + }; + + if (outlineStrokeStyle && outlineLineWidth) { + ctx.strokeStyle = outlineStrokeStyle; + ctx.lineWidth = outlineLineWidth; + ctx.setLineDash([]); + strokePath(); + } + + ctx.strokeStyle = primaryStrokeStyle; + ctx.lineWidth = primaryLineWidth; + ctx.setLineDash(dashed ? [10, 8] : []); + strokePath(); + }); + }; + + drawPolylines( + layers.road_edge, + MAP_STYLE.roadEdge, + 2.6, + false, + MAP_STYLE.roadEdgeOutline, + 4.8, + ); + drawPolylines( + layers.road_lane_left_edge, + MAP_STYLE.laneBoundary, + 1.8, + false, + MAP_STYLE.laneBoundaryOutline, + 3.2, + ); + drawPolylines( + layers.road_lane_right_edge, + MAP_STYLE.laneBoundary, + 1.8, + false, + MAP_STYLE.laneBoundaryOutline, + 3.2, + ); + drawPolylines( + layers.road_lane_center, + MAP_STYLE.laneCenterDash, + 1.4, + true, + MAP_STYLE.laneCenterOutline, + 2.8, + ); + drawPolylines( + layers.stop_line, + MAP_STYLE.stopLine, + 3.0, + false, + MAP_STYLE.stopLineOutline, + 5.4, + ); + drawPolylines( + layers.other_line, + MAP_STYLE.otherLine, + 2.0, + false, + MAP_STYLE.otherLineOutline, + 3.8, + ); + ctx.setLineDash([]); +} + +function drawTrajectory(ctx, projection, points, style) { + if (!points?.length || points.length < 2) return; + const screenPoints = points.map((point) => projection.projectPoint(point.x, point.y)); + + const strokePath = () => { + ctx.beginPath(); + screenPoints.forEach((point, index) => { + if (index === 0) { + ctx.moveTo(point.x, point.y); + } else { + ctx.lineTo(point.x, point.y); + } + }); + ctx.stroke(); + }; + + ctx.save(); + if (style.glowStyle && style.glowWidth) { + ctx.strokeStyle = style.glowStyle; + ctx.lineWidth = style.glowWidth; + ctx.lineCap = "round"; + ctx.lineJoin = "round"; + ctx.setLineDash([]); + strokePath(); + } + ctx.strokeStyle = style.strokeStyle; + ctx.lineWidth = style.lineWidth; + ctx.lineCap = "round"; + ctx.lineJoin = "round"; + ctx.setLineDash(style.dashed ? [10, 8] : []); + strokePath(); + ctx.restore(); +} + +function planStyleForBackend(backendId, selected) { + const key = String(backendId || "").toLowerCase(); + const base = + key.includes("pdm") + ? MAP_STYLE.backendPlans.pdm + : key.includes("vam") || key.includes("vla") + ? MAP_STYLE.backendPlans.vam + : MAP_STYLE.backendPlans.default; + return { + strokeStyle: base.stroke, + glowStyle: base.glow, + lineWidth: selected ? 4.2 : 2.6, + glowWidth: selected ? 10 : 6, + dashed: !selected, + }; +} + +function drawCandidatePlans(ctx, projection, candidatePlans) { + candidatePlans.forEach((candidatePlan) => { + drawTrajectory( + ctx, + projection, + candidatePlan.points || [], + planStyleForBackend(candidatePlan.backend_id, candidatePlan.selected), + ); + }); +} + +function drawActor(ctx, projection, actor, style) { + if (!actor) return; + const headingRad = ((actor.heading_deg || 0) * Math.PI) / 180; + const cos = Math.cos(headingRad); + const sin = Math.sin(headingRad); + const halfLength = style.lengthM / 2; + const halfWidth = style.widthM / 2; + const worldCorners = [ + { x: halfLength, y: halfWidth }, + { x: halfLength, y: -halfWidth }, + { x: -halfLength, y: -halfWidth }, + { x: -halfLength, y: halfWidth }, + ].map((point) => ({ + x: actor.x + point.x * cos - point.y * sin, + y: actor.y + point.x * sin + point.y * cos, + })); + const corners = worldCorners.map((point) => projection.projectPoint(point.x, point.y)); + + ctx.beginPath(); + corners.forEach((point, index) => { + if (index === 0) { + ctx.moveTo(point.x, point.y); + } else { + ctx.lineTo(point.x, point.y); + } + }); + ctx.closePath(); + ctx.fillStyle = style.fillStyle; + ctx.strokeStyle = style.strokeStyle; + ctx.lineWidth = 1.5; + ctx.fill(); + ctx.stroke(); +} + +function buildHistoryPath(snapshot) { + if (snapshot?.ego_history?.length) { + return snapshot.ego_history; + } + if (snapshot?.tick_id == null) return []; + return state.snapshotHistory + .filter((item) => item.tick_id <= snapshot.tick_id) + .map((item) => ({ x: item.ego.x, y: item.ego.y })); +} + +function defaultMapRotation(snapshot) { + const egoHeadingRad = (((snapshot?.ego?.heading_deg || 0) * Math.PI) / 180); + return Math.PI / 2 - egoHeadingRad; +} + +function applyDefaultMapView(snapshot) { + state.mapView.zoom = 1; + state.mapView.panX = 0; + state.mapView.panY = 0; + state.mapView.rotation = defaultMapRotation(snapshot); + state.mapView.hasCustomView = false; + state.mapView.isDragging = false; + state.mapView.dragMode = null; +} + +function resetMapView(snapshot = activeSnapshot()) { + applyDefaultMapView(snapshot); + renderAll(); +} + +function attachMapInteractions() { + const canvas = els.mapCanvas; + if (!canvas || canvas.dataset.interactionsAttached === "true") return; + canvas.dataset.interactionsAttached = "true"; + + canvas.addEventListener("wheel", (event) => { + event.preventDefault(); + const zoomFactor = event.deltaY < 0 ? 1.12 : 1 / 1.12; + state.mapView.zoom = Math.min(12, Math.max(0.2, state.mapView.zoom * zoomFactor)); + state.mapView.hasCustomView = true; + renderAll(); + }); + + canvas.addEventListener("mousedown", (event) => { + event.preventDefault(); + state.mapView.isDragging = true; + state.mapView.dragMode = event.shiftKey ? "rotate" : "pan"; + state.mapView.lastPointerX = event.clientX; + state.mapView.lastPointerY = event.clientY; + }); + + window.addEventListener("mousemove", (event) => { + if (!state.mapView.isDragging) return; + const deltaX = event.clientX - state.mapView.lastPointerX; + const deltaY = event.clientY - state.mapView.lastPointerY; + state.mapView.lastPointerX = event.clientX; + state.mapView.lastPointerY = event.clientY; + + if (state.mapView.dragMode === "rotate") { + state.mapView.rotation += deltaX * 0.01; + } else { + state.mapView.panX += deltaX; + state.mapView.panY += deltaY; + } + state.mapView.hasCustomView = true; + renderAll(); + }); + + const endDrag = () => { + state.mapView.isDragging = false; + state.mapView.dragMode = null; + }; + window.addEventListener("mouseup", endDrag); + canvas.addEventListener("mouseleave", endDrag); +} + +function attachSensorOverlayInteractions() { + const header = els.sensorOverlayHeader; + const overlay = els.sensorOverlay; + if (!header || !overlay || header.dataset.interactionsAttached === "true") return; + header.dataset.interactionsAttached = "true"; + + header.addEventListener("mousedown", (event) => { + if (event.target.closest("button")) return; + event.preventDefault(); + state.sensorOverlay.dragging = true; + state.sensorOverlay.lastPointerX = event.clientX; + state.sensorOverlay.lastPointerY = event.clientY; + }); + + window.addEventListener("mousemove", (event) => { + if (!state.sensorOverlay.dragging) return; + const deltaX = event.clientX - state.sensorOverlay.lastPointerX; + const deltaY = event.clientY - state.sensorOverlay.lastPointerY; + state.sensorOverlay.lastPointerX = event.clientX; + state.sensorOverlay.lastPointerY = event.clientY; + state.sensorOverlay.offsetX += deltaX; + state.sensorOverlay.offsetY += deltaY; + positionSensorOverlay(); + }); + + window.addEventListener("mouseup", () => { + state.sensorOverlay.dragging = false; + }); +} + +function positionSensorOverlay() { + const overlay = els.sensorOverlay; + if (!overlay) return; + overlay.style.transform = `translate(${state.sensorOverlay.offsetX}px, ${state.sensorOverlay.offsetY}px)`; +} + +function renderChart() { + const history = state.snapshotHistory.slice(-90); + const snapshot = activeSnapshot(); + if (els.speedCurrent) { + els.speedCurrent.textContent = snapshot ? formatSpeed(snapshot.ego.speed_mps) : "-"; + } + if (els.steeringCurrent) { + els.steeringCurrent.textContent = snapshot + ? formatSteering(snapshot.ego.front_steering_angle_rad) + : "-"; + } + renderLineChart(els.chartCanvas, history, { + yAccessor: (snapshot) => snapshot.ego.speed_mps * 3.6, + yUnit: "km/h", + strokeStyle: CHART_STYLE.speed, + fillStyle: CHART_STYLE.speedFill, + minSpan: 10, + symmetric: false, + }); + renderLineChart(els.steeringChartCanvas, history, { + yAccessor: trueFrontSteeringDeg, + yUnit: "deg", + strokeStyle: CHART_STYLE.steering, + fillStyle: CHART_STYLE.steeringFill, + minSpan: 8, + symmetric: true, + }); +} + +function trueFrontSteeringDeg(snapshot) { + return ((snapshot?.ego?.front_steering_angle_rad || 0) * 180) / Math.PI; +} + +function renderLineChart(canvas, history, style) { + if (!canvas) return; + const rect = canvas.getBoundingClientRect(); + canvas.width = Math.max(Math.floor(rect.width), 240); + canvas.height = Math.max(Math.floor(rect.height), 160); + const ctx = canvas.getContext("2d"); + ctx.clearRect(0, 0, canvas.width, canvas.height); + + const margin = { top: 16, right: 16, bottom: 26, left: 46 }; + const plotWidth = Math.max(canvas.width - margin.left - margin.right, 10); + const plotHeight = Math.max(canvas.height - margin.top - margin.bottom, 10); + + drawChartFrame(ctx, canvas, margin, plotWidth, plotHeight); + if (history.length < 2) { + drawChartEmptyState(ctx, canvas, margin); + return; + } + + const samples = history + .map((snapshot) => ({ + t: snapshot.sim_time_us / 1e6, + v: style.yAccessor(snapshot), + })) + .filter((sample) => Number.isFinite(sample.t) && Number.isFinite(sample.v)); + if (samples.length < 2) { + drawChartEmptyState(ctx, canvas, margin); + return; + } + const times = samples.map((sample) => sample.t); + const values = samples.map((sample) => sample.v); + const timeMin = times[0]; + const timeMax = times[times.length - 1]; + const timeSpan = Math.max(timeMax - timeMin, 1e-6); + + let yMin = Math.min(...values); + let yMax = Math.max(...values); + if (style.symmetric) { + const bound = Math.max(Math.abs(yMin), Math.abs(yMax), style.minSpan / 2); + yMin = -bound; + yMax = bound; + } else { + yMin = Math.min(0, yMin); + yMax = Math.max(yMax, style.minSpan); + if (yMax - yMin < style.minSpan) { + yMax = yMin + style.minSpan; + } + } + const ySpan = Math.max(yMax - yMin, 1e-6); + + const xOf = (timeSec) => margin.left + ((timeSec - timeMin) / timeSpan) * plotWidth; + const yOf = (value) => margin.top + (1 - (value - yMin) / ySpan) * plotHeight; + + drawChartGrid(ctx, margin, plotWidth, plotHeight, yMin, yMax, style); + drawChartAxes(ctx, canvas, margin, plotWidth, plotHeight, yMin, yMax, timeMin, timeMax, style); + drawChartSeries(ctx, samples, xOf, yOf, margin, plotHeight, style); +} + +function drawChartFrame(ctx, canvas, margin, plotWidth, plotHeight) { + ctx.save(); + ctx.fillStyle = "#ffffff"; + ctx.fillRect(0, 0, canvas.width, canvas.height); + ctx.strokeStyle = "#f1f5f9"; + ctx.strokeRect(margin.left, margin.top, plotWidth, plotHeight); + ctx.restore(); +} + +function drawChartEmptyState(ctx, canvas, margin) { + ctx.save(); + ctx.fillStyle = CHART_STYLE.mutedText; + ctx.font = "11px Inter, sans-serif"; + ctx.fillText("至少需要两帧数据", margin.left + 8, canvas.height / 2); + ctx.restore(); +} + +function drawChartGrid(ctx, margin, plotWidth, plotHeight, yMin, yMax, style) { + ctx.save(); + ctx.strokeStyle = CHART_STYLE.grid; + ctx.lineWidth = 1; + for (let i = 0; i <= 4; i += 1) { + const y = margin.top + (i / 4) * plotHeight; + ctx.beginPath(); + ctx.moveTo(margin.left, y); + ctx.lineTo(margin.left + plotWidth, y); + ctx.stroke(); + } + if (style.symmetric && yMin < 0 && yMax > 0) { + const zeroY = margin.top + (1 - (0 - yMin) / (yMax - yMin)) * plotHeight; + ctx.strokeStyle = "#cbd5e1"; + ctx.setLineDash([4, 4]); + ctx.beginPath(); + ctx.moveTo(margin.left, zeroY); + ctx.lineTo(margin.left + plotWidth, zeroY); + ctx.stroke(); + } + ctx.restore(); +} + +function drawChartAxes(ctx, canvas, margin, plotWidth, plotHeight, yMin, yMax, timeMin, timeMax, style) { + ctx.save(); + ctx.fillStyle = CHART_STYLE.text; + ctx.strokeStyle = CHART_STYLE.axis; + ctx.lineWidth = 1; + ctx.font = "10px Inter, sans-serif"; + + ctx.beginPath(); + ctx.moveTo(margin.left, margin.top); + ctx.lineTo(margin.left, margin.top + plotHeight); + ctx.lineTo(margin.left + plotWidth, margin.top + plotHeight); + ctx.stroke(); + + for (let i = 0; i <= 4; i += 1) { + const value = yMax - ((yMax - yMin) * i) / 4; + const y = margin.top + (i / 4) * plotHeight; + ctx.fillText(`${value.toFixed(1)}`, 4, y + 3); + } + + for (let i = 0; i <= 3; i += 1) { + const timeValue = timeMin + ((timeMax - timeMin) * i) / 3; + const x = margin.left + (i / 3) * plotWidth; + ctx.fillText(`${timeValue.toFixed(1)}s`, x - 8, canvas.height - 8); + } + + ctx.fillText(style.yUnit, 6, 12); + ctx.fillText("time", canvas.width - 34, canvas.height - 8); + ctx.restore(); +} + +function drawChartSeries(ctx, samples, xOf, yOf, margin, plotHeight, style) { + const baselineY = margin.top + plotHeight; + + ctx.save(); + ctx.beginPath(); + samples.forEach((sample, index) => { + const x = xOf(sample.t); + const y = yOf(sample.v); + if (index === 0) { + ctx.moveTo(x, y); + } else { + ctx.lineTo(x, y); + } + }); + const lastX = xOf(samples[samples.length - 1].t); + const firstX = xOf(samples[0].t); + ctx.lineTo(lastX, baselineY); + ctx.lineTo(firstX, baselineY); + ctx.closePath(); + ctx.fillStyle = style.fillStyle; + ctx.fill(); + + ctx.beginPath(); + samples.forEach((sample, index) => { + const x = xOf(sample.t); + const y = yOf(sample.v); + if (index === 0) { + ctx.moveTo(x, y); + } else { + ctx.lineTo(x, y); + } + }); + ctx.strokeStyle = style.strokeStyle; + ctx.lineWidth = 2.2; + ctx.lineJoin = "round"; + ctx.lineCap = "round"; + ctx.stroke(); + ctx.restore(); +} + +function renderSensorFrame(snapshot) { + if (!state.sessionId || !snapshot || !state.selectedSensorId) { + els.primarySensorImage.removeAttribute("src"); + } else { + els.primarySensorImage.src = + `/api/frame?session_id=${encodeURIComponent(state.sessionId)}` + + `&sensor_id=${encodeURIComponent(state.selectedSensorId)}` + + `&tick_id=${snapshot.tick_id}`; + } + + els.sensorTabs.innerHTML = ""; + state.sensors.forEach((sensor) => { + const button = document.createElement("button"); + button.className = + "rounded-md px-2 py-1 text-[10px] font-bold " + + (sensor.sensor_id === state.selectedSensorId + ? "bg-brand-50 text-brand-600" + : "bg-slate-100 text-slate-500"); + button.textContent = sensor.sensor_id.split("_")[1] || sensor.sensor_id; + button.onclick = () => { + state.selectedSensorId = sensor.sensor_id; + renderAll(); + }; + els.sensorTabs.appendChild(button); + }); +} + +function renderLists(session, snapshot) { + const decision = snapshot?.latest_decision ?? session?.latest_decision; + els.decisionReason.textContent = decision?.arbitration_reason ?? "等待数据..."; + + if (els.backendSelector) { + const availableBackends = session?.available_backend_ids?.length + ? session.available_backend_ids + : [...new Set((decision?.candidates || []).map((candidate) => candidate.backend_id))]; + const activeBackends = new Set(session?.active_backend_ids || []); + els.backendSelector.innerHTML = ""; + availableBackends.forEach((backendId) => { + const label = document.createElement("label"); + label.className = + "flex items-center justify-between rounded-md border border-slate-200 bg-white px-2 py-1 text-[10px] font-medium text-slate-600"; + label.innerHTML = ` + ${backendId} + + `; + els.backendSelector.appendChild(label); + }); + } + + els.candidateList.innerHTML = ""; + (decision?.candidates || []).forEach((candidate) => { + const item = document.createElement("div"); + item.className = + `p-2 border rounded-lg text-[10px] ${ + candidate.selected ? "border-brand-500 bg-brand-50" : "border-slate-100" + }`; + item.innerHTML = ` +
+ ${candidate.backend_id} + ${candidate.status} +
+
${candidate.candidate_id}
+
+ +
+ `; + const selectButton = item.querySelector(".candidate-select"); + if (selectButton && !candidate.selected) { + selectButton.onclick = () => + mutateSession("/api/candidates/select", { + candidate_id: candidate.candidate_id, + }).catch((error) => showError(error.message)); + } + els.candidateList.appendChild(item); + }); + + els.checkpointList.innerHTML = ""; + state.checkpoints.slice(-5).reverse().forEach((checkpoint) => { + const item = document.createElement("div"); + item.className = + "flex items-center justify-between p-2 bg-slate-50 rounded-lg border border-slate-100"; + item.innerHTML = ` + T${checkpoint.tick_id} + + `; + item.onclick = () => + mutateSession("/api/checkpoints/restore", { + checkpoint_id: checkpoint.checkpoint_id, + }).catch((error) => showError(error.message)); + els.checkpointList.appendChild(item); + }); +} + +function renderSessionList() { + if (!els.sessionList) return; + els.sessionList.innerHTML = ""; + if (!state.availableSessions.length) { + const empty = document.createElement("div"); + empty.className = "text-[10px] text-slate-400"; + empty.textContent = "无活跃会话"; + els.sessionList.appendChild(empty); + return; + } + + state.availableSessions.forEach((session) => { + const item = document.createElement("div"); + const isCurrent = state.sessionId === session.interactive_session_id; + item.className = + "flex items-center justify-between rounded-lg border p-2 " + + (isCurrent ? "border-brand-300 bg-brand-50" : "border-slate-100 bg-slate-50"); + item.innerHTML = ` +
+
${session.scene_id}
+
${session.interactive_session_id}
+
+ + `; + if (!isCurrent) { + item.querySelector("button").onclick = () => + reconnectSession(session.interactive_session_id).catch((error) => showError(error.message)); + } else { + item.querySelector("button").disabled = true; + } + els.sessionList.appendChild(item); + }); +} + +function renderAll() { + const session = state.sessionState; + const snapshot = activeSnapshot(); + const status = session?.status ?? "OFFLINE"; + + els.statusDot.className = + `w-2 h-2 rounded-full mr-2 ${ + status === "RUNNING" ? "bg-emerald-500 animate-pulse" : "bg-slate-300" + }`; + els.sessionStatus.textContent = + status === "RUNNING" ? "运行中" : status === "PAUSED" ? "已暂停" : "离线"; + els.sessionTick.textContent = snapshot?.tick_id ?? "-"; + els.sessionTime.textContent = snapshot ? `时长: ${formatSimTime(snapshot.sim_time_us)}` : "时长: -"; + els.egoSpeed.textContent = snapshot ? formatSpeed(snapshot.ego.speed_mps) : "-"; + els.sceneLabel.textContent = session?.scene_id ?? "未加载场景"; + if (els.decisionPageLink) { + const target = new URL("/decision", window.location.origin); + if (state.sessionId) target.searchParams.set("session_id", state.sessionId); + if (session?.scene_id) target.searchParams.set("scene_id", session.scene_id); + els.decisionPageLink.href = target.toString(); + } + + const history = state.snapshotHistory; + if (history.length) { + const active = snapshot ?? history.at(-1); + els.timelineSlider.min = history[0].tick_id; + els.timelineSlider.max = history.at(-1).tick_id; + els.timelineSlider.value = active.tick_id; + els.timelineCurrent.textContent = `帧 T${active.tick_id}`; + els.timelineEnd.textContent = `T${history.at(-1).tick_id}`; + } + + renderMap(snapshot); + renderSensorFrame(snapshot); + renderChart(); + renderLists(session, snapshot); + renderSessionList(); + if (els.mapCompassNeedle) { + els.mapCompassNeedle.style.transform = `rotate(${-state.mapView.rotation}rad)`; + } + if (window.lucide) window.lucide.createIcons(); +} + +async function loadScenes() { + const payload = await api("/api/scenes"); + state.availableScenes = payload.scenes || []; + renderSceneOptions(); +} + +async function loadSceneMap(sceneId) { + if (!sceneId) { + state.sceneMap = null; + return; + } + state.sceneMap = await api(`/api/map?scene_id=${encodeURIComponent(sceneId)}`); +} + +async function loadSessions() { + const payload = await api("/api/sessions"); + state.availableSessions = payload.sessions || []; + renderSessionList(); +} + +async function refresh() { + if (!state.sessionId || state.refreshInFlight) return; + state.refreshInFlight = true; + try { + clearError(); + const session = await api(`/api/session/state?session_id=${encodeURIComponent(state.sessionId)}`); + state.sessionState = session; + saveSessionId(state.sessionId); + const [sensorsPayload, checkpointsPayload] = await Promise.all([ + api(`/api/sensors?session_id=${encodeURIComponent(state.sessionId)}`), + api(`/api/checkpoints?session_id=${encodeURIComponent(state.sessionId)}`), + ]); + state.sensors = sensorsPayload.sensors || []; + if (!state.selectedSensorId) { + state.selectedSensorId = state.sensors[0]?.sensor_id || null; + } + state.checkpoints = checkpointsPayload.checkpoints || []; + if ( + session.latest_snapshot && + !state.snapshotHistory.some((snapshot) => snapshot.tick_id === session.latest_snapshot.tick_id) + ) { + state.snapshotHistory.push(session.latest_snapshot); + state.snapshotHistory.sort((left, right) => left.tick_id - right.tick_id); + } else if (session.latest_snapshot) { + state.snapshotHistory = state.snapshotHistory.map((snapshot) => + snapshot.tick_id === session.latest_snapshot.tick_id ? session.latest_snapshot : snapshot, + ); + } + if (session.latest_snapshot && !state.mapView.hasCustomView) { + applyDefaultMapView(session.latest_snapshot); + } + if (!state.sceneMap || state.sceneMap.scene_id !== session.scene_id) { + await loadSceneMap(session.scene_id); + } + renderAll(); + } catch (error) { + if (String(error.message || "").includes("Unknown interactive_session_id")) { + clearSavedSessionId(); + state.sessionId = null; + } + showError(error.message); + throw error; + } finally { + state.refreshInFlight = false; + } +} + +async function createSession() { + const sceneId = els.sceneId.value; + if (!sceneId) { + showError("请先选择场景。"); + return; + } + clearError(); + const session = await api("/api/session/create", { scene_id: sceneId }); + state.sessionId = session.interactive_session_id; + saveSessionId(state.sessionId); + state.snapshotHistory = []; + state.viewedTickId = null; + state.selectedSensorId = null; + state.sceneMap = null; + state.mapView.hasCustomView = false; + await refresh(); + await loadSessions(); +} + +async function reconnectSession(sessionId = "") { + const targetSessionId = + sessionId || + new URLSearchParams(window.location.search).get("session_id") || + loadSavedSessionId(); + if (!targetSessionId) { + return; + } + clearError(); + state.sessionId = targetSessionId; + state.snapshotHistory = []; + state.viewedTickId = null; + state.selectedSensorId = null; + state.sceneMap = null; + state.mapView.hasCustomView = false; + try { + await refresh(); + await loadSessions(); + } catch (error) { + clearSavedSessionId(); + throw error; + } +} + +async function mutateSession(path, body = {}) { + if (!state.sessionId) { + showError("当前没有已连接的会话。"); + return; + } + clearError(); + await api(path, { session_id: state.sessionId, ...body }); + await refresh(); + await loadSessions(); +} + +els.prevFrame.onclick = () => { + const current = activeSnapshot(); + const index = state.snapshotHistory.findIndex((snapshot) => snapshot.tick_id === current?.tick_id); + if (index > 0) { + state.viewedTickId = state.snapshotHistory[index - 1].tick_id; + renderAll(); + } +}; + +els.nextFrame.onclick = () => { + const current = activeSnapshot(); + const latest = state.snapshotHistory.at(-1); + if (current && latest && current.tick_id < latest.tick_id) { + const index = state.snapshotHistory.findIndex((snapshot) => snapshot.tick_id === current.tick_id); + state.viewedTickId = state.snapshotHistory[index + 1].tick_id; + renderAll(); + return; + } + mutateSession("/api/session/step").catch((error) => showError(error.message)); +}; + +els.createSession.onclick = () => createSession().catch((error) => showError(error.message)); +els.reconnectSession.onclick = () => reconnectSession().catch((error) => showError(error.message)); +els.refreshState.onclick = () => refresh().catch((error) => showError(error.message)); +els.playSession.onclick = () => mutateSession("/api/session/resume").catch((error) => showError(error.message)); +els.pauseSession.onclick = () => mutateSession("/api/session/pause").catch((error) => showError(error.message)); +els.liveJump.onclick = () => { + state.viewedTickId = null; + renderAll(); +}; +els.timelineSlider.oninput = () => { + state.viewedTickId = Number(els.timelineSlider.value); + renderAll(); +}; +els.resetMapView.onclick = () => resetMapView(); +els.applyBackends.onclick = () => { + const selected = [...document.querySelectorAll("#backend-selector input:checked")].map( + (input) => input.value, + ); + mutateSession("/api/backends/active", { backend_ids: selected }).catch((error) => + showError(error.message), + ); +}; + +attachMapInteractions(); +attachSensorOverlayInteractions(); + +Promise.all([loadScenes(), loadSessions()]) + .then(() => reconnectSession()) + .catch((error) => showError(error.message)); + +setInterval(() => { + if (state.sessionState?.status === "RUNNING") { + refresh().catch(() => {}); + } +}, 800); + +setInterval(() => { + loadSessions().catch(() => {}); +}, 2000); + +renderAll(); +lucide.createIcons(); diff --git a/src/runtime/alpasim_runtime/web_debugger/static/decision.html b/src/runtime/alpasim_runtime/web_debugger/static/decision.html new file mode 100644 index 00000000..5f715eb0 --- /dev/null +++ b/src/runtime/alpasim_runtime/web_debugger/static/decision.html @@ -0,0 +1,121 @@ + + + + + + Alpasim 决策分析 + + + + + + + +
+
+
Decision Analysis
+
+ 未连接会话 + / + T- + offline +
+
+
+ 返回主控台 + + + + +
+
+ +
+ + +
+
+
+
Selected Backend
+
-
+
model: -
+
+
+
Fallback
+
-
+
proposals: -
+
+
+
Arbitration
+
-
+
candidate: -
+
+
+
Snapshot Time
+
-
+
input: -
+
+
+ +
+
+

候选分析

+
绿: VAM / 红: PDM
+
+
+
+
+ + +
+ + + + diff --git a/src/runtime/alpasim_runtime/web_debugger/static/decision.js b/src/runtime/alpasim_runtime/web_debugger/static/decision.js new file mode 100644 index 00000000..b7c4fcda --- /dev/null +++ b/src/runtime/alpasim_runtime/web_debugger/static/decision.js @@ -0,0 +1,370 @@ +const STORAGE_KEY = "alpasim.web_debugger.session_id"; + +const state = { + sessionId: null, + session: null, + scenes: [], + snapshotHistory: [], + viewedTickId: null, + refreshInFlight: false, +}; + +const els = { + errorBanner: document.getElementById("error-banner"), + headerScene: document.getElementById("header-scene"), + headerTick: document.getElementById("header-tick"), + headerStatus: document.getElementById("header-status"), + mainPageLink: document.getElementById("main-page-link"), + refresh: document.getElementById("refresh"), + pause: document.getElementById("pause"), + resume: document.getElementById("resume"), + step: document.getElementById("step"), + reconnect: document.getElementById("reconnect"), + prevTick: document.getElementById("prev-tick"), + nextTick: document.getElementById("next-tick"), + sceneId: document.getElementById("scene-id"), + createSession: document.getElementById("create-session"), + applyBackends: document.getElementById("apply-backends"), + backendSelector: document.getElementById("backend-selector"), + timelineList: document.getElementById("timeline-list"), + selectedBackend: document.getElementById("selected-backend"), + selectedModel: document.getElementById("selected-model"), + fallbackReason: document.getElementById("fallback-reason"), + proposalCount: document.getElementById("proposal-count"), + arbitrationReason: document.getElementById("arbitration-reason"), + selectedCandidate: document.getElementById("selected-candidate"), + snapshotTime: document.getElementById("snapshot-time"), + snapshotInput: document.getElementById("snapshot-input"), + candidateList: document.getElementById("candidate-list"), + timingMetrics: document.getElementById("timing-metrics"), + qualityMetrics: document.getElementById("quality-metrics"), + selectedDebugJson: document.getElementById("selected-debug-json"), +}; + +function showError(message) { + els.errorBanner.textContent = message; + els.errorBanner.classList.remove("hidden"); +} + +function clearError() { + els.errorBanner.textContent = ""; + els.errorBanner.classList.add("hidden"); +} + +async function readResponse(response) { + if (response.ok) return response.json(); + const payload = await response.json().catch(() => ({})); + throw new Error(payload.grpc_details || payload.error || `${response.status} ${response.statusText}`); +} + +async function api(path, body = null) { + const response = await fetch( + path, + body + ? { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + } + : undefined, + ); + return readResponse(response); +} + +function loadSavedSessionId() { + return window.localStorage.getItem(STORAGE_KEY) || ""; +} + +function saveSessionId(sessionId) { + if (sessionId) window.localStorage.setItem(STORAGE_KEY, sessionId); +} + +function activeSnapshot() { + if (state.viewedTickId === null) return state.snapshotHistory.at(-1) || state.session?.latest_snapshot || null; + return state.snapshotHistory.find((snapshot) => snapshot.tick_id === state.viewedTickId) || state.snapshotHistory.at(-1) || null; +} + +function formatTime(us) { + return `${(us / 1e6).toFixed(2)}s`; +} + +function pretty(value) { + if (typeof value === "number") return Number.isInteger(value) ? String(value) : value.toFixed(2); + if (typeof value === "boolean") return value ? "true" : "false"; + if (value === null || value === undefined || value === "") return "-"; + return String(value); +} + +function backendTone(backendId) { + const key = String(backendId || "").toLowerCase(); + if (key.includes("pdm")) return { border: "border-rose-300", bg: "bg-rose-50", text: "text-rose-700" }; + if (key.includes("vam") || key.includes("vla")) return { border: "border-emerald-300", bg: "bg-emerald-50", text: "text-emerald-700" }; + return { border: "border-violet-300", bg: "bg-violet-50", text: "text-violet-700" }; +} + +function selectedCandidate(snapshot) { + const decision = snapshot?.latest_decision || state.session?.latest_decision; + if (!decision) return null; + return (decision.candidates || []).find((candidate) => candidate.selected) || null; +} + +function flattenEntries(payload, prefix = "") { + if (!payload || typeof payload !== "object") return []; + return Object.entries(payload).flatMap(([key, value]) => { + const nextKey = prefix ? `${prefix}.${key}` : key; + if (value && typeof value === "object" && !Array.isArray(value)) { + return flattenEntries(value, nextKey); + } + return [[nextKey, value]]; + }); +} + +function renderMetricTable(container, payload) { + if (!container) return; + container.innerHTML = ""; + const rows = flattenEntries(payload); + if (!rows.length) { + container.innerHTML = '
暂无数据
'; + return; + } + rows.forEach(([key, value]) => { + const row = document.createElement("div"); + row.className = "kv-row"; + row.innerHTML = `${key}${pretty(value)}`; + container.appendChild(row); + }); +} + +function renderSceneOptions() { + els.sceneId.innerHTML = ""; + const scenes = state.scenes || []; + if (!scenes.length) { + const option = document.createElement("option"); + option.value = ""; + option.textContent = "无可用场景"; + els.sceneId.appendChild(option); + return; + } + scenes.forEach((scene) => { + const option = document.createElement("option"); + option.value = scene.scene_id; + option.textContent = scene.label || scene.scene_id; + els.sceneId.appendChild(option); + }); + const preferredSceneId = new URLSearchParams(window.location.search).get("scene_id"); + els.sceneId.value = preferredSceneId || scenes[0].scene_id; +} + +function renderTimeline(snapshot) { + els.timelineList.innerHTML = ""; + const history = [...state.snapshotHistory].sort((a, b) => a.tick_id - b.tick_id); + if (!history.length) { + els.timelineList.innerHTML = '
暂无快照
'; + return; + } + history.slice().reverse().forEach((item) => { + const decision = item.latest_decision || {}; + const selected = (decision.candidates || []).find((candidate) => candidate.selected); + const isActive = snapshot?.tick_id === item.tick_id; + const tone = backendTone(selected?.backend_id); + const card = document.createElement("button"); + card.className = `w-full rounded-xl border p-3 text-left ${isActive ? "border-brand-500 bg-brand-50" : "border-slate-200 bg-white hover:bg-slate-50"}`; + card.innerHTML = ` +
+ T${item.tick_id} + ${formatTime(item.sim_time_us)} +
+
${selected?.backend_id || "-"}
+
${decision.arbitration_reason || "-"}
+ `; + card.onclick = () => { + state.viewedTickId = item.tick_id; + renderAll(); + }; + els.timelineList.appendChild(card); + }); +} + +function renderBackendSelector() { + const session = state.session; + const decision = activeSnapshot()?.latest_decision || session?.latest_decision; + const available = session?.available_backend_ids?.length + ? session.available_backend_ids + : [...new Set((decision?.candidates || []).map((candidate) => candidate.backend_id))]; + const active = new Set(session?.active_backend_ids || []); + els.backendSelector.innerHTML = ""; + available.forEach((backendId) => { + const tone = backendTone(backendId); + const label = document.createElement("label"); + label.className = `flex items-center justify-between rounded-lg border px-3 py-2 text-[11px] font-semibold ${tone.border} ${tone.bg}`; + label.innerHTML = `${backendId}`; + els.backendSelector.appendChild(label); + }); +} + +function renderCandidates(snapshot) { + const decision = snapshot?.latest_decision || state.session?.latest_decision; + els.candidateList.innerHTML = ""; + (decision?.candidates || []).forEach((candidate) => { + const tone = backendTone(candidate.backend_id); + const debug = candidate.diagnostics?.driver_debug || {}; + const card = document.createElement("div"); + card.className = `rounded-2xl border p-4 ${candidate.selected ? `${tone.border} ${tone.bg}` : "border-slate-200 bg-white"}`; + card.innerHTML = ` +
+
+
${candidate.backend_id}
+
${candidate.status}
+
+ +
+
+
candidate_id${candidate.candidate_id}
+
selected_model_type${pretty(debug.selected_model_type || candidate.diagnostics?.model_type_override)}
+
fallback_reason${pretty(debug.fallback_reason)}
+
proposal_count${pretty(debug.proposal_count)}
+
nearby_lane_count${pretty(debug.nearby_lane_count)}
+
actor_count${pretty(debug.actor_count)}
+
+ `; + const button = card.querySelector(".candidate-select"); + if (button && !candidate.selected) { + button.onclick = () => mutateSession("/api/candidates/select", { candidate_id: candidate.candidate_id }); + } + els.candidateList.appendChild(card); + }); +} + +function renderHeader(snapshot) { + const session = state.session; + const selected = selectedCandidate(snapshot); + const debug = selected?.diagnostics?.driver_debug || {}; + els.headerScene.textContent = session?.scene_id || "未连接会话"; + els.headerTick.textContent = snapshot ? `T${snapshot.tick_id}` : "T-"; + els.headerStatus.textContent = (session?.status || "OFFLINE").toLowerCase(); + const mainTarget = new URL("/", window.location.origin); + if (state.sessionId) mainTarget.searchParams.set("session_id", state.sessionId); + if (session?.scene_id) mainTarget.searchParams.set("scene_id", session.scene_id); + els.mainPageLink.href = mainTarget.toString(); + + els.selectedBackend.textContent = selected?.backend_id || "-"; + els.selectedModel.textContent = `model: ${pretty(debug.selected_model_type || selected?.diagnostics?.model_type_override)}`; + els.fallbackReason.textContent = pretty(debug.fallback_reason); + els.proposalCount.textContent = `proposals: ${pretty(debug.proposal_count)}`; + els.arbitrationReason.textContent = snapshot?.latest_decision?.arbitration_reason || session?.latest_decision?.arbitration_reason || "-"; + els.selectedCandidate.textContent = `candidate: ${snapshot?.latest_decision?.selected_candidate_id || session?.latest_decision?.selected_candidate_id || "-"}`; + els.snapshotTime.textContent = snapshot ? formatTime(snapshot.sim_time_us) : "-"; + els.snapshotInput.textContent = `input: ${snapshot?.latest_decision?.input_snapshot_id || "-"}`; + els.selectedDebugJson.textContent = JSON.stringify(selected?.diagnostics || {}, null, 2); +} + +function renderAll() { + const snapshot = activeSnapshot(); + renderHeader(snapshot); + renderBackendSelector(); + renderTimeline(snapshot); + renderCandidates(snapshot); + renderMetricTable(els.timingMetrics, snapshot?.context_diagnostics?.timing || {}); + renderMetricTable(els.qualityMetrics, snapshot?.context_diagnostics?.quality || {}); + if (window.lucide) window.lucide.createIcons(); +} + +async function loadScenes() { + const payload = await api("/api/scenes"); + state.scenes = payload.scenes || []; + renderSceneOptions(); +} + +async function refresh() { + if (!state.sessionId || state.refreshInFlight) return; + state.refreshInFlight = true; + try { + clearError(); + const session = await api(`/api/session/state?session_id=${encodeURIComponent(state.sessionId)}`); + state.session = session; + saveSessionId(state.sessionId); + if (session.latest_snapshot) { + const existing = state.snapshotHistory.findIndex((snapshot) => snapshot.tick_id === session.latest_snapshot.tick_id); + if (existing >= 0) { + state.snapshotHistory[existing] = session.latest_snapshot; + } else { + state.snapshotHistory.push(session.latest_snapshot); + state.snapshotHistory.sort((a, b) => a.tick_id - b.tick_id); + } + } + renderAll(); + } finally { + state.refreshInFlight = false; + } +} + +async function createSession() { + const sceneId = els.sceneId.value; + const session = await api("/api/session/create", { scene_id: sceneId }); + state.sessionId = session.interactive_session_id; + state.snapshotHistory = []; + state.viewedTickId = null; + await refresh(); +} + +async function reconnectSession(sessionId = "") { + const target = sessionId || new URLSearchParams(window.location.search).get("session_id") || loadSavedSessionId(); + if (!target) return; + state.sessionId = target; + state.snapshotHistory = []; + state.viewedTickId = null; + await refresh(); +} + +async function mutateSession(path, body = {}) { + if (!state.sessionId) return; + clearError(); + await api(path, { session_id: state.sessionId, ...body }); + await refresh(); +} + +els.refresh.onclick = () => refresh().catch((error) => showError(error.message)); +els.pause.onclick = () => mutateSession("/api/session/pause").catch((error) => showError(error.message)); +els.resume.onclick = () => mutateSession("/api/session/resume").catch((error) => showError(error.message)); +els.step.onclick = () => mutateSession("/api/session/step").catch((error) => showError(error.message)); +els.reconnect.onclick = () => reconnectSession().catch((error) => showError(error.message)); +els.createSession.onclick = () => createSession().catch((error) => showError(error.message)); +els.applyBackends.onclick = () => { + const backendIds = [...document.querySelectorAll("#backend-selector input:checked")].map((node) => node.value); + mutateSession("/api/backends/active", { backend_ids: backendIds }).catch((error) => showError(error.message)); +}; +els.prevTick.onclick = () => { + const snapshot = activeSnapshot(); + const history = state.snapshotHistory; + const index = history.findIndex((item) => item.tick_id === snapshot?.tick_id); + if (index > 0) { + state.viewedTickId = history[index - 1].tick_id; + renderAll(); + } +}; +els.nextTick.onclick = () => { + const snapshot = activeSnapshot(); + const history = state.snapshotHistory; + const index = history.findIndex((item) => item.tick_id === snapshot?.tick_id); + if (index >= 0 && index < history.length - 1) { + state.viewedTickId = history[index + 1].tick_id; + renderAll(); + return; + } + mutateSession("/api/session/step").catch((error) => showError(error.message)); +}; + +Promise.all([loadScenes()]) + .then(() => reconnectSession()) + .catch((error) => showError(error.message)); + +setInterval(() => { + if (state.session?.status === "RUNNING") { + refresh().catch(() => {}); + } +}, 800); + +renderAll(); +lucide.createIcons(); diff --git a/src/runtime/alpasim_runtime/web_debugger/static/index.html b/src/runtime/alpasim_runtime/web_debugger/static/index.html new file mode 100644 index 00000000..3bf5683b --- /dev/null +++ b/src/runtime/alpasim_runtime/web_debugger/static/index.html @@ -0,0 +1,175 @@ + + + + + + Alpasim 仿真控制台 (标准版) + + + + + + + + + +
+
+

Alpasim Console

+
+
+ 离线 + 当前帧: - + 时长: - +
+
+
+ + 决策分析页 + + 主车时速 + - + +
+
+ + +
+ + + + +
+
+ +
未加载场景
+
+
+
+ + +
+
N
+
+ +
+
+
滚轮缩放, 拖拽平移, Shift + 拖拽 旋转
+
+ History + Driver Plan +
+
+
+
+ 传感器视角 +
+
+
+ +
+
+
+
+ + +
+ + +
+
+
REAL-TIME
+
+ T0 + / + T0 +
+
+
+ +
+
+
+ +
+ + +
+ +
+ +
+
+ + + + diff --git a/src/runtime/alpasim_runtime/web_debugger/static/styles.css b/src/runtime/alpasim_runtime/web_debugger/static/styles.css new file mode 100644 index 00000000..a5763b2b --- /dev/null +++ b/src/runtime/alpasim_runtime/web_debugger/static/styles.css @@ -0,0 +1,80 @@ +/* Alpasim Webviz - Professional Styling */ + +:root { + --resizer-size: 4px; + --active-color: #0ea5e9; +} + +/* Layout System */ +.resizer-h.active, .resizer-v.active { + background-color: var(--active-color) !important; + opacity: 1 !important; +} + +.stack-item { + @apply p-3 rounded-xl border border-slate-100 bg-white hover:border-brand-200 hover:shadow-md transition-all flex flex-col space-y-1 shadow-sm; +} +.stack-item.selected { + @apply border-brand-500 bg-brand-50/20 ring-1 ring-brand-500/20; +} + +.compact-btn { + @apply px-2 py-1 rounded-lg bg-slate-100 hover:bg-slate-200 text-[9px] font-black text-slate-600 transition-colors uppercase tracking-widest; +} + +/* Map Elements (XODR Inspired) */ +#map-canvas { + background-color: #ffffff; + image-rendering: -webkit-optimize-contrast; +} + +/* Sensor and Images */ +#primary-sensor-image:not([src]), +#primary-sensor-image[src=""] { + display: none; +} +#primary-sensor-image[src] ~ #sensor-empty { + display: none; +} + +.sensor-tab { + @apply px-2 py-0.5 rounded-md text-[9px] font-black border border-slate-200 text-slate-400 hover:bg-slate-50 transition-all cursor-pointer uppercase; +} +.sensor-tab.active { + @apply bg-slate-900 text-white border-slate-900 shadow-sm; +} + +/* Custom Scrollbar */ +::-webkit-scrollbar { width: 4px; height: 4px; } +::-webkit-scrollbar-track { background: transparent; } +::-webkit-scrollbar-thumb { background: #e2e8f0; border-radius: 10px; } +::-webkit-scrollbar-thumb:hover { background: #cbd5e1; } + +/* Status Pulse */ +@keyframes status-pulse { + 0% { transform: scale(1); opacity: 1; } + 50% { transform: scale(1.2); opacity: 0.7; } + 100% { transform: scale(1); opacity: 1; } +} +.status-pulse { + animation: status-pulse 2s infinite ease-in-out; +} + +/* Range Input styling */ +input[type="range"]::-webkit-slider-thumb { + -webkit-appearance: none; + height: 12px; + width: 12px; + border-radius: 50%; + background: white; + border: 2px solid var(--active-color); + box-shadow: 0 2px 4px rgba(0,0,0,0.1); + cursor: pointer; + margin-top: -4px; +} +input[type="range"]::-webkit-slider-runnable-track { + width: 100%; + height: 4px; + background: #334155; + border-radius: 2px; +} diff --git a/src/runtime/alpasim_runtime/web_debugger_v2/package-lock.json b/src/runtime/alpasim_runtime/web_debugger_v2/package-lock.json new file mode 100644 index 00000000..a7979906 --- /dev/null +++ b/src/runtime/alpasim_runtime/web_debugger_v2/package-lock.json @@ -0,0 +1,1120 @@ +{ + "name": "alpasim-webviz-debugger", + "version": "2.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "alpasim-webviz-debugger", + "version": "2.0.0", + "license": "Apache-2.0", + "dependencies": { + "axios": "^1.6.0", + "express": "^4.18.2", + "http-proxy-middleware": "^2.0.6" + } + }, + "node_modules/@types/http-proxy": { + "version": "1.17.17", + "resolved": "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.17.tgz", + "integrity": "sha512-ED6LB+Z1AVylNTu7hdzuBqOgMnvG/ld6wGCG8wFnAzKX5uyW2K3WD52v0gnLCTK/VLpXtKckgWuyScYK6cSPaw==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/node": { + "version": "25.6.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.0.tgz", + "integrity": "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.19.0" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.15.0.tgz", + "integrity": "sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.11", + "form-data": "^4.0.5", + "proxy-from-env": "^2.1.0" + } + }, + "node_modules/body-parser": { + "version": "1.20.4", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.4.tgz", + "integrity": "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.14.0", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, + "node_modules/express": { + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.3", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.14.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/finalhandler": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", + "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/http-proxy": { + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.0", + "follow-redirects": "^1.0.0", + "requires-port": "^1.0.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/http-proxy-middleware": { + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.9.tgz", + "integrity": "sha512-c1IyJYLYppU574+YI7R4QyX2ystMtVXZwIdzazUIPIJsHuWNd+mho2j+bKoHftndicGj9yh+xjd+l0yj7VeT1Q==", + "license": "MIT", + "dependencies": { + "@types/http-proxy": "^1.17.8", + "http-proxy": "^1.18.1", + "is-glob": "^4.0.1", + "is-plain-obj": "^3.0.0", + "micromatch": "^4.0.2" + }, + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "@types/express": "^4.17.13" + }, + "peerDependenciesMeta": { + "@types/express": { + "optional": true + } + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-plain-obj": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-3.0.0.tgz", + "integrity": "sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", + "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", + "license": "MIT" + }, + "node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/proxy-from-env": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", + "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/qs": { + "version": "6.14.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", + "integrity": "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "license": "MIT" + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", + "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.4.1", + "range-parser": "~1.2.1", + "statuses": "~2.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", + "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "~0.19.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/undici-types": { + "version": "7.19.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.19.2.tgz", + "integrity": "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==", + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + } + } +} diff --git a/src/runtime/alpasim_runtime/web_debugger_v2/package.json b/src/runtime/alpasim_runtime/web_debugger_v2/package.json new file mode 100644 index 00000000..bca6667a --- /dev/null +++ b/src/runtime/alpasim_runtime/web_debugger_v2/package.json @@ -0,0 +1,23 @@ +{ + "name": "alpasim-webviz-debugger", + "version": "2.0.0", + "description": "Professional Webviz-style debugger for Alpasim", + "main": "server.js", + "scripts": { + "start": "node server.js", + "dev": "node server.js --dev" + }, + "dependencies": { + "express": "^4.18.2", + "http-proxy-middleware": "^2.0.6", + "axios": "^1.6.0" + }, + "keywords": [ + "autonomous-driving", + "webviz", + "debugger", + "simulation" + ], + "author": "Alpasim Team", + "license": "Apache-2.0" +} diff --git a/src/runtime/alpasim_runtime/web_debugger_v2/public/assets/app.js b/src/runtime/alpasim_runtime/web_debugger_v2/public/assets/app.js new file mode 100644 index 00000000..9f13451d --- /dev/null +++ b/src/runtime/alpasim_runtime/web_debugger_v2/public/assets/app.js @@ -0,0 +1,315 @@ +/** + * Alpasim Webviz V2 - Core Application + * Integrated Layout Management, Realistic Map Rendering, Session Persistence + */ + +const state = { + sessionId: null, + sessionState: null, + sensors: [], + selectedSensorId: null, + sceneMap: null, + snapshotHistory: [], + viewedTickId: null, + refreshTimer: null, + isEgoLocked: true, + renderQueued: false, +}; + +const STORAGE_KEY = "alpasim_last_session_id"; + +const els = { + sceneId: document.getElementById("scene-id"), + createSession: document.getElementById("create-session"), + refreshState: document.getElementById("refresh-state"), + playSession: document.getElementById("play-session"), + pauseSession: document.getElementById("pause-session"), + prevFrame: document.getElementById("prev-frame"), + nextFrame: document.getElementById("next-frame"), + liveJump: document.getElementById("live-jump"), + timelineSlider: document.getElementById("timeline-slider"), + playbackMode: document.getElementById("playback-mode"), + timelineCurrent: document.getElementById("timeline-current"), + timelineEnd: document.getElementById("timeline-end"), + statusIndicator: document.getElementById("status-indicator"), + sessionStatus: document.getElementById("session-status"), + sessionTick: document.getElementById("session-tick"), + sessionTime: document.getElementById("session-time"), + egoSpeed: document.getElementById("ego-speed"), + sceneLabel: document.getElementById("scene-label"), + mapScale: document.getElementById("map-scale"), + mapSource: document.getElementById("map-source"), + sensorTabs: document.getElementById("sensor-tabs"), + primarySensorImage: document.getElementById("primary-sensor-image"), + mapCanvas: document.getElementById("map-canvas"), + chartCanvas: document.getElementById("chart-canvas"), + candidateList: document.getElementById("candidate-list"), + checkpointList: document.getElementById("checkpoint-list"), + lockEgo: document.getElementById("lock-ego"), + sessionsContainer: document.getElementById("sessions-container"), + listSessions: document.getElementById("list-sessions"), + reconnectLast: document.getElementById("reconnect-last"), +}; + +// --- API Helpers --- +async function api(path, body = null) { + const r = await fetch(path, body ? { method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify(body) } : {}); + if (!r.ok) { + const err = await r.json().catch(() => ({})); + throw new Error(err.grpc_details || err.error || `${r.status} ${r.statusText}`); + } + return r.json(); +} + +// --- Session Persistence & Management --- +async function saveSession(sid) { + state.sessionId = sid; + localStorage.setItem(STORAGE_KEY, sid); + updateReconnectUI(); + await listSessions(); +} + +function updateReconnectUI() { + const lastId = localStorage.getItem(STORAGE_KEY); + if (lastId && lastId !== state.sessionId) { + els.reconnectLast.textContent = `重连上次: ${lastId.slice(0, 8)}...`; + els.reconnectLast.classList.remove("hidden"); + } else { + els.reconnectLast.classList.add("hidden"); + } +} + +async function attachSession(sid) { + if (!sid) return; + try { + const sess = await api(`/api/session/state?session_id=${encodeURIComponent(sid)}`); + // 如果获取状态成功，说明会话存在 + state.sessionId = sid; + state.sessionState = sess; + localStorage.setItem(STORAGE_KEY, sid); + + // 初始化该会话的数据 + const [sensors, map] = await Promise.all([ + api(`/api/sensors?session_id=${encodeURIComponent(sid)}`), + api(`/api/map?scene_id=${encodeURIComponent(sess.scene_id)}`) + ]); + state.sensors = sensors.sensors; + state.sceneMap = map; + state.selectedSensorId = sensors.sensors[0]?.sensor_id; + state.snapshotHistory = sess.latest_snapshot ? [sess.latest_snapshot] : []; + + await refresh(); // 开始同步循环 + console.log(`Successfully attached to session: ${sid}`); + } catch (e) { + console.warn(`Could not attach to session ${sid}:`, e.message); + if (sid === state.sessionId) state.sessionId = null; + updateReconnectUI(); + } +} + +async function listSessions() { + try { + const data = await api("/api/sessions"); + els.sessionsContainer.innerHTML = ""; + if (data.session_ids && data.session_ids.length > 0) { + data.session_ids.forEach(sid => { + const isActive = sid === state.sessionId; + const item = document.createElement("div"); + item.className = `session-item flex items-center justify-between p-2 rounded-lg border transition-all ${isActive ? 'bg-brand-50 border-brand-200' : 'bg-slate-50 border-slate-100 hover:border-slate-200'} group`; + item.innerHTML = ` +
+ ${sid} + + ${isActive ? '● 当前活跃' : '活跃会话'} + +
+ + `; + const btn = item.querySelector(".attach-btn"); + if (btn) btn.onclick = () => attachSession(sid); + els.sessionsContainer.appendChild(item); + }); + } else { + els.sessionsContainer.innerHTML = '
无活跃会话记录
'; + } + } catch (e) { + console.error("List sessions failed", e); + } +} + +// --- Layout & Map Engine --- +class MapEngine { + constructor(canvas) { + this.canvas = canvas; this.ctx = canvas.getContext("2d"); + this.cam = { x: 0, y: 0, zoom: 15 }; + this.initEvents(); + } + initEvents() { + this.canvas.onwheel = (e) => { e.preventDefault(); this.cam.zoom = Math.min(150, Math.max(1, this.cam.zoom * (e.deltaY > 0 ? 0.9 : 1.1))); state.isEgoLocked = false; scheduleRender(); }; + } + render(snapshot, map) { + const { ctx, canvas, cam } = this; + syncCanvasRes(canvas); + ctx.clearRect(0, 0, canvas.width, canvas.height); + if (state.isEgoLocked && snapshot) { cam.x = snapshot.ego.x; cam.y = snapshot.ego.y; } + + // Grid + ctx.strokeStyle = "#f1f5f9"; ctx.lineWidth = 0.5; + const step = 50; + for(let i=0; i ` +
+ Tick T${ck.tick_id} + +
+ `).join(""); + + if (window.lucide) window.lucide.createIcons(); +} + +function activeSnapshot() { + if (state.viewedTickId === null) return state.snapshotHistory.at(-1); + return state.snapshotHistory.find(s => s.tick_id === state.viewedTickId) || state.snapshotHistory.at(-1); +} + +function scheduleRender() { + if (state.renderQueued) return; + state.renderQueued = true; + requestAnimationFrame(() => { state.renderQueued = false; renderAll(); }); +} + +async function refresh() { + if (!state.sessionId) return; + try { + const sess = await api(`/api/session/state?session_id=${encodeURIComponent(state.sessionId)}`); + state.sessionState = sess; + const [map, cks] = await Promise.all([ + api(`/api/map?scene_id=${encodeURIComponent(sess.scene_id)}`), + api(`/api/checkpoints?session_id=${encodeURIComponent(state.sessionId)}`) + ]); + state.sceneMap = map; state.checkpoints = cks.checkpoints || []; + const snap = sess.latest_snapshot; + if (snap && !state.snapshotHistory.some(s => s.tick_id === snap.tick_id)) { + state.snapshotHistory.push(snap); state.snapshotHistory.sort((a,b) => a.tick_id - b.tick_id); + } + scheduleRender(); + updateReconnectUI(); + } catch (e) { + if (e.message.includes("404") || e.message.includes("not found")) { + console.warn("Session expired on backend"); + state.sessionId = null; + } + scheduleRender(); + } +} + +async function mutate(path, body = {}) { + if (!state.sessionId) return; + await api(path, { session_id: state.sessionId, ...body }); + await refresh(); +} + +function syncCanvasRes(c) { + const r = c.getBoundingClientRect(); + const dpr = window.devicePixelRatio || 1; + if (c.width !== Math.round(r.width * dpr) || c.height !== Math.round(r.height * dpr)) { + c.width = Math.round(r.width * dpr); c.height = Math.round(r.height * dpr); + } +} + +// --- Initialization --- +window.map = new MapEngine(els.mapCanvas); + +els.createSession.onclick = async () => { + const sess = await api("/api/session/create", { scene_id: els.sceneId.value }); + await saveSession(sess.interactive_session_id); + state.snapshotHistory = []; + await attachSession(sess.interactive_session_id); +}; + +els.reconnectLast.onclick = () => { + const lastId = localStorage.getItem(STORAGE_KEY); + if (lastId) attachSession(lastId); +}; + +els.listSessions.onclick = listSessions; +els.playSession.onclick = () => mutate("/api/session/resume"); +els.pauseSession.onclick = () => mutate("/api/session/pause"); +els.nextFrame.onclick = async () => { + const cur = activeSnapshot(); const latest = state.snapshotHistory.at(-1); + if (cur && latest && cur.tick_id < latest.tick_id) { + state.viewedTickId = state.snapshotHistory[state.snapshotHistory.findIndex(s => s.tick_id === cur.tick_id) + 1].tick_id; + renderAll(); + } else { await api("/api/session/step", { session_id: state.sessionId }); await refresh(); } +}; +els.prevFrame.onclick = () => { + const cur = activeSnapshot(); const idx = state.snapshotHistory.findIndex(s => s.tick_id === cur?.tick_id); + if (idx > 0) { state.viewedTickId = state.snapshotHistory[idx-1].tick_id; renderAll(); } +}; +els.liveJump.onclick = () => { state.viewedTickId = null; scheduleRender(); }; +els.timelineSlider.oninput = () => { state.viewedTickId = Number(els.timelineSlider.value); renderAll(); }; + +// Boot logic +(async () => { + updateReconnectUI(); + const lastId = localStorage.getItem(STORAGE_KEY); + if (lastId) { + console.log("Auto-reconnecting to last session:", lastId); + await attachSession(lastId); + } + await listSessions(); + setInterval(() => { if (state.sessionState?.status === 'RUNNING') refresh(); }, 1000); + setInterval(listSessions, 5000); // 周期性刷新列表 +})(); + +window.onresize = scheduleRender; +lucide.createIcons(); diff --git a/src/runtime/alpasim_runtime/web_debugger_v2/public/index.html b/src/runtime/alpasim_runtime/web_debugger_v2/public/index.html new file mode 100644 index 00000000..f11b813d --- /dev/null +++ b/src/runtime/alpasim_runtime/web_debugger_v2/public/index.html @@ -0,0 +1,148 @@ + + + + + Alpasim Webviz 仿真调试台 + + + + + + +
+
+
+ + Alpasim Webviz v2.0 +
+
+
+ 未连接 + 帧 ID: - + - +
+
+
+ +
+ Speed + - +
+ +
+
+ +
+ + + +
+ +
+
+ +
+ + - +
+
+ + + +
+
+ +
+ +
+
+
+ 传感器视角 +
+
+
+ +
+
+
+ 速度趋势 + +
+
+
+ +
+ + +
+ + +
+
+
实时同步
+
+ T0 + / + T0 +
+
+
+ +
+
+
+ + + + +
+ +
+
+ + + + diff --git a/src/runtime/alpasim_runtime/web_debugger_v2/server.js b/src/runtime/alpasim_runtime/web_debugger_v2/server.js new file mode 100644 index 00000000..f3a19b8e --- /dev/null +++ b/src/runtime/alpasim_runtime/web_debugger_v2/server.js @@ -0,0 +1,28 @@ +const express = require('express'); +const { createProxyMiddleware } = require('http-proxy-middleware'); +const path = require('path'); + +const app = express(); +const PORT = 3000; +const PYTHON_BACKEND = 'http://127.0.0.1:8080'; + +// 1. API Proxy - 将所有 /api 请求转发到原来的 Python server.py +app.use('/api', createProxyMiddleware({ + target: PYTHON_BACKEND, + changeOrigin: true, + logLevel: 'debug' +})); + +// 2. Static Files - 托管前端构建后的产物 +app.use(express.static(path.join(__dirname, 'public'))); + +// 3. Fallback to index.html (SPA support) +app.get('*', (req, res) => { + res.sendFile(path.join(__dirname, 'public', 'index.html')); +}); + +app.listen(PORT, () => { + console.log(`\n🚀 Alpasim Webviz Debugger V2 is running!`); + console.log(`🔗 URL: http://localhost:${PORT}`); + console.log(`📡 Proxying API to: ${PYTHON_BACKEND}\n`); +}); diff --git a/src/runtime/tests/test_config.py b/src/runtime/tests/test_config.py index 9b0d321c..c1025b45 100644 --- a/src/runtime/tests/test_config.py +++ b/src/runtime/tests/test_config.py @@ -48,5 +48,40 @@ def test_typed_parse_config_invalid_yaml(tmp_path): config.typed_parse_config(not_yaml, config.UserSimulatorConfig) +def test_simulation_config_parses_driver_backends(tmp_path): + cfg_path = tmp_path / "user.yaml" + cfg_path.write_text( + """ +endpoints: + sensorsim: {n_concurrent_rollouts: 1} + driver: {n_concurrent_rollouts: 1} + physics: {n_concurrent_rollouts: 1} + trafficsim: {n_concurrent_rollouts: 1} + controller: {n_concurrent_rollouts: 1} +simulation_config: + n_sim_steps: 1 + n_rollouts: 1 + observation_cache_size: 12 + observation_window_summary_size: 6 + driver_backends: + - backend_id: ar1_backend + model_type: ar1 + priority: 5 + - backend_id: pdm_backend + model_type: pdm + priority: 1 +scenes: + - scene_id: clipgt-a +""" + ) + + user_cfg = config.typed_parse_config(cfg_path, config.UserSimulatorConfig) + + assert len(user_cfg.simulation_config.driver_backends) == 2 + assert user_cfg.simulation_config.driver_backends[1].model_type == "pdm" + assert user_cfg.simulation_config.observation_cache_size == 12 + assert user_cfg.simulation_config.observation_window_summary_size == 6 + + # TODO(mwatson, mtyszkiewicz): What should happen when the config is empty? Currently, # no error is raised, and we return an empty config object. Is this the desired behavior? diff --git a/src/runtime/tests/test_daemon_engine.py b/src/runtime/tests/test_daemon_engine.py index 13c1522a..73e02729 100644 --- a/src/runtime/tests/test_daemon_engine.py +++ b/src/runtime/tests/test_daemon_engine.py @@ -39,6 +39,9 @@ async def test_engine_startup_gathers_versions_and_validates_scenes( version_ids = MagicMock() eval_config = MagicMock() worker_runtime = SimpleNamespace(stop=AsyncMock()) + scheduler_pools = {"driver": MagicMock()} + interactive_pools = {"driver": MagicMock()} + captured_interactive_pools = None class _FakeScheduler: def __init__( @@ -66,10 +69,19 @@ async def _fake_build_runtime_context(*args, **kwargs): eval_config=eval_config, version_ids=version_ids, scene_id_to_artifact_path={"clipgt-a": "/tmp/scene-a.usdz"}, - pools={"driver": MagicMock()}, + pools=scheduler_pools, max_in_flight=1, ) + class _FakeInteractiveSessionManager: + def __init__(self, *, pools, **kwargs) -> None: + del kwargs + nonlocal captured_interactive_pools + captured_interactive_pools = pools + + async def close_all(self) -> None: + return None + monkeypatch.setattr( "alpasim_runtime.daemon.engine.build_runtime_context", _fake_build_runtime_context, @@ -82,6 +94,14 @@ async def _fake_build_runtime_context(*args, **kwargs): "alpasim_runtime.daemon.engine.start_worker_runtime", _fake_start_worker_runtime ) monkeypatch.setattr("alpasim_runtime.daemon.engine.DaemonScheduler", _FakeScheduler) + monkeypatch.setattr( + "alpasim_runtime.daemon.engine.create_address_pools", + lambda *_args, **_kwargs: interactive_pools, + ) + monkeypatch.setattr( + "alpasim_runtime.daemon.engine.InteractiveSessionManager", + _FakeInteractiveSessionManager, + ) engine = DaemonEngine( user_config="u.yaml", @@ -93,6 +113,8 @@ async def _fake_build_runtime_context(*args, **kwargs): await engine.startup() assert engine.version_ids is version_ids + assert captured_interactive_pools is interactive_pools + assert captured_interactive_pools is not scheduler_pools await engine.shutdown() @@ -104,6 +126,7 @@ async def test_engine_startup_skips_config_scene_validation_when_disabled( version_ids = MagicMock() eval_config = MagicMock() worker_runtime = SimpleNamespace(stop=AsyncMock()) + interactive_pools = {"driver": MagicMock()} class _FakeScheduler: def __init__( @@ -147,6 +170,14 @@ async def _fake_build_runtime_context(*args, **kwargs): "alpasim_runtime.daemon.engine.start_worker_runtime", _fake_start_worker_runtime ) monkeypatch.setattr("alpasim_runtime.daemon.engine.DaemonScheduler", _FakeScheduler) + monkeypatch.setattr( + "alpasim_runtime.daemon.engine.create_address_pools", + lambda *_args, **_kwargs: interactive_pools, + ) + monkeypatch.setattr( + "alpasim_runtime.daemon.engine.InteractiveSessionManager", + lambda **_kwargs: SimpleNamespace(close_all=AsyncMock()), + ) engine = DaemonEngine( user_config="u.yaml", diff --git a/src/runtime/tests/test_daemon_main.py b/src/runtime/tests/test_daemon_main.py index e31827bd..208ea4a4 100644 --- a/src/runtime/tests/test_daemon_main.py +++ b/src/runtime/tests/test_daemon_main.py @@ -242,7 +242,11 @@ def test_simulate_parser_rejects_removed_grpc_shutdown_flag() -> None: async def test_runtime_daemon_app_run_starts_and_stops_server( monkeypatch: pytest.MonkeyPatch, ) -> None: - engine = SimpleNamespace(startup=AsyncMock(), shutdown=AsyncMock()) + engine = SimpleNamespace( + startup=AsyncMock(), + shutdown=AsyncMock(), + interactive_session_manager=object(), + ) class _FakeServer: def __init__(self) -> None: @@ -268,6 +272,10 @@ async def stop(self, grace: float) -> None: "alpasim_runtime.daemon.app.runtime_pb2_grpc.add_RuntimeServiceServicer_to_server", lambda _servicer, _server: None, ) + monkeypatch.setattr( + "alpasim_runtime.daemon.app.interactive_runtime_pb2_grpc.add_InteractiveRuntimeServiceServicer_to_server", + lambda _servicer, _server: None, + ) app = RuntimeDaemonApp( engine=engine, @@ -288,7 +296,11 @@ async def stop(self, grace: float) -> None: async def test_runtime_daemon_app_servicer_shutdown_request_stops_run( monkeypatch: pytest.MonkeyPatch, ) -> None: - engine = SimpleNamespace(startup=AsyncMock(), shutdown=AsyncMock()) + engine = SimpleNamespace( + startup=AsyncMock(), + shutdown=AsyncMock(), + interactive_session_manager=object(), + ) class _FakeServer: def __init__(self) -> None: @@ -320,6 +332,10 @@ def register_servicer(servicer: RuntimeDaemonServicer, _server: object) -> None: "alpasim_runtime.daemon.app.runtime_pb2_grpc.add_RuntimeServiceServicer_to_server", register_servicer, ) + monkeypatch.setattr( + "alpasim_runtime.daemon.app.interactive_runtime_pb2_grpc.add_InteractiveRuntimeServiceServicer_to_server", + lambda _servicer, _server: None, + ) app = RuntimeDaemonApp( engine=engine, @@ -342,7 +358,11 @@ def register_servicer(servicer: RuntimeDaemonServicer, _server: object) -> None: async def test_runtime_daemon_app_shutdowns_engine_when_server_start_fails( monkeypatch: pytest.MonkeyPatch, ) -> None: - engine = SimpleNamespace(startup=AsyncMock(), shutdown=AsyncMock()) + engine = SimpleNamespace( + startup=AsyncMock(), + shutdown=AsyncMock(), + interactive_session_manager=object(), + ) class _FakeServer: def add_insecure_port(self, listen_address: str) -> None: @@ -362,6 +382,10 @@ async def stop(self, grace: float) -> None: "alpasim_runtime.daemon.app.runtime_pb2_grpc.add_RuntimeServiceServicer_to_server", lambda _servicer, _server: None, ) + monkeypatch.setattr( + "alpasim_runtime.daemon.app.interactive_runtime_pb2_grpc.add_InteractiveRuntimeServiceServicer_to_server", + lambda _servicer, _server: None, + ) app = RuntimeDaemonApp( engine=engine, @@ -379,7 +403,11 @@ async def stop(self, grace: float) -> None: async def test_runtime_daemon_app_shutdowns_engine_when_server_stop_fails( monkeypatch: pytest.MonkeyPatch, ) -> None: - engine = SimpleNamespace(startup=AsyncMock(), shutdown=AsyncMock()) + engine = SimpleNamespace( + startup=AsyncMock(), + shutdown=AsyncMock(), + interactive_session_manager=object(), + ) class _FakeServer: def add_insecure_port(self, listen_address: str) -> None: @@ -399,6 +427,10 @@ async def stop(self, grace: float) -> None: "alpasim_runtime.daemon.app.runtime_pb2_grpc.add_RuntimeServiceServicer_to_server", lambda _servicer, _server: None, ) + monkeypatch.setattr( + "alpasim_runtime.daemon.app.interactive_runtime_pb2_grpc.add_InteractiveRuntimeServiceServicer_to_server", + lambda _servicer, _server: None, + ) app = RuntimeDaemonApp( engine=engine, diff --git a/src/runtime/tests/test_decision.py b/src/runtime/tests/test_decision.py new file mode 100644 index 00000000..58865710 --- /dev/null +++ b/src/runtime/tests/test_decision.py @@ -0,0 +1,287 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from alpasim_runtime.decision import ( + BackendMetadata, + CandidateDecision, + CandidateStatus, + DecisionBundle, + DecisionSnapshot, + DriverServiceBackendAdapter, + DriverBackendRegistry, + MultiBackendDriverOrchestrator, + build_input_snapshot_id, + select_candidate_in_bundle, +) +from alpasim_utils.geometry import Pose, Trajectory +import numpy as np +from unittest.mock import AsyncMock + + +def _make_candidate(step_id: int, snapshot_id: str, backend_id: str) -> CandidateDecision: + trajectory = Trajectory.from_poses( + timestamps=np.array([100_000], dtype=np.uint64), + poses=[ + Pose( + position=np.array([0.0, 0.0, 0.0], dtype=np.float32), + quaternion=np.array([0.0, 0.0, 0.0, 1.0], dtype=np.float32), + ) + ], + ) + return CandidateDecision( + candidate_id=f"{snapshot_id}:{backend_id}:0", + step_id=step_id, + input_snapshot_id=snapshot_id, + backend_id=backend_id, + status=CandidateStatus.READY, + trajectory=trajectory, + generated_at_us=100_000, + ) + + +class _FakeBackend: + def __init__(self, backend_id: str, priority: int, *, supports_restore: bool = False) -> None: + self.metadata = BackendMetadata( + backend_id=backend_id, + backend_type="fake", + supports_parallel=True, + supports_restore=supports_restore, + priority=priority, + ) + self.infer = AsyncMock() + self.capture_backend_state = MagicMock(return_value={"backend_id": backend_id}) + self.restore_backend_state = AsyncMock() + + +def _make_snapshot(step_id: int = 1) -> DecisionSnapshot: + snapshot_id = build_input_snapshot_id( + step_id=step_id, + time_now_us=100_000, + time_query_us=200_000, + planner_context={"ego": {"yaw": 0.0}}, + route_waypoints_in_rig=[[0.0, 0.0, 0.0]], + traffic_actor_ids=["actor_a"], + ego_pose_history_timestamps_us=[0, 100_000], + camera_frame_timestamps_us={"cam_front": 100_000}, + renderer_data=b"", + ) + return DecisionSnapshot( + step_id=step_id, + input_snapshot_id=snapshot_id, + time_now_us=100_000, + time_query_us=200_000, + ego_pose_history_timestamps_us=[0, 100_000], + traffic_actor_ids=["actor_a"], + route_waypoints_in_rig=[[0.0, 0.0, 0.0]], + planner_context={"ego": {"yaw": 0.0}}, + renderer_data=b"", + camera_frame_timestamps_us={"cam_front": 100_000}, + ) + + +def test_build_input_snapshot_id_is_deterministic() -> None: + first = build_input_snapshot_id( + step_id=1, + time_now_us=100_000, + time_query_us=200_000, + planner_context={"a": 1}, + route_waypoints_in_rig=[[0.0, 1.0, 2.0]], + traffic_actor_ids=["b", "a"], + ego_pose_history_timestamps_us=[0, 100_000], + camera_frame_timestamps_us={"cam_front": 100_000}, + renderer_data=b"payload", + ) + second = build_input_snapshot_id( + step_id=1, + time_now_us=100_000, + time_query_us=200_000, + planner_context={"a": 1}, + route_waypoints_in_rig=[[0.0, 1.0, 2.0]], + traffic_actor_ids=["b", "a"], + ego_pose_history_timestamps_us=[0, 100_000], + camera_frame_timestamps_us={"cam_front": 100_000}, + renderer_data=b"payload", + ) + assert first == second + + +@pytest.mark.asyncio +async def test_multi_backend_orchestrator_selects_highest_priority_ready_candidate() -> None: + snapshot = _make_snapshot() + backend_fast = _FakeBackend("fast", priority=10) + backend_safe = _FakeBackend("safe", priority=1) + backend_fast.infer.return_value = _make_candidate(snapshot.step_id, snapshot.input_snapshot_id, "fast") + backend_safe.infer.return_value = _make_candidate(snapshot.step_id, snapshot.input_snapshot_id, "safe") + + orchestrator = MultiBackendDriverOrchestrator( + DriverBackendRegistry([backend_fast, backend_safe]) + ) + bundle = await orchestrator.generate_candidates(snapshot) + selected = await orchestrator.select_candidate(bundle) + + assert len(bundle.candidates) == 2 + assert selected.backend_id == "safe" + assert selected.status == CandidateStatus.SELECTED + + +@pytest.mark.asyncio +async def test_multi_backend_orchestrator_filters_requested_backend_ids() -> None: + snapshot = _make_snapshot() + backend_fast = _FakeBackend("fast", priority=10) + backend_safe = _FakeBackend("safe", priority=1) + backend_fast.infer.return_value = _make_candidate(snapshot.step_id, snapshot.input_snapshot_id, "fast") + backend_safe.infer.return_value = _make_candidate(snapshot.step_id, snapshot.input_snapshot_id, "safe") + + orchestrator = MultiBackendDriverOrchestrator( + DriverBackendRegistry([backend_fast, backend_safe]) + ) + bundle = await orchestrator.generate_candidates(snapshot, backend_ids=["fast"]) + + assert [candidate.backend_id for candidate in bundle.candidates] == ["fast"] + backend_fast.infer.assert_awaited_once() + backend_safe.infer.assert_not_called() + + +@pytest.mark.asyncio +async def test_driver_service_backend_adapter_sets_next_model_override() -> None: + driver = SimpleNamespace( + drive=AsyncMock(return_value=_make_candidate(1, "snapshot", "backend").trajectory), + set_next_model_for_next_drive=MagicMock(), + set_planner_context_for_next_drive=MagicMock(), + ) + adapter = DriverServiceBackendAdapter( + driver, + backend_id="pdm_backend", + model_type_override="pdm", + ) + snapshot = _make_snapshot() + + candidate = await adapter.infer(snapshot) + + driver.set_next_model_for_next_drive.assert_called_once_with("pdm") + driver.set_planner_context_for_next_drive.assert_called_once() + assert candidate.backend_id == "pdm_backend" + assert candidate.diagnostics["model_type_override"] == "pdm" + + +@pytest.mark.asyncio +async def test_driver_service_backend_adapter_injects_observation_window_summary() -> None: + driver = SimpleNamespace( + drive=AsyncMock(return_value=_make_candidate(1, "snapshot", "backend").trajectory), + set_next_model_for_next_drive=MagicMock(), + set_planner_context_for_next_drive=MagicMock(), + ) + adapter = DriverServiceBackendAdapter( + driver, + backend_id="vla_backend", + observation_window_summary_getter=lambda snapshot_id, window_size: { + "anchor_snapshot_id": snapshot_id, + "window_size": window_size, + "available_frames": 2, + "frames": [{"input_snapshot_id": "prev"}, {"input_snapshot_id": snapshot_id}], + }, + observation_window_summary_size=4, + ) + snapshot = _make_snapshot() + + await adapter.infer(snapshot) + + planner_context = driver.set_planner_context_for_next_drive.call_args.args[0] + assert ( + planner_context["decision_metadata"]["observation_window"]["anchor_snapshot_id"] + == snapshot.input_snapshot_id + ) + assert planner_context["decision_metadata"]["observation_window"]["window_size"] == 4 + + +def test_multi_backend_orchestrator_captures_restoreable_backend_state() -> None: + backend_fast = _FakeBackend("fast", priority=10, supports_restore=False) + backend_safe = _FakeBackend("safe", priority=1, supports_restore=True) + orchestrator = MultiBackendDriverOrchestrator( + DriverBackendRegistry([backend_fast, backend_safe]) + ) + + checkpoint, unsupported = orchestrator.capture_backend_checkpoint() + + assert checkpoint == {"safe": {"backend_id": "safe"}} + assert unsupported == ["fast"] + backend_safe.capture_backend_state.assert_called_once() + backend_fast.capture_backend_state.assert_not_called() + + +@pytest.mark.asyncio +async def test_multi_backend_orchestrator_restores_restoreable_backend_state() -> None: + backend_safe = _FakeBackend("safe", priority=1, supports_restore=True) + orchestrator = MultiBackendDriverOrchestrator( + DriverBackendRegistry([backend_safe]) + ) + + await orchestrator.restore_backend_checkpoint({"safe": {"checkpoint": 1}}) + + backend_safe.restore_backend_state.assert_awaited_once_with({"checkpoint": 1}) + + +@pytest.mark.asyncio +async def test_multi_backend_orchestrator_recompute_appends_new_candidate_and_stales_old() -> None: + snapshot = _make_snapshot() + backend_safe = _FakeBackend("safe", priority=1) + original = _make_candidate(snapshot.step_id, snapshot.input_snapshot_id, "safe") + backend_safe.infer.return_value = original + orchestrator = MultiBackendDriverOrchestrator( + DriverBackendRegistry([backend_safe]) + ) + bundle = await orchestrator.generate_candidates(snapshot) + backend_safe.infer.return_value = _make_candidate( + snapshot.step_id, snapshot.input_snapshot_id, "safe" + ) + + updated = await orchestrator.recompute_candidate(bundle, "safe") + + assert [candidate.status for candidate in updated.candidates] == [ + CandidateStatus.STALE, + CandidateStatus.READY, + ] + assert updated.candidates[-1].candidate_id.endswith(":1") + assert updated.candidates[-1].recompute_count == 1 + + +def test_select_candidate_in_bundle_marks_previous_selection_rejected() -> None: + snapshot = _make_snapshot() + bundle = DecisionBundle( + snapshot=snapshot, + candidates=[ + CandidateDecision( + candidate_id=f"{snapshot.input_snapshot_id}:safe:0", + step_id=snapshot.step_id, + input_snapshot_id=snapshot.input_snapshot_id, + backend_id="safe", + status=CandidateStatus.SELECTED, + trajectory=None, + ), + CandidateDecision( + candidate_id=f"{snapshot.input_snapshot_id}:fast:0", + step_id=snapshot.step_id, + input_snapshot_id=snapshot.input_snapshot_id, + backend_id="fast", + status=CandidateStatus.READY, + trajectory=None, + ), + ], + selected_candidate_id=f"{snapshot.input_snapshot_id}:safe:0", + arbitration_reason="priority", + ) + + updated = select_candidate_in_bundle(bundle, f"{snapshot.input_snapshot_id}:fast:0") + + assert updated.selected_candidate_id == f"{snapshot.input_snapshot_id}:fast:0" + assert updated.arbitration_reason == "manual_selection" + assert [candidate.status for candidate in updated.candidates] == [ + CandidateStatus.REJECTED, + CandidateStatus.SELECTED, + ] diff --git a/src/runtime/tests/test_event_loop.py b/src/runtime/tests/test_event_loop.py index dbf7c34c..68db9b9f 100644 --- a/src/runtime/tests/test_event_loop.py +++ b/src/runtime/tests/test_event_loop.py @@ -5,10 +5,15 @@ from types import SimpleNamespace from typing import Any, cast -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock +import pytest +from alpasim_runtime.config import DriverBackendConfig +from alpasim_runtime.decision import MultiBackendDriverOrchestrator from alpasim_runtime.event_loop import EventBasedRollout -from alpasim_runtime.events.base import SimulationEndEvent +from alpasim_runtime.events.base import Event, EventQueue, SimulationEndEvent +from alpasim_runtime.events.state import RolloutState, ServiceBundle +from alpasim_runtime.events.step import StepEvent from alpasim_runtime.events.policy import PolicyEvent @@ -38,3 +43,131 @@ def test_initial_event_schedule_matches_control_timestamps() -> None: assert policy.timestamp_us == 200 assert end.timestamp_us == 300 + + +class _MarkerEvent(Event): + """Simple event used to assert loop ordering in tests.""" + + priority = 5 + + def __init__(self, timestamp_us: int, marker: list[int]): + super().__init__(timestamp_us) + self._marker = marker + + async def handle(self, rollout_state: RolloutState, queue: EventQueue) -> None: + del rollout_state, queue + self._marker.append(self.timestamp_us) + + +@pytest.mark.asyncio +async def test_run_until_step_commit_stops_after_first_step_event( + rollout_state: RolloutState, + service_bundle: ServiceBundle, +) -> None: + rollout = cast(Any, object.__new__(EventBasedRollout)) + marker: list[int] = [] + + rollout._initialized = True + rollout._closed = False + rollout._state = rollout_state + rollout._async_stack = None + rollout._loop_start_time = 0.0 + rollout._rollout_start_time = 0.0 + rollout._event_queue = EventQueue.init_from_sequence( + [ + _MarkerEvent(timestamp_us=50, marker=marker), + StepEvent( + timestamp_us=100, + control_timestep_us=100, + services=service_bundle, + ), + _MarkerEvent(timestamp_us=150, marker=marker), + ] + ) + rollout_state.step_context = None + + result = await rollout.run_until_step_commit() + + assert marker == [50] + assert result.step_committed is True + assert result.simulation_finished is False + assert result.committed_step_timestamp_us == 100 + + pending_timestamps = sorted(event.timestamp_us for event in rollout._event_queue.queue) + assert pending_timestamps == [150, 200] + + +def test_create_service_bundle_builds_multi_backend_orchestrator_from_config() -> None: + rollout = cast(Any, object.__new__(EventBasedRollout)) + rollout.unbound = SimpleNamespace( + driver_backends=[ + DriverBackendConfig(backend_id="ar1_backend", model_type="ar1", priority=5), + DriverBackendConfig(backend_id="pdm_backend", model_type="pdm", priority=1), + ] + ) + rollout.driver = MagicMock() + rollout.controller = MagicMock() + rollout.physics = MagicMock() + rollout.trafficsim = MagicMock() + rollout.broadcaster = MagicMock() + rollout.planner_delay_buffer = MagicMock() + rollout._default_driver_orchestrator = None + + bundle = rollout._create_service_bundle() + + assert isinstance(bundle.driver_orchestrator, MultiBackendDriverOrchestrator) + + +@pytest.mark.asyncio +async def test_capture_and_restore_runtime_checkpoint_preserves_state_and_queue_async( + rollout_state: RolloutState, +) -> None: + rollout = cast(Any, object.__new__(EventBasedRollout)) + handler = MagicMock() + rollout._initialized = True + rollout._closed = False + rollout._default_driver_orchestrator = None + rollout._build_default_driver_orchestrator = MagicMock(return_value=None) + rollout._state = rollout_state + rollout._event_queue = EventQueue.init_from_sequence( + [ + _MarkerEvent(timestamp_us=50, marker=[]), + SimulationEndEvent(timestamp_us=300), + ] + ) + rollout_state.rendered_images_handler = handler + rollout_state.last_decision_step_id = 7 + checkpoint = rollout.capture_runtime_checkpoint() + + rollout_state.last_decision_step_id = 99 + rollout._event_queue.pop() + + await rollout.restore_runtime_checkpoint(checkpoint) + + assert rollout.current_state.last_decision_step_id == 7 + assert rollout.current_state.rendered_images_handler is handler + assert sorted(event.timestamp_us for event in rollout._event_queue.queue) == [50, 300] + + +@pytest.mark.asyncio +async def test_restore_runtime_checkpoint_restores_backend_state( + rollout_state: RolloutState, +) -> None: + rollout = cast(Any, object.__new__(EventBasedRollout)) + orchestrator = SimpleNamespace( + capture_backend_checkpoint=MagicMock(return_value=({"safe": {"cp": 1}}, [])), + restore_backend_checkpoint=AsyncMock(), + ) + rollout._initialized = True + rollout._closed = False + rollout._default_driver_orchestrator = orchestrator + rollout._build_default_driver_orchestrator = MagicMock(return_value=orchestrator) + rollout._state = rollout_state + rollout._event_queue = EventQueue.init_from_sequence([SimulationEndEvent(timestamp_us=300)]) + rollout_state.rendered_images_handler = MagicMock() + + checkpoint = rollout.capture_runtime_checkpoint() + + await rollout.restore_runtime_checkpoint(checkpoint) + + orchestrator.restore_backend_checkpoint.assert_awaited_once_with({"safe": {"cp": 1}}) diff --git a/src/runtime/tests/test_event_policy.py b/src/runtime/tests/test_event_policy.py index 75ce188c..2068fad6 100644 --- a/src/runtime/tests/test_event_policy.py +++ b/src/runtime/tests/test_event_policy.py @@ -14,6 +14,7 @@ from alpasim_runtime.config import PhysicsUpdateMode from alpasim_runtime.events.base import EventQueue from alpasim_runtime.events.physics import PhysicsEvent, PhysicsTarget +from alpasim_runtime.observation_cache import ObservationCache from alpasim_runtime.events.policy import ( PolicyEvent, assert_sensors_up_to_date, @@ -21,6 +22,7 @@ ) from alpasim_runtime.events.state import RolloutState, ServiceBundle, StepContext from alpasim_utils.geometry import Polyline, Pose, Trajectory +from alpasim_utils.types import ImageWithMetadata # --------------------------------------------------------------------------- # PolicyEvent tests @@ -126,8 +128,68 @@ async def test_run_fills_step_context_timing( assert rollout_state.step_context is not None assert rollout_state.step_context.step_start_us == 200_000 assert rollout_state.step_context.target_time_us == 300_000 + assert rollout_state.step_context.decision_bundle is not None + assert rollout_state.step_context.decision_bundle.snapshot.step_id == 2 + assert rollout_state.step_context.decision_bundle.selected_candidate_id is not None assert rollout_state.step_context.driver_trajectory is not None + @pytest.mark.asyncio + async def test_run_builds_stable_input_snapshot_identity( + self, + policy_event: PolicyEvent, + rollout_state: RolloutState, + mock_driver: AsyncMock, + simple_trajectory: Trajectory, + ) -> None: + mock_driver.drive.return_value = simple_trajectory.clip(200_000, 300_001) + + await policy_event.run(rollout_state, EventQueue()) + first_bundle = rollout_state.step_context.decision_bundle + assert first_bundle is not None + + rollout_state.step_context = StepContext() + rollout_state.data_sensorsim_to_driver = None + mock_driver.drive.reset_mock() + mock_driver.drive.return_value = simple_trajectory.clip(200_000, 300_001) + + await policy_event.run(rollout_state, EventQueue()) + second_bundle = rollout_state.step_context.decision_bundle + assert second_bundle is not None + assert ( + first_bundle.snapshot.input_snapshot_id + == second_bundle.snapshot.input_snapshot_id + ) + + @pytest.mark.asyncio + async def test_run_appends_shared_observation_frame( + self, + policy_event: PolicyEvent, + rollout_state: RolloutState, + mock_driver: AsyncMock, + simple_trajectory: Trajectory, + ) -> None: + rollout_state.observation_cache = ObservationCache(max_frames=4) + rollout_state.last_rendered_images = { + "cam_front": ImageWithMetadata( + start_timestamp_us=167_000, + end_timestamp_us=200_000, + image_bytes=b"frame-bytes", + camera_logical_id="cam_front", + ) + } + rollout_state.data_sensorsim_to_driver = b"renderer-bytes" + mock_driver.drive.return_value = simple_trajectory.clip(200_000, 300_001) + + await policy_event.run(rollout_state, EventQueue()) + + cached = rollout_state.observation_cache.latest() + assert cached is not None + assert cached.input_snapshot_id == ( + rollout_state.step_context.decision_bundle.snapshot.input_snapshot_id + ) + assert cached.rendered_images["cam_front"].image_bytes == b"frame-bytes" + assert cached.renderer_data == b"renderer-bytes" + @pytest.mark.asyncio async def test_drains_outstanding_tasks_before_drive( self, diff --git a/src/runtime/tests/test_event_step.py b/src/runtime/tests/test_event_step.py index 4b8b4b34..f58da8fa 100644 --- a/src/runtime/tests/test_event_step.py +++ b/src/runtime/tests/test_event_step.py @@ -9,6 +9,12 @@ import numpy as np import pytest +from alpasim_runtime.decision import ( + CandidateDecision, + CandidateStatus, + DecisionBundle, + DecisionSnapshot, +) from alpasim_runtime.events.base import EventQueue from alpasim_runtime.events.state import RolloutState, ServiceBundle, StepContext from alpasim_runtime.events.step import StepEvent @@ -127,3 +133,57 @@ async def test_replaces_step_context_with_fresh_one( assert rollout_state.step_context is not None assert rollout_state.step_context.outstanding_tasks == [] assert rollout_state.step_context.step_start_us == 0 # fresh defaults + + @pytest.mark.asyncio + async def test_persists_last_committed_decision_bundle( + self, + rollout_state: RolloutState, + service_bundle: ServiceBundle, + ) -> None: + ego_traj = _make_trajectory(300_000, 300_000, 100_000) + zero_dynamics = np.zeros((len(ego_traj), 12), dtype=np.float64) + dynamic_traj = DynamicTrajectory.from_trajectory_and_dynamics( + ego_traj, zero_dynamics + ) + snapshot = DecisionSnapshot( + step_id=2, + input_snapshot_id="snapshot-2", + time_now_us=200_000, + time_query_us=300_000, + ego_pose_history_timestamps_us=[0, 200_000], + traffic_actor_ids=[], + route_waypoints_in_rig=[], + planner_context=None, + renderer_data=None, + camera_frame_timestamps_us={}, + ) + candidate = CandidateDecision( + candidate_id="snapshot-2:default:0", + step_id=2, + input_snapshot_id="snapshot-2", + backend_id="default", + status=CandidateStatus.SELECTED, + trajectory=ego_traj, + ) + decision_bundle = DecisionBundle( + snapshot=snapshot, + candidates=[candidate], + selected_candidate_id=candidate.candidate_id, + arbitration_reason="test", + ) + + old_ctx = StepContext(step_start_us=200_000, target_time_us=300_000) + old_ctx.decision_bundle = decision_bundle + old_ctx.ego_true = dynamic_traj + old_ctx.ego_estimated = dynamic_traj + old_ctx.corrected_ego_trajectory = ego_traj + rollout_state.step_context = old_ctx + + event = StepEvent( + timestamp_us=200_000, + control_timestep_us=100_000, + services=service_bundle, + ) + await event.run(rollout_state, EventQueue()) + + assert rollout_state.last_committed_decision_bundle == decision_bundle diff --git a/src/runtime/tests/test_interactive_models.py b/src/runtime/tests/test_interactive_models.py new file mode 100644 index 00000000..5678a12f --- /dev/null +++ b/src/runtime/tests/test_interactive_models.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +from __future__ import annotations + +import numpy as np + +from alpasim_runtime.decision import ( + CandidateDecision, + CandidateStatus, + DecisionBundle, + DecisionSnapshot, +) +from alpasim_runtime.interactive.session_runner import _decision_summary_from_bundle +from alpasim_utils.geometry import Pose, Trajectory + + +def test_decision_summary_from_bundle_marks_selected_candidate() -> None: + trajectory = Trajectory.from_poses( + timestamps=np.array([100_000], dtype=np.uint64), + poses=[ + Pose( + position=np.array([0.0, 0.0, 0.0], dtype=np.float32), + quaternion=np.array([0.0, 0.0, 0.0, 1.0], dtype=np.float32), + ) + ], + ) + snapshot = DecisionSnapshot( + step_id=1, + input_snapshot_id="input-1", + time_now_us=100_000, + time_query_us=200_000, + ego_pose_history_timestamps_us=[0, 100_000], + traffic_actor_ids=[], + route_waypoints_in_rig=[], + planner_context=None, + renderer_data=None, + camera_frame_timestamps_us={"cam_front": 100_000}, + ) + fast = CandidateDecision( + candidate_id="input-1:fast:0", + step_id=1, + input_snapshot_id="input-1", + backend_id="fast", + status=CandidateStatus.READY, + trajectory=trajectory, + ) + safe = CandidateDecision( + candidate_id="input-1:safe:0", + step_id=1, + input_snapshot_id="input-1", + backend_id="safe", + status=CandidateStatus.SELECTED, + trajectory=trajectory, + ) + summary = _decision_summary_from_bundle( + DecisionBundle( + snapshot=snapshot, + candidates=[fast, safe], + selected_candidate_id=safe.candidate_id, + arbitration_reason="priority", + ) + ) + + assert summary.step_id == 1 + assert summary.selected_candidate_id == "input-1:safe:0" + assert [candidate.backend_id for candidate in summary.candidates] == ["fast", "safe"] + assert [candidate.selected for candidate in summary.candidates] == [False, True] diff --git a/src/runtime/tests/test_interactive_servicer.py b/src/runtime/tests/test_interactive_servicer.py new file mode 100644 index 00000000..44990600 --- /dev/null +++ b/src/runtime/tests/test_interactive_servicer.py @@ -0,0 +1,356 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from alpasim_runtime.daemon.interactive_servicer import ( + InteractiveRuntimeServicer, + _snapshot_to_proto, + _state_to_proto, +) +from alpasim_runtime.interactive.models import ( + CandidateSummaryModel, + CheckpointSummaryModel, + DecisionSummaryModel, + EgoStateModel, + SessionSnapshotModel, + SessionStateModel, +) +from alpasim_grpc.v0.common_pb2 import DynamicState, Pose, Quat, Vec3 +from alpasim_grpc.v1 import interactive_runtime_pb2 + + +def _make_pose() -> Pose: + return Pose(vec=Vec3(x=1.0, y=2.0, z=0.0), quat=Quat(w=1.0, x=0.0, y=0.0, z=0.0)) + + +def _make_decision() -> DecisionSummaryModel: + return DecisionSummaryModel( + step_id=3, + input_snapshot_id="input-3", + selected_candidate_id="input-3:safe:0", + candidates=[ + CandidateSummaryModel( + candidate_id="input-3:fast:0", + backend_id="fast", + status="READY", + selected=False, + diagnostics={"driver_debug": {"selected_model_type": "fast"}}, + ), + CandidateSummaryModel( + candidate_id="input-3:safe:0", + backend_id="safe", + status="SELECTED", + selected=True, + diagnostics={"driver_debug": {"selected_model_type": "safe"}}, + ), + ], + arbitration_reason="priority", + ) + + +def test_state_to_proto_includes_latest_decision() -> None: + decision = _make_decision() + snapshot = SessionSnapshotModel( + interactive_session_id="sess-1", + tick_id=4, + sim_time_us=400_000, + ego=EgoStateModel( + pose=_make_pose(), + dynamics=DynamicState(), + ), + actors=[], + frame_refs=[], + latest_decision=decision, + context_diagnostics={"timing": {"ego_age_ms": 0.0}}, + ) + state = SessionStateModel( + interactive_session_id="sess-1", + rollout_uuid="rollout-1", + scene_id="scene-a", + status="PAUSED", + current_tick_id=4, + current_sim_time_us=400_000, + latest_snapshot=snapshot, + latest_decision=decision, + active_backend_ids=["fast", "safe"], + ) + + proto = _state_to_proto(state) + + assert proto.latest_decision.step_id == 3 + assert proto.latest_decision.selected_candidate_id == "input-3:safe:0" + assert len(proto.latest_decision.candidates) == 2 + assert proto.latest_decision.candidates[0].diagnostics_json != "" + assert proto.latest_snapshot.latest_decision.step_id == 3 + assert proto.latest_snapshot.context_diagnostics_json != "" + assert list(proto.active_backend_ids) == ["fast", "safe"] + + +def test_snapshot_to_proto_includes_latest_decision() -> None: + snapshot = SessionSnapshotModel( + interactive_session_id="sess-1", + tick_id=4, + sim_time_us=400_000, + ego=EgoStateModel( + pose=_make_pose(), + dynamics=DynamicState(), + front_steering_angle_rad=0.15, + ), + actors=[], + frame_refs=[], + latest_decision=_make_decision(), + ) + + proto = _snapshot_to_proto(snapshot) + + assert proto.latest_decision.input_snapshot_id == "input-3" + assert proto.latest_decision.candidates[1].selected is True + assert proto.ego.front_steering_angle_rad == pytest.approx(0.15) + + +class _AbortContext: + def __init__(self) -> None: + self.code = None + self.details = None + + async def abort(self, code, details): + self.code = code + self.details = details + raise RuntimeError("aborted") + + +@pytest.mark.asyncio +async def test_list_candidates_returns_candidate_summaries() -> None: + manager = SimpleNamespace( + list_candidates=AsyncMock( + return_value=_make_decision().candidates + ) + ) + servicer = InteractiveRuntimeServicer(manager=manager) + + response = await servicer.ListCandidates( + interactive_runtime_pb2.ListCandidatesRequest( + interactive_session_id="sess-1" + ), + _AbortContext(), + ) + + manager.list_candidates.assert_awaited_once_with("sess-1") + assert len(response.candidates) == 2 + assert response.candidates[1].backend_id == "safe" + assert response.candidates[1].selected is True + + +@pytest.mark.asyncio +async def test_list_sessions_returns_session_states() -> None: + manager = SimpleNamespace( + list_sessions=AsyncMock( + return_value=[ + SessionStateModel( + interactive_session_id="sess-1", + rollout_uuid="rollout-1", + scene_id="scene-a", + status="PAUSED", + current_tick_id=4, + current_sim_time_us=400_000, + latest_snapshot=None, + latest_decision=None, + active_backend_ids=["safe"], + ) + ] + ) + ) + servicer = InteractiveRuntimeServicer(manager=manager) + + response = await servicer.ListSessions( + interactive_runtime_pb2.ListSessionsRequest(), + _AbortContext(), + ) + + manager.list_sessions.assert_awaited_once_with() + assert len(response.sessions) == 1 + assert response.sessions[0].interactive_session_id == "sess-1" + assert response.sessions[0].scene_id == "scene-a" + + +@pytest.mark.asyncio +async def test_set_active_backends_returns_updated_state() -> None: + manager = SimpleNamespace( + set_active_backends=AsyncMock( + return_value=SessionStateModel( + interactive_session_id="sess-1", + rollout_uuid="rollout-1", + scene_id="scene-a", + status="PAUSED", + current_tick_id=4, + current_sim_time_us=400_000, + latest_snapshot=None, + latest_decision=None, + active_backend_ids=["safe"], + ) + ) + ) + servicer = InteractiveRuntimeServicer(manager=manager) + + response = await servicer.SetActiveBackends( + interactive_runtime_pb2.SetActiveBackendsRequest( + interactive_session_id="sess-1", + backend_ids=["safe"], + ), + _AbortContext(), + ) + + manager.set_active_backends.assert_awaited_once_with("sess-1", ["safe"]) + assert list(response.active_backend_ids) == ["safe"] + + +@pytest.mark.asyncio +async def test_list_checkpoints_returns_checkpoint_summaries() -> None: + manager = SimpleNamespace( + list_checkpoints=AsyncMock( + return_value=[ + CheckpointSummaryModel( + checkpoint_id="tick-4", + tick_id=4, + sim_time_us=400_000, + status="PAUSED", + restore_supported=False, + unsupported_backend_ids=["default_driver"], + ) + ] + ) + ) + servicer = InteractiveRuntimeServicer(manager=manager) + + response = await servicer.ListCheckpoints( + interactive_runtime_pb2.ListCheckpointsRequest( + interactive_session_id="sess-1" + ), + _AbortContext(), + ) + + manager.list_checkpoints.assert_awaited_once_with("sess-1") + assert len(response.checkpoints) == 1 + assert response.checkpoints[0].checkpoint_id == "tick-4" + assert response.checkpoints[0].restore_supported is False + assert list(response.checkpoints[0].unsupported_backend_ids) == ["default_driver"] + + +@pytest.mark.asyncio +async def test_recompute_candidate_returns_updated_state() -> None: + manager = SimpleNamespace( + recompute_candidate=AsyncMock( + return_value=SessionStateModel( + interactive_session_id="sess-1", + rollout_uuid="rollout-1", + scene_id="scene-a", + status="PAUSED", + current_tick_id=4, + current_sim_time_us=400_000, + latest_snapshot=None, + latest_decision=None, + active_backend_ids=["safe"], + ) + ) + ) + servicer = InteractiveRuntimeServicer(manager=manager) + + response = await servicer.RecomputeCandidate( + interactive_runtime_pb2.RecomputeCandidateRequest( + interactive_session_id="sess-1", + backend_id="safe", + ), + _AbortContext(), + ) + + manager.recompute_candidate.assert_awaited_once_with("sess-1", "safe") + assert response.current_tick_id == 4 + + +@pytest.mark.asyncio +async def test_select_candidate_returns_updated_state() -> None: + manager = SimpleNamespace( + select_candidate=AsyncMock( + return_value=SessionStateModel( + interactive_session_id="sess-1", + rollout_uuid="rollout-1", + scene_id="scene-a", + status="PAUSED", + current_tick_id=4, + current_sim_time_us=400_000, + latest_snapshot=None, + latest_decision=None, + active_backend_ids=["safe"], + ) + ) + ) + servicer = InteractiveRuntimeServicer(manager=manager) + + response = await servicer.SelectCandidate( + interactive_runtime_pb2.SelectCandidateRequest( + interactive_session_id="sess-1", + candidate_id="input-4:fast:0", + ), + _AbortContext(), + ) + + manager.select_candidate.assert_awaited_once_with("sess-1", "input-4:fast:0") + assert response.current_tick_id == 4 + + +@pytest.mark.asyncio +async def test_restore_checkpoint_returns_updated_state() -> None: + manager = SimpleNamespace( + restore_checkpoint=AsyncMock( + return_value=SessionStateModel( + interactive_session_id="sess-1", + rollout_uuid="rollout-1", + scene_id="scene-a", + status="PAUSED", + current_tick_id=4, + current_sim_time_us=400_000, + latest_snapshot=None, + latest_decision=None, + active_backend_ids=["safe"], + ) + ) + ) + servicer = InteractiveRuntimeServicer(manager=manager) + + response = await servicer.RestoreCheckpoint( + interactive_runtime_pb2.RestoreCheckpointRequest( + interactive_session_id="sess-1", + checkpoint_id="tick-4", + ), + _AbortContext(), + ) + + manager.restore_checkpoint.assert_awaited_once_with("sess-1", "tick-4") + assert response.current_tick_id == 4 + + +@pytest.mark.asyncio +async def test_restore_checkpoint_returns_failed_precondition_for_unsupported_restore() -> None: + manager = SimpleNamespace( + restore_checkpoint=AsyncMock( + side_effect=RuntimeError("Checkpoint restore is not supported for backends: default_driver") + ) + ) + servicer = InteractiveRuntimeServicer(manager=manager) + context = _AbortContext() + + with pytest.raises(RuntimeError, match="aborted"): + await servicer.RestoreCheckpoint( + interactive_runtime_pb2.RestoreCheckpointRequest( + interactive_session_id="sess-1", + checkpoint_id="tick-4", + ), + context, + ) + + assert context.code.name == "FAILED_PRECONDITION" diff --git a/src/runtime/tests/test_interactive_session_runner.py b/src/runtime/tests/test_interactive_session_runner.py new file mode 100644 index 00000000..6cb4b878 --- /dev/null +++ b/src/runtime/tests/test_interactive_session_runner.py @@ -0,0 +1,304 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +from __future__ import annotations + +from collections import OrderedDict +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from alpasim_runtime.decision import ( + CandidateDecision, + CandidateStatus, + DecisionBundle, + DecisionSnapshot, +) +from alpasim_runtime.interactive.frame_store import FrameStore +from alpasim_runtime.interactive.models import ( + CandidateSummaryModel, + CheckpointSummaryModel, + DecisionSummaryModel, + EgoStateModel, + FrameDataModel, + SessionSnapshotModel, +) +from alpasim_runtime.interactive.session_runner import InteractiveSessionRunner +from alpasim_grpc.v0.common_pb2 import DynamicState, Pose, Quat, Vec3 + + +class _AsyncLock: + async def __aenter__(self): + return None + + async def __aexit__(self, exc_type, exc, tb): + return False + + +def _make_pose() -> Pose: + return Pose(vec=Vec3(x=1.0, y=2.0, z=0.0), quat=Quat(w=1.0, x=0.0, y=0.0, z=0.0)) + + +def _make_snapshot() -> SessionSnapshotModel: + return SessionSnapshotModel( + interactive_session_id="sess-1", + tick_id=4, + sim_time_us=400_000, + ego=EgoStateModel(pose=_make_pose(), dynamics=DynamicState()), + actors=[], + frame_refs=[], + latest_decision=DecisionSummaryModel( + step_id=4, + input_snapshot_id="input-4", + selected_candidate_id="input-4:safe:0", + candidates=[ + CandidateSummaryModel( + candidate_id="input-4:safe:0", + backend_id="safe", + status="SELECTED", + selected=True, + ) + ], + ), + ) + + +@pytest.mark.asyncio +async def test_list_and_restore_checkpoint_updates_state() -> None: + runner = object.__new__(InteractiveSessionRunner) + runner._lock = _AsyncLock() + runner.pause = AsyncMock() + runner._publish = AsyncMock() + runner._build_state = MagicMock( + return_value=SimpleNamespace( + interactive_session_id="sess-1", + rollout_uuid="rollout-1", + scene_id="scene-a", + status="PAUSED", + current_tick_id=4, + current_sim_time_us=400_000, + latest_snapshot=_make_snapshot(), + latest_decision=_make_snapshot().latest_decision, + active_backend_ids=["safe"], + error="", + ) + ) + runner._capture_images = AsyncMock() + runner._frame_store = FrameStore(max_retained_ticks=4) + frame = FrameDataModel( + sensor_id="cam_front", + frame_start_us=300_000, + frame_end_us=333_000, + frame_encoding="JPEG", + content_type="image/jpeg", + image_bytes=b"abc", + ) + runner._frame_store.restore(OrderedDict([(4, {"cam_front": frame})])) + runner._pending_frames = [frame] + runner._tick_id = 4 + runner._latest_snapshot = _make_snapshot() + runner._status = "COMPLETED" + runner._error = "stale" + checkpoint = SimpleNamespace( + checkpoint_id="tick-4", + tick_id=4, + sim_time_us=400_000, + runtime_checkpoint=SimpleNamespace(unsupported_backend_ids=[]), + frame_store_snapshot=OrderedDict([(4, {"cam_front": frame})]), + latest_snapshot=_make_snapshot(), + pending_frames=[frame], + status="PAUSED", + ) + runner._checkpoints = OrderedDict([("tick-4", checkpoint)]) + rollout = SimpleNamespace( + restore_runtime_checkpoint=AsyncMock(), + current_state=SimpleNamespace(rendered_images_handler=None), + ) + runner._rollout = rollout + + checkpoints = await runner.list_checkpoints() + restored_state = await runner.restore_checkpoint("tick-4") + + assert checkpoints == [ + CheckpointSummaryModel( + checkpoint_id="tick-4", + tick_id=4, + sim_time_us=400_000, + status="PAUSED", + restore_supported=True, + unsupported_backend_ids=[], + ) + ] + runner.pause.assert_awaited_once() + rollout.restore_runtime_checkpoint.assert_awaited_once_with(checkpoint.runtime_checkpoint) + assert runner._tick_id == 4 + assert runner._status == "PAUSED" + assert runner._error == "" + assert runner._frame_store.get_frame("cam_front", 4).image_bytes == b"abc" + assert restored_state.status == "PAUSED" + + +@pytest.mark.asyncio +async def test_restore_checkpoint_rejects_unsupported_backend_restore() -> None: + runner = object.__new__(InteractiveSessionRunner) + runner._lock = _AsyncLock() + runner.pause = AsyncMock() + runner._publish = AsyncMock() + runner._rollout = SimpleNamespace() + runner._checkpoints = OrderedDict( + [ + ( + "tick-4", + SimpleNamespace( + checkpoint_id="tick-4", + tick_id=4, + sim_time_us=400_000, + runtime_checkpoint=SimpleNamespace( + unsupported_backend_ids=["default_driver"] + ), + frame_store_snapshot=OrderedDict(), + latest_snapshot=None, + pending_frames=[], + status="PAUSED", + ), + ) + ] + ) + + with pytest.raises(RuntimeError, match="default_driver"): + await runner.restore_checkpoint("tick-4") + + +@pytest.mark.asyncio +async def test_recompute_candidate_updates_latest_decision_bundle() -> None: + runner = object.__new__(InteractiveSessionRunner) + runner._lock = _AsyncLock() + runner.pause = AsyncMock() + runner._publish = AsyncMock() + runner._build_state = MagicMock(return_value=SimpleNamespace(status="PAUSED")) + runner._record_checkpoint_locked = MagicMock() + runner._latest_snapshot = _make_snapshot() + snapshot = DecisionSnapshot( + step_id=4, + input_snapshot_id="input-4", + time_now_us=400_000, + time_query_us=500_000, + ego_pose_history_timestamps_us=[300_000, 400_000], + traffic_actor_ids=[], + route_waypoints_in_rig=[], + planner_context=None, + renderer_data=None, + camera_frame_timestamps_us={}, + ) + old_candidate = CandidateDecision( + candidate_id="input-4:safe:0", + step_id=4, + input_snapshot_id="input-4", + backend_id="safe", + status=CandidateStatus.SELECTED, + trajectory=None, + ) + updated_bundle = DecisionBundle( + snapshot=snapshot, + candidates=[ + old_candidate, + CandidateDecision( + candidate_id="input-4:safe:1", + step_id=4, + input_snapshot_id="input-4", + backend_id="safe", + status=CandidateStatus.READY, + trajectory=None, + recompute_count=1, + ), + ], + selected_candidate_id="input-4:safe:0", + arbitration_reason="recomputed:safe", + ) + orchestrator = SimpleNamespace(recompute_candidate=AsyncMock(return_value=updated_bundle)) + rollout = SimpleNamespace( + _build_default_driver_orchestrator=MagicMock(return_value=orchestrator), + driver=SimpleNamespace(), + current_state=SimpleNamespace( + last_committed_decision_bundle=DecisionBundle( + snapshot=snapshot, + candidates=[old_candidate], + selected_candidate_id="input-4:safe:0", + arbitration_reason="priority", + ), + available_driver_backend_ids=["safe"], + ), + ) + runner._rollout = rollout + + state = await runner.recompute_candidate("safe") + + runner.pause.assert_awaited_once() + orchestrator.recompute_candidate.assert_awaited_once() + assert runner._rollout.current_state.last_committed_decision_bundle == updated_bundle + assert runner._latest_snapshot.latest_decision is not None + assert len(runner._latest_snapshot.latest_decision.candidates) == 2 + assert runner._latest_snapshot.context_diagnostics["candidate_count"] == 2 + runner._record_checkpoint_locked.assert_called_once_with(runner._latest_snapshot) + assert state.status == "PAUSED" + + +@pytest.mark.asyncio +async def test_select_candidate_updates_latest_decision_bundle() -> None: + runner = object.__new__(InteractiveSessionRunner) + runner._lock = _AsyncLock() + runner.pause = AsyncMock() + runner._publish = AsyncMock() + runner._build_state = MagicMock(return_value=SimpleNamespace(status="PAUSED")) + runner._record_checkpoint_locked = MagicMock() + runner._latest_snapshot = _make_snapshot() + snapshot = DecisionSnapshot( + step_id=4, + input_snapshot_id="input-4", + time_now_us=400_000, + time_query_us=500_000, + ego_pose_history_timestamps_us=[300_000, 400_000], + traffic_actor_ids=[], + route_waypoints_in_rig=[], + planner_context=None, + renderer_data=None, + camera_frame_timestamps_us={}, + ) + rollout = SimpleNamespace( + current_state=SimpleNamespace( + last_committed_decision_bundle=DecisionBundle( + snapshot=snapshot, + candidates=[ + CandidateDecision( + candidate_id="input-4:safe:0", + step_id=4, + input_snapshot_id="input-4", + backend_id="safe", + status=CandidateStatus.SELECTED, + trajectory=None, + ), + CandidateDecision( + candidate_id="input-4:fast:0", + step_id=4, + input_snapshot_id="input-4", + backend_id="fast", + status=CandidateStatus.READY, + trajectory=None, + ), + ], + selected_candidate_id="input-4:safe:0", + arbitration_reason="priority", + ) + ) + ) + runner._rollout = rollout + + state = await runner.select_candidate("input-4:fast:0") + + runner.pause.assert_awaited_once() + assert runner._rollout.current_state.last_committed_decision_bundle.selected_candidate_id == "input-4:fast:0" + assert runner._latest_snapshot.latest_decision is not None + assert runner._latest_snapshot.latest_decision.selected_candidate_id == "input-4:fast:0" + assert runner._latest_snapshot.context_diagnostics["selected_candidate_id"] == "input-4:fast:0" + runner._record_checkpoint_locked.assert_called_once_with(runner._latest_snapshot) + assert state.status == "PAUSED" diff --git a/src/runtime/tests/test_observation_cache.py b/src/runtime/tests/test_observation_cache.py new file mode 100644 index 00000000..c5adb85c --- /dev/null +++ b/src/runtime/tests/test_observation_cache.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +from __future__ import annotations + +import numpy as np +from alpasim_runtime.observation_cache import ( + ObservationCache, + ObservationFrame, +) +from alpasim_utils.geometry import DynamicTrajectory, Pose, Trajectory +from alpasim_utils.scenario import TrafficObjects +from alpasim_utils.types import ImageWithMetadata + + +def _make_dynamic_trajectory(timestamp_us: int) -> DynamicTrajectory: + trajectory = Trajectory.from_poses( + timestamps=np.array([timestamp_us], dtype=np.uint64), + poses=[ + Pose( + np.array([float(timestamp_us) / 1e6, 0.0, 0.0], dtype=np.float32), + np.array([0.0, 0.0, 0.0, 1.0], dtype=np.float32), + ) + ], + ) + return DynamicTrajectory.from_trajectory_and_dynamics( + trajectory, + np.zeros((1, 12), dtype=np.float64), + ) + + +def _make_frame(step_id: int) -> ObservationFrame: + timestamp_us = step_id * 100_000 + return ObservationFrame( + step_id=step_id, + input_snapshot_id=f"snapshot-{step_id}", + time_now_us=timestamp_us, + time_query_us=timestamp_us + 100_000, + camera_frame_timestamps_us={"cam_front": timestamp_us}, + rendered_images={ + "cam_front": ImageWithMetadata( + start_timestamp_us=timestamp_us - 33_000, + end_timestamp_us=timestamp_us, + image_bytes=f"img-{step_id}".encode(), + camera_logical_id="cam_front", + ) + }, + renderer_data=f"renderer-{step_id}".encode(), + ego_trajectory=_make_dynamic_trajectory(timestamp_us), + ego_trajectory_estimate=_make_dynamic_trajectory(timestamp_us), + traffic_objs=TrafficObjects(), + ego_pose_history_timestamps_us=[timestamp_us - 100_000, timestamp_us], + route_waypoints_in_rig=[[0.0, 0.0, 0.0]], + planner_context={"step_id": step_id}, + active_backend_ids=["default_driver"], + ) + + +def test_observation_cache_window_and_checkpoint_restore() -> None: + cache = ObservationCache(max_frames=4) + for step_id in range(1, 5): + cache.append(_make_frame(step_id)) + + window = cache.get_window("snapshot-4", window_size=2) + checkpoint = cache.checkpoint() + + assert [frame.input_snapshot_id for frame in window.frames] == [ + "snapshot-3", + "snapshot-4", + ] + assert window.frames[-1].rendered_images["cam_front"].image_bytes == b"img-4" + + cache.append(_make_frame(5)) + cache.restore(checkpoint) + + assert cache.latest() is not None + assert cache.latest().input_snapshot_id == "snapshot-4" + assert cache.get("snapshot-4").renderer_data == b"renderer-4" diff --git a/src/runtime/tests/test_runtime_context.py b/src/runtime/tests/test_runtime_context.py new file mode 100644 index 00000000..8c7132c2 --- /dev/null +++ b/src/runtime/tests/test_runtime_context.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from alpasim_runtime.config import UserSimulatorConfig +from alpasim_runtime.runtime_context import build_runtime_context +from omegaconf import OmegaConf + + +def _make_user_config_dictconfig(): + return OmegaConf.merge( + OmegaConf.structured(UserSimulatorConfig), + { + "simulation_config": { + "n_sim_steps": 1, + "n_rollouts": 1, + }, + "scenes": [{"scene_id": "clipgt-a"}], + "endpoints": { + "driver": {"n_concurrent_rollouts": 1}, + "sensorsim": {"n_concurrent_rollouts": 1}, + "physics": {"n_concurrent_rollouts": 1}, + "trafficsim": {"n_concurrent_rollouts": 1}, + "controller": {"n_concurrent_rollouts": 1}, + }, + "nr_workers": 1, + }, + ) + + +@pytest.mark.asyncio +async def test_build_runtime_context_skips_scene_validation_without_dataclass_replace( + monkeypatch: pytest.MonkeyPatch, +) -> None: + user_config = _make_user_config_dictconfig() + network_config = SimpleNamespace() + version_ids = SimpleNamespace() + eval_config = SimpleNamespace() + validate_scenarios = AsyncMock() + + monkeypatch.setattr( + "alpasim_runtime.runtime_context.parse_simulator_config", + lambda *_args, **_kwargs: SimpleNamespace( + user=user_config, + network=network_config, + ), + ) + monkeypatch.setattr( + "alpasim_runtime.runtime_context.typed_parse_config", + lambda *_args, **_kwargs: eval_config, + ) + monkeypatch.setattr( + "alpasim_runtime.runtime_context.gather_versions_from_addresses", + AsyncMock(return_value=version_ids), + ) + monkeypatch.setattr( + "alpasim_runtime.runtime_context.validate_scenarios", + validate_scenarios, + ) + monkeypatch.setattr( + "alpasim_runtime.runtime_context.Artifact.discover_from_glob", + lambda *_args, **_kwargs: {}, + ) + monkeypatch.setattr( + "alpasim_runtime.runtime_context.create_address_pools", + lambda *_args, **_kwargs: {}, + ) + monkeypatch.setattr( + "alpasim_runtime.runtime_context.compute_max_in_flight", + lambda *_args, **_kwargs: 1, + ) + + context = await build_runtime_context( + user_config_path="u.yaml", + network_config_path="n.yaml", + eval_config_path="e.yaml", + usdz_glob="/tmp/*.usdz", + validate_config_scenes=False, + ) + + validate_scenarios.assert_awaited_once() + validated_config = validate_scenarios.await_args.args[0] + assert validated_config.user.scenes == [] + assert context.config.user.scenes[0].scene_id == "clipgt-a" diff --git a/src/runtime/tests/test_web_debugger_server.py b/src/runtime/tests/test_web_debugger_server.py new file mode 100644 index 00000000..a659f1b2 --- /dev/null +++ b/src/runtime/tests/test_web_debugger_server.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +from __future__ import annotations + +from pathlib import Path + +from alpasim_runtime.web_debugger.server import _load_available_scene_ids + + +class _StubMapProvider: + def __init__(self, scene_ids: list[str], fail: bool = False) -> None: + self._scene_ids = scene_ids + self._fail = fail + + def list_scene_ids(self) -> list[str]: + if self._fail: + raise RuntimeError("discovery failed") + return list(self._scene_ids) + + +def test_load_available_scene_ids_prefers_discovered_union(tmp_path: Path) -> None: + user_config = tmp_path / "user-config.yaml" + user_config.write_text( + """ +scenes: + - scene_id: scene-from-config + - scene_id: shared-scene +""".strip(), + encoding="utf-8", + ) + + scene_ids = _load_available_scene_ids( + _StubMapProvider(["scene-from-glob", "shared-scene"]), + str(user_config), + ) + + assert scene_ids == ["scene-from-config", "scene-from-glob", "shared-scene"] + + +def test_load_available_scene_ids_falls_back_to_user_config(tmp_path: Path) -> None: + user_config = tmp_path / "user-config.yaml" + user_config.write_text( + """ +scenes: + - scene_id: scene-from-config +""".strip(), + encoding="utf-8", + ) + + scene_ids = _load_available_scene_ids( + _StubMapProvider([], fail=True), + str(user_config), + ) + + assert scene_ids == ["scene-from-config"] diff --git a/src/wizard/configs/deploy/local_external_mtgs_sensorsim.yaml b/src/wizard/configs/deploy/local_external_mtgs_sensorsim.yaml new file mode 100644 index 00000000..0c6b5ada --- /dev/null +++ b/src/wizard/configs/deploy/local_external_mtgs_sensorsim.yaml @@ -0,0 +1,65 @@ +# @package _global_ +# Local deployment override for running MTGS sensorsim outside Docker. +# +# Start the MTGS gRPC service in the host conda `mtgs` environment on +# localhost:50053, then run the wizard with `+deploy=local_external_mtgs_sensorsim`. + +defaults: + - local + - _self_ + +scenes: + scene_ids: + - mtgs-road_block-331220_4690660_331190_4690710 + local_usdz_dir: ${repo-relative:"data/nre-artifacts/road_block-331220_4690660_331190_4690710"} + +wizard: + debug_flags: + use_localhost: true + run_sim_services: + ["driver", "physics", "trafficsim", "controller", "runtime"] + external_services: + sensorsim: + - "localhost:50053" + +runtime: + extra_cameras: [] + endpoints: + do_shutdown: false + sensorsim: + n_concurrent_rollouts: 1 + skip: false + physics: + skip: true + trafficsim: + skip: true + simulation_config: + cameras: + - logical_id: CAM_F0 + height: 180 + width: 320 + frame_interval_us: 100000 + shutter_duration_us: 30000 + first_frame_offset_us: -30000 + n_sim_steps: 100 + n_rollouts: 1 + group_render_requests: false + min_traffic_duration_us: 0 + +driver: + inference: + use_cameras: ["CAM_F0"] + rectification: null + +eval: + scorers: + image: + camera_logical_id: CAM_F0 + video: + camera_id_to_render: CAM_F0 + +services: + driver: + gpus: null + physics: + gpus: null diff --git a/src/wizard/configs/deploy/local_mtgs_sensorsim.yaml b/src/wizard/configs/deploy/local_mtgs_sensorsim.yaml new file mode 100644 index 00000000..bfcbd6b1 --- /dev/null +++ b/src/wizard/configs/deploy/local_mtgs_sensorsim.yaml @@ -0,0 +1,60 @@ +# @package _global_ +# Local deployment override for using MTGS as the SensorsimService. +# +# It keeps the AlpaSim runtime contract unchanged and only replaces the +# sensorsim container command. The mounted MTGS repo must provide the `mtgs` +# Python package plus Nerfstudio/CUDA dependencies in the selected image/env. + +defaults: + - local + - _self_ + +scenes: + scene_ids: + - mtgs-road_block-331220_4690660_331190_4690710 + local_usdz_dir: ${repo-relative:"data/nre-artifacts/road_block-331220_4690660_331190_4690710"} + +runtime: + endpoints: + sensorsim: + n_concurrent_rollouts: 1 + physics: + skip: true + trafficsim: + skip: true + +services: + sensorsim: + image: alpasim-base:0.1.4 + external_image: false + volumes: + - ${repo-relative:"src"}:/repo/src + - ${repo-relative:"MTGS"}:/mnt/mtgs + - ${repo-relative:"data/nre-artifacts/road_block-331220_4690660_331190_4690710"}:/mnt/mtgs-artifact + environments: + - OMP_NUM_THREADS=1 + - MTGS_REPO=/mnt/mtgs + - PYTHONPATH=/repo/src/mtgs_sensorsim:/repo/src/grpc:/mnt/mtgs + workdir: /repo + command: + - "uv run --no-sync python -m alpasim_mtgs_sensorsim" + - "--host=0.0.0.0" + - "--port={port}" + - "--config=/mnt/mtgs/experiments/main_mt/MTGS/road_block-331220_4690660_331190_4690710/config.yml" + - "--artifact-dir=/mnt/mtgs-artifact" + - "--scene-id=mtgs-road_block-331220_4690660_331190_4690710" + - "--travel-id=7" + - "--native-height=1080" + - "--native-width=1920" + - "--warmup-renders=1" + - "--warmup-height=180" + - "--warmup-width=320" + - "--log-level=${wizard.log_level}" + replicas_per_container: 1 + gpus: [0] + + driver: + gpus: null + + physics: + gpus: null diff --git a/src/wizard/configs/driver/pdm.yaml b/src/wizard/configs/driver/pdm.yaml new file mode 100644 index 00000000..eba2fd38 --- /dev/null +++ b/src/wizard/configs/driver/pdm.yaml @@ -0,0 +1,49 @@ +# Should be used in defaults list, e.g. +# - /driver: pdm +# Type validation happens at driver runtime via OmegaConf.structured merge + +defaults: + - _self_ # YAML values override schema defaults + - pdm_runtime_configs # Runtime overrides suitable for PDMClosed-style planner + +# PDM Driver Configuration for Alpasim + +# Logging level (uses wizard's global setting) +log_level: ${wizard.log_level} + +# Model configuration +model: + model_type: pdm + # The PDM backend is dependency-free and does not require weights. + checkpoint_path: "unused" + # Current implementation is CPU-friendly; can be moved to CUDA later if needed. + device: "cpu" + +# Server configuration +host: "0.0.0.0" +port: ??? + +# Inference configuration +inference: + # Current driver interface still requires camera streams. + use_cameras: ["camera_front_wide_120fov"] + max_batch_size: 1 + subsample_factor: 1 + context_length: 1 + output_frequency_hz: 10 + +# Route configuration +route: + default_command: 2 # Default command: 0=right, 1=left, 2=straight + use_waypoint_commands: true + command_distance_threshold: 3.0 + min_lookahead_distance: 20.0 + +# Output configuration +output_dir: "/mnt/output/driver" + +# Optional smoothing/retiming can be enabled later if needed +trajectory_optimizer: + enabled: false + +plot_debug_images: false diff --git a/src/wizard/configs/driver/pdm_runtime_configs.yaml b/src/wizard/configs/driver/pdm_runtime_configs.yaml new file mode 100644 index 00000000..1508a32a --- /dev/null +++ b/src/wizard/configs/driver/pdm_runtime_configs.yaml @@ -0,0 +1,22 @@ +# @package _global_ + +runtime: + simulation_config: + force_gt_duration_us: 2_000_000 + n_sim_steps: 100 + n_rollouts: 1 + control_timestep_us: 100_000 # 10 Hz driver drive() calls + time_start_offset_us: 500_000 + + cameras: + - height: 540 + width: 960 + logical_id: camera_front_wide_120fov + frame_interval_us: 100_000 + shutter_duration_us: 30_000 + first_frame_offset_us: -30_000 + +services: + driver: + gpus: null + replicas_per_container: 1