ci: add automated GCP sandbox integration test workflow triggered daily and on releases

.github/workflows/auto_draft_release.yaml

@@ -6,13 +6,14 @@ on:
       - main
 
 jobs:
-  draft-release:
+  check-bump:
     runs-on: ubuntu-latest
-    permissions:
-      contents: write
+    outputs:
+      is_bump: ${{ steps.check-bump.outputs.is_bump }}
+      version: ${{ steps.check-bump.outputs.version }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Check if version bump commit
         id: check-bump
@@ -29,14 +30,65 @@ jobs:
             echo "Not a version bump commit."
           fi
 
-      - name: Create Draft Release
-        if: steps.check-bump.outputs.is_bump == 'true'
-        uses: softprops/action-gh-release@v2
+  gcp-integration-test:
+    needs: check-bump
+    if: needs.check-bump.outputs.is_bump == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      - name: Authenticate to GCP
+        uses: google-github-actions/auth@v2
         with:
-          tag_name: v${{ steps.check-bump.outputs.version }}
-          name: v${{ steps.check-bump.outputs.version }}
-          draft: true
-          prerelease: ${{ contains(steps.check-bump.outputs.version, 'rc') }}
-          generate_release_notes: true
+          workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER }}
+          service_account: ${{ secrets.GCP_WIF_SA_EMAIL }}
+
+      - name: Set up GCP Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          project_id: datcom-ci
+
+      - name: Install uv and Set up Python
+        uses: astral-sh/setup-uv@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install Dependencies
+        run: uv sync --locked
+
+      - name: Run GCP Integration Test
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DC_API_KEY: ${{ secrets.DC_API_KEY }}
+        run: |
+          TAG="${{ needs.check-bump.outputs.version }}"
+
+          uv run python3 tests/datacommons-integration-tests/gcp_test_runner.py \
+            --project-id datcom-ci \
+            --dcp-version "$TAG" \
+            --dc-api-key "$DC_API_KEY"
+
+  hermetic-integration-test:
+    needs: check-bump
+    if: needs.check-bump.outputs.is_bump == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      - name: Install uv and Set up Python
+        uses: astral-sh/setup-uv@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install Dependencies
+        run: uv sync --locked
+
+      - name: Run hermetic integration test
+        run: uv run pytest -s tests/datacommons-integration-tests/run_local_integration_test.py
+
+
+

.github/workflows/gcp_integration_test.yaml

@@ -0,0 +1,92 @@
+name: GCP Sandbox Integration Test
+
+on:
+  schedule:
+    # Run weekdays (Monday through Thursday) at 02:00 UTC (tests latest images)
+    - cron: '0 2 * * 1-4'
+
+  release:
+    types: [published]
+  workflow_dispatch:
+    # Allow manual trigger via GitHub UI with optional tag override
+    inputs:
+      version_tag:
+        description: 'Image tag to test (e.g. latest, stable, 1.1.0). If empty, defaults to latest.'
+        required: false
+        type: string
+        default: 'latest'
+      confirm_resource_cost:
+        description: 'Type "yes" to authorize the high GCP sandbox resource cost of this run.'
+        required: true
+        type: string
+        default: 'no'
+
+permissions:
+  contents: 'read'
+  id-token: 'write'
+
+jobs:
+  gcp-integration-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Verify Confirmation (Manual Runs Only)
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          CONFIRMATION="${{ github.event.inputs.confirm_resource_cost }}"
+          if [ "${CONFIRMATION,,}" != "yes" ]; then
+            echo "Error: High-cost execution confirmation was not authorized (must type 'yes')."
+            exit 1
+          fi
+
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      - name: Authenticate to GCP
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER }}
+          service_account: ${{ secrets.GCP_WIF_SA_EMAIL }}
+
+      - name: Set up GCP Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          project_id: datcom-ci
+
+      - name: Install uv and Set up Python
+        uses: astral-sh/setup-uv@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install Dependencies
+        run: uv sync --locked
+
+      - name: Determine Image Tag
+        id: image-tag
+        run: |
+          if [ "${{ github.event_name }}" = "release" ]; then
+            # Strip 'v' prefix if present (e.g. v1.1.1 -> 1.1.1)
+            RAW_TAG="${{ github.event.release.tag_name }}"
+            TAG="${RAW_TAG#v}"
+            echo "Using release image tag: $TAG"
+            echo "tag=$TAG" >> $GITHUB_OUTPUT
+          elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            TAG="${{ github.event.inputs.version_tag }}"
+            echo "Using manual input image tag: $TAG"
+            echo "tag=$TAG" >> $GITHUB_OUTPUT
+          else
+            echo "Using default cron image tag: latest"
+            echo "tag=latest" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Run GCP Integration Test
+        env:
+          DC_API_KEY: ${{ secrets.DC_API_KEY }}
+        run: |
+          TAG="${{ steps.image-tag.outputs.tag }}"
+
+          uv run python3 tests/datacommons-integration-tests/gcp_test_runner.py \
+            --project-id datcom-ci \
+            --dcp-version "$TAG" \
+            --dc-api-key "$DC_API_KEY"
+
+

tests/datacommons-integration-tests/README.md

@@ -60,30 +60,51 @@ You can modify the flags inside `custom_feature_flags.yaml` if you need to tweak
 
 In addition to local emulated tests, this package includes a script to run automated, end-to-end integration tests on GCP real resources. This validates Terraform scaffolding, GCS, Spanner database seeding, Cloud Workflow executions, and Cloud Run web server APIs.
 
-### Running the GCP Tests
+### Execution Modes
+
+The GCP integration runner supports two execution modes:
+
+#### 1. Heavy Mode (Full Provisioning & E2E Validation)
+Pulls the latest templates, provisions a completely new isolated sandbox stack in GCP via Terraform (Spanner DB, GCS bucket, Cloud Workflows, and Cloud Run service), performs ingestion, runs the tests, and destroys the sandbox on completion (unless `--keep-sandbox` is specified).
+* **Usage:**
+  ```bash
+  uv run python tests/datacommons-integration-tests/gcp_test_runner.py \
+      --project-id datcom-ci \
+      --dc-api-key "YOUR_API_KEY"
+  ```
+
+#### 2. Lightweight Mode (Fast Debugging Loop)
+Skips the slow Terraform setup and teardown stages entirely. It reads resource configurations from an existing `/tmp/workspace-<namespace>` directory, clears the database tables, re-uploads local test data files, triggers the ingestion workflow, and runs `pytest`. This reduces iteration time **from 10+ minutes down to 1-2 minutes**.
+* **Usage:**
+  1. First, create a persistent sandbox and keep it alive:
+     ```bash
+     uv run python tests/datacommons-integration-tests/gcp_test_runner.py --namespace my-debug-sandbox --keep-sandbox
+     ```
+  2. Run subsequent test runs in lightweight mode:
+     ```bash
+     uv run python tests/datacommons-integration-tests/gcp_test_runner.py --namespace my-debug-sandbox --reuse-sandbox
+     ```
 
-To run the GCP integration test suite (requires authenticated GCP CLI access):
-```bash
-uv run python tests/datacommons-integration-tests/run_gcp_integration_test.py \
-    --project-id <YOUR_GCP_PROJECT_ID> \
-    --dc-api-key "YOUR_GOOGLE_DATA_COMMONS_API_KEY"
-```
-
-Alternatively, you can pass the API key via an environment variable:
-```bash
-DC_API_KEY="YOUR_API_KEY" uv run python tests/datacommons-integration-tests/run_gcp_integration_test.py \
-    --project-id <YOUR_GCP_PROJECT_ID>
-```
 
 ### Script Arguments
-*   `--project-id`: GCP Project ID (default: `datcom-ci`).
-*   `--dc-api-key`: Data Commons API Key.
-*   `--region`: GCP region (default: `us-central1`).
-*   `--namespace`: Custom resource naming namespace (default: randomized `itest-XXXX`).
-*   `--keep-sandbox`: Do not destroy sandbox GCP resources on completion (useful for debugging).
-*   `--reuse-sandbox`: Reuse existing sandbox resources (requires passing persistent `--namespace` and having run with `--keep-sandbox` previously).
-*   `--tf-git-ref`: Git reference for the Terraform templates repository (default: `main`).
-*   `--services-image`, `--preprocessing-image`, `--helper-image`: Override container images deployed during provisioning.
+*   `--project-id`: The Google Cloud Project ID where the sandbox resources should be provisioned (default: `datcom-ci`).
+*   `--dc-api-key`: Google Data Commons API Key needed to authenticate and configure sandbox clients.
+*   `--region`: The GCP region to deploy resources (default: `us-central1`).
+*   `--namespace`: Custom naming namespace for resources (default: randomized `itest-XXXX`).
+*   `--keep-sandbox`: Do not destroy sandbox GCP resources on completion/failure. Useful for debugging active instances.
+*   `--reuse-sandbox`: Reuse existing local workspace and GCP sandbox resources if they exist. Requires passing a persistent, custom `--namespace` (e.g. `--namespace itest-9611`) and having run with `--keep-sandbox` in the previous run.
+*   `--tf-git-ref`: Git reference branch/tag/commit for the GCP Terraform templates repository (default: `main`).
+*   `--dcp-version`: Override default DCP version (controls all images and templates) (default: `latest`).
+
+
+### E2E Verification Stages & Philosophy
+
+Once the GCP sandbox stack is up and seeded with dataset MCFs, the script triggers the entire API verification suite concurrently. This validates three core capabilities of the deployed Data Commons Platform backend:
+
+*   **Stage A: V2 Observations & Custom Data Retrieval**
+    *   *Philosophy:* Validates E2E query routing and data loading for custom variables. It asserts observations values on custom seeded OECD wages metrics to verify local database mapping, loading, and seeding.
+*   **Stage B: V2 Embeddings & Natural Language Resolution**
+    *   *Philosophy:* Validates custom search index and embeddings generation. This checks that the Ingestion Helper successfully generated vector coordinates from custom MCFs, loaded them into Spanner, and that the Resolver can map natural language queries (like "wages") back to these custom database entities.