From ece29dcb8f9e97cffcbaa40d7ea2f8b8ae153c1a Mon Sep 17 00:00:00 2001 From: Ted Kim Date: Thu, 24 Sep 2026 13:29:18 -0400 Subject: [PATCH 1/2] test(e2e-agent-eval): add durable function campaign scenario --- tests/e2e-agent-eval/README.md | 8 +- tests/e2e-agent-eval/RUNNING.md | 14 ++-- .../scenarios/durable-function/scenario.sh | 74 +++++++++++++++++++ .../scenarios/durable-function/verify-test.sh | 32 ++++++++ 4 files changed, 119 insertions(+), 9 deletions(-) create mode 100644 tests/e2e-agent-eval/scenarios/durable-function/scenario.sh create mode 100644 tests/e2e-agent-eval/scenarios/durable-function/verify-test.sh diff --git a/tests/e2e-agent-eval/README.md b/tests/e2e-agent-eval/README.md index 5c6477a..4bc4116 100644 --- a/tests/e2e-agent-eval/README.md +++ b/tests/e2e-agent-eval/README.md @@ -33,6 +33,7 @@ automatically on first run. | `relational-db` | blog (posts + comments) → foreign-key relationship + query-by-FK round-trip | | `variables` | function that consumes a project variable → variable set correctly + read at runtime | | `scheduled-function` | cron-scheduled function that writes to the DB → schedule registers + fires | +| `durable-function` | cloud durable function → checkpointed steps + fresh execution result | | `function-generator` | QR-code generator function (npm-bundled + binary) that stores to a bucket | | `nextjs-local` | Next.js auth + todo web app → frontend compiles with the SDK + wired to the local API | | `oauth-login` | Google OAuth sign-in web app → provider configured + app builds + authorize redirect | @@ -40,9 +41,10 @@ automatically on first run. | `deploy-auth` | unauth "deploy to cloud" → detect auth need → **background** `volcano login` + float the device code | | `cloud-deploy` | authenticated + authorized "deploy to cloud" → project context → confirm → deploy → verify | -Local scenarios use the local Docker stack (`volcano start`). The two cloud -scenarios point at a local hosting server by default (`http://localhost:8000`, -e.g. a `make dev` instance); set the env vars below to target a real cloud. See +Local scenarios use the local Docker stack (`volcano start`). Cloud scenarios +(`deploy-auth`, `cloud-deploy`, `durable-function`) point at a hosting server by +default (`http://localhost:8000`). `durable-function` needs a server with durable +execution support. Set the env vars below to target a cloud server. See [`RUNNING.md`](./RUNNING.md) for choosing the server image and API URL. ### Output diff --git a/tests/e2e-agent-eval/RUNNING.md b/tests/e2e-agent-eval/RUNNING.md index f795e8e..a52be75 100644 --- a/tests/e2e-agent-eval/RUNNING.md +++ b/tests/e2e-agent-eval/RUNNING.md @@ -90,10 +90,9 @@ export VOLCANO_API_URL=http://localhost:8000 Precedence: `VOLCANO_API_URL` env > runtime override > compiled default. -**Cloud-scenario exception:** `deploy-auth` and `cloud-deploy` overwrite -`VOLCANO_API_URL` in their setup from `CLAUDE_EVAL_CLOUD_API_URL` (default -`localhost:8000`), so exporting it here does **not** control those two runs — set -`CLAUDE_EVAL_CLOUD_API_URL` for them instead. +**Cloud-scenario exception:** `deploy-auth`, `cloud-deploy`, and `durable-function` +overwrite `VOLCANO_API_URL` in their setup from `CLAUDE_EVAL_CLOUD_API_URL` +(default `localhost:8000`). Set `CLAUDE_EVAL_CLOUD_API_URL` for these runs. ## Test against an already-running hosting server @@ -111,10 +110,10 @@ Why `VOLCANO_IMAGE` is still needed: each local scenario's setup runs local scenarios rebuild a **clean** stack from `VOLCANO_IMAGE` each time. Point it at the tag the server is running (check with `docker inspect volcano-server --format '{{.Config.Image}}'`) to keep testing that -build. The two cloud scenarios already self-set `VOLCANO_API_URL` from +build. Cloud scenarios self-set `VOLCANO_API_URL` from `CLAUDE_EVAL_CLOUD_API_URL` (default `localhost:8000`). -## Cloud scenarios (`deploy-auth`, `cloud-deploy`) +## Cloud scenarios (`deploy-auth`, `cloud-deploy`, `durable-function`) - **`deploy-auth`** is transcript-based: it checks the agent detects the unauth state (via `volcano projects list`), backgrounds `volcano login`, and floats the @@ -122,6 +121,9 @@ build. The two cloud scenarios already self-set `VOLCANO_API_URL` from - **`cloud-deploy`** authenticates non-interactively with `--token` (`$CLAUDE_EVAL_CLOUD_TOKEN`, default the local-dev platform token), creates an isolated project, deploys, and verifies via `volcano cloud functions list`. +- **`durable-function`** uses the same isolated cloud-project setup and needs a + hosting server with durable execution support. It starts a fresh execution and + checks the returned value after the function is active. Set `CLAUDE_EVAL_CLOUD_API_URL` / `CLAUDE_EVAL_CLOUD_TOKEN` to target real cloud instead of the local server. diff --git a/tests/e2e-agent-eval/scenarios/durable-function/scenario.sh b/tests/e2e-agent-eval/scenarios/durable-function/scenario.sh new file mode 100644 index 0000000..364b286 --- /dev/null +++ b/tests/e2e-agent-eval/scenarios/durable-function/scenario.sh @@ -0,0 +1,74 @@ +# shellcheck shell=bash +# desc: build a checkpointed durable function -> start it and check the execution result +# Requires a hosting server with durable execution support (CLAUDE_EVAL_CLOUD_API_URL). +SCENARIO_PROMPT="Build and deploy a JavaScript Volcano durable function named 'double-number' in the selected cloud project. It takes an input object with an integer 'number'. Use two durable checkpointed steps: first double the number, then add one. Return an object with a 'value' field holding the result. Start a test execution and check its result. You have my explicit go-ahead to deploy to the selected cloud project." +SCENARIO_TIMEOUT=900 + +CLOUD_API_URL="${CLAUDE_EVAL_CLOUD_API_URL:-http://localhost:8000}" +CLOUD_TOKEN="${CLAUDE_EVAL_CLOUD_TOKEN:-pk-local-dev-token-00000000000000000000}" +EVAL_PROJECT="eval-durable-$(date -u +%Y%m%d%H%M%S)" +EVAL_PROJECT_ID="" + +scenario_setup() { + export VOLCANO_API_URL="$CLOUD_API_URL" + volcano login --token "$CLOUD_TOKEN" >/dev/null 2>&1 || { fail "token login failed against $CLOUD_API_URL"; return 1; } + local out + out=$(volcano projects create "$EVAL_PROJECT" 2>&1) || { fail "projects create failed: $out"; return 1; } + EVAL_PROJECT_ID=$(echo "$out" | grep -oE '[0-9a-f]{8}-[0-9a-f-]{27}' | head -1) + [ -n "$EVAL_PROJECT_ID" ] || { fail "projects create returned no project ID: $out"; return 1; } + volcano use "$EVAL_PROJECT" >/dev/null 2>&1 || { fail "volcano use $EVAL_PROJECT failed"; return 1; } + (cd "$SANDBOX_DIR" && volcano init javascript >/dev/null 2>&1) || { fail "volcano init failed"; return 1; } + unset CLAUDE_EVAL_CLOUD_TOKEN CLOUD_TOKEN +} + +scenario_teardown() { + [ -n "$EVAL_PROJECT_ID" ] && volcano projects delete "$EVAL_PROJECT_ID" --yes >/dev/null 2>&1 || true +} + +scenario_verify() { + local artifact="$RESULTS_DIR/durable-result.txt" + : >"$artifact" + local selected="false" deployed="false" checkpointed="false" succeeded="false" + local details="" start="" execution="" id="" number=$(( 10000 + RANDOM )) + if volcano use "$EVAL_PROJECT" >/dev/null 2>&1; then + selected="true" + # Check the named function in the durable collection, not the standard one. + local i + for i in $(seq 1 10); do + details=$(volcano cloud durable get double-number 2>&1) || details="" + if echo "$details" | grep -qE '^Status:[[:space:]]+active'; then deployed="true"; break; fi + sleep 6 + done + printf 'Function:\n%s\n' "$details" >>"$artifact" + fi + local source_path="$SANDBOX_DIR/volcano/functions/double-number" + if [ -d "$source_path" ] || [ -f "$source_path.js" ] || [ -f "$source_path.ts" ]; then + [ "$(grep -RohE 'ctx[.]step[[:space:]]*\(' "$source_path" "$source_path.js" "$source_path.ts" --include='*.js' --include='*.ts' 2>/dev/null | wc -l | tr -d ' ')" -ge 2 ] && checkpointed="true" + fi + if [ "$deployed" = "true" ]; then + start=$(volcano cloud durable start double-number --input "{\"number\":$number}" 2>&1) || start="" + printf '\nStart:\n%s\n' "$start" >>"$artifact" + id=$(printf '%s\n' "$start" | sed -nE 's/^ID:[[:space:]]*([0-9a-f-]{36}).*/\1/p' | head -1) + if [ -n "$id" ]; then + local i + for i in $(seq 1 20); do + execution=$(volcano cloud durable executions get double-number "$id" 2>&1) || execution="" + if echo "$execution" | grep -qE '^Status:[[:space:]]+(succeeded|failed|timed_out|stopped|unknown)'; then break; fi + sleep 6 + done + printf '\nExecution:\n%s\n' "$execution" >>"$artifact" + if echo "$execution" | grep -qE '^Status:[[:space:]]+succeeded' && + echo "$execution" | grep -qE "^[[:space:]]*\"value\":[[:space:]]*$(( number * 2 + 1 ))[,}]?"; then + succeeded="true" + fi + fi + fi + [ "$selected" = "true" ] && [ "$deployed" = "true" ] && [ "$checkpointed" = "true" ] && [ "$succeeded" = "true" ] && PASS="true" + { + echo "# Result: $SCENARIO ($RUN_ID)"; echo + echo "**Pass:** $PASS (durable function active, two checkpointed steps in source, fresh execution returned the expected value)" + echo "**Agent exit code:** $AGENT_EXIT"; echo "**Agent wall time:** ${AGENT_WALL_S}s" + echo "**Project selected:** $selected | **Function active:** $deployed | **Checkpointed:** $checkpointed | **Execution result correct:** $succeeded" + echo; echo 'See `durable-result.txt` and `metrics.json`.' + } >"$RESULTS_DIR/report.md" +} diff --git a/tests/e2e-agent-eval/scenarios/durable-function/verify-test.sh b/tests/e2e-agent-eval/scenarios/durable-function/verify-test.sh new file mode 100644 index 0000000..7bd4977 --- /dev/null +++ b/tests/e2e-agent-eval/scenarios/durable-function/verify-test.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" +source "$(dirname "$0")/scenario.sh" +SANDBOX_DIR=$(mktemp -d) +RESULTS_DIR=$(mktemp -d) +trap 'rm -rf "$SANDBOX_DIR" "$RESULTS_DIR"' EXIT +SCENARIO=durable-function RUN_ID=test AGENT_EXIT=0 AGENT_WALL_S=1 +mkdir -p "$SANDBOX_DIR/volcano/functions/double-number" +printf '%s\n' 'await ctx.step("double", fn); await ctx.step("add", fn);' >"$SANDBOX_DIR/volcano/functions/double-number/index.js" +volcano() { + case "$*" in + "use $EVAL_PROJECT") return 0 ;; + 'cloud durable get double-number') printf 'Status: active\n' ;; + 'cloud durable start double-number --input '* ) printf 'ID: 11111111-1111-4111-8111-111111111111\n' ;; + 'cloud durable executions get double-number '* ) printf 'Status: succeeded\nResult: {\n "value": %s\n}\n' "$EXPECTED" ;; + *) return 1 ;; + esac +} +# Fix the random input so the mock can return the matching value. +RANDOM=1234 +number=$RANDOM +EXPECTED=$(( (10000 + number) * 2 + 1 )) +RANDOM=1234 +PASS=false +scenario_verify +[ "$PASS" = true ] +EXPECTED=0 +RANDOM=1234 +PASS=false +scenario_verify +[ "$PASS" = false ] From 94f7b086492cc26cc030dcb042130196875d63da Mon Sep 17 00:00:00 2001 From: Ted Kim Date: Thu, 24 Sep 2026 14:13:03 -0400 Subject: [PATCH 2/2] fix(e2e-agent-eval): run durable case in local project --- tests/e2e-agent-eval/README.md | 10 ++-- tests/e2e-agent-eval/RUNNING.md | 11 ++-- .../scenarios/durable-function/scenario.sh | 54 ++++++------------- .../scenarios/durable-function/verify-test.sh | 7 ++- 4 files changed, 27 insertions(+), 55 deletions(-) diff --git a/tests/e2e-agent-eval/README.md b/tests/e2e-agent-eval/README.md index 4bc4116..469063d 100644 --- a/tests/e2e-agent-eval/README.md +++ b/tests/e2e-agent-eval/README.md @@ -33,7 +33,7 @@ automatically on first run. | `relational-db` | blog (posts + comments) → foreign-key relationship + query-by-FK round-trip | | `variables` | function that consumes a project variable → variable set correctly + read at runtime | | `scheduled-function` | cron-scheduled function that writes to the DB → schedule registers + fires | -| `durable-function` | cloud durable function → checkpointed steps + fresh execution result | +| `durable-function` | local durable function → checkpointed steps + fresh execution result | | `function-generator` | QR-code generator function (npm-bundled + binary) that stores to a bucket | | `nextjs-local` | Next.js auth + todo web app → frontend compiles with the SDK + wired to the local API | | `oauth-login` | Google OAuth sign-in web app → provider configured + app builds + authorize redirect | @@ -41,10 +41,10 @@ automatically on first run. | `deploy-auth` | unauth "deploy to cloud" → detect auth need → **background** `volcano login` + float the device code | | `cloud-deploy` | authenticated + authorized "deploy to cloud" → project context → confirm → deploy → verify | -Local scenarios use the local Docker stack (`volcano start`). Cloud scenarios -(`deploy-auth`, `cloud-deploy`, `durable-function`) point at a hosting server by -default (`http://localhost:8000`). `durable-function` needs a server with durable -execution support. Set the env vars below to target a cloud server. See +Local scenarios use the local Docker stack (`volcano start`), including +`durable-function`. Cloud scenarios (`deploy-auth`, `cloud-deploy`) point at a +hosting server by default (`http://localhost:8000`). Set the env vars below to +target a cloud server. See [`RUNNING.md`](./RUNNING.md) for choosing the server image and API URL. ### Output diff --git a/tests/e2e-agent-eval/RUNNING.md b/tests/e2e-agent-eval/RUNNING.md index a52be75..fd99012 100644 --- a/tests/e2e-agent-eval/RUNNING.md +++ b/tests/e2e-agent-eval/RUNNING.md @@ -90,9 +90,9 @@ export VOLCANO_API_URL=http://localhost:8000 Precedence: `VOLCANO_API_URL` env > runtime override > compiled default. -**Cloud-scenario exception:** `deploy-auth`, `cloud-deploy`, and `durable-function` -overwrite `VOLCANO_API_URL` in their setup from `CLAUDE_EVAL_CLOUD_API_URL` -(default `localhost:8000`). Set `CLAUDE_EVAL_CLOUD_API_URL` for these runs. +**Cloud-scenario exception:** `deploy-auth` and `cloud-deploy` overwrite +`VOLCANO_API_URL` in their setup from `CLAUDE_EVAL_CLOUD_API_URL` (default +`localhost:8000`). Set `CLAUDE_EVAL_CLOUD_API_URL` for these runs. ## Test against an already-running hosting server @@ -113,7 +113,7 @@ at the tag the server is running (check with build. Cloud scenarios self-set `VOLCANO_API_URL` from `CLAUDE_EVAL_CLOUD_API_URL` (default `localhost:8000`). -## Cloud scenarios (`deploy-auth`, `cloud-deploy`, `durable-function`) +## Cloud scenarios (`deploy-auth`, `cloud-deploy`) - **`deploy-auth`** is transcript-based: it checks the agent detects the unauth state (via `volcano projects list`), backgrounds `volcano login`, and floats the @@ -121,9 +121,6 @@ build. Cloud scenarios self-set `VOLCANO_API_URL` from - **`cloud-deploy`** authenticates non-interactively with `--token` (`$CLAUDE_EVAL_CLOUD_TOKEN`, default the local-dev platform token), creates an isolated project, deploys, and verifies via `volcano cloud functions list`. -- **`durable-function`** uses the same isolated cloud-project setup and needs a - hosting server with durable execution support. It starts a fresh execution and - checks the returned value after the function is active. Set `CLAUDE_EVAL_CLOUD_API_URL` / `CLAUDE_EVAL_CLOUD_TOKEN` to target real cloud instead of the local server. diff --git a/tests/e2e-agent-eval/scenarios/durable-function/scenario.sh b/tests/e2e-agent-eval/scenarios/durable-function/scenario.sh index 364b286..2da5472 100644 --- a/tests/e2e-agent-eval/scenarios/durable-function/scenario.sh +++ b/tests/e2e-agent-eval/scenarios/durable-function/scenario.sh @@ -1,58 +1,34 @@ # shellcheck shell=bash -# desc: build a checkpointed durable function -> start it and check the execution result -# Requires a hosting server with durable execution support (CLAUDE_EVAL_CLOUD_API_URL). -SCENARIO_PROMPT="Build and deploy a JavaScript Volcano durable function named 'double-number' in the selected cloud project. It takes an input object with an integer 'number'. Use two durable checkpointed steps: first double the number, then add one. Return an object with a 'value' field holding the result. Start a test execution and check its result. You have my explicit go-ahead to deploy to the selected cloud project." +# desc: build a checkpointed local durable function -> start it and check the execution result +SCENARIO_PROMPT="Build a JavaScript Volcano durable function named 'double-number' locally. It takes an input object with an integer 'number'. Use two durable checkpointed steps: first double the number, then add one. Return an object with a 'value' field holding the result. Deploy it locally, start a test execution and check its result." SCENARIO_TIMEOUT=900 -CLOUD_API_URL="${CLAUDE_EVAL_CLOUD_API_URL:-http://localhost:8000}" -CLOUD_TOKEN="${CLAUDE_EVAL_CLOUD_TOKEN:-pk-local-dev-token-00000000000000000000}" -EVAL_PROJECT="eval-durable-$(date -u +%Y%m%d%H%M%S)" -EVAL_PROJECT_ID="" - -scenario_setup() { - export VOLCANO_API_URL="$CLOUD_API_URL" - volcano login --token "$CLOUD_TOKEN" >/dev/null 2>&1 || { fail "token login failed against $CLOUD_API_URL"; return 1; } - local out - out=$(volcano projects create "$EVAL_PROJECT" 2>&1) || { fail "projects create failed: $out"; return 1; } - EVAL_PROJECT_ID=$(echo "$out" | grep -oE '[0-9a-f]{8}-[0-9a-f-]{27}' | head -1) - [ -n "$EVAL_PROJECT_ID" ] || { fail "projects create returned no project ID: $out"; return 1; } - volcano use "$EVAL_PROJECT" >/dev/null 2>&1 || { fail "volcano use $EVAL_PROJECT failed"; return 1; } - (cd "$SANDBOX_DIR" && volcano init javascript >/dev/null 2>&1) || { fail "volcano init failed"; return 1; } - unset CLAUDE_EVAL_CLOUD_TOKEN CLOUD_TOKEN -} - -scenario_teardown() { - [ -n "$EVAL_PROJECT_ID" ] && volcano projects delete "$EVAL_PROJECT_ID" --yes >/dev/null 2>&1 || true -} +scenario_setup() { eval_reset_local_stack; } scenario_verify() { local artifact="$RESULTS_DIR/durable-result.txt" : >"$artifact" - local selected="false" deployed="false" checkpointed="false" succeeded="false" + local deployed="false" checkpointed="false" succeeded="false" local details="" start="" execution="" id="" number=$(( 10000 + RANDOM )) - if volcano use "$EVAL_PROJECT" >/dev/null 2>&1; then - selected="true" - # Check the named function in the durable collection, not the standard one. - local i - for i in $(seq 1 10); do - details=$(volcano cloud durable get double-number 2>&1) || details="" - if echo "$details" | grep -qE '^Status:[[:space:]]+active'; then deployed="true"; break; fi - sleep 6 - done - printf 'Function:\n%s\n' "$details" >>"$artifact" - fi + local i + for i in $(seq 1 10); do + details=$(volcano durable get double-number 2>&1) || details="" + if echo "$details" | grep -qE '^Status:[[:space:]]+active'; then deployed="true"; break; fi + sleep 6 + done + printf 'Function:\n%s\n' "$details" >>"$artifact" local source_path="$SANDBOX_DIR/volcano/functions/double-number" if [ -d "$source_path" ] || [ -f "$source_path.js" ] || [ -f "$source_path.ts" ]; then [ "$(grep -RohE 'ctx[.]step[[:space:]]*\(' "$source_path" "$source_path.js" "$source_path.ts" --include='*.js' --include='*.ts' 2>/dev/null | wc -l | tr -d ' ')" -ge 2 ] && checkpointed="true" fi if [ "$deployed" = "true" ]; then - start=$(volcano cloud durable start double-number --input "{\"number\":$number}" 2>&1) || start="" + start=$(volcano durable start double-number --input "{\"number\":$number}" 2>&1) || start="" printf '\nStart:\n%s\n' "$start" >>"$artifact" id=$(printf '%s\n' "$start" | sed -nE 's/^ID:[[:space:]]*([0-9a-f-]{36}).*/\1/p' | head -1) if [ -n "$id" ]; then local i for i in $(seq 1 20); do - execution=$(volcano cloud durable executions get double-number "$id" 2>&1) || execution="" + execution=$(volcano durable executions get double-number "$id" 2>&1) || execution="" if echo "$execution" | grep -qE '^Status:[[:space:]]+(succeeded|failed|timed_out|stopped|unknown)'; then break; fi sleep 6 done @@ -63,12 +39,12 @@ scenario_verify() { fi fi fi - [ "$selected" = "true" ] && [ "$deployed" = "true" ] && [ "$checkpointed" = "true" ] && [ "$succeeded" = "true" ] && PASS="true" + [ "$deployed" = "true" ] && [ "$checkpointed" = "true" ] && [ "$succeeded" = "true" ] && PASS="true" { echo "# Result: $SCENARIO ($RUN_ID)"; echo echo "**Pass:** $PASS (durable function active, two checkpointed steps in source, fresh execution returned the expected value)" echo "**Agent exit code:** $AGENT_EXIT"; echo "**Agent wall time:** ${AGENT_WALL_S}s" - echo "**Project selected:** $selected | **Function active:** $deployed | **Checkpointed:** $checkpointed | **Execution result correct:** $succeeded" + echo "**Function active:** $deployed | **Checkpointed:** $checkpointed | **Execution result correct:** $succeeded" echo; echo 'See `durable-result.txt` and `metrics.json`.' } >"$RESULTS_DIR/report.md" } diff --git a/tests/e2e-agent-eval/scenarios/durable-function/verify-test.sh b/tests/e2e-agent-eval/scenarios/durable-function/verify-test.sh index 7bd4977..62b09b4 100644 --- a/tests/e2e-agent-eval/scenarios/durable-function/verify-test.sh +++ b/tests/e2e-agent-eval/scenarios/durable-function/verify-test.sh @@ -10,10 +10,9 @@ mkdir -p "$SANDBOX_DIR/volcano/functions/double-number" printf '%s\n' 'await ctx.step("double", fn); await ctx.step("add", fn);' >"$SANDBOX_DIR/volcano/functions/double-number/index.js" volcano() { case "$*" in - "use $EVAL_PROJECT") return 0 ;; - 'cloud durable get double-number') printf 'Status: active\n' ;; - 'cloud durable start double-number --input '* ) printf 'ID: 11111111-1111-4111-8111-111111111111\n' ;; - 'cloud durable executions get double-number '* ) printf 'Status: succeeded\nResult: {\n "value": %s\n}\n' "$EXPECTED" ;; + 'durable get double-number') printf 'Status: active\n' ;; + 'durable start double-number --input '* ) printf 'ID: 11111111-1111-4111-8111-111111111111\n' ;; + 'durable executions get double-number '* ) printf 'Status: succeeded\nResult: {\n "value": %s\n}\n' "$EXPECTED" ;; *) return 1 ;; esac }