From a4190f2df6860cef0215efdc1c33f6400018e474 Mon Sep 17 00:00:00 2001 From: Dylan Volz Date: Sat, 7 Nov 2026 02:34:21 +0000 Subject: [PATCH 1/2] ci(samples): retry transient LLM 503/429 errors and propagate runner exit codes Wrap sample test invocations in run_with_retry (up to 3 attempts with exponential backoff and DB table reset) when transient upstream LLM errors (503 UNAVAILABLE / 429 RESOURCE_EXHAUSTED) occur, and add matching retry helpers in the Go sample test harnesses. Also replace the piped find | while loop with process substitution and capture test exit codes before cleanup so subshell failures are never masked by rm -rf. --- .ci/sample_tests/run_tests.sh | 95 ++++++++++++++----- .../pre-post-processing/go/agent_test.go | 55 ++++++++++- .../quickstart/go/quickstart_test.go | 55 ++++++++++- 3 files changed, 171 insertions(+), 34 deletions(-) diff --git a/.ci/sample_tests/run_tests.sh b/.ci/sample_tests/run_tests.sh index 5c1695a70c4b..5f042e63b956 100644 --- a/.ci/sample_tests/run_tests.sh +++ b/.ci/sample_tests/run_tests.sh @@ -95,24 +95,27 @@ run_python_test() { local name=$(basename "$dir") echo "--- Running Python Test: $name ---" ( - cd "$dir" - python3 -m venv .venv - source .venv/bin/activate - pip install -q -r requirements.txt pytest + cd "$dir" || exit 1 + rm -rf .venv + python3 -m venv .venv || exit 1 + source .venv/bin/activate || exit 1 + pip install -q -r requirements.txt pytest || exit 1 - cd .. + cd .. || exit 1 local test_file=$(find . -maxdepth 1 -name "*test.py" | head -n 1) + local rc=0 if [ -n "$test_file" ]; then echo "Found native test: $test_file. Running pytest..." export ORCH_NAME="$name" export PYTHONPATH="../" - pytest "$test_file" + pytest "$test_file" || rc=$? else echo "No native test found. running agent directly..." export PYTHONPATH="../" - python3 "${name}/${AGENT_FILE_PATTERN}" + python3 "${name}/${AGENT_FILE_PATTERN}" || rc=$? fi rm -rf "${name}/.venv" + exit "$rc" ) } @@ -121,21 +124,23 @@ run_js_test() { local name=$(basename "$dir") echo "--- Running JS Test: $name ---" ( - cd "$dir" - if [ -f "package-lock.json" ]; then npm ci -q; else npm install -q; fi + cd "$dir" || exit 1 + if [ -f "package-lock.json" ]; then npm ci -q || exit 1; else npm install -q || exit 1; fi - cd .. + cd .. || exit 1 # Looking for a JS test file in the parent directory local test_file=$(find . -maxdepth 1 -name "*test.js" | head -n 1) + local rc=0 if [ -n "$test_file" ]; then echo "Found native test: $test_file. Running node --test..." export ORCH_NAME="$name" - node --test "$test_file" + node --test "$test_file" || rc=$? else echo "No native test found. running agent directly..." - node "${name}/${AGENT_FILE_PATTERN}" + node "${name}/${AGENT_FILE_PATTERN}" || rc=$? fi rm -rf "${name}/node_modules" + exit "$rc" ) } @@ -146,30 +151,72 @@ run_go_test() { # TODO: Re-enable 'langchain' Go test once tmc/langchaingo supports thought_signature on Gemini 3+ (#1464) if [ "$name" == "openAI" ] || [ "$name" == "langchain" ]; then echo -e "\nSkipping framework '${name}': Temporarily excluded." - return + return 0 fi echo "--- Running Go Test: $name ---" ( - cd "$dir" + cd "$dir" || exit 1 if [ -f "go.mod" ]; then - go mod tidy + go mod tidy || exit 1 fi - cd .. + cd .. || exit 1 local test_file=$(find . -maxdepth 1 -name "*test.go" | head -n 1) + local rc=0 if [ -n "$test_file" ]; then echo "Found native test: $test_file. Running go test..." export ORCH_NAME="$name" - go test -v ./... + export SAMPLE_TEST_MAX_ATTEMPTS=1 + go test -v ./... || rc=$? else echo "No native test found. running agent directly..." - cd "$name" - go run "." + cd "$name" || exit 1 + go run "." || rc=$? fi + exit "$rc" ) } +is_transient_llm_error() { + local log_file=$1 + grep -qiE '(Error 503|503 Service Unavailable|Status: UNAVAILABLE|"status":[[:space:]]*"UNAVAILABLE"|experiencing high demand|Error 429|429 Too Many Requests|RESOURCE_EXHAUSTED|overloaded)' "$log_file" +} + +run_with_retry() { + local max_attempts=3 + local delay=10 + local attempt=1 + local log_file + log_file=$(mktemp) + + while [ "$attempt" -le "$max_attempts" ]; do + local rc=0 + set +e + "$@" 2>&1 | tee "$log_file" + rc=${PIPESTATUS[0]} + set -e + + if [ "$rc" -eq 0 ]; then + rm -f "$log_file" + return 0 + fi + + if [ "$attempt" -lt "$max_attempts" ] && is_transient_llm_error "$log_file"; then + echo "WARN: Transient upstream LLM error detected (attempt ${attempt}/${max_attempts}). Resetting DB table and retrying in ${delay}s..." + setup_db_table + sleep "$delay" + delay=$((delay * 2)) + attempt=$((attempt + 1)) + : > "$log_file" + continue + fi + + rm -f "$log_file" + return "$rc" + done +} + cleanup() { echo "Cleaning up background processes..." [ -n "$TOOLBOX_PID" ] && kill "$TOOLBOX_PID" || true @@ -191,13 +238,13 @@ setup_db_table echo "Scanning $TARGET_ROOT for tests with pattern $AGENT_FILE_PATTERN..." -find "$TARGET_ROOT" -name "$AGENT_FILE_PATTERN" | while read -r agent_file; do +while read -r agent_file; do sample_dir=$(dirname "$agent_file") if [[ "$TARGET_LANG" == "python" ]]; then - run_python_test "$sample_dir" + run_with_retry run_python_test "$sample_dir" elif [[ "$TARGET_LANG" == "js" ]]; then - run_js_test "$sample_dir" + run_with_retry run_js_test "$sample_dir" elif [[ "$TARGET_LANG" == "go" ]]; then - run_go_test "$sample_dir" + run_with_retry run_go_test "$sample_dir" fi -done +done < <(find "$TARGET_ROOT" -name "$AGENT_FILE_PATTERN" | sort) diff --git a/docs/en/documentation/configuration/pre-post-processing/go/agent_test.go b/docs/en/documentation/configuration/pre-post-processing/go/agent_test.go index d8bf3ab7a6a0..20b4d29c0e8a 100644 --- a/docs/en/documentation/configuration/pre-post-processing/go/agent_test.go +++ b/docs/en/documentation/configuration/pre-post-processing/go/agent_test.go @@ -19,10 +19,38 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "testing" + "time" ) +func isTransientLLMError(output string) bool { + lower := strings.ToLower(output) + for _, marker := range []string{ + "error 503", + "status: unavailable", + "experiencing high demand", + "error 429", + "resource_exhausted", + "overloaded", + } { + if strings.Contains(lower, marker) { + return true + } + } + return false +} + +func maxSampleAttempts() int { + if raw := os.Getenv("SAMPLE_TEST_MAX_ATTEMPTS"); raw != "" { + if n, err := strconv.Atoi(raw); err == nil && n >= 1 { + return n + } + } + return 3 +} + func TestQuickstartSample(t *testing.T) { framework := os.Getenv("ORCH_NAME") if framework == "" { @@ -40,13 +68,30 @@ func TestQuickstartSample(t *testing.T) { t.Fatalf("Test setup failed: directory for framework '%s' not found.", framework) } - cmd := exec.Command("go", "run", ".") - cmd.Dir = sampleDir + maxAttempts := maxSampleAttempts() + backoff := 5 * time.Second var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr + var err error + for attempt := 1; attempt <= maxAttempts; attempt++ { + stdout.Reset() + stderr.Reset() + cmd := exec.Command("go", "run", ".") + cmd.Dir = sampleDir + cmd.Stdout = &stdout + cmd.Stderr = &stderr - err := cmd.Run() + err = cmd.Run() + if err == nil { + break + } + if attempt < maxAttempts && isTransientLLMError(stderr.String()+"\n"+stdout.String()) { + t.Logf("Transient upstream LLM error on attempt %d/%d; retrying in %s...\n%s", attempt, maxAttempts, backoff, stderr.String()) + time.Sleep(backoff) + backoff *= 2 + continue + } + break + } actualOutput := stdout.String() if err != nil { diff --git a/docs/en/documentation/getting-started/quickstart/go/quickstart_test.go b/docs/en/documentation/getting-started/quickstart/go/quickstart_test.go index ac3f5a8b2af5..6a2a4baac19f 100644 --- a/docs/en/documentation/getting-started/quickstart/go/quickstart_test.go +++ b/docs/en/documentation/getting-started/quickstart/go/quickstart_test.go @@ -19,12 +19,40 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "testing" + "time" ) var goldenKeywords = []string{"Hilton Basel", "Hyatt Regency", "book"} +func isTransientLLMError(output string) bool { + lower := strings.ToLower(output) + for _, marker := range []string{ + "error 503", + "status: unavailable", + "experiencing high demand", + "error 429", + "resource_exhausted", + "overloaded", + } { + if strings.Contains(lower, marker) { + return true + } + } + return false +} + +func maxSampleAttempts() int { + if raw := os.Getenv("SAMPLE_TEST_MAX_ATTEMPTS"); raw != "" { + if n, err := strconv.Atoi(raw); err == nil && n >= 1 { + return n + } + } + return 3 +} + func TestQuickstartSample(t *testing.T) { framework := os.Getenv("ORCH_NAME") if framework == "" { @@ -48,13 +76,30 @@ func TestQuickstartSample(t *testing.T) { t.Fatalf("Test setup failed: directory for framework '%s' not found.", framework) } - cmd := exec.Command("go", "run", ".") - cmd.Dir = sampleDir + maxAttempts := maxSampleAttempts() + backoff := 5 * time.Second var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr + var err error + for attempt := 1; attempt <= maxAttempts; attempt++ { + stdout.Reset() + stderr.Reset() + cmd := exec.Command("go", "run", ".") + cmd.Dir = sampleDir + cmd.Stdout = &stdout + cmd.Stderr = &stderr - err := cmd.Run() + err = cmd.Run() + if err == nil { + break + } + if attempt < maxAttempts && isTransientLLMError(stderr.String()+"\n"+stdout.String()) { + t.Logf("Transient upstream LLM error on attempt %d/%d; retrying in %s...\n%s", attempt, maxAttempts, backoff, stderr.String()) + time.Sleep(backoff) + backoff *= 2 + continue + } + break + } actualOutput := stdout.String() if err != nil { From 35362b8f9176a605544ff9cd6c7921636e5727f5 Mon Sep 17 00:00:00 2001 From: Dylan Volz Date: Sat, 7 Nov 2026 02:51:11 +0000 Subject: [PATCH 2/2] ci(samples): align Go test transient LLM error markers with run_tests.sh --- .../configuration/pre-post-processing/go/agent_test.go | 4 ++++ .../getting-started/quickstart/go/quickstart_test.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/docs/en/documentation/configuration/pre-post-processing/go/agent_test.go b/docs/en/documentation/configuration/pre-post-processing/go/agent_test.go index 20b4d29c0e8a..302d2daff629 100644 --- a/docs/en/documentation/configuration/pre-post-processing/go/agent_test.go +++ b/docs/en/documentation/configuration/pre-post-processing/go/agent_test.go @@ -29,9 +29,13 @@ func isTransientLLMError(output string) bool { lower := strings.ToLower(output) for _, marker := range []string{ "error 503", + "503 service unavailable", "status: unavailable", + "\"status\":\"unavailable\"", + "\"status\": \"unavailable\"", "experiencing high demand", "error 429", + "429 too many requests", "resource_exhausted", "overloaded", } { diff --git a/docs/en/documentation/getting-started/quickstart/go/quickstart_test.go b/docs/en/documentation/getting-started/quickstart/go/quickstart_test.go index 6a2a4baac19f..2dccde29554a 100644 --- a/docs/en/documentation/getting-started/quickstart/go/quickstart_test.go +++ b/docs/en/documentation/getting-started/quickstart/go/quickstart_test.go @@ -31,9 +31,13 @@ func isTransientLLMError(output string) bool { lower := strings.ToLower(output) for _, marker := range []string{ "error 503", + "503 service unavailable", "status: unavailable", + "\"status\":\"unavailable\"", + "\"status\": \"unavailable\"", "experiencing high demand", "error 429", + "429 too many requests", "resource_exhausted", "overloaded", } {