diff --git a/indexer/pkg/solingest/logger_test.go b/indexer/pkg/solingest/logger_test.go index 4aab8f22..bbb7180f 100644 --- a/indexer/pkg/solingest/logger_test.go +++ b/indexer/pkg/solingest/logger_test.go @@ -30,6 +30,12 @@ func TestTemporalLoggerErrorLevel(t *testing.T) { err: errors.New("Code: 62. DB::Exception: Syntax error"), want: slog.LevelError, }, + { + name: "third-party http 503 demoted to warn", + msg: "Activity error.", + err: errors.New("validatorsapp refresh: failed to get validators: unexpected status code 503:

503 Service Unavailable

"), + want: slog.LevelWarn, + }, { name: "context cancellation demoted to warn", msg: "Activity error.", diff --git a/utils/pkg/dberror/dberror.go b/utils/pkg/dberror/dberror.go index e95655b5..e44fd743 100644 --- a/utils/pkg/dberror/dberror.go +++ b/utils/pkg/dberror/dberror.go @@ -13,19 +13,21 @@ import ( // eofRe matches "eof" as a standalone word in a lowercased error message. var eofRe = regexp.MustCompile(`\beof\b`) -// awsRespErrRe matches the AWS SDK v2 "https response error StatusCode: ..." -// shape (lowercased by Classify before matching) for a transient S3 status: a -// 200 (S3's documented "200 OK with an error mid-body" blip, which the SDK -// surfaces as a response error but does not retry internally — transient per -// S3's own guidance) or the SDK's DefaultRetryableHTTPStatusCodes set of -// {500, 502, 503, 504}. 501 NotImplemented and 505 are deliberately not -// matched: they are permanent endpoint/configuration failures, not blips. The -// "https response error statuscode:" prefix scopes the match to AWS SDK v2 -// messages, so ClickHouse, Neo4j, and Influx errors never hit it. Actionable -// 4xx (403 AccessDenied, 404 NoSuchBucket/NoSuchKey) are also excluded so -// they keep paging. The trailing \b prevents matching a status inside a -// longer digit run (e.g. "statuscode: 2001"). -var awsRespErrRe = regexp.MustCompile(`https response error statuscode: (200|500|502|503|504)\b`) +// awsRespErrRe matches the AWS SDK v2 "https response error StatusCode: 200" +// shape (lowercased by Classify): S3's documented "200 OK with an error +// mid-body" blip, which the SDK surfaces as a response error but does not +// retry internally — transient per S3's own guidance. The prefix keeps 200 +// scoped to AWS SDK messages; a bare "status code 200" elsewhere stays +// non-transient. Retryable 5xx are handled shape-independently by +// httpStatusRe. The trailing \b prevents matching a longer digit run. +var awsRespErrRe = regexp.MustCompile(`https response error statuscode: 200\b`) + +// httpStatusRe captures the first "status code NNN" mention in an error +// string — the failed request's own status, since wrapping prepends; a status +// quoted later (e.g. in a response body) must not classify. Classify treats +// the SDK-retryable set {500, 502, 503, 504} as transient; 501/505 are +// permanent endpoint failures and 4xx are actionable, so all keep paging. +var httpStatusRe = regexp.MustCompile(`status[ _]?code[:= ]?\s*(\d{3})\b`) // ErrTransient is a sentinel that explicitly marks an error as transient for // IsTransient, independent of its message. Wrap a return with it (e.g. via @@ -111,12 +113,21 @@ func Classify(err error) ErrorType { return ErrorTypeConnectivity } - // AWS SDK v2 transient S3 responses (200-with-embedded-error, retryable - // 5xx server errors) — self-healing blips, not actionable. + // AWS SDK v2 transient 200-with-embedded-error responses — self-healing + // blips, not actionable. if awsRespErrRe.MatchString(errStr) { return ErrorTypeConnectivity } + // Retryable 5xx from any HTTP upstream; a non-5xx first status falls + // through to the remaining patterns. + if m := httpStatusRe.FindStringSubmatch(errStr); m != nil { + switch m[1] { + case "500", "502", "503", "504": + return ErrorTypeConnectivity + } + } + // Connection/connectivity patterns connectivityPatterns := []string{ "connectivityerror", diff --git a/utils/pkg/dberror/dberror_test.go b/utils/pkg/dberror/dberror_test.go index 4651fba8..c1d1d5b2 100644 --- a/utils/pkg/dberror/dberror_test.go +++ b/utils/pkg/dberror/dberror_test.go @@ -59,6 +59,17 @@ func TestClassifyAndIsTransient(t *testing.T) { // A non-AWS message mentioning statuscode: 200 without the SDK prefix must not match. {"non-aws statuscode 200", errors.New("handler returned statuscode: 200 but body was empty"), dberror.ErrorTypeUnknown, false}, + // Retryable 5xx in the "status code NNN" shape any HTTP client emits (the + // validators.app shape that paged on 2026-08-30); 4xx and 501 stay actionable. + {"validatorsapp 503", errors.New(`validatorsapp refresh: failed to get validators: unexpected status code 503:

503 Service Unavailable

`), dberror.ErrorTypeConnectivity, true}, + {"generic status code 500", errors.New("unexpected status code 500: internal server error"), dberror.ErrorTypeConnectivity, true}, + {"generic status_code variant", errors.New("request failed: status_code=502"), dberror.ErrorTypeConnectivity, true}, + {"generic status code 400 stays actionable", errors.New("unexpected status code 400: bad request"), dberror.ErrorTypeUnknown, false}, + {"generic status code 501 stays actionable", errors.New("unexpected status code 501: not implemented"), dberror.ErrorTypeUnknown, false}, + // The request's own status decides — wrapping prepends, so the first + // mention is it. A 4xx whose body quotes a 5xx stays actionable. + {"embedded 5xx inside a 4xx", errors.New(`unexpected status code 400: {"err":"upstream returned status code 503"}`), dberror.ErrorTypeUnknown, false}, + // Non-transient: real, actionable failures should still escalate to ERROR. {"syntax error", errors.New("Code: 62. DB::Exception: Syntax error"), dberror.ErrorTypeQuery, false}, {"access denied", errors.New("access denied for user"), dberror.ErrorTypeAuth, false},