From 75d21dd4de60779f2e5731926edc5439bbd74ed8 Mon Sep 17 00:00:00 2001 From: Ben Blier Date: Sun, 30 Aug 2026 10:06:15 -0400 Subject: [PATCH 1/2] logging: classify generic http 5xx errors as transient dberror.Classify only recognized the AWS SDK 5xx shape, so validators.app 503s ("unexpected status code 503") stayed non-transient and every Temporal retry attempt paged via the solingest logger. The workflow-side escalator already pages sustained outages at ~15 min; the per-attempt lines are noise. --- indexer/pkg/solingest/logger_test.go | 6 ++++++ utils/pkg/dberror/dberror.go | 11 +++++++++++ utils/pkg/dberror/dberror_test.go | 9 +++++++++ 3 files changed, 26 insertions(+) diff --git a/indexer/pkg/solingest/logger_test.go b/indexer/pkg/solingest/logger_test.go index 4aab8f22..bbb7180f 100644 --- a/indexer/pkg/solingest/logger_test.go +++ b/indexer/pkg/solingest/logger_test.go @@ -30,6 +30,12 @@ func TestTemporalLoggerErrorLevel(t *testing.T) { err: errors.New("Code: 62. DB::Exception: Syntax error"), want: slog.LevelError, }, + { + name: "third-party http 503 demoted to warn", + msg: "Activity error.", + err: errors.New("validatorsapp refresh: failed to get validators: unexpected status code 503:

503 Service Unavailable

"), + want: slog.LevelWarn, + }, { name: "context cancellation demoted to warn", msg: "Activity error.", diff --git a/utils/pkg/dberror/dberror.go b/utils/pkg/dberror/dberror.go index e95655b5..a2c506cf 100644 --- a/utils/pkg/dberror/dberror.go +++ b/utils/pkg/dberror/dberror.go @@ -27,6 +27,12 @@ var eofRe = regexp.MustCompile(`\beof\b`) // longer digit run (e.g. "statuscode: 2001"). var awsRespErrRe = regexp.MustCompile(`https response error statuscode: (200|500|502|503|504)\b`) +// httpStatus5xxRe matches the "status code NNN" shape non-AWS HTTP clients use +// (e.g. validators.app's "unexpected status code 503") for the same retryable +// set as awsRespErrRe. 501/505 are permanent endpoint failures and 4xx are +// actionable, so all keep paging. +var httpStatus5xxRe = regexp.MustCompile(`status[ _]?code[:= ]?\s*(500|502|503|504)\b`) + // ErrTransient is a sentinel that explicitly marks an error as transient for // IsTransient, independent of its message. Wrap a return with it (e.g. via // errors.Join or fmt.Errorf("...: %w", ErrTransient)) when the caller knows a @@ -117,6 +123,11 @@ func Classify(err error) ErrorType { return ErrorTypeConnectivity } + // Retryable 5xx from any other HTTP upstream. + if httpStatus5xxRe.MatchString(errStr) { + return ErrorTypeConnectivity + } + // Connection/connectivity patterns connectivityPatterns := []string{ "connectivityerror", diff --git a/utils/pkg/dberror/dberror_test.go b/utils/pkg/dberror/dberror_test.go index 4651fba8..084e6956 100644 --- a/utils/pkg/dberror/dberror_test.go +++ b/utils/pkg/dberror/dberror_test.go @@ -59,6 +59,15 @@ func TestClassifyAndIsTransient(t *testing.T) { // A non-AWS message mentioning statuscode: 200 without the SDK prefix must not match. {"non-aws statuscode 200", errors.New("handler returned statuscode: 200 but body was empty"), dberror.ErrorTypeUnknown, false}, + // Non-AWS HTTP clients format retryable 5xx as "status code NNN" (the + // validators.app shape that paged on 2026-08-30). Same 5xx set as the AWS + // shape; 4xx and 501 stay actionable. + {"validatorsapp 503", errors.New(`validatorsapp refresh: failed to get validators: unexpected status code 503:

503 Service Unavailable

`), dberror.ErrorTypeConnectivity, true}, + {"generic status code 500", errors.New("unexpected status code 500: internal server error"), dberror.ErrorTypeConnectivity, true}, + {"generic status_code variant", errors.New("request failed: status_code=502"), dberror.ErrorTypeConnectivity, true}, + {"generic status code 400 stays actionable", errors.New("unexpected status code 400: bad request"), dberror.ErrorTypeUnknown, false}, + {"generic status code 501 stays actionable", errors.New("unexpected status code 501: not implemented"), dberror.ErrorTypeUnknown, false}, + // Non-transient: real, actionable failures should still escalate to ERROR. {"syntax error", errors.New("Code: 62. DB::Exception: Syntax error"), dberror.ErrorTypeQuery, false}, {"access denied", errors.New("access denied for user"), dberror.ErrorTypeAuth, false}, From da53e1f80a02198759bfad6d71e3e6609e6e4842 Mon Sep 17 00:00:00 2001 From: Ben Blier Date: Mon, 31 Aug 2026 07:55:03 -0400 Subject: [PATCH 2/2] logging: decide on the request's own status and scope aws matching to 200 Review notes: the first status-code mention decides classification, so a 4xx whose body quotes a 5xx stays actionable, and awsRespErrRe shrinks to the 200-with-embedded-error case now that retryable 5xx match shape-independently. --- utils/pkg/dberror/dberror.go | 48 +++++++++++++++---------------- utils/pkg/dberror/dberror_test.go | 8 ++++-- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/utils/pkg/dberror/dberror.go b/utils/pkg/dberror/dberror.go index a2c506cf..e44fd743 100644 --- a/utils/pkg/dberror/dberror.go +++ b/utils/pkg/dberror/dberror.go @@ -13,25 +13,21 @@ import ( // eofRe matches "eof" as a standalone word in a lowercased error message. var eofRe = regexp.MustCompile(`\beof\b`) -// awsRespErrRe matches the AWS SDK v2 "https response error StatusCode: ..." -// shape (lowercased by Classify before matching) for a transient S3 status: a -// 200 (S3's documented "200 OK with an error mid-body" blip, which the SDK -// surfaces as a response error but does not retry internally — transient per -// S3's own guidance) or the SDK's DefaultRetryableHTTPStatusCodes set of -// {500, 502, 503, 504}. 501 NotImplemented and 505 are deliberately not -// matched: they are permanent endpoint/configuration failures, not blips. The -// "https response error statuscode:" prefix scopes the match to AWS SDK v2 -// messages, so ClickHouse, Neo4j, and Influx errors never hit it. Actionable -// 4xx (403 AccessDenied, 404 NoSuchBucket/NoSuchKey) are also excluded so -// they keep paging. The trailing \b prevents matching a status inside a -// longer digit run (e.g. "statuscode: 2001"). -var awsRespErrRe = regexp.MustCompile(`https response error statuscode: (200|500|502|503|504)\b`) - -// httpStatus5xxRe matches the "status code NNN" shape non-AWS HTTP clients use -// (e.g. validators.app's "unexpected status code 503") for the same retryable -// set as awsRespErrRe. 501/505 are permanent endpoint failures and 4xx are -// actionable, so all keep paging. -var httpStatus5xxRe = regexp.MustCompile(`status[ _]?code[:= ]?\s*(500|502|503|504)\b`) +// awsRespErrRe matches the AWS SDK v2 "https response error StatusCode: 200" +// shape (lowercased by Classify): S3's documented "200 OK with an error +// mid-body" blip, which the SDK surfaces as a response error but does not +// retry internally — transient per S3's own guidance. The prefix keeps 200 +// scoped to AWS SDK messages; a bare "status code 200" elsewhere stays +// non-transient. Retryable 5xx are handled shape-independently by +// httpStatusRe. The trailing \b prevents matching a longer digit run. +var awsRespErrRe = regexp.MustCompile(`https response error statuscode: 200\b`) + +// httpStatusRe captures the first "status code NNN" mention in an error +// string — the failed request's own status, since wrapping prepends; a status +// quoted later (e.g. in a response body) must not classify. Classify treats +// the SDK-retryable set {500, 502, 503, 504} as transient; 501/505 are +// permanent endpoint failures and 4xx are actionable, so all keep paging. +var httpStatusRe = regexp.MustCompile(`status[ _]?code[:= ]?\s*(\d{3})\b`) // ErrTransient is a sentinel that explicitly marks an error as transient for // IsTransient, independent of its message. Wrap a return with it (e.g. via @@ -117,15 +113,19 @@ func Classify(err error) ErrorType { return ErrorTypeConnectivity } - // AWS SDK v2 transient S3 responses (200-with-embedded-error, retryable - // 5xx server errors) — self-healing blips, not actionable. + // AWS SDK v2 transient 200-with-embedded-error responses — self-healing + // blips, not actionable. if awsRespErrRe.MatchString(errStr) { return ErrorTypeConnectivity } - // Retryable 5xx from any other HTTP upstream. - if httpStatus5xxRe.MatchString(errStr) { - return ErrorTypeConnectivity + // Retryable 5xx from any HTTP upstream; a non-5xx first status falls + // through to the remaining patterns. + if m := httpStatusRe.FindStringSubmatch(errStr); m != nil { + switch m[1] { + case "500", "502", "503", "504": + return ErrorTypeConnectivity + } } // Connection/connectivity patterns diff --git a/utils/pkg/dberror/dberror_test.go b/utils/pkg/dberror/dberror_test.go index 084e6956..c1d1d5b2 100644 --- a/utils/pkg/dberror/dberror_test.go +++ b/utils/pkg/dberror/dberror_test.go @@ -59,14 +59,16 @@ func TestClassifyAndIsTransient(t *testing.T) { // A non-AWS message mentioning statuscode: 200 without the SDK prefix must not match. {"non-aws statuscode 200", errors.New("handler returned statuscode: 200 but body was empty"), dberror.ErrorTypeUnknown, false}, - // Non-AWS HTTP clients format retryable 5xx as "status code NNN" (the - // validators.app shape that paged on 2026-08-30). Same 5xx set as the AWS - // shape; 4xx and 501 stay actionable. + // Retryable 5xx in the "status code NNN" shape any HTTP client emits (the + // validators.app shape that paged on 2026-08-30); 4xx and 501 stay actionable. {"validatorsapp 503", errors.New(`validatorsapp refresh: failed to get validators: unexpected status code 503:

503 Service Unavailable

`), dberror.ErrorTypeConnectivity, true}, {"generic status code 500", errors.New("unexpected status code 500: internal server error"), dberror.ErrorTypeConnectivity, true}, {"generic status_code variant", errors.New("request failed: status_code=502"), dberror.ErrorTypeConnectivity, true}, {"generic status code 400 stays actionable", errors.New("unexpected status code 400: bad request"), dberror.ErrorTypeUnknown, false}, {"generic status code 501 stays actionable", errors.New("unexpected status code 501: not implemented"), dberror.ErrorTypeUnknown, false}, + // The request's own status decides — wrapping prepends, so the first + // mention is it. A 4xx whose body quotes a 5xx stays actionable. + {"embedded 5xx inside a 4xx", errors.New(`unexpected status code 400: {"err":"upstream returned status code 503"}`), dberror.ErrorTypeUnknown, false}, // Non-transient: real, actionable failures should still escalate to ERROR. {"syntax error", errors.New("Code: 62. DB::Exception: Syntax error"), dberror.ErrorTypeQuery, false},