fix: scope retry health by model and region

apps/gateway/src/api.spec.ts

@@ -2783,11 +2783,11 @@ describe("api", () => {
 			expect(logs[0].hasError).toBe(true);
 			expect(logs[0].errorDetails?.statusCode).toBe(401);
 
-			expect(isTrackedKeyHealthy("provider-key-id-stream-auth-error")).toBe(
-				false,
-			);
 			expect(
-				getTrackedKeyMetrics("provider-key-id-stream-auth-error"),
+				isTrackedKeyHealthy("provider-key-id-stream-auth-error", "custom"),
+			).toBe(false);
+			expect(
+				getTrackedKeyMetrics("provider-key-id-stream-auth-error", "custom"),
 			).toMatchObject({
 				permanentlyBlacklisted: true,
 				totalRequests: 1,

apps/gateway/src/chat/chat.ts

@@ -151,6 +151,7 @@ import {
 	MAX_RETRIES,
 	providerRetryKey,
 	selectNextProvider,
+	shouldRetryAlternateKey,
 	shouldRetryRequest,
 } from "./tools/retry-with-fallback.js";
 import {
@@ -1484,7 +1485,6 @@ chat.openapi(completions, async (c) => {
 		const customProviderKey = await findCustomProviderKey(
 			project.organizationId,
 			customProviderName,
-			requestId,
 		);
 		if (!customProviderKey) {
 			throw new HTTPException(400, {
@@ -1867,7 +1867,7 @@ chat.openapi(completions, async (c) => {
 				const providerKey = await findProviderKey(
 					project.organizationId,
 					usedProvider,
-					requestId,
+					modelInfo.id || stripRegionFromModelName(usedModel, usedRegion),
 				);
 				lockedRegion = providerKey
 					? resolveExplicitRegionFromProviderKey(providerKey)
@@ -2528,7 +2528,7 @@ chat.openapi(completions, async (c) => {
 				const providerKey = await findProviderKey(
 					project.organizationId,
 					requestedProvider,
-					requestId,
+					modelInfo.id || stripRegionFromModelName(usedModel, usedRegion),
 				);
 				explicitDirectRegion = providerKey
 					? resolveExplicitRegionFromProviderKey(providerKey)
@@ -2766,13 +2766,13 @@ chat.openapi(completions, async (c) => {
 			providerKey = await findCustomProviderKey(
 				project.organizationId,
 				customProviderName,
-				requestId,
+				baseModelName,
 			);
 		} else {
 			providerKey = await findProviderKey(
 				project.organizationId,
 				usedProvider,
-				requestId,
+				baseModelName,
 			);
 		}
 
@@ -2830,7 +2830,9 @@ chat.openapi(completions, async (c) => {
 			});
 		}
 
-		const envResult = getProviderEnv(usedProvider);
+		const envResult = getProviderEnv(usedProvider, {
+			selectionScope: baseModelName,
+		});
 		usedToken = envResult.token;
 		configIndex = envResult.configIndex;
 		envVarName = envResult.envVarName;
@@ -2848,13 +2850,13 @@ chat.openapi(completions, async (c) => {
 			providerKey = await findCustomProviderKey(
 				project.organizationId,
 				customProviderName,
-				requestId,
+				baseModelName,
 			);
 		} else {
 			providerKey = await findProviderKey(
 				project.organizationId,
 				usedProvider,
-				requestId,
+				baseModelName,
 			);
 		}
 
@@ -2905,7 +2907,9 @@ chat.openapi(completions, async (c) => {
 				});
 			}
 
-			const envResult = getProviderEnv(usedProvider);
+			const envResult = getProviderEnv(usedProvider, {
+				selectionScope: baseModelName,
+			});
 			usedToken = envResult.token;
 			configIndex = envResult.configIndex;
 			envVarName = envResult.envVarName;
@@ -4787,10 +4791,21 @@ chat.openapi(completions, async (c) => {
 
 							// Report key health for the selected token source
 							if (envVarName !== undefined) {
-								reportKeyError(envVarName, configIndex, 0);
+								reportKeyError(
+									envVarName,
+									configIndex,
+									0,
+									undefined,
+									baseModelName,
+								);
 							}
 							if (providerKey?.id) {
-								reportTrackedKeyError(providerKey.id, 0);
+								reportTrackedKeyError(
+									providerKey.id,
+									0,
+									undefined,
+									baseModelName,
+								);
 							}
 
 							if (willRetrySameProvider && sameProviderRetryContext) {
@@ -4899,7 +4914,13 @@ chat.openapi(completions, async (c) => {
 						let sameProviderRetryContext: Awaited<
 							ReturnType<typeof resolveProviderContext>
 						> | null = null;
-						if (isRetryableErrorType(finishReason)) {
+						if (
+							shouldRetryAlternateKey(
+								finishReason,
+								res.status,
+								errorResponseText,
+							)
+						) {
 							rememberFailedKey(usedProvider, usedRegion, {
 								envVarName,
 								configIndex,
@@ -5023,13 +5044,15 @@ chat.openapi(completions, async (c) => {
 								configIndex,
 								res.status,
 								errorResponseText,
+								baseModelName,
 							);
 						}
 						if (providerKey?.id && finishReason !== "content_filter") {
 							reportTrackedKeyError(
 								providerKey.id,
 								res.status,
 								errorResponseText,
+								baseModelName,
 							);
 						}
 
@@ -5168,7 +5191,13 @@ chat.openapi(completions, async (c) => {
 						let sameProviderRetryContext: Awaited<
 							ReturnType<typeof resolveProviderContext>
 						> | null = null;
-						if (isRetryableErrorType(errorType)) {
+						if (
+							shouldRetryAlternateKey(
+								errorType,
+								inferredStatusCode,
+								errorResponseText,
+							)
+						) {
 							rememberFailedKey(usedProvider, usedRegion, {
 								envVarName,
 								configIndex,
@@ -5277,13 +5306,15 @@ chat.openapi(completions, async (c) => {
 								configIndex,
 								inferredStatusCode,
 								errorResponseText,
+								baseModelName,
 							);
 						}
 						if (providerKey?.id && errorType !== "content_filter") {
 							reportTrackedKeyError(
 								providerKey.id,
 								inferredStatusCode,
 								errorResponseText,
+								baseModelName,
 							);
 						}
 
@@ -7631,16 +7662,27 @@ chat.openapi(completions, async (c) => {
 					// Report key health for the selected token source
 					if (envVarName !== undefined) {
 						if (streamingError !== null) {
-							reportKeyError(envVarName, configIndex, streamingErrorStatusCode);
+							reportKeyError(
+								envVarName,
+								configIndex,
+								streamingErrorStatusCode,
+								undefined,
+								baseModelName,
+							);
 						} else {
-							reportKeySuccess(envVarName, configIndex);
+							reportKeySuccess(envVarName, configIndex, baseModelName);
 						}
 					}
 					if (providerKey?.id) {
 						if (streamingError !== null) {
-							reportTrackedKeyError(providerKey.id, streamingErrorStatusCode);
+							reportTrackedKeyError(
+								providerKey.id,
+								streamingErrorStatusCode,
+								undefined,
+								baseModelName,
+							);
 						} else {
-							reportTrackedKeySuccess(providerKey.id);
+							reportTrackedKeySuccess(providerKey.id, baseModelName);
 						}
 					}
 
@@ -7980,10 +8022,10 @@ chat.openapi(completions, async (c) => {
 
 			// Report key health for the selected token source
 			if (envVarName !== undefined) {
-				reportKeyError(envVarName, configIndex, 0);
+				reportKeyError(envVarName, configIndex, 0, undefined, baseModelName);
 			}
 			if (providerKey?.id) {
-				reportTrackedKeyError(providerKey.id, 0);
+				reportTrackedKeyError(providerKey.id, 0, undefined, baseModelName);
 			}
 
 			if (willRetrySameProvider && sameProviderRetryContext) {
@@ -8339,7 +8381,9 @@ chat.openapi(completions, async (c) => {
 			let sameProviderRetryContext: Awaited<
 				ReturnType<typeof resolveProviderContext>
 			> | null = null;
-			if (isRetryableErrorType(finishReason)) {
+			if (
+				shouldRetryAlternateKey(finishReason, res.status, errorResponseText)
+			) {
 				rememberFailedKey(usedProvider, usedRegion, {
 					envVarName,
 					configIndex,
@@ -8464,10 +8508,21 @@ chat.openapi(completions, async (c) => {
 			// Report key health for the selected token source
 			// Don't report content_filter as a key error - it's intentional provider behavior
 			if (envVarName !== undefined && finishReason !== "content_filter") {
-				reportKeyError(envVarName, configIndex, res.status, errorResponseText);
+				reportKeyError(
+					envVarName,
+					configIndex,
+					res.status,
+					errorResponseText,
+					baseModelName,
+				);
 			}
 			if (providerKey?.id && finishReason !== "content_filter") {
-				reportTrackedKeyError(providerKey.id, res.status, errorResponseText);
+				reportTrackedKeyError(
+					providerKey.id,
+					res.status,
+					errorResponseText,
+					baseModelName,
+				);
 			}
 
 			if (willRetrySameProvider && sameProviderRetryContext) {
@@ -9205,10 +9260,10 @@ chat.openapi(completions, async (c) => {
 	// Report key health for the selected token source
 	// Note: We don't report empty responses as key errors since they're not upstream errors
 	if (envVarName !== undefined) {
-		reportKeySuccess(envVarName, configIndex);
+		reportKeySuccess(envVarName, configIndex, baseModelName);
 	}
 	if (providerKey?.id) {
-		reportTrackedKeySuccess(providerKey.id);
+		reportTrackedKeySuccess(providerKey.id, baseModelName);
 	}
 
 	if (cachingEnabled && cacheKey && !stream && !hasEmptyNonStreamingResponse) {

apps/gateway/src/chat/tools/get-finish-reason-from-error.spec.ts

@@ -112,6 +112,15 @@ describe("getFinishReasonFromError", () => {
 		expect(getFinishReasonFromError(403)).toBe("gateway_error");
 	});
 
+	it("returns gateway_error for 400 invalid API key payloads", () => {
+		expect(
+			getFinishReasonFromError(
+				400,
+				'{"error":{"message":"API key not valid. Please pass a valid API key.","type":"authentication_error","code":"invalid_api_key"}}',
+			),
+		).toBe("gateway_error");
+	});
+
 	it("returns client_error when no error text provided for other 4xx", () => {
 		expect(getFinishReasonFromError(400)).toBe("client_error");
 		expect(getFinishReasonFromError(422)).toBe("client_error");

apps/gateway/src/chat/tools/get-finish-reason-from-error.ts

@@ -1,3 +1,5 @@
+import { hasInvalidProviderCredentialError } from "@/lib/provider-auth-errors.js";
+
 /**
  * Determines the appropriate finish reason based on HTTP status code and error message
  * 5xx status codes indicate upstream provider errors
@@ -55,8 +57,12 @@ export function getFinishReasonFromError(
 		return "content_filter";
 	}
 
-	// 401/403 usually indicate invalid or unauthorized provider credentials
-	if (statusCode === 401 || statusCode === 403) {
+	// 401/403 and known provider credential payloads indicate bad provider keys.
+	if (
+		statusCode === 401 ||
+		statusCode === 403 ||
+		hasInvalidProviderCredentialError(errorText)
+	) {
 		return "gateway_error";
 	}
 

apps/gateway/src/chat/tools/get-provider-env.spec.ts

@@ -1,5 +1,6 @@
 import { afterEach, beforeEach, describe, expect, it } from "vitest";
 
+import { reportKeyError, resetKeyHealth } from "@/lib/api-key-health.js";
 import { resetRoundRobinCounters } from "@/lib/round-robin-env.js";
 
 import { getProviderEnv } from "./get-provider-env.js";
@@ -9,6 +10,7 @@ describe("getProviderEnv", () => {
 
 	beforeEach(() => {
 		resetRoundRobinCounters();
+		resetKeyHealth();
 		process.env.LLM_OPENAI_API_KEY = "sk-openai-a,sk-openai-b,sk-openai-c";
 	});
 
@@ -55,4 +57,20 @@ describe("getProviderEnv", () => {
 		expect(thirdKey.token).toBe("sk-openai-c");
 		expect(thirdKey.configIndex).toBe(2);
 	});
+
+	it("passes selection scope through to env key health", () => {
+		reportKeyError("LLM_OPENAI_API_KEY", 0, 500, undefined, "gpt-4");
+		reportKeyError("LLM_OPENAI_API_KEY", 0, 500, undefined, "gpt-4");
+		reportKeyError("LLM_OPENAI_API_KEY", 0, 500, undefined, "gpt-4");
+
+		const gpt4Selection = getProviderEnv("openai", {
+			selectionScope: "gpt-4",
+		});
+		const claudeSelection = getProviderEnv("openai", {
+			selectionScope: "claude-3-5-sonnet",
+		});
+
+		expect(gpt4Selection.configIndex).toBe(1);
+		expect(claudeSelection.configIndex).toBe(0);
+	});
 });

apps/gateway/src/chat/tools/get-provider-env.ts

@@ -20,6 +20,7 @@ export interface ProviderEnvResult {
 interface GetProviderEnvOptions {
 	advanceRoundRobin?: boolean;
 	excludedIndices?: ReadonlySet<number>;
+	selectionScope?: string;
 }
 
 /**
@@ -62,9 +63,10 @@ export function getProviderEnv(
 
 	const advanceRoundRobin = options.advanceRoundRobin ?? true;
 	const excludedIndices = options.excludedIndices;
+	const selectionScope = options.selectionScope;
 	const result = advanceRoundRobin
-		? getRoundRobinValue(envVar, envValue, excludedIndices)
-		: peekRoundRobinValue(envVar, envValue, excludedIndices);
+		? getRoundRobinValue(envVar, envValue, selectionScope, excludedIndices)
+		: peekRoundRobinValue(envVar, envValue, selectionScope, excludedIndices);
 
 	return { token: result.value, configIndex: result.index, envVarName: envVar };
 }