Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions lib/ai/__tests__/providers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import {
getModelCutoffDate,
getModelDisplayName,
isAnthropicModel,
isDeepSeekModel,
isKimiModel,
createTrackedProvider,
myProvider,
Expand Down Expand Up @@ -41,6 +42,13 @@ describe("provider registry", () => {
(myProvider.languageModel("model-grok-4.6-pro") as { modelId: string })
.modelId,
).toBe("x-ai/grok-4.6");
expect(
(
myProvider.languageModel("model-deepseek-v4-pro-0813") as {
modelId: string;
}
).modelId,
).toBe("deepseek/deepseek-v4-pro-0813");
expect(
(myProvider.languageModel("model-glm-5.2") as { modelId: string })
.modelId,
Expand Down Expand Up @@ -72,6 +80,12 @@ describe("provider registry", () => {
expect(getModelDisplayName("model-grok-4.5-pro")).toBe("xAI Grok 4.6");
expect(getModelDisplayName("model-grok-4.6-pro")).toBe("xAI Grok 4.6");
expect(getModelCutoffDate("model-grok-4.6-pro")).toBe("August 2026");
expect(getModelDisplayName("model-deepseek-v4-pro-0813")).toBe(
"DeepSeek V4 Pro 0813",
);
expect(getModelCutoffDate("model-deepseek-v4-pro-0813")).toBe(
"August 2026",
);
expect(getModelDisplayName("model-glm-5.2")).toBe("Z.ai GLM 5.2");
expect(getModelDisplayName("model-kimi-k3")).toBe("Moonshot Kimi K3");
expect(getModelCutoffDate("model-opus-4.6")).toBe("July 2026");
Expand All @@ -87,6 +101,11 @@ describe("provider registry", () => {
expect(isAnthropicModel("anthropic/claude-opus-4.6")).toBe(true);
});

it("classifies both DeepSeek V4 Pro experiment routes as DeepSeek", () => {
expect(isDeepSeekModel("model-deepseek-v4-pro")).toBe(true);
expect(isDeepSeekModel("model-deepseek-v4-pro-0813")).toBe(true);
});

it("keeps tracked free routes split by mode", () => {
const provider = createTrackedProvider();
expect(
Expand Down Expand Up @@ -273,6 +292,9 @@ describe("supportsMultimodalToolResults", () => {
it("rejects text-only DeepSeek model keys", () => {
expect(supportsMultimodalToolResults("agent-model-free")).toBe(false);
expect(supportsMultimodalToolResults("model-deepseek-v4-pro")).toBe(false);
expect(supportsMultimodalToolResults("model-deepseek-v4-pro-0813")).toBe(
false,
);
});

it.each([
Expand Down
12 changes: 10 additions & 2 deletions lib/ai/providers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,8 @@ type OpenRouterInstance = typeof openrouter;
export const KIMI_K3_SLUG = "moonshotai/kimi-k3";
export const GLM_5_2_SLUG = "z-ai/glm-5.2";
export const GROK_4_6_SLUG = "x-ai/grok-4.6";
export const DEEPSEEK_V4_PRO_SLUG = "deepseek/deepseek-v4-pro";
export const DEEPSEEK_V4_PRO_0813_SLUG = "deepseek/deepseek-v4-pro-0813";
export const DEEPSEEK_V4_FLASH_SLUG = "deepseek/deepseek-v4-flash-0731";
export const DEEPSEEK_V4_FLASH_PREVIOUS_SLUG = "deepseek/deepseek-v4-flash";
const TITLE_GENERATOR_DEEPSEEK_SLUG = "deepseek/deepseek-v4-flash";
Expand Down Expand Up @@ -209,7 +211,10 @@ const buildProviderMap = (
"model-grok-4.5": or(GROK_4_6_SLUG),
"model-grok-4.5-pro": or(GROK_4_6_SLUG),
"model-grok-4.6-pro": or(GROK_4_6_SLUG),
"model-deepseek-v4-pro": or("deepseek/deepseek-v4-pro"),
"model-deepseek-v4-pro": or(DEEPSEEK_V4_PRO_SLUG),
// HAC-68 treatment alias. Auto and Standard keep their persisted tier
// while the server-side experiment chooses the provider route per user.
"model-deepseek-v4-pro-0813": or(DEEPSEEK_V4_PRO_0813_SLUG),
// Keep the persisted Max compatibility key while routing new requests to
// Kimi K3. Renaming the key would invalidate existing stored selections.
"model-opus-4.6": or(KIMI_K3_SLUG),
Expand All @@ -236,6 +241,7 @@ export const modelCutoffDates: Partial<Record<ModelName, string>> &
"model-grok-4.5-pro": "August 2026",
"model-grok-4.6-pro": "August 2026",
"model-deepseek-v4-pro": "May 2025",
"model-deepseek-v4-pro-0813": "August 2026",
"model-opus-4.6": "July 2026",
"model-glm-5.2": "June 2026",
"fallback-agent-model": "August 2026",
Expand All @@ -255,6 +261,7 @@ export const modelDisplayNames: Record<ModelName, string> &
"model-grok-4.5-pro": "xAI Grok 4.6",
"model-grok-4.6-pro": "xAI Grok 4.6",
"model-deepseek-v4-pro": "DeepSeek V4 Pro",
"model-deepseek-v4-pro-0813": "DeepSeek V4 Pro 0813",
"model-opus-4.6": "Moonshot Kimi K3",
"model-glm-5.2": "Z.ai GLM 5.2",
"model-kimi-k3": "Moonshot Kimi K3",
Expand Down Expand Up @@ -283,7 +290,8 @@ export function isDeepSeekModel(modelName: string): boolean {
return (
modelName === "ask-model-free" ||
modelName === "agent-model-free" ||
modelName === "model-deepseek-v4-pro"
modelName === "model-deepseek-v4-pro" ||
modelName === "model-deepseek-v4-pro-0813"
);
}

Expand Down
21 changes: 21 additions & 0 deletions lib/api/__tests__/chat-stream-helpers-fallback.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ const DEEPSEEK_FLASH_CANONICAL_SLUG = "deepseek/deepseek-v4-flash-20260731";
const DEEPSEEK_FLASH_PREVIOUS_SLUG = "deepseek/deepseek-v4-flash";
const DEEPSEEK_FLASH_PREVIOUS_CANONICAL_SLUG =
"deepseek/deepseek-v4-flash-20260423";
const DEEPSEEK_V4_PRO_SLUG = "deepseek/deepseek-v4-pro";
const MEDIUM_GROK_PRIMARY_MODELS = [
"ask-model",
"agent-model",
Expand All @@ -43,6 +44,7 @@ const HIGH_GROK_PRIMARY_OR_FALLBACK_MODELS = [
"model-grok-4.5-pro",
"model-grok-4.6-pro",
"model-deepseek-v4-pro",
"model-deepseek-v4-pro-0813",
"model-opus-4.6",
"model-glm-5.2",
"model-kimi-k3",
Expand Down Expand Up @@ -288,6 +290,19 @@ describe("buildProviderOptions fallback chain", () => {
});
});

it("falls back from DeepSeek V4 Pro 0813 through the control route, Grok, then Kimi K3", () => {
const opts = buildProviderOptions(
false,
"user-1",
"model-deepseek-v4-pro-0813",
"ask",
);
expect(opts.openrouter).toMatchObject({
models: [DEEPSEEK_V4_PRO_SLUG, GROK_SLUG, KIMI_K3_SLUG],
user: "user-1",
});
});

it("falls back from the Grok-backed paid Ask image route to Kimi K3", () => {
const opts = buildProviderOptions(false, "user-1", "ask-model", "ask");
expect(opts.openrouter).toMatchObject({
Expand Down Expand Up @@ -632,6 +647,12 @@ describe("getRetryFallbackModel", () => {
);
});

it("retries the DeepSeek 0813 treatment with the current DeepSeek route", () => {
expect(getRetryFallbackModel("model-deepseek-v4-pro-0813", "ask")).toBe(
"model-deepseek-v4-pro",
);
});

it("retries Kimi K3 with the active Grok route", () => {
expect(getRetryFallbackModel("model-kimi-k3", "agent")).toBe(
"model-grok-4.6",
Expand Down
39 changes: 39 additions & 0 deletions lib/api/chat-handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,12 @@ import { phLogger } from "@/lib/posthog/server";
import { PAID_FUNNEL_EVENTS } from "@/lib/analytics/paid-funnel";
import { readAnalyticsRequestContext } from "@/lib/analytics/request-context";
import { buildAgentStepLimitTelemetry } from "@/lib/analytics/agent-step-limit-telemetry";
import {
captureDeepSeekV4Pro0813ExperimentExposure,
evaluateDeepSeekV4Pro0813Experiment,
getActiveDeepSeekV4Pro0813ExperimentAssignment,
getDeepSeekV4Pro0813ExperimentContext,
} from "@/lib/experiments/deepseek-v4-pro-0813";
import {
capturePaidDailyFreeAllowanceServerEvent,
createPaidDailyFreeAllowanceBudgetSnapshot,
Expand Down Expand Up @@ -424,6 +430,17 @@ export const createChatHandler = () => {
);
}

const deepSeekV4Pro0813Experiment =
await evaluateDeepSeekV4Pro0813Experiment({
posthog: (posthog ??= PostHogClient()),
userId,
selectedModel,
requestId: req.headers.get("x-vercel-id") ?? undefined,
});
if (deepSeekV4Pro0813Experiment) {
selectedModel = deepSeekV4Pro0813Experiment.modelKey;
}

const notesEnabled =
(subscription !== "free" || isAgentMode(mode)) &&
(userCustomization?.include_notes ?? true);
Expand Down Expand Up @@ -559,6 +576,15 @@ export const createChatHandler = () => {
});
}

const activeDeepSeekV4Pro0813Experiment =
getActiveDeepSeekV4Pro0813ExperimentAssignment(
deepSeekV4Pro0813Experiment,
selectedModel,
);
const routingExperimentContext = getDeepSeekV4Pro0813ExperimentContext(
activeDeepSeekV4Pro0813Experiment,
);

const freeMonthlyBudgetSnapshot =
subscription === "free"
? await checkFreeMonthlyCostLimit(freeUsageSubject)
Expand Down Expand Up @@ -1110,6 +1136,7 @@ export const createChatHandler = () => {
usage: usageCostRecord,
responseModel: state.responseModel,
analyticsRequestContext,
experiment: routingExperimentContext,
fallbackServed:
state.responseModel && retryUsedFallbackModel
? true
Expand Down Expand Up @@ -1315,6 +1342,16 @@ export const createChatHandler = () => {

let result;
try {
captureDeepSeekV4Pro0813ExperimentExposure({
posthog,
userId,
subscription,
mode,
selectedModelOverride,
selectedModel,
configuredModel: configuredModelId,
assignment: activeDeepSeekV4Pro0813Experiment,
});
result = await createStream(selectedModel);
} catch (error) {
// If provider returns an API error before streaming, retry with fallback.
Expand Down Expand Up @@ -1678,6 +1715,7 @@ export const createChatHandler = () => {
finishReason: state.streamFinishReason,
budgetAbortDetails: state.budgetAbortDetails,
isAutoContinue: !!isAutoContinue,
experiment: routingExperimentContext,
stepLimitTelemetry:
buildAgentStepLimitTelemetry({
configuredMaxSteps:
Expand Down Expand Up @@ -1993,6 +2031,7 @@ export const createChatHandler = () => {
finishReason: state.streamFinishReason,
budgetAbortDetails: state.budgetAbortDetails,
isAutoContinue: !!isAutoContinue,
experiment: routingExperimentContext,
stepLimitTelemetry: buildAgentStepLimitTelemetry({
configuredMaxSteps: state.configuredMaxSteps,
stepCount: state.agentStepCount,
Expand Down
11 changes: 11 additions & 0 deletions lib/api/chat-stream-helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -541,6 +541,13 @@ const AGENT_TEXT_FALLBACK_CHAIN = [
"model-kimi-k3",
] as const satisfies readonly ModelName[];

// HAC-68 treatment falls back through today's DeepSeek route before the
// existing Grok/Kimi recovery chain.
const DEEPSEEK_V4_PRO_0813_FALLBACK_CHAIN = [
"model-deepseek-v4-pro",
...AGENT_TEXT_FALLBACK_CHAIN,
] as const satisfies readonly ModelName[];

// HackerAI Pro uses Grok 4.6 for every request. GLM 5.2 remains its first
// fallback, followed by Kimi K3 so media requests still have a multimodal final
// recovery path if both primary providers are unavailable.
Expand All @@ -553,6 +560,7 @@ const MODEL_FALLBACK_CHAIN: Partial<Record<ModelName, readonly ModelName[]>> = {
"ask-model-free": AGENT_TEXT_FALLBACK_CHAIN,
"agent-model-free": AGENT_TEXT_FALLBACK_CHAIN,
"model-deepseek-v4-pro": AGENT_TEXT_FALLBACK_CHAIN,
"model-deepseek-v4-pro-0813": DEEPSEEK_V4_PRO_0813_FALLBACK_CHAIN,
"ask-model": GROK_4_6_FALLBACK_CHAIN,
"agent-model": GROK_4_6_FALLBACK_CHAIN,
"model-grok-4.6": GROK_4_6_FALLBACK_CHAIN,
Expand Down Expand Up @@ -642,6 +650,9 @@ export function getRetryFallbackModel(
modelName: ModelName,
_mode: ChatMode,
): ModelName {
if (modelName === "model-deepseek-v4-pro-0813") {
return "model-deepseek-v4-pro";
}
if (
modelName === "model-grok-4.6-pro" ||
modelName === "model-grok-4.5-pro"
Expand Down
122 changes: 122 additions & 0 deletions lib/experiments/__tests__/deepseek-v4-pro-0813.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
import {
captureDeepSeekV4Pro0813ExperimentExposure,
DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY,
DEEPSEEK_V4_PRO_0813_EXPOSURE_EVENT,
DEEPSEEK_V4_PRO_0813_FEATURE_PROPERTY,
evaluateDeepSeekV4Pro0813Experiment,
getActiveDeepSeekV4Pro0813ExperimentAssignment,
getDeepSeekV4Pro0813ExperimentContext,
isEligibleForDeepSeekV4Pro0813Experiment,
} from "@/lib/experiments/deepseek-v4-pro-0813";

describe("DeepSeek V4 Pro 0813 experiment", () => {
it("only evaluates requests already resolved to the current DeepSeek V4 Pro route", () => {
expect(
isEligibleForDeepSeekV4Pro0813Experiment("model-deepseek-v4-pro"),
).toBe(true);
expect(isEligibleForDeepSeekV4Pro0813Experiment("model-grok-4.5")).toBe(
false,
);
expect(isEligibleForDeepSeekV4Pro0813Experiment("agent-model-free")).toBe(
false,
);
});

it.each([
["control", "model-deepseek-v4-pro"],
["test", "model-deepseek-v4-pro-0813"],
] as const)("maps %s to %s", async (variant, modelKey) => {
const evaluateFlags = jest.fn(async () => ({ getFlag: () => variant }));

await expect(
evaluateDeepSeekV4Pro0813Experiment({
posthog: { evaluateFlags } as never,
userId: "user-1",
selectedModel: "model-deepseek-v4-pro",
}),
).resolves.toEqual({
key: DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY,
variant,
modelKey,
});
});

it("does not evaluate ineligible routes", async () => {
const evaluateFlags = jest.fn();

await expect(
evaluateDeepSeekV4Pro0813Experiment({
posthog: { evaluateFlags } as never,
userId: "user-1",
selectedModel: "model-grok-4.5",
}),
).resolves.toBeUndefined();
expect(evaluateFlags).not.toHaveBeenCalled();
});

it("fails closed when evaluation returns an unknown value", async () => {
await expect(
evaluateDeepSeekV4Pro0813Experiment({
posthog: {
evaluateFlags: jest.fn(async () => ({ getFlag: () => true })),
} as never,
userId: "user-1",
selectedModel: "model-deepseek-v4-pro",
}),
).resolves.toBeUndefined();
});

it("captures a privacy-safe custom exposure from the provider surface", () => {
const capture = jest.fn();
const assignment = {
key: DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY,
variant: "test" as const,
modelKey: "model-deepseek-v4-pro-0813" as const,
};

captureDeepSeekV4Pro0813ExperimentExposure({
posthog: { capture } as never,
userId: "user-1",
subscription: "pro",
mode: "agent",
selectedModelOverride: "hackerai-standard",
selectedModel: assignment.modelKey,
configuredModel: "deepseek/deepseek-v4-pro-0813",
assignment,
});

expect(capture).toHaveBeenCalledWith({
distinctId: "user-1",
event: DEEPSEEK_V4_PRO_0813_EXPOSURE_EVENT,
properties: {
experiment_key: DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY,
experiment_variant: "test",
[DEEPSEEK_V4_PRO_0813_FEATURE_PROPERTY]: "test",
subscription: "pro",
subscription_tier: "pro",
mode: "agent",
selected_model: "model-deepseek-v4-pro-0813",
selected_model_override: "hackerai-standard",
configured_model: "deepseek/deepseek-v4-pro-0813",
exposure_surface: "provider_request",
$process_person_profile: false,
},
});
expect(getDeepSeekV4Pro0813ExperimentContext(assignment)).toEqual({
key: DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY,
variant: "test",
});
expect(
getActiveDeepSeekV4Pro0813ExperimentAssignment(
assignment,
"model-deepseek-v4-pro-0813",
),
).toBe(assignment);
expect(
getActiveDeepSeekV4Pro0813ExperimentAssignment(
assignment,
"model-grok-4.5",
),
).toBeUndefined();
});
});
Loading