Skip to content

feat: bill anthropic cache write #2171

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
steebchen merged 13 commits into from
May 7, 2026
Merged

feat: bill anthropic cache write #2171

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
2838ac1
feat: track cache write costs
RATCHAW May 5, 2026
96fb83b
feat: add cacheWriteInputPrice1h for improved pricing accuracy
RATCHAW May 5, 2026
535c3a7
test: enhance pricing tests for Anthropic models
RATCHAW May 5, 2026
141bc76
fix(gateway): extract cache-write tokens from cached streaming replays
RATCHAW May 5, 2026
d27744a
fix(gateway): preserve cache_control ttl on inbound requests
RATCHAW May 6, 2026
9fc65d4
feat(gateway): surface 5m/1h cache_creation breakdown
RATCHAW May 6, 2026
9b940b3
fix(admin): expose cache-write tokens and cost in metrics
RATCHAW May 6, 2026
c3e7654
fix(gateway): emit cache_write_tokens in streaming chunks
RATCHAW May 6, 2026
b6e7722
chore: regenerate api types for cache-write fields
RATCHAW May 6, 2026
5b0b2a4
fix(gateway): forward 1h cache-write tokens on cached replay
RATCHAW May 6, 2026
7c81bfa
chore(db): consolidate cache-write migrations into one
RATCHAW May 6, 2026
44da5ea
Merge branch 'main' into feat/anthropic-cache-write-costs
smakosh May 6, 2026
ba9fe24
Merge branch 'main' of https://github.com/theopenco/llmgateway into f…
RATCHAW May 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions apps/api/src/routes/activity.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ const dailyActivitySchema = z.object({
inputTokens: z.number(),
outputTokens: z.number(),
cachedTokens: z.number(),
cacheWriteTokens: z.number(),
totalTokens: z.number(),
cost: z.number(),
inputCost: z.number(),
Expand All @@ -50,6 +51,7 @@ const dailyActivitySchema = z.object({
imageOutputCost: z.number(),
videoOutputCost: z.number(),
cachedInputCost: z.number(),
cacheWriteInputCost: z.number(),
errorCount: z.number(),
errorRate: z.number(),
cacheCount: z.number(),
Expand Down Expand Up @@ -262,10 +264,18 @@ activity.openapi(getActivity, async (c) => {
sql<number>`COALESCE(SUM(CAST(${apiKeyHourlyStats.cachedTokens} AS NUMERIC)), 0)`.as(
"cachedTokens",
),
cacheWriteTokens:
sql<number>`COALESCE(SUM(CAST(${apiKeyHourlyStats.cacheWriteTokens} AS NUMERIC)), 0)`.as(
"cacheWriteTokens",
),
cachedInputCost:
sql<number>`COALESCE(SUM(${apiKeyHourlyStats.cachedInputCost}), 0)`.as(
"cachedInputCost",
),
cacheWriteInputCost:
sql<number>`COALESCE(SUM(${apiKeyHourlyStats.cacheWriteInputCost}), 0)`.as(
"cacheWriteInputCost",
),
creditsRequestCount:
sql<number>`COALESCE(SUM(${apiKeyHourlyStats.creditsRequestCount}), 0)`.as(
"creditsRequestCount",
Expand Down Expand Up @@ -388,6 +398,7 @@ activity.openapi(getActivity, async (c) => {
const inputTokens = Number(day.inputTokens);
const outputTokens = Number(day.outputTokens);
const cachedTokens = Number(day.cachedTokens);
const cacheWriteTokens = Number(day.cacheWriteTokens);
const totalTokens = Number(day.totalTokens);
const cost = Number(day.cost);
const inputCost = Number(day.inputCost);
Expand All @@ -401,6 +412,7 @@ activity.openapi(getActivity, async (c) => {
const imageOutputCost = Number(day.imageOutputCost);
const videoOutputCost = Number(day.videoOutputCost);
const cachedInputCost = Number(day.cachedInputCost);
const cacheWriteInputCost = Number(day.cacheWriteInputCost);

const creditsRequestCount = Number(day.creditsRequestCount);
const apiKeysRequestCount = Number(day.apiKeysRequestCount);
Expand All @@ -420,6 +432,7 @@ activity.openapi(getActivity, async (c) => {
inputTokens,
outputTokens,
cachedTokens,
cacheWriteTokens,
totalTokens,
cost,
inputCost,
Expand All @@ -430,6 +443,7 @@ activity.openapi(getActivity, async (c) => {
imageOutputCost,
videoOutputCost,
cachedInputCost,
cacheWriteInputCost,
errorCount,
errorRate,
cacheCount,
Expand Down Expand Up @@ -476,6 +490,10 @@ activity.openapi(getActivity, async (c) => {
sql<number>`COALESCE(SUM(CAST(${projectHourlyStats.cachedTokens} AS NUMERIC)), 0)`.as(
"cachedTokens",
),
cacheWriteTokens:
sql<number>`COALESCE(SUM(CAST(${projectHourlyStats.cacheWriteTokens} AS NUMERIC)), 0)`.as(
"cacheWriteTokens",
),
totalTokens:
sql<number>`COALESCE(SUM(CAST(${projectHourlyStats.totalTokens} AS NUMERIC)), 0)`.as(
"totalTokens",
Expand Down Expand Up @@ -515,6 +533,10 @@ activity.openapi(getActivity, async (c) => {
sql<number>`COALESCE(SUM(${projectHourlyStats.cachedInputCost}), 0)`.as(
"cachedInputCost",
),
cacheWriteInputCost:
sql<number>`COALESCE(SUM(${projectHourlyStats.cacheWriteInputCost}), 0)`.as(
"cacheWriteInputCost",
),
errorCount:
sql<number>`COALESCE(SUM(${projectHourlyStats.errorCount}), 0)`.as(
"errorCount",
Expand Down Expand Up @@ -649,6 +671,7 @@ activity.openapi(getActivity, async (c) => {
const inputTokens = Number(day.inputTokens);
const outputTokens = Number(day.outputTokens);
const cachedTokens = Number(day.cachedTokens);
const cacheWriteTokens = Number(day.cacheWriteTokens);
const totalTokens = Number(day.totalTokens);
const cost = Number(day.cost);
const inputCost = Number(day.inputCost);
Expand All @@ -659,6 +682,7 @@ activity.openapi(getActivity, async (c) => {
const imageOutputCost = Number(day.imageOutputCost);
const videoOutputCost = Number(day.videoOutputCost);
const cachedInputCost = Number(day.cachedInputCost);
const cacheWriteInputCost = Number(day.cacheWriteInputCost);
const errorCount = Number(day.errorCount);
const cacheCount = Number(day.cacheCount);
const discountSavings = Number(day.discountSavings);
Expand All @@ -679,6 +703,7 @@ activity.openapi(getActivity, async (c) => {
inputTokens,
outputTokens,
cachedTokens,
cacheWriteTokens,
totalTokens,
cost,
inputCost,
Expand All @@ -689,6 +714,7 @@ activity.openapi(getActivity, async (c) => {
imageOutputCost,
videoOutputCost,
cachedInputCost,
cacheWriteInputCost,
errorCount,
errorRate,
cacheCount,
Expand Down
44 changes: 44 additions & 0 deletions apps/api/src/routes/admin.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,8 @@ const orgMetricsSchema = z.object({
outputCost: z.number(),
cachedTokens: z.number(),
cachedCost: z.number(),
cacheWriteTokens: z.number(),
cacheWriteCost: z.number(),
mostUsedModel: z.string().nullable(),
mostUsedProvider: z.string().nullable(),
mostUsedModelCost: z.number(),
Expand Down Expand Up @@ -1003,6 +1005,8 @@ admin.openapi(getOrganizationMetrics, async (c) => {
let outputCost = 0;
let cachedTokens = 0;
let cachedCost = 0;
let cacheWriteTokens = 0;
let cacheWriteCost = 0;
let discountSavings = 0;
let mostUsedModel: string | null = null;
let mostUsedProvider: string | null = null;
Expand All @@ -1028,6 +1032,10 @@ admin.openapi(getOrganizationMetrics, async (c) => {
sql<number>`COALESCE(SUM(CAST(${projectHourlyStats.cachedTokens} AS INTEGER)), 0)`.as(
"cachedTokens",
),
cacheWriteTokens:
sql<number>`COALESCE(SUM(CAST(${projectHourlyStats.cacheWriteTokens} AS INTEGER)), 0)`.as(
"cacheWriteTokens",
),
totalTokens:
sql<number>`COALESCE(SUM(CAST(${projectHourlyStats.totalTokens} AS INTEGER)), 0)`.as(
"totalTokens",
Expand All @@ -1051,6 +1059,10 @@ admin.openapi(getOrganizationMetrics, async (c) => {
sql<number>`COALESCE(SUM(${projectHourlyStats.cachedInputCost}), 0)`.as(
"cachedInputCost",
),
cacheWriteInputCost:
sql<number>`COALESCE(SUM(${projectHourlyStats.cacheWriteInputCost}), 0)`.as(
"cacheWriteInputCost",
),
})
.from(projectHourlyStats)
.where(
Expand All @@ -1071,6 +1083,8 @@ admin.openapi(getOrganizationMetrics, async (c) => {
outputCost = Number(totals.outputCost) || 0;
cachedTokens = Number(totals.cachedTokens) || 0;
cachedCost = Number(totals.cachedInputCost) || 0;
cacheWriteTokens = Number(totals.cacheWriteTokens) || 0;
cacheWriteCost = Number(totals.cacheWriteInputCost) || 0;
discountSavings = Number(totals.discountSavings) || 0;
}

Expand Down Expand Up @@ -1130,6 +1144,8 @@ admin.openapi(getOrganizationMetrics, async (c) => {
outputCost,
cachedTokens,
cachedCost,
cacheWriteTokens,
cacheWriteCost,
mostUsedModel,
mostUsedProvider,
mostUsedModelCost,
Expand Down Expand Up @@ -1386,6 +1402,8 @@ const projectMetricsSchema = z.object({
outputCost: z.number(),
cachedTokens: z.number(),
cachedCost: z.number(),
cacheWriteTokens: z.number(),
cacheWriteCost: z.number(),
mostUsedModel: z.string().nullable(),
mostUsedProvider: z.string().nullable(),
mostUsedModelCost: z.number(),
Expand Down Expand Up @@ -1462,6 +1480,8 @@ admin.openapi(getProjectMetrics, async (c) => {
let outputCost = 0;
let cachedTokens = 0;
let cachedCost = 0;
let cacheWriteTokens = 0;
let cacheWriteCost = 0;
let discountSavings = 0;
let mostUsedModel: string | null = null;
let mostUsedProvider: string | null = null;
Expand All @@ -1485,6 +1505,10 @@ admin.openapi(getProjectMetrics, async (c) => {
sql<number>`COALESCE(SUM(CAST(${projectHourlyStats.cachedTokens} AS INTEGER)), 0)`.as(
"cachedTokens",
),
cacheWriteTokens:
sql<number>`COALESCE(SUM(CAST(${projectHourlyStats.cacheWriteTokens} AS INTEGER)), 0)`.as(
"cacheWriteTokens",
),
totalTokens:
sql<number>`COALESCE(SUM(CAST(${projectHourlyStats.totalTokens} AS INTEGER)), 0)`.as(
"totalTokens",
Expand All @@ -1508,6 +1532,10 @@ admin.openapi(getProjectMetrics, async (c) => {
sql<number>`COALESCE(SUM(${projectHourlyStats.cachedInputCost}), 0)`.as(
"cachedInputCost",
),
cacheWriteInputCost:
sql<number>`COALESCE(SUM(${projectHourlyStats.cacheWriteInputCost}), 0)`.as(
"cacheWriteInputCost",
),
})
.from(projectHourlyStats)
.where(
Expand All @@ -1528,6 +1556,8 @@ admin.openapi(getProjectMetrics, async (c) => {
outputCost = Number(totals.outputCost) || 0;
cachedTokens = Number(totals.cachedTokens) || 0;
cachedCost = Number(totals.cachedInputCost) || 0;
cacheWriteTokens = Number(totals.cacheWriteTokens) || 0;
cacheWriteCost = Number(totals.cacheWriteInputCost) || 0;
discountSavings = Number(totals.discountSavings) || 0;
}

Expand Down Expand Up @@ -1584,6 +1614,8 @@ admin.openapi(getProjectMetrics, async (c) => {
outputCost,
cachedTokens,
cachedCost,
cacheWriteTokens,
cacheWriteCost,
mostUsedModel,
mostUsedProvider,
mostUsedModelCost,
Expand All @@ -1608,12 +1640,14 @@ const logEntrySchema = z.object({
totalTokens: z.string().nullable(),
reasoningTokens: z.string().nullable(),
cachedTokens: z.string().nullable(),
cacheWriteTokens: z.string().nullable(),
imageInputTokens: z.string().nullable(),
imageOutputTokens: z.string().nullable(),
cost: z.number().nullable(),
inputCost: z.number().nullable(),
outputCost: z.number().nullable(),
cachedInputCost: z.number().nullable(),
cacheWriteInputCost: z.number().nullable(),
requestCost: z.number().nullable(),
webSearchCost: z.number().nullable(),
imageInputCost: z.number().nullable(),
Expand Down Expand Up @@ -1793,12 +1827,14 @@ admin.openapi(getProjectLogs, async (c) => {
totalTokens: tables.log.totalTokens,
reasoningTokens: tables.log.reasoningTokens,
cachedTokens: tables.log.cachedTokens,
cacheWriteTokens: tables.log.cacheWriteTokens,
imageInputTokens: tables.log.imageInputTokens,
imageOutputTokens: tables.log.imageOutputTokens,
cost: tables.log.cost,
inputCost: tables.log.inputCost,
outputCost: tables.log.outputCost,
cachedInputCost: tables.log.cachedInputCost,
cacheWriteInputCost: tables.log.cacheWriteInputCost,
requestCost: tables.log.requestCost,
webSearchCost: tables.log.webSearchCost,
imageInputCost: tables.log.imageInputCost,
Expand Down Expand Up @@ -1866,6 +1902,7 @@ admin.openapi(getProjectLogs, async (c) => {
totalTokens: l.totalTokens ? String(l.totalTokens) : null,
reasoningTokens: l.reasoningTokens ? String(l.reasoningTokens) : null,
cachedTokens: l.cachedTokens ? String(l.cachedTokens) : null,
cacheWriteTokens: l.cacheWriteTokens ? String(l.cacheWriteTokens) : null,
Copy link
Copy Markdown
Contributor

@coderabbitai coderabbitai Bot May 6, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor | ⚡ Quick win

Preserve zero cache-write tokens in log responses.

This truthy check turns 0 into null, so clients cannot distinguish “no cache writes” from “field missing”. Check for null/undefined explicitly instead.

🩹 Suggested fix 
-			cacheWriteTokens: l.cacheWriteTokens ? String(l.cacheWriteTokens) : null,
+			cacheWriteTokens:
+				l.cacheWriteTokens === null || l.cacheWriteTokens === undefined
+					? null
+					: String(l.cacheWriteTokens),
🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@apps/api/src/routes/admin.ts` at line 1905, The mapping currently treats
falsy values as missing so cacheWriteTokens becomes null when it's 0; in the
object where cacheWriteTokens is assigned (the property named cacheWriteTokens
in the admin route response builder), change the falsy check to an explicit
null/undefined check—e.g. if l.cacheWriteTokens is null or undefined return
null, otherwise return String(l.cacheWriteTokens)—so a value of 0 is preserved
in responses.

Copy link
Copy Markdown
Contributor Author

@RATCHAW RATCHAW May 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cacheWriteTokens is a Drizzle decimal() column
decimal() defaults to mode: "string"
The string "0" is truthy in javascript

Copy link
Copy Markdown
Contributor

@coderabbitai coderabbitai Bot May 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@RATCHAW, you're absolutely right — thank you for the correction. Since decimal() in Drizzle defaults to mode: "string", the value arrives as "0" (a string) rather than the number 0, and "0" is truthy in JavaScript. The existing truthy check is therefore safe and consistent with all the other token columns (cachedTokens, reasoningTokens, etc.) in this same response builder. My concern was invalid.

🧠 Learnings used 
Learnt from: smakosh
Repo: theopenco/llmgateway PR: 1810
File: apps/api/src/routes/internal-models.ts:177-185
Timestamp: 2026-03-10T22:28:59.974Z
Learning: In apps/api/src/routes/internal-models.ts, ensure that the mapping.discount value returned by the /internal/models endpoint is overwritten with the resolved effective discount using globalDiscount ?? mapping.discount. This is intentional because the gateway computes/reads discount differently (from static model definitions and getEffectiveDiscount), so the internal API should standardize on the effective discount for UI clients. This guideline applies specifically to this endpoint/file and should be kept as a targeted, file-specific rule unless a broader review reveals a consistent pattern across similar routes.

imageInputTokens: l.imageInputTokens ? String(l.imageInputTokens) : null,
imageOutputTokens: l.imageOutputTokens
? String(l.imageOutputTokens)
Expand Down Expand Up @@ -5043,6 +5080,8 @@ const mappingDetailSchema = z.object({
inputPrice: z.string().nullable(),
outputPrice: z.string().nullable(),
cachedInputPrice: z.string().nullable(),
cacheWriteInputPrice: z.string().nullable(),
cacheWriteInputPrice1h: z.string().nullable(),
imageInputPrice: z.string().nullable(),
requestPrice: z.string().nullable(),
contextSize: z.number().nullable(),
Expand Down Expand Up @@ -5099,6 +5138,9 @@ admin.openapi(getMappingDetail, async (c) => {
inputPrice: tables.modelProviderMapping.inputPrice,
outputPrice: tables.modelProviderMapping.outputPrice,
cachedInputPrice: tables.modelProviderMapping.cachedInputPrice,
cacheWriteInputPrice: tables.modelProviderMapping.cacheWriteInputPrice,
cacheWriteInputPrice1h:
tables.modelProviderMapping.cacheWriteInputPrice1h,
imageInputPrice: tables.modelProviderMapping.imageInputPrice,
requestPrice: tables.modelProviderMapping.requestPrice,
contextSize: tables.modelProviderMapping.contextSize,
Expand Down Expand Up @@ -5186,6 +5228,8 @@ admin.openapi(getMappingDetail, async (c) => {
inputPrice: m.inputPrice,
outputPrice: m.outputPrice,
cachedInputPrice: m.cachedInputPrice,
cacheWriteInputPrice: m.cacheWriteInputPrice,
cacheWriteInputPrice1h: m.cacheWriteInputPrice1h,
imageInputPrice: m.imageInputPrice,
requestPrice: m.requestPrice,
contextSize: m.contextSize,
Expand Down
2 changes: 2 additions & 0 deletions apps/api/src/routes/internal-models.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ const modelProviderMappingSchema = z.object({
inputPrice: z.string().nullable(),
outputPrice: z.string().nullable(),
cachedInputPrice: z.string().nullable(),
cacheWriteInputPrice: z.string().nullable(),
cacheWriteInputPrice1h: z.string().nullable(),
imageInputPrice: z.string().nullable(),
imageOutputPrice: z.string().nullable(),
imageInputTokensByResolution: z.record(z.number()).nullable(),
Expand Down
2 changes: 2 additions & 0 deletions apps/api/src/routes/logs.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ const logSchema = z.object({
completionTokens: z.string().nullable(),
totalTokens: z.string().nullable(),
reasoningTokens: z.string().nullable(),
cacheWriteTokens: z.string().nullable().optional(),
messages: z.any(),
temperature: z.number().nullable(),
maxTokens: z.number().nullable(),
Expand All @@ -131,6 +132,7 @@ const logSchema = z.object({
outputCost: z.number().nullable(),
requestCost: z.number().nullable(),
cachedInputCost: z.number().nullable().optional(),
cacheWriteInputCost: z.number().nullable().optional(),
webSearchCost: z.number().nullable().optional(),
imageInputTokens: z.string().nullable(),
imageOutputTokens: z.string().nullable(),
Expand Down
8 changes: 8 additions & 0 deletions apps/api/src/testing.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,10 @@ function getCommonAggregationFields() {
sql<string>`coalesce(sum(cast(${tables.log.cachedTokens} as numeric)), 0)`.as(
"cachedTokens",
),
cacheWriteTokens:
sql<string>`coalesce(sum(cast(${tables.log.cacheWriteTokens} as numeric)), 0)`.as(
"cacheWriteTokens",
),
cost: sql<number>`coalesce(sum(${tables.log.cost}), 0)`.as("cost"),
inputCost: sql<number>`coalesce(sum(${tables.log.inputCost}), 0)`.as(
"inputCost",
Expand Down Expand Up @@ -163,6 +167,10 @@ function getCommonAggregationFields() {
sql<number>`coalesce(sum(${tables.log.cachedInputCost}), 0)`.as(
"cachedInputCost",
),
cacheWriteInputCost:
sql<number>`coalesce(sum(${tables.log.cacheWriteInputCost}), 0)`.as(
"cacheWriteInputCost",
),
// Per-mode breakdowns
creditsRequestCount:
sql<number>`sum(case when ${tables.log.usedMode} = 'credits' then 1 else 0 end)::int`.as(
Expand Down
Loading
Loading