feat(i18n): prefetch translations through OpenRouter batch endpoints

Add a --batch mode to the translate script. A silent dry run collects every
request the real run would send, those go to the model's :batch endpoint at
half the live price, and the real run consumes each answer once before
falling back to the live model for validation retries, truncation, and
anything a batch did not answer within TRANSLATE_BATCH_WAIT_MINUTES.
Batch-side failures never fail the run; they only cost the live price.
The translate workflow now runs in batch mode.

Author: berry-13

.github/workflows/translate_docs.yml

@@ -171,7 +171,11 @@ jobs:
           if [ "${{ inputs.force }}" = "true" ]; then
             FLAGS="--force"
           fi
-          npx tsx scripts/translate.ts $FLAGS
+          # --batch sends every uncached block through the model's half-price
+          # `:batch` endpoint first and runs only the leftovers live. It waits up to
+          # TRANSLATE_BATCH_WAIT_MINUTES (default 180) so the live tail and the
+          # commit still fit in the job's 6-hour limit.
+          npx tsx scripts/translate.ts --batch $FLAGS
 
       # `!cancelled()` so a translate step that failed part-way still commits the
       # pages it did finish; gated on the prepare step so a failure in checkout or

lib/i18n/batch.test.ts

@@ -0,0 +1,185 @@
+import { describe, it, expect, beforeEach } from 'vitest'
+import { mkdtemp, mkdir, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { createPrefetchedModel, prefetchTranslations, requestKey, type BatchAnswer } from './batch'
+import {
+  buildTranslateRequest,
+  TruncatedOutputError,
+  type TranslateModel,
+  type TranslateRequest,
+} from './engine'
+import { runTranslation } from './run'
+
+const echoSource = (prompt: string) => prompt.split(/Translate the following[^\n]*:\n/).pop() ?? ''
+
+function countingModel(): TranslateModel & { calls: number } {
+  const m = {
+    calls: 0,
+    generate: async ({ prompt }: TranslateRequest) => {
+      m.calls++
+      return echoSource(prompt)
+    },
+  }
+  return m
+}
+
+const req = (text: string): TranslateRequest =>
+  buildTranslateRequest({ text, locale: 'de', kind: 'block' })
+
+function json(body: unknown, status = 200): Response {
+  return new Response(JSON.stringify(body), { status })
+}
+
+const baseOpts = {
+  apiKey: 'k',
+  model: 'anthropic/claude-haiku-5.5',
+  chunkSize: 1000,
+  waitMs: 60_000,
+  pollMs: 1000,
+  log: () => {},
+  sleep: async () => {},
+}
+
+describe('createPrefetchedModel', () => {
+  it('serves a prefetched answer once, then falls back to the live model', async () => {
+    const live = countingModel()
+    const r = req('Hello')
+    const model = createPrefetchedModel(
+      new Map([[requestKey(r), { text: 'Hallo', finishReason: 'stop' }]]),
+      live,
+    )
+    expect(await model.generate(r)).toBe('Hallo')
+    expect(live.calls).toBe(0)
+    // A validation retry of the same request must reach the live model.
+    expect(await model.generate(r)).toBe('Hello')
+    expect(live.calls).toBe(1)
+  })
+
+  it('rejects a truncated batch answer like the live path does', async () => {
+    const r = req('Hello')
+    const model = createPrefetchedModel(
+      new Map([[requestKey(r), { text: 'Hal', finishReason: 'length' }]]),
+      countingModel(),
+    )
+    await expect(model.generate(r)).rejects.toBeInstanceOf(TruncatedOutputError)
+  })
+})
+
+describe('prefetchTranslations', () => {
+  it('submits distinct requests with `requests` last and harvests completed answers', async () => {
+    const a = req('One')
+    const b = req('Two')
+    const bodies: Record<string, unknown>[] = []
+    let polls = 0
+    const fakeFetch = (async (url: string, init?: RequestInit) => {
+      if (init?.method === 'POST') {
+        bodies.push(JSON.parse(String(init.body)))
+        return json({ id: 'b1', status: 'validating' })
+      }
+      polls++
+      if (polls === 1) return json({ id: 'b1', status: 'in_progress', results: null })
+      return json({
+        id: 'b1',
+        status: 'completed',
+        results: [
+          {
+            custom_id: requestKey(a),
+            response: {
+              status_code: 200,
+              body: { choices: [{ message: { content: 'Eins' }, finish_reason: 'stop' }] },
+            },
+            error: null,
+          },
+          { custom_id: requestKey(b), response: null, error: { message: 'boom' } },
+        ],
+      })
+    }) as typeof fetch
+
+    const answers = await prefetchTranslations([a, b, a], { ...baseOpts, fetch: fakeFetch })
+
+    expect(bodies).toHaveLength(1)
+    expect(Object.keys(bodies[0]).at(-1)).toBe('requests')
+    expect(bodies[0].model).toBe('anthropic/claude-haiku-5.5:batch')
+    expect((bodies[0].requests as unknown[]).length).toBe(2)
+    expect(answers.get(requestKey(a))).toEqual({ text: 'Eins', finishReason: 'stop' })
+    // An errored result has no answer, so that request runs live.
+    expect(answers.has(requestKey(b))).toBe(false)
+  })
+
+  it('splits requests into chunks', async () => {
+    let posts = 0
+    const fakeFetch = (async (_url: string, init?: RequestInit) => {
+      if (init?.method === 'POST') return json({ id: `b${++posts}`, status: 'validating' })
+      return json({ id: 'x', status: 'completed', results: [] })
+    }) as typeof fetch
+    await prefetchTranslations([req('1'), req('2'), req('3')], {
+      ...baseOpts,
+      chunkSize: 2,
+      fetch: fakeFetch,
+    })
+    expect(posts).toBe(2)
+  })
+
+  it('stops waiting at the deadline and returns what it has', async () => {
+    let t = 0
+    const fakeFetch = (async (_url: string, init?: RequestInit) =>
+      init?.method === 'POST'
+        ? json({ id: 'b1', status: 'validating' })
+        : json({ id: 'b1', status: 'in_progress', results: null })) as typeof fetch
+    const answers = await prefetchTranslations([req('One')], {
+      ...baseOpts,
+      fetch: fakeFetch,
+      now: () => t,
+      sleep: async (ms) => {
+        t += ms
+      },
+    })
+    expect(answers.size).toBe(0)
+    expect(t).toBeGreaterThanOrEqual(baseOpts.waitMs)
+  })
+
+  it('does not throw when a submit fails', async () => {
+    const fakeFetch = (async () => json({ error: 'no :batch endpoint' }, 400)) as typeof fetch
+    const answers = await prefetchTranslations([req('One')], { ...baseOpts, fetch: fakeFetch })
+    expect(answers.size).toBe(0)
+  })
+})
+
+describe('batch prefetch with runTranslation', () => {
+  let content: string
+  let cache: string
+  beforeEach(async () => {
+    const base = await mkdtemp(join(tmpdir(), 'batch-'))
+    content = join(base, 'docs')
+    cache = join(base, 'cache')
+    await mkdir(content, { recursive: true })
+    await writeFile(
+      join(content, 'index.mdx'),
+      `---\ntitle: Hello\n---\n\n# Hello\n\nA paragraph.\n\nA paragraph.\n`,
+    )
+    await writeFile(join(content, 'meta.json'), JSON.stringify({ title: 'Docs', pages: ['index'] }))
+  })
+
+  it('answers every request a real run makes, so nothing runs live', async () => {
+    const opts = { contentDir: content, cacheDir: cache, locales: ['de', 'fr'] }
+    const requests: TranslateRequest[] = []
+    await runTranslation({
+      ...opts,
+      model: countingModel(),
+      dryRun: true,
+      onMiss: (input) => requests.push(buildTranslateRequest(input)),
+    })
+    expect(requests.length).toBeGreaterThan(0)
+
+    const answers = new Map<string, BatchAnswer>()
+    for (const r of requests)
+      answers.set(requestKey(r), { text: echoSource(r.prompt), finishReason: 'stop' })
+
+    const live = countingModel()
+    const stats = await runTranslation({ ...opts, model: createPrefetchedModel(answers, live) })
+    expect(live.calls).toBe(0)
+    expect(stats.translatedBlocks).toBeGreaterThan(0)
+    expect(stats.skipped).toEqual([])
+  })
+})

lib/i18n/batch.ts

@@ -0,0 +1,189 @@
+import { createHash } from 'node:crypto'
+import {
+  assertNotTruncated,
+  TRANSLATE_MODEL_SETTINGS,
+  type TranslateModel,
+  type TranslateRequest,
+} from './engine'
+
+/**
+ * Batch prefetch for the translation pipeline. OpenRouter's `:batch` endpoints bill
+ * at half the live rate but answer asynchronously (up to a 24h window), which does
+ * not fit the per-block translate/validate/retry loop in run.ts. So the batch only
+ * answers each request's first attempt: the run then consumes those answers and
+ * falls back to the live model for everything else (validation retries, truncation,
+ * requests a batch could not answer before the deadline). The run's caching,
+ * validation, and give-up logic stay exactly as they are.
+ *
+ * https://openrouter.ai/docs/projects/docs/batch-quickstart
+ */
+
+const BATCHES_URL = 'https://openrouter.ai/api/v1/batches'
+const TERMINAL = new Set(['completed', 'failed', 'expired', 'cancelled'])
+
+export interface BatchAnswer {
+  text: string
+  finishReason: string
+}
+
+/** Stable id for one request; doubles as the batch `custom_id`. */
+export function requestKey(req: Parameters<TranslateModel['generate']>[0]): string {
+  return createHash('sha256')
+    .update(JSON.stringify([req.system, req.prompt, req.maxOutputTokens]))
+    .digest('hex')
+    .slice(0, 32)
+}
+
+/**
+ * Serves each prefetched answer once, then defers to the live model. Once, so a
+ * validation retry of the same request reaches the live model instead of being
+ * handed the same rejected answer again.
+ */
+export function createPrefetchedModel(
+  prefetched: Map<string, BatchAnswer>,
+  live: TranslateModel,
+): TranslateModel {
+  return {
+    async generate(req) {
+      const key = requestKey(req)
+      const hit = prefetched.get(key)
+      if (!hit) return live.generate(req)
+      prefetched.delete(key)
+      // Same contract as the live path: a length-limited answer is not a translation.
+      assertNotTruncated(hit.finishReason, req.maxOutputTokens)
+      return hit.text
+    },
+  }
+}
+
+interface BatchObject {
+  id: string
+  status: string
+  request_counts?: { total?: number; completed?: number; failed?: number }
+  results?: Array<{
+    custom_id: string
+    response?: {
+      status_code?: number
+      body?: { choices?: Array<{ message?: { content?: unknown }; finish_reason?: unknown }> }
+    } | null
+    error?: unknown
+  }> | null
+}
+
+export interface PrefetchOptions {
+  apiKey: string
+  /** Live model slug; `:batch` is appended. */
+  model: string
+  /** Requests per submitted batch. */
+  chunkSize: number
+  /** Give up waiting after this long; unanswered requests run live. */
+  waitMs: number
+  pollMs: number
+  log: (message: string) => void
+  fetch?: typeof fetch
+  sleep?: (ms: number) => Promise<void>
+  now?: () => number
+}
+
+/**
+ * Submit every distinct request as OpenRouter batches, wait for them up to
+ * `waitMs`, and return the answers keyed by `requestKey`. Never throws for a
+ * batch-side failure: a request without an answer simply runs live, so the worst
+ * case is the old cost, not a failed run.
+ */
+export async function prefetchTranslations(
+  requests: TranslateRequest[],
+  opts: PrefetchOptions,
+): Promise<Map<string, BatchAnswer>> {
+  const doFetch = opts.fetch ?? fetch
+  const sleep = opts.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms)))
+  const now = opts.now ?? Date.now
+  const headers = {
+    Authorization: `Bearer ${opts.apiKey}`,
+    'Content-Type': 'application/json',
+    'HTTP-Referer': 'https://www.librechat.ai',
+    'X-Title': 'LibreChat',
+  }
+
+  const unique = new Map<string, TranslateRequest>()
+  for (const req of requests) unique.set(requestKey(req), req)
+  const answers = new Map<string, BatchAnswer>()
+  if (unique.size === 0) return answers
+
+  const entries = [...unique]
+  const pending: string[] = []
+  for (let i = 0; i < entries.length; i += opts.chunkSize) {
+    const chunk = entries.slice(i, i + opts.chunkSize)
+    // Key order matters: OpenRouter rejects a body whose `requests` comes first.
+    const body = {
+      endpoint: '/v1/chat/completions',
+      model: `${opts.model}:batch`,
+      completion_window: '24h',
+      requests: chunk.map(([key, req]) => ({
+        custom_id: key,
+        body: {
+          messages: [
+            { role: 'system', content: req.system },
+            { role: 'user', content: req.prompt },
+          ],
+          max_tokens: req.maxOutputTokens,
+          ...TRANSLATE_MODEL_SETTINGS,
+        },
+      })),
+    }
+    try {
+      const res = await doFetch(BATCHES_URL, {
+        method: 'POST',
+        headers,
+        body: JSON.stringify(body),
+      })
+      if (!res.ok) throw new Error(`HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)
+      const created = (await res.json()) as BatchObject
+      pending.push(created.id)
+      opts.log(`submitted batch ${created.id} (${chunk.length} request(s))`)
+    } catch (e) {
+      opts.log(
+        `batch submit failed, ${chunk.length} request(s) will run live: ${(e as Error).message}`,
+      )
+    }
+  }
+
+  const deadline = now() + opts.waitMs
+  while (pending.length > 0) {
+    for (const id of [...pending]) {
+      let batch: BatchObject
+      try {
+        const res = await doFetch(`${BATCHES_URL}/${id}`, { headers })
+        if (!res.ok) throw new Error(`HTTP ${res.status}`)
+        batch = (await res.json()) as BatchObject
+      } catch (e) {
+        // A failed poll is not a failed batch; try again next round.
+        opts.log(`batch ${id} poll failed: ${(e as Error).message}`)
+        continue
+      }
+      if (!TERMINAL.has(batch.status)) continue
+      pending.splice(pending.indexOf(id), 1)
+      let harvested = 0
+      for (const r of batch.results ?? []) {
+        const choice = r.response?.status_code === 200 ? r.response.body?.choices?.[0] : undefined
+        const text = choice?.message?.content
+        if (typeof text !== 'string') continue
+        answers.set(r.custom_id, {
+          text,
+          finishReason: typeof choice?.finish_reason === 'string' ? choice.finish_reason : 'stop',
+        })
+        harvested++
+      }
+      opts.log(`batch ${id} ${batch.status}: ${harvested} answer(s)`)
+    }
+    if (pending.length === 0) break
+    if (now() >= deadline) {
+      opts.log(
+        `stopped waiting on ${pending.length} batch(es) after ${Math.round(opts.waitMs / 60000)} min; their requests will run live`,
+      )
+      break
+    }
+    await sleep(opts.pollMs)
+  }
+  return answers
+}