From 20974c801da24238b0117ecaf264f1142ac37ed8 Mon Sep 17 00:00:00 2001 From: Cali Castle Date: Sun, 23 Aug 2026 15:34:04 +0800 Subject: [PATCH 1/3] perf(ama): drain durable operations inline, relax cron cadence The */5 AMA work cron kept the Neon endpoint awake around the clock: each run issues an unconditional hold-release UPDATE plus a claimDue query, resetting the 5-minute autosuspend timer every cycle (~183 CU-hrs/month of always-on compute). Mutating endpoints now kick a background drain of the durable operations queue via waitUntil in the same invocation that enqueued the work, so booking emails, Finalizing Booking recovery, refunds, and admin-triggered retries start immediately instead of waiting for the next sweep. The lease-based runner already tolerates concurrent drains, so the kick and the scheduled sweep cannot double-execute. With request-triggered work handled inline, the cron becomes a pure clock fallback (reminders, retry backoff, expired Slot Hold bookkeeping) and drops to */30, with media reconcile hourly and aligned so wakeups consolidate. Reminders and retries may now land up to 30 minutes later; everything guest-facing gets faster, and the database can suspend between wakeups. --- .../admin/ama/bookings/[bookingId]/route.ts | 7 ++++++- .../ama/operations/[operationId]/route.ts | 7 ++++++- app/api/ama/manage/[token]/cancel/route.ts | 14 ++++++++++++-- app/api/ama/manage/[token]/reschedule/route.ts | 14 ++++++++++++-- app/api/ama/stripe/webhook/route.ts | 11 +++++++++-- lib/ama/booking/server.ts | 18 ++++++++++++++++++ vercel.json | 4 ++-- 7 files changed, 65 insertions(+), 10 deletions(-) diff --git a/app/api/admin/ama/bookings/[bookingId]/route.ts b/app/api/admin/ama/bookings/[bookingId]/route.ts index 6b93dcd61..e8f41f105 100644 --- a/app/api/admin/ama/bookings/[bookingId]/route.ts +++ b/app/api/admin/ama/bookings/[bookingId]/route.ts @@ -3,6 +3,9 @@ import { getAmaAdminServices, ownerRequestAuthenticator, } from '~/lib/ama/admin/server' +import { kickAmaOperations } from '~/lib/ama/booking/server' + +export const maxDuration = 60 export async function POST( request: Request, @@ -10,10 +13,12 @@ export async function POST( ) { const { bookingId } = await params const { bookingAdmin, security, baseUrl } = getAmaAdminServices() - return createAdminBookingActionHandler({ + const response = await createAdminBookingActionHandler({ authenticator: ownerRequestAuthenticator, service: bookingAdmin, security, baseUrl, })(request, bookingId) + if (response.ok) kickAmaOperations() + return response } diff --git a/app/api/admin/ama/operations/[operationId]/route.ts b/app/api/admin/ama/operations/[operationId]/route.ts index 16a8b77f1..6b3078710 100644 --- a/app/api/admin/ama/operations/[operationId]/route.ts +++ b/app/api/admin/ama/operations/[operationId]/route.ts @@ -3,6 +3,9 @@ import { getAmaAdminServices, ownerRequestAuthenticator, } from '~/lib/ama/admin/server' +import { kickAmaOperations } from '~/lib/ama/booking/server' + +export const maxDuration = 60 export async function POST( request: Request, @@ -10,10 +13,12 @@ export async function POST( ) { const { operationId } = await params const { bookingAdmin, security, baseUrl } = getAmaAdminServices() - return createAdminOperationActionHandler({ + const response = await createAdminOperationActionHandler({ authenticator: ownerRequestAuthenticator, service: bookingAdmin, security, baseUrl, })(request, operationId) + if (response.ok) kickAmaOperations() + return response } diff --git a/app/api/ama/manage/[token]/cancel/route.ts b/app/api/ama/manage/[token]/cancel/route.ts index 2a9a196ac..b4beb7579 100644 --- a/app/api/ama/manage/[token]/cancel/route.ts +++ b/app/api/ama/manage/[token]/cancel/route.ts @@ -1,7 +1,12 @@ import { createManageCancelHandler } from '~/lib/ama/booking/http' -import { getAmaBookingServices } from '~/lib/ama/booking/server' +import { + getAmaBookingServices, + kickAmaOperations, +} from '~/lib/ama/booking/server' import { protectAmaLaunchBoundary } from '~/lib/ama/security/launch-boundary-server' +export const maxDuration = 60 + export async function POST( request: Request, { params }: { params: Promise<{ token: string }> }, @@ -13,5 +18,10 @@ export async function POST( if (blocked) return blocked const { token } = await params const { manage, guard } = getAmaBookingServices() - return createManageCancelHandler({ manage, guard })(request, token) + const response = await createManageCancelHandler({ manage, guard })( + request, + token, + ) + if (response.ok) kickAmaOperations() + return response } diff --git a/app/api/ama/manage/[token]/reschedule/route.ts b/app/api/ama/manage/[token]/reschedule/route.ts index 7d0b51cff..999b1d3a8 100644 --- a/app/api/ama/manage/[token]/reschedule/route.ts +++ b/app/api/ama/manage/[token]/reschedule/route.ts @@ -1,7 +1,12 @@ import { createManageRescheduleHandler } from '~/lib/ama/booking/http' -import { getAmaBookingServices } from '~/lib/ama/booking/server' +import { + getAmaBookingServices, + kickAmaOperations, +} from '~/lib/ama/booking/server' import { protectAmaLaunchBoundary } from '~/lib/ama/security/launch-boundary-server' +export const maxDuration = 60 + export async function POST( request: Request, { params }: { params: Promise<{ token: string }> }, @@ -13,5 +18,10 @@ export async function POST( if (blocked) return blocked const { token } = await params const { manage, guard } = getAmaBookingServices() - return createManageRescheduleHandler({ manage, guard })(request, token) + const response = await createManageRescheduleHandler({ manage, guard })( + request, + token, + ) + if (response.ok) kickAmaOperations() + return response } diff --git a/app/api/ama/stripe/webhook/route.ts b/app/api/ama/stripe/webhook/route.ts index 269c7457b..a80bc190f 100644 --- a/app/api/ama/stripe/webhook/route.ts +++ b/app/api/ama/stripe/webhook/route.ts @@ -1,14 +1,21 @@ import { createStripeWebhookHandler, json } from '~/lib/ama/booking/http' -import { getAmaBookingServices } from '~/lib/ama/booking/server' +import { + getAmaBookingServices, + kickAmaOperations, +} from '~/lib/ama/booking/server' import { protectAmaLaunchBoundary } from '~/lib/ama/security/launch-boundary-server' +export const maxDuration = 60 + export async function POST(request: Request) { const blocked = protectAmaLaunchBoundary(request, ['payments']) if (blocked) return blocked const { booking, stripeWebhookSecret } = getAmaBookingServices() if (!stripeWebhookSecret) return json(503, { error: 'feature_disabled' }) - return createStripeWebhookHandler({ + const response = await createStripeWebhookHandler({ service: booking, signingSecret: stripeWebhookSecret, })(request) + if (response.ok) kickAmaOperations() + return response } diff --git a/lib/ama/booking/server.ts b/lib/ama/booking/server.ts index ab03dec9b..cf19788e8 100644 --- a/lib/ama/booking/server.ts +++ b/lib/ama/booking/server.ts @@ -1,5 +1,7 @@ import 'server-only' +import { waitUntil } from '@vercel/functions' + import { createRateLimiter } from '~/lib/rate-limit/server' import { availabilityRepository } from '../availability/repository' @@ -227,3 +229,19 @@ export function getAmaBookingServices() { services ??= createServices() return services } + +/** + * Starts a background drain of due durable operations inside the current + * invocation, so work enqueued by a mutation (booking emails, Finalizing + * Booking recovery, refunds) begins immediately instead of waiting for the + * next scheduled sweep. The scheduled endpoint remains the sole driver for + * reminders, retry backoff, and expired Slot Hold release. + */ +export function kickAmaOperations() { + const { runner } = getAmaBookingServices() + waitUntil( + runner.run().catch((error) => { + console.error('ama: inline operations drain failed', error) + }), + ) +} diff --git a/vercel.json b/vercel.json index b1aac97bb..9452d405b 100644 --- a/vercel.json +++ b/vercel.json @@ -6,11 +6,11 @@ "crons": [ { "path": "/api/internal/media/reconcile", - "schedule": "*/15 * * * *" + "schedule": "0 * * * *" }, { "path": "/api/internal/ama/work", - "schedule": "*/5 * * * *" + "schedule": "*/30 * * * *" } ] } From 3604c252f5dd0c4925589581d818b6133922d825 Mon Sep 17 00:00:00 2001 From: Cali Castle Date: Sun, 23 Aug 2026 15:44:57 +0800 Subject: [PATCH 2/3] fix(ama): gate webhook kick on outcome, drain past a full batch Address review feedback on the inline operations drain: - The Stripe webhook handler now surfaces the processed outcome via an onOutcome callback, and the route kicks the drain only for booking_created. Duplicate, ignored, orphaned, booking-exists, and hold-release deliveries return 200 without enqueueing work, so they no longer start a runner pass. - A single runner pass caps at its batch size, so older due work could crowd out the operation the triggering mutation just enqueued. The kick now re-runs the drain while full batches come back, bounded by a 30-second inline budget; leftovers stay with the scheduled sweep. --- app/api/ama/stripe/webhook/route.ts | 9 ++++++--- lib/ama/booking/http.ts | 9 ++++++++- lib/ama/booking/server.ts | 20 +++++++++++++++++++- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/app/api/ama/stripe/webhook/route.ts b/app/api/ama/stripe/webhook/route.ts index a80bc190f..93c0a092a 100644 --- a/app/api/ama/stripe/webhook/route.ts +++ b/app/api/ama/stripe/webhook/route.ts @@ -12,10 +12,13 @@ export async function POST(request: Request) { if (blocked) return blocked const { booking, stripeWebhookSecret } = getAmaBookingServices() if (!stripeWebhookSecret) return json(503, { error: 'feature_disabled' }) - const response = await createStripeWebhookHandler({ + return createStripeWebhookHandler({ service: booking, signingSecret: stripeWebhookSecret, + // Only booking creation enqueues durable work; duplicate, ignored, + // orphaned, and hold-release deliveries return 200 without any. + onOutcome: (outcome) => { + if (outcome === 'booking_created') kickAmaOperations() + }, })(request) - if (response.ok) kickAmaOperations() - return response } diff --git a/lib/ama/booking/http.ts b/lib/ama/booking/http.ts index 7bbc88521..605a9d4b1 100644 --- a/lib/ama/booking/http.ts +++ b/lib/ama/booking/http.ts @@ -9,7 +9,7 @@ import { import type { SecurityRateLimiter } from '../security/service' import { verifyStripeWebhook } from '../stripe/webhook' import type { ManageService } from './manage' -import type { BookingService } from './service' +import type { BookingService, WebhookOutcome } from './service' const MAX_JSON_BODY_BYTES = 32 * 1024 @@ -253,12 +253,18 @@ type WebhookDependencies = { service: Pick signingSecret: string clock?: { now(): Date } + /** + * Observes the processed outcome so callers can react to the ones that + * enqueued durable work without re-parsing the response body. + */ + onOutcome?: (outcome: WebhookOutcome) => void } export function createStripeWebhookHandler({ service, signingSecret, clock = { now: () => new Date() }, + onOutcome, }: WebhookDependencies) { return async function POST(request: Request) { let payload: string @@ -276,6 +282,7 @@ export function createStripeWebhookHandler({ if (!event) return json(400, { error: 'invalid_signature' }) try { const outcome = await service.processWebhookEvent(event) + onOutcome?.(outcome) return json(200, { received: true, outcome }) } catch { // Signal Stripe to redeliver; the persisted provider event makes the diff --git a/lib/ama/booking/server.ts b/lib/ama/booking/server.ts index cf19788e8..4eb499420 100644 --- a/lib/ama/booking/server.ts +++ b/lib/ama/booking/server.ts @@ -32,6 +32,10 @@ import { bookingRepository } from './repository' import { createBookingService } from './service' const PROVIDER_REQUEST_TIMEOUT_MS = 8_000 +const OPERATIONS_BATCH_SIZE = 10 +// Leaves headroom under the mutating routes' maxDuration for the drain pass +// already in flight when the budget runs out. +const INLINE_DRAIN_BUDGET_MS = 30_000 let services: ReturnType | undefined @@ -188,6 +192,7 @@ function createServices() { clock, }) const runner = createOperationsRunner({ + batchSize: OPERATIONS_BATCH_SIZE, operations: durableOperationsRepository, handler: createOperationHandlers({ repository: bookingRepository, @@ -240,7 +245,20 @@ export function getAmaBookingServices() { export function kickAmaOperations() { const { runner } = getAmaBookingServices() waitUntil( - runner.run().catch((error) => { + (async () => { + // A full batch means older due work may have crowded out the operation + // this mutation enqueued, so keep draining until a partial batch shows + // the due queue is empty or the inline budget is spent. Anything left + // over stays with the scheduled sweep. + const startedAtMs = Date.now() + let result = await runner.run() + while ( + result.claimed >= OPERATIONS_BATCH_SIZE && + Date.now() - startedAtMs < INLINE_DRAIN_BUDGET_MS + ) { + result = await runner.run() + } + })().catch((error) => { console.error('ama: inline operations drain failed', error) }), ) From 26ce34035770fe44620c686cb21eda7c33ee9700 Mon Sep 17 00:00:00 2001 From: Cali Castle Date: Sun, 23 Aug 2026 15:52:24 +0800 Subject: [PATCH 3/3] fix(ama): give the inline drain room to outlast a slow backlog A single slow first pass (ten due operations with provider calls) could exhaust the 30-second outer budget before the drain reached the operation the triggering mutation enqueued, deferring it to the scheduled sweep. Raise the mutating routes to the platform-default 300s maxDuration (the explicit 60 had actually lowered it) and extend the drain budget to 240s, so 240s of passes plus one worst-case 45s pass still finish inside the ceiling. Passes are claim-ordered by nextAttemptAt, so each one moves the queue strictly toward the newest operation. --- app/api/admin/ama/bookings/[bookingId]/route.ts | 2 +- app/api/admin/ama/operations/[operationId]/route.ts | 2 +- app/api/ama/manage/[token]/cancel/route.ts | 2 +- app/api/ama/manage/[token]/reschedule/route.ts | 2 +- app/api/ama/stripe/webhook/route.ts | 2 +- lib/ama/booking/server.ts | 8 +++++--- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/app/api/admin/ama/bookings/[bookingId]/route.ts b/app/api/admin/ama/bookings/[bookingId]/route.ts index e8f41f105..c93e03eb2 100644 --- a/app/api/admin/ama/bookings/[bookingId]/route.ts +++ b/app/api/admin/ama/bookings/[bookingId]/route.ts @@ -5,7 +5,7 @@ import { } from '~/lib/ama/admin/server' import { kickAmaOperations } from '~/lib/ama/booking/server' -export const maxDuration = 60 +export const maxDuration = 300 export async function POST( request: Request, diff --git a/app/api/admin/ama/operations/[operationId]/route.ts b/app/api/admin/ama/operations/[operationId]/route.ts index 6b3078710..c604dc3cd 100644 --- a/app/api/admin/ama/operations/[operationId]/route.ts +++ b/app/api/admin/ama/operations/[operationId]/route.ts @@ -5,7 +5,7 @@ import { } from '~/lib/ama/admin/server' import { kickAmaOperations } from '~/lib/ama/booking/server' -export const maxDuration = 60 +export const maxDuration = 300 export async function POST( request: Request, diff --git a/app/api/ama/manage/[token]/cancel/route.ts b/app/api/ama/manage/[token]/cancel/route.ts index b4beb7579..82ea00cec 100644 --- a/app/api/ama/manage/[token]/cancel/route.ts +++ b/app/api/ama/manage/[token]/cancel/route.ts @@ -5,7 +5,7 @@ import { } from '~/lib/ama/booking/server' import { protectAmaLaunchBoundary } from '~/lib/ama/security/launch-boundary-server' -export const maxDuration = 60 +export const maxDuration = 300 export async function POST( request: Request, diff --git a/app/api/ama/manage/[token]/reschedule/route.ts b/app/api/ama/manage/[token]/reschedule/route.ts index 999b1d3a8..d99109984 100644 --- a/app/api/ama/manage/[token]/reschedule/route.ts +++ b/app/api/ama/manage/[token]/reschedule/route.ts @@ -5,7 +5,7 @@ import { } from '~/lib/ama/booking/server' import { protectAmaLaunchBoundary } from '~/lib/ama/security/launch-boundary-server' -export const maxDuration = 60 +export const maxDuration = 300 export async function POST( request: Request, diff --git a/app/api/ama/stripe/webhook/route.ts b/app/api/ama/stripe/webhook/route.ts index 93c0a092a..6467d881e 100644 --- a/app/api/ama/stripe/webhook/route.ts +++ b/app/api/ama/stripe/webhook/route.ts @@ -5,7 +5,7 @@ import { } from '~/lib/ama/booking/server' import { protectAmaLaunchBoundary } from '~/lib/ama/security/launch-boundary-server' -export const maxDuration = 60 +export const maxDuration = 300 export async function POST(request: Request) { const blocked = protectAmaLaunchBoundary(request, ['payments']) diff --git a/lib/ama/booking/server.ts b/lib/ama/booking/server.ts index 4eb499420..96a07fe3b 100644 --- a/lib/ama/booking/server.ts +++ b/lib/ama/booking/server.ts @@ -33,9 +33,11 @@ import { createBookingService } from './service' const PROVIDER_REQUEST_TIMEOUT_MS = 8_000 const OPERATIONS_BATCH_SIZE = 10 -// Leaves headroom under the mutating routes' maxDuration for the drain pass -// already in flight when the budget runs out. -const INLINE_DRAIN_BUDGET_MS = 30_000 +// Budget for starting drain passes. The mutating routes set maxDuration to +// 300s; 240s here plus one worst-case 45s pass still finishes inside that +// ceiling, and covers enough passes that a backlog cannot starve the +// operation the triggering mutation enqueued. +const INLINE_DRAIN_BUDGET_MS = 240_000 let services: ReturnType | undefined