diff --git a/packages/realm-server/handlers/create-realm.ts b/packages/realm-server/handlers/create-realm.ts new file mode 100644 index 00000000000..7e9e5f7f425 --- /dev/null +++ b/packages/realm-server/handlers/create-realm.ts @@ -0,0 +1,403 @@ +import type Koa from 'koa'; +import { resolve, join } from 'path'; +import { ensureDirSync, writeJSONSync } from 'fs-extra'; +import * as Sentry from '@sentry/node'; +import type { + DBAdapter, + Realm, + RealmInfo, + VirtualNetwork, +} from '@cardstack/runtime-common'; +import { + createResponse, + DEFAULT_PERMISSIONS, + insertPermissions, + logger, + param, + query, + SupportedMimeType, + userInitiatedPriority, +} from '@cardstack/runtime-common'; +import { getMatrixUsername } from '@cardstack/runtime-common/matrix-client'; +import { insertSourceRealmInRegistry } from '../lib/realm-registry-writes'; +import type { RealmRegistryReconciler } from '../lib/realm-registry-reconciler'; +import { + fetchRequestFromContext, + sendResponseForBadRequest, + sendResponseForSystemError, + setContextResponse, +} from '../middleware'; +import type { RealmServerTokenClaim } from '../utils/jwt'; + +const log = logger('realm-server'); + +export type CreateRealmDeps = { + serverURL: URL; + realms: Realm[]; + dbAdapter: DBAdapter; + virtualNetwork: VirtualNetwork; + realmsRootPath: string; + reconciler: RealmRegistryReconciler; +}; + +export type CreateRealmInput = { + // matrix userIDs look like "@mango:boxel.ai" + ownerUserId: string; + endpoint: string; + name: string; + backgroundURL?: string; + iconURL?: string; +}; + +export type CreateRealmResult = { + url: string; + realm: Realm; + info: Partial; +}; + +interface RealmCreationJSON { + data: { + type: 'realm'; + attributes: { + endpoint: string; + name: string; + backgroundURL?: string; + iconURL?: string; + }; + }; +} + +export async function createRealm( + deps: CreateRealmDeps, + { ownerUserId, endpoint, name, backgroundURL, iconURL }: CreateRealmInput, +): Promise { + let { + serverURL, + realms, + dbAdapter, + virtualNetwork, + realmsRootPath, + reconciler, + } = deps; + + // Server-root collision check. Read realms[] AND realm_registry — + // every production realm has a registry row, but test fixtures + // construct CLI-style realms via runTestRealmServer that don't + // mirror to the registry. Either source matching the origin is a + // collision. + let serverRootUrl = serverURL.origin + '/'; + let realmAtServerRoot = realms.find((r) => { + let realmUrl = new URL(r.url); + return ( + realmUrl.href.replace(/\/$/, '') === realmUrl.origin && + realmUrl.hostname === serverURL.hostname + ); + }); + if (realmAtServerRoot) { + throw errorWithStatus( + 400, + `Cannot create a realm: a realm is already mounted at the origin of this server: ${realmAtServerRoot.url}`, + ); + } + let serverRootRows = (await query(dbAdapter, [ + `SELECT url FROM realm_registry WHERE url =`, + param(serverRootUrl), + ])) as { url: string }[]; + if (serverRootRows.length > 0) { + throw errorWithStatus( + 400, + `Cannot create a realm: a realm is already mounted at the origin of this server: ${serverRootRows[0].url}`, + ); + } + if (!endpoint.match(/^[a-z0-9-]+$/)) { + throw errorWithStatus( + 400, + `realm endpoint '${endpoint}' contains invalid characters`, + ); + } + + let ownerUsername = getMatrixUsername(ownerUserId); + let url = new URL( + `${serverURL.pathname.replace(/\/$/, '')}/${ownerUsername}/${endpoint}/`, + serverURL, + ).href; + + let existingRows = (await query(dbAdapter, [ + `SELECT url FROM realm_registry WHERE url =`, + param(url), + ])) as { url: string }[]; + if (existingRows.length > 0) { + throw errorWithStatus(400, `realm '${url}' already exists on this server`); + } + + let realmPath = resolve(join(realmsRootPath, ownerUsername, endpoint)); + ensureDirSync(realmPath); + + let info: Partial = { + name, + ...(iconURL ? { iconURL } : {}), + ...(backgroundURL ? { backgroundURL } : {}), + publishable: true, + }; + + // Serialize against any other caller of withWriteLock for this + // same URL (concurrent createRealm for the same endpoint, or a + // concurrent publish/unpublish/delete). This is almost never a real + // concurrency concern — the endpoint was already checked above for + // collision. + await dbAdapter.withWriteLock(url, async () => { + await insertPermissions(dbAdapter, new URL(url), { + [ownerUserId]: DEFAULT_PERMISSIONS, + }); + + // CS-10053: publishable lives in realm_metadata now, not the + // sidecar. The legacy .realm.json is no longer written here; + // hostHome/interactHome (still sidecar-owned until CS-10055) + // are absent on a fresh realm and don't need a placeholder file. + // Reset all mutable metadata columns on conflict so a stale row + // (e.g. left over from a previous realm at the same URL whose + // delete didn't clean up) doesn't bleed into the new realm. + await query(dbAdapter, [ + `INSERT INTO realm_metadata (url, publishable, show_as_catalog) VALUES (`, + param(url), + `,`, + param(true), + `,`, + param(null), + `) ON CONFLICT (url) DO UPDATE SET publishable = true, show_as_catalog = NULL, updated_at = now()`, + ]); + writeJSONSync(join(realmPath, 'realm.json'), { + data: { + type: 'card', + attributes: { + cardInfo: { name }, + ...(iconURL ? { iconURL } : {}), + ...(backgroundURL ? { backgroundURL } : {}), + }, + meta: { + adoptsFrom: { + module: 'https://cardstack.com/base/realm-config', + name: 'RealmConfig', + }, + }, + }, + }); + writeJSONSync(join(realmPath, 'index.json'), { + data: { + type: 'card', + meta: { + adoptsFrom: { + module: 'https://cardstack.com/base/cards-grid', + name: 'CardsGrid', + }, + }, + }, + }); + + // Register the source realm in realm_registry. The INSERT emits + // NOTIFY realm_registry; the reconciler on every instance picks + // up the row, and the realm is lazy-mounted on first request. + await insertSourceRealmInRegistry(dbAdapter, { + url, + diskId: `${ownerUsername}/${endpoint}`, + ownerUsername, + }); + }); + + // virtualNetwork URL mapping was historically bridged here so a + // virtual realm URL (e.g. cardstack.com/base/) routed to the + // physical localhost URL. For dynamically-created realms via + // /_create-realm, the URL is already a physical + // serverURL-rooted URL (no remap needed), but preserve the + // detection-and-add for any environment that maps it. + let actualRealmURL = virtualNetwork.mapURL(url, 'virtual-to-real'); + if (actualRealmURL && actualRealmURL.href !== url) { + virtualNetwork.addURLMapping(new URL(url), actualRealmURL); + } + + // Mount the realm on the *handling* instance and let the mount + // pipeline itself drive the one-and-only from-scratch-index, at + // userInitiatedPriority so a backed-up queue of system-priority + // jobs (e.g. a deploy-triggered reindex storm) does not stall realm + // creation. lookupOrMount → ensureMounted → realm.start → #startup + // sees `isNewIndex = true` for a freshly-registered realm and + // enqueues exactly one job via publishFullIndex, which also updates + // the realm's in-memory #stats / #ignoreData / #ignoreDataVersion + // when the job completes. + // + // The 202 response with status:'pending' is for sibling instances — + // they pick up the realm via NOTIFY realm_registry and lazy-mount + // on first request. Mounting eagerly here also drains the queue + // locally so the test framework's teardown (close server → drain + // runner → close DB) doesn't race a worker mid-fetch on the now- + // closed HTTP listener. + let realm = await reconciler.lookupOrMount(url, { + fromScratchIndexPriority: userInitiatedPriority, + }); + if (!realm) { + throw new Error( + `expected realm ${url} to be mounted after createRealm — registry row missing or mount failed`, + ); + } + + return { url, realm, info }; +} + +export default function handleCreateRealmRequest( + deps: CreateRealmDeps, +): (ctxt: Koa.Context, next: Koa.Next) => Promise { + return async function (ctxt: Koa.Context, _next: Koa.Next) { + let token = ctxt.state.token as RealmServerTokenClaim; + if (!token) { + await sendResponseForSystemError( + ctxt, + 'token is required to create realm', + ); + return; + } + + let { user: ownerUserId } = token; + let request = await fetchRequestFromContext(ctxt); + let body = await request.text(); + let json: Record; + try { + json = JSON.parse(body); + } catch (e) { + await sendResponseForBadRequest( + ctxt, + 'Request body is not valid JSON-API - invalid JSON', + ); + return; + } + try { + assertIsRealmCreationJSON(json); + } catch (e: any) { + await sendResponseForBadRequest( + ctxt, + `Request body is not valid JSON-API - ${e.message}`, + ); + return; + } + + let url: string | undefined; + let realm: Realm | undefined; + let info: Partial | undefined; + let start = Date.now(); + try { + let result = await createRealm(deps, { + ownerUserId, + ...json.data.attributes, + }); + url = result.url; + realm = result.realm; + info = result.info; + log.debug(`created new realm ${url} in ${Date.now() - start} ms`); + } catch (e: any) { + if ('status' in e && e.status === 400) { + await sendResponseForBadRequest(ctxt, e.message); + } else { + log.error( + `Error creating realm '${json.data.attributes.name}' for user ${ownerUserId}`, + e, + ); + await sendResponseForSystemError(ctxt, `${e.message}: at ${e.stack}`); + } + return; + } finally { + let creationTimeMs = Date.now() - start; + if (creationTimeMs > 30_000) { + let msg = `it took a long time, ${creationTimeMs} ms, to create realm for ${ownerUserId}, ${JSON.stringify( + json.data.attributes, + )}`; + console.error(msg); + Sentry.captureMessage(msg); + } + } + + // Phase 3 PR 2: createRealm wrote the realm directory + the + // realm_registry row, then mounted + started the realm via the + // reconciler so it's fully indexed on this instance. The 202 + + // status:'pending' is for sibling instances — they pick up the + // realm via NOTIFY realm_registry and lazy-mount on first + // request. Clients should poll //_readiness-check before + // treating the realm as ready globally. + let response = createResponse({ + body: JSON.stringify( + { + data: { + type: 'realm', + id: url, + attributes: { + ...json.data.attributes, + ...info, + status: 'pending', + }, + }, + }, + null, + 2, + ), + init: { + status: 202, + headers: { + 'content-type': SupportedMimeType.JSONAPI, + }, + }, + requestContext: { + realm, + permissions: { + [ownerUserId]: DEFAULT_PERMISSIONS, + }, + }, + }); + await setContextResponse(ctxt, response); + return; + }; +} + +function assertIsRealmCreationJSON( + json: any, +): asserts json is RealmCreationJSON { + if (typeof json !== 'object') { + throw new Error(`json must be an object`); + } + if (!('data' in json) || typeof json.data !== 'object') { + throw new Error(`json is missing "data" object`); + } + let { data } = json; + if (!('type' in data) || data.type !== 'realm') { + throw new Error('json.data.type must be "realm"'); + } + if (!('attributes' in data || typeof data.attributes !== 'object')) { + throw new Error(`json.data is missing "attributes" object`); + } + let { attributes } = data; + if (!('name' in attributes) || typeof attributes.name !== 'string') { + throw new Error( + `json.data.attributes.name is required and must be a string`, + ); + } + if (!('endpoint' in attributes) || typeof attributes.endpoint !== 'string') { + throw new Error( + `json.data.attributes.endpoint is required and must be a string`, + ); + } + if ( + 'backgroundURL' in attributes && + typeof attributes.backgroundURL !== 'string' + ) { + throw new Error(`json.data.attributes.backgroundURL must be a string`); + } + if ('iconURL' in attributes && typeof attributes.iconURL !== 'string') { + throw new Error(`json.data.attributes.iconURL must be a string`); + } +} + +function errorWithStatus( + status: number, + message: string, +): Error & { status: number } { + let error = new Error(message); + (error as Error & { status: number }).status = status; + return error as Error & { status: number }; +} diff --git a/packages/realm-server/handlers/handle-create-realm.ts b/packages/realm-server/handlers/handle-create-realm.ts deleted file mode 100644 index 38b38c66734..00000000000 --- a/packages/realm-server/handlers/handle-create-realm.ts +++ /dev/null @@ -1,181 +0,0 @@ -import type { RealmServerTokenClaim } from '../utils/jwt'; -import { - fetchRequestFromContext, - sendResponseForBadRequest, - sendResponseForSystemError, - setContextResponse, -} from '../middleware'; -import type Koa from 'koa'; -import type { Realm, RealmInfo } from '@cardstack/runtime-common'; -import { - createResponse, - DEFAULT_PERMISSIONS, - logger, - SupportedMimeType, -} from '@cardstack/runtime-common'; -import * as Sentry from '@sentry/node'; -import type { CreateRoutesArgs } from '../routes'; - -interface RealmCreationJSON { - data: { - type: 'realm'; - attributes: { - endpoint: string; - name: string; - backgroundURL?: string; - iconURL?: string; - }; - }; -} - -const log = logger('realm-server'); - -export default function handleCreateRealmRequest({ - createRealm, -}: CreateRoutesArgs): (ctxt: Koa.Context, next: Koa.Next) => Promise { - return async function (ctxt: Koa.Context, _next: Koa.Next) { - let token = ctxt.state.token as RealmServerTokenClaim; - if (!token) { - await sendResponseForSystemError( - ctxt, - 'token is required to create realm', - ); - return; - } - - let { user: ownerUserId } = token; - let request = await fetchRequestFromContext(ctxt); - let body = await request.text(); - let json: Record; - try { - json = JSON.parse(body); - } catch (e) { - await sendResponseForBadRequest( - ctxt, - 'Request body is not valid JSON-API - invalid JSON', - ); - return; - } - try { - assertIsRealmCreationJSON(json); - } catch (e: any) { - await sendResponseForBadRequest( - ctxt, - `Request body is not valid JSON-API - ${e.message}`, - ); - return; - } - - let url: string | undefined; - let realm: Realm | undefined; - let info: Partial | undefined; - let start = Date.now(); - try { - let result = await createRealm({ - ownerUserId, - ...json.data.attributes, - }); - url = result.url; - realm = result.realm; - info = result.info; - log.debug(`created new realm ${url} in ${Date.now() - start} ms`); - } catch (e: any) { - if ('status' in e && e.status === 400) { - await sendResponseForBadRequest(ctxt, e.message); - } else { - log.error( - `Error creating realm '${json.data.attributes.name}' for user ${ownerUserId}`, - e, - ); - await sendResponseForSystemError(ctxt, `${e.message}: at ${e.stack}`); - } - return; - } finally { - let creationTimeMs = Date.now() - start; - if (creationTimeMs > 30_000) { - let msg = `it took a long time, ${creationTimeMs} ms, to create realm for ${ownerUserId}, ${JSON.stringify( - json.data.attributes, - )}`; - console.error(msg); - Sentry.captureMessage(msg); - } - } - - // Phase 3 PR 2: createRealm wrote the realm directory + the - // realm_registry row, then mounted + started the realm via the - // reconciler so it's fully indexed on this instance. The 202 + - // status:'pending' is for sibling instances — they pick up the - // realm via NOTIFY realm_registry and lazy-mount on first - // request. Clients should poll //_readiness-check before - // treating the realm as ready globally. - let response = createResponse({ - body: JSON.stringify( - { - data: { - type: 'realm', - id: url, - attributes: { - ...json.data.attributes, - ...info, - status: 'pending', - }, - }, - }, - null, - 2, - ), - init: { - status: 202, - headers: { - 'content-type': SupportedMimeType.JSONAPI, - }, - }, - requestContext: { - realm, - permissions: { - [ownerUserId]: DEFAULT_PERMISSIONS, - }, - }, - }); - await setContextResponse(ctxt, response); - return; - }; -} - -function assertIsRealmCreationJSON( - json: any, -): asserts json is RealmCreationJSON { - if (typeof json !== 'object') { - throw new Error(`json must be an object`); - } - if (!('data' in json) || typeof json.data !== 'object') { - throw new Error(`json is missing "data" object`); - } - let { data } = json; - if (!('type' in data) || data.type !== 'realm') { - throw new Error('json.data.type must be "realm"'); - } - if (!('attributes' in data || typeof data.attributes !== 'object')) { - throw new Error(`json.data is missing "attributes" object`); - } - let { attributes } = data; - if (!('name' in attributes) || typeof attributes.name !== 'string') { - throw new Error( - `json.data.attributes.name is required and must be a string`, - ); - } - if (!('endpoint' in attributes) || typeof attributes.endpoint !== 'string') { - throw new Error( - `json.data.attributes.endpoint is required and must be a string`, - ); - } - if ( - 'backgroundURL' in attributes && - typeof attributes.backgroundURL !== 'string' - ) { - throw new Error(`json.data.attributes.backgroundURL must be a string`); - } - if ('iconURL' in attributes && typeof attributes.iconURL !== 'string') { - throw new Error(`json.data.attributes.iconURL must be a string`); - } -} diff --git a/packages/realm-server/handlers/send-event.ts b/packages/realm-server/handlers/send-event.ts new file mode 100644 index 00000000000..3e54c00259a --- /dev/null +++ b/packages/realm-server/handlers/send-event.ts @@ -0,0 +1,37 @@ +import type { DBAdapter } from '@cardstack/runtime-common'; +import { fetchSessionRoom } from '@cardstack/runtime-common'; +import type { MatrixClient } from '@cardstack/runtime-common/matrix-client'; +import { APP_BOXEL_REALM_SERVER_EVENT_MSGTYPE } from '@cardstack/runtime-common/matrix-constants'; + +export type SendEventDeps = { + matrixClient: MatrixClient; + dbAdapter: DBAdapter; +}; + +export type SendEvent = ( + user: string, + eventType: string, + data?: Record, +) => Promise; + +export function createSendEvent({ + matrixClient, + dbAdapter, +}: SendEventDeps): SendEvent { + return async function sendEvent(user, eventType, data) { + if (!matrixClient.isLoggedIn()) { + await matrixClient.login(); + } + let roomId = await fetchSessionRoom(dbAdapter, user); + if (!roomId) { + console.error( + `Failed to send event: ${eventType}, cannot find session room for user: ${user}`, + ); + } + + await matrixClient.sendEvent(roomId!, 'm.room.message', { + body: JSON.stringify({ eventType, data }), + msgtype: APP_BOXEL_REALM_SERVER_EVENT_MSGTYPE, + }); + }; +} diff --git a/packages/realm-server/handlers/serve-from-realm.ts b/packages/realm-server/handlers/serve-from-realm.ts new file mode 100644 index 00000000000..27f68686f3f --- /dev/null +++ b/packages/realm-server/handlers/serve-from-realm.ts @@ -0,0 +1,66 @@ +import type Koa from 'koa'; +import type { + DBAdapter, + Realm, + VirtualNetwork, +} from '@cardstack/runtime-common'; +import { logger } from '@cardstack/runtime-common'; +import { fetchRequestFromContext, setContextResponse } from '../middleware'; +import { setupCloseHandler } from '../node-realm'; +import { findOrMountRealm } from '../lib/realm-routing'; +import type { RealmRegistryReconciler } from '../lib/realm-registry-reconciler'; + +export type ServeFromRealmDeps = { + realms: Realm[]; + reconciler: RealmRegistryReconciler; + dbAdapter: DBAdapter; + virtualNetwork: VirtualNetwork; +}; + +const log = logger('realm-server'); + +export function createServeFromRealm( + deps: ServeFromRealmDeps, +): (ctxt: Koa.Context, next: Koa.Next) => Promise { + let { virtualNetwork } = deps; + return async function serveFromRealm(ctxt: Koa.Context, _next: Koa.Next) { + if (ctxt.request.path === '/_boom') { + throw new Error('boom'); + } + let request = await fetchRequestFromContext(ctxt); + // Phase 3 lazy mount: trigger findOrMountRealm before dispatching to + // virtualNetwork.handle so non-pinned realms (source/published) mount + // on first request. virtualNetwork.handle returns 404 for any URL + // whose handle isn't registered, which is exactly what happens for + // a realm that the reconciler knows about (knownByUrl) but hasn't + // mounted yet. findOrMountRealm walks knownByUrl, calls + // reconciler.lookupOrMount() on a prefix match, and that + // synchronously publishes the realm into virtualNetwork before the + // dispatch below. Mount failures throw — the catch turns them into + // 503 so the next request retries from scratch (ensureMounted's + // failure path clears mounted/pendingMounts). + let requestURL = new URL( + `${ctxt.protocol}://${ctxt.host}${ctxt.originalUrl}`, + ); + try { + await findOrMountRealm(requestURL, deps); + } catch (err: any) { + log.warn( + `failed to mount realm for request ${requestURL.href}: ${err?.message ?? err}`, + ); + ctxt.status = 503; + ctxt.body = `Realm mount failed: ${err?.message ?? err}`; + return; + } + let realmResponse = await virtualNetwork.handle( + request, + (mappedRequest) => { + // Setup this handler only after the request has been mapped because + // the *mapped request* is the one that gets closed, not the original one + setupCloseHandler(ctxt.res, mappedRequest); + }, + ); + + await setContextResponse(ctxt, realmResponse); + }; +} diff --git a/packages/realm-server/handlers/serve-index.ts b/packages/realm-server/handlers/serve-index.ts new file mode 100644 index 00000000000..a9ca704c7e2 --- /dev/null +++ b/packages/realm-server/handlers/serve-index.ts @@ -0,0 +1,468 @@ +import type Koa from 'koa'; +import { JSDOM } from 'jsdom'; +import merge from 'lodash/merge'; +import type { DBAdapter, Realm } from '@cardstack/runtime-common'; +import { + Deferred, + hasExtension, + logger, + param, + query, + sanitizeHeadHTMLToString, +} from '@cardstack/runtime-common'; +import type { MatrixClient } from '@cardstack/runtime-common/matrix-client'; +import { + ensureSingleTitle, + injectHeadHTML, + injectIsolatedHTML, + retrieveHeadHTML, + retrieveIsolatedHTML, +} from '../lib/index-html-injection'; +import { retrieveScopedCSS } from '../lib/retrieve-scoped-css'; +import { + getPublishedRealmInfo, + hasPublicPermissions, + isIndexedCardInstance, + type RealmRoutingDeps, +} from '../lib/realm-routing'; +import type { RealmRegistryReconciler } from '../lib/realm-registry-reconciler'; + +export type ServeIndexDeps = { + serverURL: URL; + assetsURL: URL; + realms: Realm[]; + reconciler: RealmRegistryReconciler; + dbAdapter: DBAdapter; + matrixClient: MatrixClient; + getIndexHTML: () => Promise; + cardSizeLimitBytes: number; + fileSizeLimitBytes: number; +}; + +export type ServeIndexHandlers = { + serveIndex: (ctxt: Koa.Context, next: Koa.Next) => Promise; + serveHostApp: (ctxt: Koa.Context, next: Koa.Next) => Promise; + // Exposed for tests that exercise the index-HTML rewriting in + // isolation. Same closure backs `serveIndex` / `serveHostApp` so the + // production cache behaviour is preserved. + retrieveIndexHTML: () => Promise; +}; + +const log = logger('realm-server'); +const headLog = logger('realm-server:head'); +const isolatedLog = logger('realm-server:isolated'); +const scopedCSSLog = logger('realm-server:scoped-css'); + +export function createServeIndex(deps: ServeIndexDeps): ServeIndexHandlers { + let { + serverURL, + assetsURL, + dbAdapter, + matrixClient, + getIndexHTML, + cardSizeLimitBytes, + fileSizeLimitBytes, + } = deps; + + let routingDeps: RealmRoutingDeps = { + realms: deps.realms, + reconciler: deps.reconciler, + dbAdapter: deps.dbAdapter, + }; + + // Production cache of the rewritten index.html plus a short hash used + // for ETag invalidation. Dev mode (assetsURL.hostname === 'localhost') + // bypasses the cache so shell changes are reflected immediately. + let promiseForIndexHTML: Promise | undefined; + let indexHTMLHash: string | undefined; + + async function retrieveIndexHTML(): Promise { + let isDev = assetsURL.hostname === 'localhost'; + + if (!isDev && promiseForIndexHTML) { + return promiseForIndexHTML; + } + + let deferred = new Deferred(); + + if (!isDev) { + promiseForIndexHTML = deferred.promise; + } + + let rewriteRealmURL = (url?: string) => { + if (!url) { + return url; + } + + let parsed = new URL(url); + return new URL( + `${parsed.pathname}${parsed.search}${parsed.hash}`, + serverURL, + ).href; + }; + + let indexHTML = (await getIndexHTML()).replace( + /()/, + (_match, g1, g2, g3) => { + let config = JSON.parse(decodeURIComponent(g2)); + + if (config.publishedRealmBoxelSpaceDomain === 'localhost:4201') { + // if this is the default, this needs to be the realm server’s host + // to work in Matrix tests, since publishedRealmBoxelSpaceDomain is currently + // the default domain for publishing a realm + config.publishedRealmBoxelSpaceDomain = serverURL.host; + } + + if (config.publishedRealmBoxelSiteDomain === 'localhost:4201') { + // if this is the default, this needs to be the realm server’s host + // to work in Matrix tests, since publishedRealmBoxelSiteDomain is currently + // the default domain for publishing a realm + config.publishedRealmBoxelSiteDomain = serverURL.host; + } + + config = merge({}, config, { + hostsOwnAssets: false, + assetsURL: assetsURL.href, + matrixURL: matrixClient.matrixURL.href.replace(/\/$/, ''), + matrixServerName: + process.env.MATRIX_SERVER_NAME || matrixClient.matrixURL.hostname, + realmServerURL: serverURL.href, + resolvedBaseRealmURL: rewriteRealmURL(config.resolvedBaseRealmURL), + resolvedCatalogRealmURL: rewriteRealmURL( + config.resolvedCatalogRealmURL, + ), + resolvedLegacyCatalogRealmURL: rewriteRealmURL( + config.resolvedLegacyCatalogRealmURL, + ), + resolvedSkillsRealmURL: rewriteRealmURL( + config.resolvedSkillsRealmURL, + ), + resolvedOpenRouterRealmURL: rewriteRealmURL( + config.resolvedOpenRouterRealmURL, + ), + defaultSystemCardId: rewriteRealmURL(config.defaultSystemCardId), + defaultFieldSpecId: rewriteRealmURL(config.defaultFieldSpecId), + cardSizeLimitBytes, + fileSizeLimitBytes, + publishedRealmDomainOverrides: + process.env.PUBLISHED_REALM_DOMAIN_OVERRIDES ?? + config.publishedRealmDomainOverrides, + }); + return `${g1}${encodeURIComponent(JSON.stringify(config))}${g3}`; + }, + ); + + indexHTML = indexHTML.replace(/(src|href)="\//g, `$1="${assetsURL.href}`); + + // Strip any static favicon/apple-touch-icon links from the base HTML + // since these are now dynamically injected between the head markers + indexHTML = indexHTML + .replace(/]*\brel="icon"[^>]*\/?>/gi, '') + .replace(/]*\brel="apple-touch-icon"[^>]*\/?>/gi, ''); + + // Recompute the hash in dev mode (where index.html is not cached) so + // that changes to the shell are reflected in the ETag. + if (!indexHTMLHash || isDev) { + let { createHash } = await import('crypto'); + indexHTMLHash = createHash('md5') + .update(indexHTML) + .digest('hex') + .slice(0, 8); + } + + deferred.fulfill(indexHTML); + return indexHTML; + } + + function defaultIconLinks(): string[] { + let faviconURL = new URL('boxel-favicon.png', assetsURL).href; + let webclipURL = new URL('boxel-webclip.png', assetsURL).href; + return [ + ``, + ``, + ]; + } + + let serveIndex = async (ctxt: Koa.Context, next: Koa.Next) => { + let acceptHeader = ctxt.header.accept ?? ''; + let lowerAcceptHeader = acceptHeader.toLowerCase(); + let includesVndMimeType = lowerAcceptHeader.includes('application/vnd.'); + let includesHtmlMimeType = lowerAcceptHeader.includes('text/html'); + + let requestURL = new URL( + `${ctxt.protocol}://${ctxt.host}${ctxt.originalUrl}`, + ); + + // Track published realm info from routing checks to avoid redundant + // DB queries in the ETag logic below. + let publishedRealmInfo: { lastPublishedAt: string | null } | null = null; + let publishedRealmInfoFetched = false; + + if (includesHtmlMimeType) { + if (includesVndMimeType) { + publishedRealmInfo = await getPublishedRealmInfo( + requestURL, + routingDeps, + ); + publishedRealmInfoFetched = true; + + if (publishedRealmInfo) { + return next(); + } + } + } else { + if (includesVndMimeType) { + return next(); + } + + if (hasExtension(requestURL.pathname)) { + return next(); + } + + publishedRealmInfo = await getPublishedRealmInfo(requestURL, routingDeps); + publishedRealmInfoFetched = true; + + if (!publishedRealmInfo) { + return next(); + } + + // For published realms with generic Accept headers (like */*), we need to + // distinguish card URLs from module URLs. Module imports (e.g., "./person") + // resolve to URLs without extensions and would incorrectly get HTML served. + // Only serve HTML if: + // 1. This is a directory index request (path ends with /), OR + // 2. The URL corresponds to an indexed card instance + let isIndexRequest = requestURL.pathname.endsWith('/'); + if (!isIndexRequest) { + let cardURL = requestURL; + let isCardInstance = await isIndexedCardInstance(cardURL, routingDeps); + if (!isCardInstance) { + return next(); + } + } + } + + // If this is a /connect iframe request, is the origin a valid published realm? + let connectMatch = ctxt.request.path.match(/\/connect\/(.+)$/); + + if (connectMatch) { + try { + let originParameter = new URL(decodeURIComponent(connectMatch[1])).href; + + let publishedRealms = await query(dbAdapter, [ + `SELECT url FROM realm_registry WHERE kind = 'published' AND url LIKE `, + param(`${originParameter}%`), + ]); + + if (publishedRealms.length === 0) { + ctxt.status = 404; + ctxt.body = `Not Found: No published realm found for origin ${originParameter}`; + + log.debug( + `Ignoring /connect request for origin ${originParameter}: no matching published realm`, + ); + + return; + } + + ctxt.set( + 'Content-Security-Policy', + `frame-ancestors ${originParameter}`, + ); + } catch (error) { + ctxt.status = 400; + ctxt.body = 'Bad Request'; + + log.info(`Error processing /connect request: ${error}`); + + return; + } + } + + ctxt.type = 'html'; + + let cardURL = requestURL; + let isIndexRequest = requestURL.pathname.endsWith('/'); + if (isIndexRequest) { + cardURL = new URL('index', requestURL); + } + + // Retrieve index HTML early so the shell hash is available for ETag. + // This is memoized in production, so it's cheap after the first call. + let indexHTML = await retrieveIndexHTML(); + + // For published realms, support HTTP caching via ETag. + // The ETag includes both last_published_at and a hash of the host app + // shell, so a deploy that changes index.html invalidates cached responses. + if (!publishedRealmInfoFetched) { + publishedRealmInfo = await getPublishedRealmInfo(requestURL, routingDeps); + } + let lastPublishedAt = publishedRealmInfo?.lastPublishedAt; + let etag = + lastPublishedAt && indexHTMLHash + ? `"${lastPublishedAt}-${indexHTMLHash}"` + : null; + + if (etag) { + let ifNoneMatch = ctxt.get('If-None-Match'); + if ( + ifNoneMatch === '*' || + ifNoneMatch + .split(',') + .some((t) => t.trim().replace(/^W\//, '') === etag) + ) { + ctxt.status = 304; + ctxt.set('ETag', etag); + ctxt.set('Cache-Control', 'public, max-age=0, must-revalidate'); + ctxt.vary('Accept'); + return; + } + } + let publicPermissions = await hasPublicPermissions(cardURL, routingDeps); + + if (!publicPermissions) { + ctxt.body = injectHeadHTML( + indexHTML, + `Boxel\n${defaultIconLinks().join('\n')}`, + ); + return; + } + + headLog.debug(`Fetching head HTML for ${cardURL.href}`); + isolatedLog.debug(`Fetching isolated HTML for ${cardURL.href}`); + scopedCSSLog.debug(`Fetching scoped CSS for ${cardURL.href}`); + + let [headHTML, isolatedHTML, scopedCSS] = await Promise.all([ + retrieveHeadHTML({ + cardURL, + dbAdapter, + log: headLog, + }), + retrieveIsolatedHTML({ + cardURL, + dbAdapter, + log: isolatedLog, + }), + retrieveScopedCSS({ + cardURL, + dbAdapter, + log: scopedCSSLog, + }), + ]); + + let doc = new JSDOM().window.document; + if (headHTML != null) { + let sanitized = sanitizeHeadHTMLToString(headHTML, doc); + if (sanitized !== null) { + headHTML = sanitized; + } else { + headHTML = null; + } + } + + if (headHTML != null) { + headLog.debug( + `Injecting head HTML for ${cardURL.href} (length ${headHTML.length})\n${truncateLogLines( + headHTML, + )}`, + ); + } else { + headLog.debug( + `No head HTML found for ${cardURL.href}, serving base index.html`, + ); + } + + if (scopedCSS != null) { + scopedCSSLog.debug( + `Using scoped CSS for ${cardURL.href} (length ${scopedCSS.length})`, + ); + } else { + scopedCSSLog.debug( + `No scoped CSS returned from database for ${cardURL.href}`, + ); + } + + let responseHTML = indexHTML; + let headFragments: string[] = []; + + if (headHTML != null) { + headFragments.push(ensureSingleTitle(headHTML)); + } else { + headFragments.push('Boxel'); + } + + if (scopedCSS != null) { + scopedCSSLog.debug(`Injecting scoped CSS for ${cardURL.href}`); + headFragments.push( + ``, + ); + } + + let hasFavicon = false; + let hasAppleTouchIcon = false; + if (headHTML != null) { + let fragment = doc.createRange().createContextualFragment(headHTML); + hasFavicon = fragment.querySelector('link[rel~="icon"]') != null; + hasAppleTouchIcon = + fragment.querySelector('link[rel~="apple-touch-icon"]') != null; + } + let faviconURL = new URL('boxel-favicon.png', assetsURL).href; + let webclipURL = new URL('boxel-webclip.png', assetsURL).href; + if (!hasFavicon) { + headFragments.push(``); + } + if (!hasAppleTouchIcon) { + headFragments.push( + ``, + ); + } + + if (headFragments.length > 0) { + responseHTML = injectHeadHTML(responseHTML, headFragments.join('\n')); + } + + if (isolatedHTML != null) { + isolatedLog.debug( + `Injecting isolated HTML for ${cardURL.href} (length ${isolatedHTML.length})\n${truncateLogLines( + isolatedHTML, + )}`, + ); + responseHTML = injectIsolatedHTML(responseHTML, isolatedHTML); + } + + if (etag) { + ctxt.set('ETag', etag); + ctxt.set('Cache-Control', 'public, max-age=0, must-revalidate'); + ctxt.vary('Accept'); + } + + ctxt.body = responseHTML; + return; + }; + + let serveHostApp = async (ctxt: Koa.Context, next: Koa.Next) => { + let acceptHeader = (ctxt.header.accept ?? '').toLowerCase(); + let isHead = ctxt.method === 'HEAD'; + if (!isHead && !acceptHeader.includes('text/html')) { + return next(); + } + + ctxt.type = 'html'; + ctxt.body = injectHeadHTML( + await retrieveIndexHTML(), + `Boxel\n${defaultIconLinks().join('\n')}`, + ); + }; + + return { serveIndex, serveHostApp, retrieveIndexHTML }; +} + +function truncateLogLines(value: string, maxLines = 3): string { + let lines = value.split(/\r?\n/); + if (lines.length <= maxLines) { + return value; + } + let truncated = lines.slice(0, maxLines); + truncated[maxLines - 1] = `${truncated[maxLines - 1]} ...`; + return truncated.join('\n'); +} diff --git a/packages/realm-server/lib/realm-registry-reconciler.ts b/packages/realm-server/lib/realm-registry-reconciler.ts index 96ca413dd40..a2e828fcd02 100644 --- a/packages/realm-server/lib/realm-registry-reconciler.ts +++ b/packages/realm-server/lib/realm-registry-reconciler.ts @@ -291,7 +291,10 @@ export class RealmRegistryReconciler { // would receive a not-yet-started Realm; routing it through the // in-flight promise instead lets the caller await start() like the // original requester. - async lookupOrMount(url: string): Promise { + async lookupOrMount( + url: string, + opts?: { fromScratchIndexPriority?: number }, + ): Promise { const inflight = this.pendingMounts.get(url); if (inflight) { return inflight; @@ -308,7 +311,7 @@ export class RealmRegistryReconciler { } this.knownByUrl.set(url, row); } - return this.ensureMounted(row); + return this.ensureMounted(row, opts); } async #lookupRow(url: string): Promise { @@ -347,7 +350,10 @@ export class RealmRegistryReconciler { // rollout safety relies on this signal — Loki/Grafana extract cold- // mount latency, mount failure rate, and pinned-vs-lazy ratios from // these lines. - async ensureMounted(row: RealmRegistryRow): Promise { + async ensureMounted( + row: RealmRegistryRow, + opts?: { fromScratchIndexPriority?: number }, + ): Promise { // pendingMounts checked before mounted: see lookupOrMount() above. // The Realm is published into mounted synchronously before its // start() promise resolves, so a caller hitting the mounted @@ -386,7 +392,7 @@ export class RealmRegistryReconciler { this.#reconcilerOwned.add(row.url); const promise = (async () => { try { - await realm.start(); + await realm.start(opts); log.info( `mount ok url=%s kind=%s pinned=%s duration_ms=%d`, row.url, diff --git a/packages/realm-server/lib/realm-routing.ts b/packages/realm-server/lib/realm-routing.ts new file mode 100644 index 00000000000..82f434a891d --- /dev/null +++ b/packages/realm-server/lib/realm-routing.ts @@ -0,0 +1,247 @@ +import { join } from 'path'; +import { existsSync } from 'fs-extra'; +import type { DBAdapter, Realm } from '@cardstack/runtime-common'; +import { + executableExtensions, + fetchRealmPermissions, + hasExtension, + param, + query, + RealmPaths, +} from '@cardstack/runtime-common'; +import { + indexCandidateExpressions, + indexURLCandidates, +} from './index-url-utils'; +import type { RealmRegistryReconciler } from './realm-registry-reconciler'; + +export type RealmRoutingDeps = { + realms: Realm[]; + reconciler: RealmRegistryReconciler; + dbAdapter: DBAdapter; +}; + +// Resolves a request URL to a mounted Realm, lazy-mounting via the +// reconciler if the request is the first hit on a non-pinned realm +// (Phase 3 lazy-mount semantics). Returns undefined when no realm in the +// registry matches the request — caller should respond 404. +// +// Lookup order: +// 1. realms[] — covers (a) realms whose mountFromRow has already +// published them to this array but whose start() is still awaiting +// fullIndex; the worker processing that fullIndex re-enters this +// resolver to fetch /_mtimes and must hit the published +// realm rather than reconciler.ensureMounted(), which would +// return the same in-flight promise and deadlock the boot path; +// and (b) handler-created realms in Phase 3 PR 1 (publish/copy +// push directly to realms[]; the reconciler may not have +// observed them via NOTIFY/reconcile yet). Phase 3 PR 2 collapses +// (b) onto the reconciler. +// 2. reconciler.knownByUrl — the Phase 3 source of truth for never- +// mounted realms. Iterates registry rows, finds the one whose URL +// prefix contains the request, delegates to lookupOrMount() which +// constructs+mounts via mountFromRow on the cold first request. +export async function findOrMountRealm( + requestURL: URL, + { realms, reconciler, dbAdapter }: RealmRoutingDeps, +): Promise { + let legacy = realms.find((candidate) => { + let realmURL = new URL(candidate.url); + realmURL.protocol = requestURL.protocol; + return new RealmPaths(realmURL).inRealm(requestURL); + }); + if (legacy) { + return legacy; + } + for (const url of reconciler.knownByUrl.keys()) { + let realmURL = new URL(url); + realmURL.protocol = requestURL.protocol; + if (new RealmPaths(realmURL).inRealm(requestURL)) { + return await reconciler.lookupOrMount(url); + } + } + // Phase 3: knownByUrl is populated by reconciler.reconcile() on + // boot + LISTEN/poll. A request that arrives between a sibling + // instance's POST /_create-realm (or /_publish-realm) and this + // instance's reconciler picking up NOTIFY would otherwise 404. + // Fall through to a direct registry probe — match on every path + // prefix and let Postgres pick the longest URL so a request to + // `/foo/bar/baz/file.json` resolves to `/foo/bar/baz/` if that's + // registered, not `/foo/` (both prefixes are valid candidates). + let candidatePaths = candidateRealmURLs(requestURL); + if (candidatePaths.length === 0) { + return undefined; + } + let inClause: (string | ReturnType)[] = ['(']; + candidatePaths.forEach((u, idx) => { + if (idx > 0) inClause.push(','); + inClause.push(param(u)); + }); + inClause.push(')'); + let rows = (await query(dbAdapter, [ + `SELECT url FROM realm_registry WHERE url IN`, + ...inClause, + `ORDER BY LENGTH(url) DESC LIMIT 1`, + ])) as { url: string }[]; + if (rows.length === 0) { + return undefined; + } + return await reconciler.lookupOrMount(rows[0].url); +} + +export async function getPublishedRealmInfo( + requestURL: URL, + deps: RealmRoutingDeps, +): Promise<{ lastPublishedAt: string | null } | null> { + let realm = await findOrMountRealm(requestURL, deps); + if (!realm) { + return null; + } + + let rows = await query(deps.dbAdapter, [ + `SELECT last_published_at FROM realm_registry WHERE kind = 'published' AND url =`, + param(realm.url), + ]); + + if (rows.length === 0) { + return null; + } + + return { + lastPublishedAt: (rows[0].last_published_at as string) ?? null, + }; +} + +// Check if the URL corresponds to an indexed card instance. +// This is used to distinguish card URLs from module URLs when deciding +// whether to serve HTML for published realms. +// +// IMPORTANT: Card instances have their file_alias set to the URL without +// the .json extension. This means an instance at /foo/bar.json has +// file_alias /foo/bar. When a module request comes in for /foo/bar (no +// extension), we must check if it's actually a module before assuming it's +// an instance. Modules take precedence over instance aliases. +export async function isIndexedCardInstance( + cardURL: URL, + deps: RealmRoutingDeps, +): Promise { + let candidates = indexURLCandidates(cardURL); + if (candidates.length === 0) { + return false; + } + + // First check if there's a module at this URL - modules take precedence + // over instance aliases. This handles the case where: + // - Module: /foo/bar.gts (file_alias: /foo/bar) + // - Instance: /foo/bar.json (file_alias: /foo/bar) + // A request for /foo/bar should serve the module, not HTML for the instance. + // Prefer the modules table here because copied/published realms do not + // carry module rows in boxel_index. + let moduleRows = await query(deps.dbAdapter, [ + ` + SELECT 1 + FROM modules + WHERE + `, + ...indexCandidateExpressions(candidates), + ` + LIMIT 1 + `, + ]); + + if (moduleRows.length > 0) { + return false; + } + + let rows = await query(deps.dbAdapter, [ + ` + SELECT 1 + FROM boxel_index + WHERE type = 'instance' + AND is_deleted IS NOT TRUE + AND + `, + ...indexCandidateExpressions(candidates), + ` + LIMIT 1 + `, + ]); + + if (rows.length === 0) { + return false; + } + + // During publish/copy index races, module rows can lag behind source files. + // Only do filesystem probing after we've identified an instance candidate + // to avoid extra IO on the hot request path. + if (await hasExtensionlessSourceModule(cardURL, deps)) { + return false; + } + + return true; +} + +export async function hasExtensionlessSourceModule( + cardURL: URL, + deps: RealmRoutingDeps, +): Promise { + let realm = await findOrMountRealm(cardURL, deps); + if (!realm?.dir) { + return false; + } + + let localPath: string; + try { + localPath = realm.paths.local(cardURL); + } catch { + return false; + } + + if (!localPath || hasExtension(localPath)) { + return false; + } + + for (let extension of executableExtensions) { + if (existsSync(join(realm.dir, `${localPath}${extension}`))) { + return true; + } + if (existsSync(join(realm.dir, localPath, `index${extension}`))) { + return true; + } + } + + return false; +} + +export async function hasPublicPermissions( + cardURL: URL, + deps: RealmRoutingDeps, +): Promise { + let realm = await findOrMountRealm(cardURL, deps); + + if (!realm) { + return false; + } + + let permissions = await fetchRealmPermissions( + deps.dbAdapter, + new URL(realm.url), + ); + + return permissions['*']?.includes('read') ?? false; +} + +// Build candidate realm URLs from a request URL by trimming the +// pathname segment-by-segment. Used by findOrMountRealm's registry +// fallback when knownByUrl is stale. Includes the origin-only form +// (root realm) and every prefix that ends with a slash. +export function candidateRealmURLs(requestURL: URL): string[] { + let segments = requestURL.pathname.split('/').filter(Boolean); + let candidates: string[] = []; + // Try longest-prefix first. + for (let i = segments.length; i >= 0; i--) { + let path = i === 0 ? '/' : '/' + segments.slice(0, i).join('/') + '/'; + candidates.push(`${requestURL.origin}${path}`); + } + return [...new Set(candidates)]; +} diff --git a/packages/realm-server/routes.ts b/packages/realm-server/routes.ts index 3113f81f0c5..f3bd90c4719 100644 --- a/packages/realm-server/routes.ts +++ b/packages/realm-server/routes.ts @@ -1,4 +1,3 @@ -import type { RealmInfo } from '@cardstack/runtime-common'; import type { DBAdapter, DefinitionLookup, @@ -11,7 +10,9 @@ import type { MatrixClient } from '@cardstack/runtime-common/matrix-client'; import Router from '@koa/router'; import { createRequire } from 'module'; import handleCreateSessionRequest from './handlers/handle-create-session'; -import handleCreateRealmRequest from './handlers/handle-create-realm'; +import handleCreateRealmRequest, { + type CreateRealmDeps, +} from './handlers/create-realm'; import handleDeleteRealm from './handlers/handle-delete-realm'; import handleFetchCatalogRealmsRequest from './handlers/handle-fetch-catalog-realms'; import handleFetchUserRequest from './handlers/handle-fetch-user'; @@ -84,19 +85,6 @@ export type CreateRoutesArgs = { reconciler: RealmRegistryReconciler; realmsRootPath: string; getMatrixRegistrationSecret: () => Promise; - createRealm: ({ - ownerUserId, - endpoint, - name, - backgroundURL, - iconURL, - }: { - ownerUserId: string; - endpoint: string; - name: string; - backgroundURL?: string; - iconURL?: string; - }) => Promise<{ url: string; realm: Realm; info: Partial }>; serveHostApp: (ctxt: Koa.Context, next: Koa.Next) => Promise; serveIndex: (ctxt: Koa.Context, next: Koa.Next) => Promise; serveFromRealm: (ctxt: Koa.Context, next: Koa.Next) => Promise; @@ -129,6 +117,15 @@ export function createRoutes(args: CreateRoutesArgs) { // worst-case memory under a synthetic-jobId flood. let searchCache = new JobScopedSearchCache(); + let createRealmDeps: CreateRealmDeps = { + serverURL: new URL(args.serverURL), + realms: args.realms, + dbAdapter: args.dbAdapter, + virtualNetwork: args.virtualNetwork, + realmsRootPath: args.realmsRootPath, + reconciler: args.reconciler, + }; + router.get( '/', healthCheck, @@ -141,7 +138,7 @@ export function createRoutes(args: CreateRoutesArgs) { router.post( '/_create-realm', jwtMiddleware(args.realmSecretSeed), - handleCreateRealmRequest(args), + handleCreateRealmRequest(createRealmDeps), ); router.delete( '/_delete-realm', diff --git a/packages/realm-server/server.ts b/packages/realm-server/server.ts index 4a9230e76cc..cbaee4274ae 100644 --- a/packages/realm-server/server.ts +++ b/packages/realm-server/server.ts @@ -1,75 +1,38 @@ import Koa from 'koa'; import cors from '@koa/cors'; import { Memoize } from 'typescript-memoize'; -import type { - DefinitionLookup, - Realm, - RealmInfo, -} from '@cardstack/runtime-common'; +import type { DefinitionLookup, Realm } from '@cardstack/runtime-common'; import { logger, SupportedMimeType, - insertPermissions, - fetchRealmPermissions, - param, - query, - Deferred, type VirtualNetwork, type DBAdapter, type QueuePublisher, - DEFAULT_PERMISSIONS, DEFAULT_CARD_SIZE_LIMIT_BYTES, DEFAULT_FILE_SIZE_LIMIT_BYTES, - RealmPaths, - fetchSessionRoom, - hasExtension, - executableExtensions, - userInitiatedPriority, } from '@cardstack/runtime-common'; -import { enqueueReindexRealmJob } from '@cardstack/runtime-common/jobs/reindex-realm'; -import { ensureDirSync, writeJSONSync, existsSync } from 'fs-extra'; -import { setupCloseHandler } from './node-realm'; +import { ensureDirSync } from 'fs-extra'; import { httpLogging, ecsMetadata, - setContextResponse, - fetchRequestFromContext, methodOverrideSupport, proxyAsset, } from './middleware'; import convertAcceptHeaderQueryParam from './middleware/convert-accept-header-qp'; -import { resolve, join } from 'path'; -import merge from 'lodash/merge'; import { extractSupportedMimeType } from '@cardstack/runtime-common/router'; import * as Sentry from '@sentry/node'; import type { MatrixClient } from '@cardstack/runtime-common/matrix-client'; -import { getMatrixUsername } from '@cardstack/runtime-common/matrix-client'; import { createRoutes } from './routes'; -import { APP_BOXEL_REALM_SERVER_EVENT_MSGTYPE } from '@cardstack/runtime-common/matrix-constants'; +import { createSendEvent } from './handlers/send-event'; +import { createServeFromRealm } from './handlers/serve-from-realm'; +import { createServeIndex } from './handlers/serve-index'; +import { findOrMountRealm } from './lib/realm-routing'; import type { Prerenderer } from '@cardstack/runtime-common'; -import { retrieveScopedCSS } from './lib/retrieve-scoped-css'; -import { insertSourceRealmInRegistry } from './lib/realm-registry-writes'; import type { RealmRegistryReconciler } from './lib/realm-registry-reconciler'; -import { - indexURLCandidates, - indexCandidateExpressions, -} from './lib/index-url-utils'; -import { - retrieveHeadHTML, - retrieveIsolatedHTML, - injectHeadHTML, - injectIsolatedHTML, - ensureSingleTitle, -} from './lib/index-html-injection'; -import { sanitizeHeadHTMLToString } from '@cardstack/runtime-common'; -import { JSDOM } from 'jsdom'; export class RealmServer { private log = logger('realm-server'); - private headLog = logger('realm-server:head'); - private isolatedLog = logger('realm-server:isolated'); - private scopedCSSLog = logger('realm-server:scoped-css'); private realms: Realm[]; private virtualNetwork: VirtualNetwork; private matrixClient: MatrixClient; @@ -85,8 +48,6 @@ export class RealmServer { private getIndexHTML: () => Promise; private serverURL: URL; private matrixRegistrationSecret: string | undefined; - private promiseForIndexHTML: Promise | undefined; - private indexHTMLHash: string | undefined; private getRegistrationSecret: | (() => Promise) | undefined; @@ -185,6 +146,28 @@ export class RealmServer { @Memoize() get app() { + let { serveIndex, serveHostApp } = createServeIndex({ + serverURL: this.serverURL, + assetsURL: this.assetsURL, + realms: this.realms, + reconciler: this.reconciler, + dbAdapter: this.dbAdapter, + matrixClient: this.matrixClient, + getIndexHTML: this.getIndexHTML, + cardSizeLimitBytes: this.cardSizeLimitBytes, + fileSizeLimitBytes: this.fileSizeLimitBytes, + }); + let serveFromRealm = createServeFromRealm({ + realms: this.realms, + reconciler: this.reconciler, + dbAdapter: this.dbAdapter, + virtualNetwork: this.virtualNetwork, + }); + let sendEvent = createSendEvent({ + matrixClient: this.matrixClient, + dbAdapter: this.dbAdapter, + }); + let app = new Koa() .use(httpLogging) .use(ecsMetadata) @@ -235,11 +218,10 @@ export class RealmServer { realmSecretSeed: this.realmSecretSeed, grafanaSecret: this.grafanaSecret, virtualNetwork: this.virtualNetwork, - createRealm: this.createRealm, - serveHostApp: this.serveHostApp, - serveIndex: this.serveIndex, - serveFromRealm: this.serveFromRealm, - sendEvent: this.sendEvent, + serveHostApp, + serveIndex, + serveFromRealm, + sendEvent, queue: this.queue, realms: this.realms, assetsURL: this.assetsURL, @@ -257,8 +239,8 @@ export class RealmServer { }, }), ) - .use(this.serveIndex) - .use(this.serveFromRealm); + .use(serveIndex) + .use(serveFromRealm); app.on('error', (err, ctx) => { console.error(`Unhandled server error`, err); @@ -322,7 +304,11 @@ export class RealmServer { // lazy-mount integration tests can drive findOrMountRealm directly // without spinning up an HTTP listener + mocked Koa context. testingOnlyFindOrMountRealm(requestURL: URL): Promise { - return this.findOrMountRealm(requestURL); + return findOrMountRealm(requestURL, { + realms: this.realms, + reconciler: this.reconciler, + dbAdapter: this.dbAdapter, + }); } // Test-only synchronous reconcile pass. The production reconciler @@ -332,844 +318,6 @@ export class RealmServer { return this.reconciler.reconcile(); } - private serveIndex = async (ctxt: Koa.Context, next: Koa.Next) => { - let acceptHeader = ctxt.header.accept ?? ''; - let lowerAcceptHeader = acceptHeader.toLowerCase(); - let includesVndMimeType = lowerAcceptHeader.includes('application/vnd.'); - let includesHtmlMimeType = lowerAcceptHeader.includes('text/html'); - - let requestURL = new URL( - `${ctxt.protocol}://${ctxt.host}${ctxt.originalUrl}`, - ); - - // Track published realm info from routing checks to avoid redundant - // DB queries in the ETag logic below. - let publishedRealmInfo: { lastPublishedAt: string | null } | null = null; - let publishedRealmInfoFetched = false; - - if (includesHtmlMimeType) { - if (includesVndMimeType) { - publishedRealmInfo = await this.getPublishedRealmInfo(requestURL); - publishedRealmInfoFetched = true; - - if (publishedRealmInfo) { - return next(); - } - } - } else { - if (includesVndMimeType) { - return next(); - } - - if (hasExtension(requestURL.pathname)) { - return next(); - } - - publishedRealmInfo = await this.getPublishedRealmInfo(requestURL); - publishedRealmInfoFetched = true; - - if (!publishedRealmInfo) { - return next(); - } - - // For published realms with generic Accept headers (like */*), we need to - // distinguish card URLs from module URLs. Module imports (e.g., "./person") - // resolve to URLs without extensions and would incorrectly get HTML served. - // Only serve HTML if: - // 1. This is a directory index request (path ends with /), OR - // 2. The URL corresponds to an indexed card instance - let isIndexRequest = requestURL.pathname.endsWith('/'); - if (!isIndexRequest) { - let cardURL = requestURL; - let isCardInstance = await this.isIndexedCardInstance(cardURL); - if (!isCardInstance) { - return next(); - } - } - } - - // If this is a /connect iframe request, is the origin a valid published realm? - - let connectMatch = ctxt.request.path.match(/\/connect\/(.+)$/); - - if (connectMatch) { - try { - let originParameter = new URL(decodeURIComponent(connectMatch[1])).href; - - let publishedRealms = await query(this.dbAdapter, [ - `SELECT url FROM realm_registry WHERE kind = 'published' AND url LIKE `, - param(`${originParameter}%`), - ]); - - if (publishedRealms.length === 0) { - ctxt.status = 404; - ctxt.body = `Not Found: No published realm found for origin ${originParameter}`; - - this.log.debug( - `Ignoring /connect request for origin ${originParameter}: no matching published realm`, - ); - - return; - } - - ctxt.set( - 'Content-Security-Policy', - `frame-ancestors ${originParameter}`, - ); - } catch (error) { - ctxt.status = 400; - ctxt.body = 'Bad Request'; - - this.log.info(`Error processing /connect request: ${error}`); - - return; - } - } - - ctxt.type = 'html'; - - let cardURL = requestURL; - let isIndexRequest = requestURL.pathname.endsWith('/'); - if (isIndexRequest) { - cardURL = new URL('index', requestURL); - } - - // Retrieve index HTML early so the shell hash is available for ETag. - // This is memoized in production, so it's cheap after the first call. - let indexHTML = await this.retrieveIndexHTML(); - - // For published realms, support HTTP caching via ETag. - // The ETag includes both last_published_at and a hash of the host app - // shell, so a deploy that changes index.html invalidates cached responses. - if (!publishedRealmInfoFetched) { - publishedRealmInfo = await this.getPublishedRealmInfo(requestURL); - } - let lastPublishedAt = publishedRealmInfo?.lastPublishedAt; - let etag = - lastPublishedAt && this.indexHTMLHash - ? `"${lastPublishedAt}-${this.indexHTMLHash}"` - : null; - - if (etag) { - let ifNoneMatch = ctxt.get('If-None-Match'); - if ( - ifNoneMatch === '*' || - ifNoneMatch - .split(',') - .some((t) => t.trim().replace(/^W\//, '') === etag) - ) { - ctxt.status = 304; - ctxt.set('ETag', etag); - ctxt.set('Cache-Control', 'public, max-age=0, must-revalidate'); - ctxt.vary('Accept'); - return; - } - } - let hasPublicPermissions = await this.hasPublicPermissions(cardURL); - - if (!hasPublicPermissions) { - ctxt.body = injectHeadHTML( - indexHTML, - `Boxel\n${this.defaultIconLinks().join('\n')}`, - ); - return; - } - - this.headLog.debug(`Fetching head HTML for ${cardURL.href}`); - this.isolatedLog.debug(`Fetching isolated HTML for ${cardURL.href}`); - this.scopedCSSLog.debug(`Fetching scoped CSS for ${cardURL.href}`); - - let [headHTML, isolatedHTML, scopedCSS] = await Promise.all([ - retrieveHeadHTML({ - cardURL, - dbAdapter: this.dbAdapter, - log: this.headLog, - }), - retrieveIsolatedHTML({ - cardURL, - dbAdapter: this.dbAdapter, - log: this.isolatedLog, - }), - retrieveScopedCSS({ - cardURL, - dbAdapter: this.dbAdapter, - log: this.scopedCSSLog, - }), - ]); - - let doc = new JSDOM().window.document; - if (headHTML != null) { - let sanitized = sanitizeHeadHTMLToString(headHTML, doc); - if (sanitized !== null) { - headHTML = sanitized; - } else { - headHTML = null; - } - } - - if (headHTML != null) { - this.headLog.debug( - `Injecting head HTML for ${cardURL.href} (length ${headHTML.length})\n${this.truncateLogLines( - headHTML, - )}`, - ); - } else { - this.headLog.debug( - `No head HTML found for ${cardURL.href}, serving base index.html`, - ); - } - - if (scopedCSS != null) { - this.scopedCSSLog.debug( - `Using scoped CSS for ${cardURL.href} (length ${scopedCSS.length})`, - ); - } else { - this.scopedCSSLog.debug( - `No scoped CSS returned from database for ${cardURL.href}`, - ); - } - - let responseHTML = indexHTML; - let headFragments: string[] = []; - - if (headHTML != null) { - headFragments.push(ensureSingleTitle(headHTML)); - } else { - headFragments.push('Boxel'); - } - - if (scopedCSS != null) { - this.scopedCSSLog.debug(`Injecting scoped CSS for ${cardURL.href}`); - headFragments.push( - ``, - ); - } - - let hasFavicon = false; - let hasAppleTouchIcon = false; - if (headHTML != null) { - let fragment = doc.createRange().createContextualFragment(headHTML); - hasFavicon = fragment.querySelector('link[rel~="icon"]') != null; - hasAppleTouchIcon = - fragment.querySelector('link[rel~="apple-touch-icon"]') != null; - } - let faviconURL = new URL('boxel-favicon.png', this.assetsURL).href; - let webclipURL = new URL('boxel-webclip.png', this.assetsURL).href; - if (!hasFavicon) { - headFragments.push(``); - } - if (!hasAppleTouchIcon) { - headFragments.push( - ``, - ); - } - - if (headFragments.length > 0) { - responseHTML = injectHeadHTML(responseHTML, headFragments.join('\n')); - } - - if (isolatedHTML != null) { - this.isolatedLog.debug( - `Injecting isolated HTML for ${cardURL.href} (length ${isolatedHTML.length})\n${this.truncateLogLines( - isolatedHTML, - )}`, - ); - responseHTML = injectIsolatedHTML(responseHTML, isolatedHTML); - } - - if (etag) { - ctxt.set('ETag', etag); - ctxt.set('Cache-Control', 'public, max-age=0, must-revalidate'); - ctxt.vary('Accept'); - } - - ctxt.body = responseHTML; - return; - }; - - private serveHostApp = async (ctxt: Koa.Context, next: Koa.Next) => { - let acceptHeader = (ctxt.header.accept ?? '').toLowerCase(); - let isHead = ctxt.method === 'HEAD'; - if (!isHead && !acceptHeader.includes('text/html')) { - return next(); - } - - ctxt.type = 'html'; - ctxt.body = injectHeadHTML( - await this.retrieveIndexHTML(), - `Boxel\n${this.defaultIconLinks().join('\n')}`, - ); - }; - - // Resolves a request URL to a mounted Realm, lazy-mounting via the - // reconciler if the request is the first hit on a non-pinned realm - // (Phase 3 lazy-mount semantics). Returns undefined when no realm in the - // registry matches the request — caller should respond 404. - // - // Lookup order: - // 1. this.realms — covers (a) realms whose mountFromRow has already - // published them to this array but whose start() is still awaiting - // fullIndex; the worker processing that fullIndex re-enters this - // resolver to fetch /_mtimes and must hit the published - // realm rather than reconciler.ensureMounted(), which would - // return the same in-flight promise and deadlock the boot path; - // and (b) handler-created realms in Phase 3 PR 1 (publish/copy - // push directly to this.realms; the reconciler may not have - // observed them via NOTIFY/reconcile yet). Phase 3 PR 2 collapses - // (b) onto the reconciler. - // 2. reconciler.knownByUrl — the Phase 3 source of truth for never- - // mounted realms. Iterates registry rows, finds the one whose URL - // prefix contains the request, delegates to lookupOrMount() which - // constructs+mounts via mountFromRow on the cold first request. - private async findOrMountRealm(requestURL: URL): Promise { - let legacy = this.realms.find((candidate) => { - let realmURL = new URL(candidate.url); - realmURL.protocol = requestURL.protocol; - return new RealmPaths(realmURL).inRealm(requestURL); - }); - if (legacy) { - return legacy; - } - for (const url of this.reconciler.knownByUrl.keys()) { - let realmURL = new URL(url); - realmURL.protocol = requestURL.protocol; - if (new RealmPaths(realmURL).inRealm(requestURL)) { - return await this.reconciler.lookupOrMount(url); - } - } - // Phase 3: knownByUrl is populated by reconciler.reconcile() on - // boot + LISTEN/poll. A request that arrives between a sibling - // instance's POST /_create-realm (or /_publish-realm) and this - // instance's reconciler picking up NOTIFY would otherwise 404. - // Fall through to a direct registry probe — match on every path - // prefix and let Postgres pick the longest URL so a request to - // `/foo/bar/baz/file.json` resolves to `/foo/bar/baz/` if that's - // registered, not `/foo/` (both prefixes are valid candidates). - let candidatePaths = candidateRealmURLs(requestURL); - if (candidatePaths.length === 0) { - return undefined; - } - let inClause: (string | ReturnType)[] = ['(']; - candidatePaths.forEach((u, idx) => { - if (idx > 0) inClause.push(','); - inClause.push(param(u)); - }); - inClause.push(')'); - let rows = (await query(this.dbAdapter, [ - `SELECT url FROM realm_registry WHERE url IN`, - ...inClause, - `ORDER BY LENGTH(url) DESC LIMIT 1`, - ])) as { url: string }[]; - if (rows.length === 0) { - return undefined; - } - return await this.reconciler.lookupOrMount(rows[0].url); - } - - private async getPublishedRealmInfo( - requestURL: URL, - ): Promise<{ lastPublishedAt: string | null } | null> { - let realm = await this.findOrMountRealm(requestURL); - if (!realm) { - return null; - } - - let rows = await query(this.dbAdapter, [ - `SELECT last_published_at FROM realm_registry WHERE kind = 'published' AND url =`, - param(realm.url), - ]); - - if (rows.length === 0) { - return null; - } - - return { - lastPublishedAt: (rows[0].last_published_at as string) ?? null, - }; - } - - // Check if the URL corresponds to an indexed card instance. - // This is used to distinguish card URLs from module URLs when deciding - // whether to serve HTML for published realms. - // - // IMPORTANT: Card instances have their file_alias set to the URL without - // the .json extension. This means an instance at /foo/bar.json has - // file_alias /foo/bar. When a module request comes in for /foo/bar (no - // extension), we must check if it's actually a module before assuming it's - // an instance. Modules take precedence over instance aliases. - private async isIndexedCardInstance(cardURL: URL): Promise { - let candidates = indexURLCandidates(cardURL); - if (candidates.length === 0) { - return false; - } - - // First check if there's a module at this URL - modules take precedence - // over instance aliases. This handles the case where: - // - Module: /foo/bar.gts (file_alias: /foo/bar) - // - Instance: /foo/bar.json (file_alias: /foo/bar) - // A request for /foo/bar should serve the module, not HTML for the instance. - // Prefer the modules table here because copied/published realms do not - // carry module rows in boxel_index. - let moduleRows = await query(this.dbAdapter, [ - ` - SELECT 1 - FROM modules - WHERE - `, - ...indexCandidateExpressions(candidates), - ` - LIMIT 1 - `, - ]); - - if (moduleRows.length > 0) { - return false; - } - - let rows = await query(this.dbAdapter, [ - ` - SELECT 1 - FROM boxel_index - WHERE type = 'instance' - AND is_deleted IS NOT TRUE - AND - `, - ...indexCandidateExpressions(candidates), - ` - LIMIT 1 - `, - ]); - - if (rows.length === 0) { - return false; - } - - // During publish/copy index races, module rows can lag behind source files. - // Only do filesystem probing after we've identified an instance candidate - // to avoid extra IO on the hot request path. - if (await this.hasExtensionlessSourceModule(cardURL)) { - return false; - } - - return true; - } - - private async hasExtensionlessSourceModule(cardURL: URL): Promise { - let realm = await this.findOrMountRealm(cardURL); - if (!realm?.dir) { - return false; - } - - let localPath: string; - try { - localPath = realm.paths.local(cardURL); - } catch { - return false; - } - - if (!localPath || hasExtension(localPath)) { - return false; - } - - for (let extension of executableExtensions) { - if (existsSync(join(realm.dir, `${localPath}${extension}`))) { - return true; - } - if (existsSync(join(realm.dir, localPath, `index${extension}`))) { - return true; - } - } - - return false; - } - - private async hasPublicPermissions(cardURL: URL): Promise { - let realm = await this.findOrMountRealm(cardURL); - - if (!realm) { - return false; - } - - let permissions = await fetchRealmPermissions( - this.dbAdapter, - new URL(realm.url), - ); - - return permissions['*']?.includes('read') ?? false; - } - - private async retrieveIndexHTML(): Promise { - // Cache index.html in production only - let isDev = this.assetsURL.hostname === 'localhost'; - - if (!isDev && this.promiseForIndexHTML) { - return this.promiseForIndexHTML; - } - - let deferred = new Deferred(); - - if (!isDev) { - this.promiseForIndexHTML = deferred.promise; - } - - let rewriteRealmURL = (url?: string) => { - if (!url) { - return url; - } - - let parsed = new URL(url); - return new URL( - `${parsed.pathname}${parsed.search}${parsed.hash}`, - this.serverURL, - ).href; - }; - - let indexHTML = (await this.getIndexHTML()).replace( - /()/, - (_match, g1, g2, g3) => { - let config = JSON.parse(decodeURIComponent(g2)); - - if (config.publishedRealmBoxelSpaceDomain === 'localhost:4201') { - // if this is the default, this needs to be the realm server’s host - // to work in Matrix tests, since publishedRealmBoxelSpaceDomain is currently - // the default domain for publishing a realm - config.publishedRealmBoxelSpaceDomain = this.serverURL.host; - } - - if (config.publishedRealmBoxelSiteDomain === 'localhost:4201') { - // if this is the default, this needs to be the realm server’s host - // to work in Matrix tests, since publishedRealmBoxelSiteDomain is currently - // the default domain for publishing a realm - config.publishedRealmBoxelSiteDomain = this.serverURL.host; - } - - config = merge({}, config, { - hostsOwnAssets: false, - assetsURL: this.assetsURL.href, - matrixURL: this.matrixClient.matrixURL.href.replace(/\/$/, ''), - matrixServerName: - process.env.MATRIX_SERVER_NAME || - this.matrixClient.matrixURL.hostname, - realmServerURL: this.serverURL.href, - resolvedBaseRealmURL: rewriteRealmURL(config.resolvedBaseRealmURL), - resolvedCatalogRealmURL: rewriteRealmURL( - config.resolvedCatalogRealmURL, - ), - resolvedLegacyCatalogRealmURL: rewriteRealmURL( - config.resolvedLegacyCatalogRealmURL, - ), - resolvedSkillsRealmURL: rewriteRealmURL( - config.resolvedSkillsRealmURL, - ), - resolvedOpenRouterRealmURL: rewriteRealmURL( - config.resolvedOpenRouterRealmURL, - ), - defaultSystemCardId: rewriteRealmURL(config.defaultSystemCardId), - defaultFieldSpecId: rewriteRealmURL(config.defaultFieldSpecId), - cardSizeLimitBytes: this.cardSizeLimitBytes, - fileSizeLimitBytes: this.fileSizeLimitBytes, - publishedRealmDomainOverrides: - process.env.PUBLISHED_REALM_DOMAIN_OVERRIDES ?? - config.publishedRealmDomainOverrides, - }); - return `${g1}${encodeURIComponent(JSON.stringify(config))}${g3}`; - }, - ); - - indexHTML = indexHTML.replace( - /(src|href)="\//g, - `$1="${this.assetsURL.href}`, - ); - - // Strip any static favicon/apple-touch-icon links from the base HTML - // since these are now dynamically injected between the head markers - indexHTML = indexHTML - .replace(/]*\brel="icon"[^>]*\/?>/gi, '') - .replace(/]*\brel="apple-touch-icon"[^>]*\/?>/gi, ''); - - // Recompute the hash in dev mode (where index.html is not cached) so - // that changes to the shell are reflected in the ETag. - if (!this.indexHTMLHash || isDev) { - let { createHash } = await import('crypto'); - this.indexHTMLHash = createHash('md5') - .update(indexHTML) - .digest('hex') - .slice(0, 8); - } - - deferred.fulfill(indexHTML); - return indexHTML; - } - - private defaultIconLinks(): string[] { - let faviconURL = new URL('boxel-favicon.png', this.assetsURL).href; - let webclipURL = new URL('boxel-webclip.png', this.assetsURL).href; - return [ - ``, - ``, - ]; - } - - private truncateLogLines(value: string, maxLines = 3): string { - let lines = value.split(/\r?\n/); - if (lines.length <= maxLines) { - return value; - } - let truncated = lines.slice(0, maxLines); - truncated[maxLines - 1] = `${truncated[maxLines - 1]} ...`; - return truncated.join('\n'); - } - - private serveFromRealm = async (ctxt: Koa.Context, _next: Koa.Next) => { - if (ctxt.request.path === '/_boom') { - throw new Error('boom'); - } - let request = await fetchRequestFromContext(ctxt); - // Phase 3 lazy mount: trigger findOrMountRealm before dispatching to - // virtualNetwork.handle so non-pinned realms (source/published) mount - // on first request. virtualNetwork.handle returns 404 for any URL - // whose handle isn't registered, which is exactly what happens for - // a realm that the reconciler knows about (knownByUrl) but hasn't - // mounted yet. findOrMountRealm walks knownByUrl, calls - // reconciler.lookupOrMount() on a prefix match, and that - // synchronously publishes the realm into virtualNetwork before the - // dispatch below. Mount failures throw — the catch turns them into - // 503 so the next request retries from scratch (ensureMounted's - // failure path clears mounted/pendingMounts). - let requestURL = new URL( - `${ctxt.protocol}://${ctxt.host}${ctxt.originalUrl}`, - ); - try { - await this.findOrMountRealm(requestURL); - } catch (err: any) { - this.log.warn( - `failed to mount realm for request ${requestURL.href}: ${err?.message ?? err}`, - ); - ctxt.status = 503; - ctxt.body = `Realm mount failed: ${err?.message ?? err}`; - return; - } - let realmResponse = await this.virtualNetwork.handle( - request, - (mappedRequest) => { - // Setup this handler only after the request has been mapped because - // the *mapped request* is the one that gets closed, not the original one - setupCloseHandler(ctxt.res, mappedRequest); - }, - ); - - await setContextResponse(ctxt, realmResponse); - }; - - private createRealm = async ({ - ownerUserId, - endpoint, - name, - backgroundURL, - iconURL, - }: { - ownerUserId: string; // note matrix userIDs look like "@mango:boxel.ai" - endpoint: string; - name: string; - backgroundURL?: string; - iconURL?: string; - }): Promise<{ url: string; realm: Realm; info: Partial }> => { - // Server-root collision check. Read realms[] AND realm_registry — - // every production realm has a registry row, but test fixtures - // construct CLI-style realms via runTestRealmServer that don't - // mirror to the registry. Either source matching the origin is a - // collision. (Phase 3 PR 2: handlers don't *mutate* realms[]; read - // is fine.) - let serverRootUrl = this.serverURL.origin + '/'; - let realmAtServerRoot = this.realms.find((r) => { - let realmUrl = new URL(r.url); - return ( - realmUrl.href.replace(/\/$/, '') === realmUrl.origin && - realmUrl.hostname === this.serverURL.hostname - ); - }); - if (realmAtServerRoot) { - throw errorWithStatus( - 400, - `Cannot create a realm: a realm is already mounted at the origin of this server: ${realmAtServerRoot.url}`, - ); - } - let serverRootRows = (await query(this.dbAdapter, [ - `SELECT url FROM realm_registry WHERE url =`, - param(serverRootUrl), - ])) as { url: string }[]; - if (serverRootRows.length > 0) { - throw errorWithStatus( - 400, - `Cannot create a realm: a realm is already mounted at the origin of this server: ${serverRootRows[0].url}`, - ); - } - if (!endpoint.match(/^[a-z0-9-]+$/)) { - throw errorWithStatus( - 400, - `realm endpoint '${endpoint}' contains invalid characters`, - ); - } - - let ownerUsername = getMatrixUsername(ownerUserId); - let url = new URL( - `${this.serverURL.pathname.replace( - /\/$/, - '', - )}/${ownerUsername}/${endpoint}/`, - this.serverURL, - ).href; - - let existingRows = (await query(this.dbAdapter, [ - `SELECT url FROM realm_registry WHERE url =`, - param(url), - ])) as { url: string }[]; - if (existingRows.length > 0) { - throw errorWithStatus( - 400, - `realm '${url}' already exists on this server`, - ); - } - - let realmPath = resolve(join(this.realmsRootPath, ownerUsername, endpoint)); - ensureDirSync(realmPath); - - let info = { - name, - ...(iconURL ? { iconURL } : {}), - ...(backgroundURL ? { backgroundURL } : {}), - publishable: true, - }; - - // Serialize against any other caller of withWriteLock for this - // same URL (concurrent createRealm for the same endpoint, or a - // concurrent publish/unpublish/delete). This is almost never a real - // concurrency concern — the endpoint was already checked above for - // collision. - await this.dbAdapter.withWriteLock(url, async () => { - await insertPermissions(this.dbAdapter, new URL(url), { - [ownerUserId]: DEFAULT_PERMISSIONS, - }); - - // CS-10053: publishable lives in realm_metadata now, not the - // sidecar. The legacy .realm.json is no longer written here; - // hostHome/interactHome (still sidecar-owned until CS-10055) - // are absent on a fresh realm and don't need a placeholder file. - // Reset all mutable metadata columns on conflict so a stale row - // (e.g. left over from a previous realm at the same URL whose - // delete didn't clean up) doesn't bleed into the new realm. - await query(this.dbAdapter, [ - `INSERT INTO realm_metadata (url, publishable, show_as_catalog) VALUES (`, - param(url), - `,`, - param(true), - `,`, - param(null), - `) ON CONFLICT (url) DO UPDATE SET publishable = true, show_as_catalog = NULL, updated_at = now()`, - ]); - writeJSONSync(join(realmPath, 'realm.json'), { - data: { - type: 'card', - attributes: { - cardInfo: { name }, - ...(iconURL ? { iconURL } : {}), - ...(backgroundURL ? { backgroundURL } : {}), - }, - meta: { - adoptsFrom: { - module: 'https://cardstack.com/base/realm-config', - name: 'RealmConfig', - }, - }, - }, - }); - writeJSONSync(join(realmPath, 'index.json'), { - data: { - type: 'card', - meta: { - adoptsFrom: { - module: 'https://cardstack.com/base/cards-grid', - name: 'CardsGrid', - }, - }, - }, - }); - - // Register the source realm in realm_registry. The INSERT emits - // NOTIFY realm_registry; the reconciler on every instance picks - // up the row, and the realm is lazy-mounted on first request. - await insertSourceRealmInRegistry(this.dbAdapter, { - url, - diskId: `${ownerUsername}/${endpoint}`, - ownerUsername, - }); - }); - - // virtualNetwork URL mapping was historically bridged here so a - // virtual realm URL (e.g. cardstack.com/base/) routed to the - // physical localhost URL. For dynamically-created realms via - // /_create-realm, the URL is already a physical - // serverURL-rooted URL (no remap needed), but preserve the - // detection-and-add for any environment that maps it. - let actualRealmURL = this.virtualNetwork.mapURL(url, 'virtual-to-real'); - if (actualRealmURL && actualRealmURL.href !== url) { - this.virtualNetwork.addURLMapping(new URL(url), actualRealmURL); - } - - // Phase 3: enqueue the from-scratch-index job at userInitiatedPriority - // so the canonical (post-coalesce) job carries that priority — even - // if reconciler.lookupOrMount below also enqueues one at the default - // systemInitiatedPriority via realm.start(). The chooseFromScratch - // coalesce JOINs same-realm jobs and keeps maxPriority. - await enqueueReindexRealmJob( - url, - ownerUsername, - this.queue, - this.dbAdapter, - userInitiatedPriority, - ); - - // Synchronously mount + start the realm on the *handling* instance. - // The 202 response with status:'pending' is for sibling instances — - // they pick up the realm via NOTIFY realm_registry and lazy-mount - // on first request. On this instance the realm is fully ready by - // the time we return: ensureMounted publishes into realms[] / - // virtualNetwork via prepareRealmFromRow and awaits realm.start(), - // which awaits the from-scratch-index job. Mounting eagerly here - // also drains the queue locally so the test framework's teardown - // (close server → drain runner → close DB) doesn't race a worker - // mid-fetch on the now-closed HTTP listener. - let realm = await this.reconciler.lookupOrMount(url); - if (!realm) { - throw new Error( - `expected realm ${url} to be mounted after createRealm — registry row missing or mount failed`, - ); - } - - return { url, realm, info }; - }; - - private sendEvent = async ( - user: string, - eventType: string, - data?: Record, - ) => { - if (!this.matrixClient.isLoggedIn()) { - await this.matrixClient.login(); - } - let roomId = await fetchSessionRoom(this.dbAdapter, user); - if (!roomId) { - console.error( - `Failed to send event: ${eventType}, cannot find session room for user: ${user}`, - ); - } - - await this.matrixClient.sendEvent(roomId!, 'm.room.message', { - body: JSON.stringify({ eventType, data }), - msgtype: APP_BOXEL_REALM_SERVER_EVENT_MSGTYPE, - }); - }; - // we use a function to get the matrix registration secret because matrix // client tests leverage a synapse instance that changes multiple times per // realm lifespan, and each new synapse instance has a unique registration @@ -1216,27 +364,3 @@ function detectRealmCollision(realms: Realm[]): void { ); } } - -function errorWithStatus( - status: number, - message: string, -): Error & { status: number } { - let error = new Error(message); - (error as Error & { status: number }).status = status; - return error as Error & { status: number }; -} - -// Build candidate realm URLs from a request URL by trimming the -// pathname segment-by-segment. Used by findOrMountRealm's registry -// fallback when knownByUrl is stale. Includes the origin-only form -// (root realm) and every prefix that ends with a slash. -function candidateRealmURLs(requestURL: URL): string[] { - let segments = requestURL.pathname.split('/').filter(Boolean); - let candidates: string[] = []; - // Try longest-prefix first. - for (let i = segments.length; i >= 0; i--) { - let path = i === 0 ? '/' : '/' + segments.slice(0, i).join('/') + '/'; - candidates.push(`${requestURL.origin}${path}`); - } - return [...new Set(candidates)]; -} diff --git a/packages/realm-server/tests/server-config-test.ts b/packages/realm-server/tests/server-config-test.ts index 0cc955733aa..f88e3283f9a 100644 --- a/packages/realm-server/tests/server-config-test.ts +++ b/packages/realm-server/tests/server-config-test.ts @@ -1,34 +1,24 @@ import { module, test } from 'qunit'; import { basename } from 'path'; -import { dirSync } from 'tmp'; -import { RealmServer } from '../server'; +import { createServeIndex } from '../handlers/serve-index'; module(basename(__filename), function () { test('prefers MATRIX_SERVER_NAME over matrix URL hostname in host config', async function (assert) { let originalMatrixServerName = process.env.MATRIX_SERVER_NAME; - let tempDir = dirSync({ unsafeCleanup: true }); process.env.MATRIX_SERVER_NAME = 'stack.cards'; try { - let server = new RealmServer({ + let { retrieveIndexHTML } = createServeIndex({ serverURL: new URL('http://127.0.0.1:4448'), + assetsURL: new URL('http://example.com/notional-assets-host/'), realms: [], reconciler: {} as any, - virtualNetwork: {} as any, + dbAdapter: {} as any, matrixClient: { matrixURL: new URL('http://localhost:8008/'), } as any, - realmServerSecretSeed: 'test-realm-server-secret', - realmSecretSeed: 'test-realm-secret', - grafanaSecret: 'test-grafana-secret', - realmsRootPath: tempDir.name, - dbAdapter: {} as any, - queue: {} as any, - definitionLookup: {} as any, - assetsURL: new URL('http://example.com/notional-assets-host/'), - matrixRegistrationSecret: 'test-matrix-registration-secret', getIndexHTML: async () => ``, + cardSizeLimitBytes: 0, + fileSizeLimitBytes: 0, }); - let html = await (server as any).retrieveIndexHTML(); + let html = await retrieveIndexHTML(); let match = html.match( //, ); @@ -60,7 +52,6 @@ module(basename(__filename), function () { } else { process.env.MATRIX_SERVER_NAME = originalMatrixServerName; } - tempDir.removeCallback(); } }); }); diff --git a/packages/realm-server/tests/server-endpoints/realm-lifecycle-test.ts b/packages/realm-server/tests/server-endpoints/realm-lifecycle-test.ts index 5acebfb65ed..cb23e4c6d73 100644 --- a/packages/realm-server/tests/server-endpoints/realm-lifecycle-test.ts +++ b/packages/realm-server/tests/server-endpoints/realm-lifecycle-test.ts @@ -133,13 +133,14 @@ module(`server-endpoints/${basename(__filename)}`, function () { let jobs = (await context.dbAdapter.execute( `SELECT priority FROM jobs WHERE job_type = 'from-scratch-index' AND args->>'realmURL' = '${json.data.id}'`, )) as { priority: number }[]; - assert.ok( - jobs.length > 0, - 'found from-scratch index job for created realm', - ); - assert.ok( - jobs.some((j) => j.priority === userInitiatedPriority), - 'user initiated realm indexing uses high priority queue', + // Contract: realm creation enqueues exactly one + // from-scratch-index job, at userInitiatedPriority. A second + // job at default priority would block creation behind any + // backlog of lower-priority indexing work. + assert.deepEqual( + jobs.map((j) => j.priority), + [userInitiatedPriority], + 'realm creation enqueues exactly one from-scratch index job at userInitiatedPriority', ); let permissions = await fetchRealmPermissions( diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index d04568faaf8..e6e024128d3 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -1296,8 +1296,14 @@ export class Realm { }); } - async start() { - this.#startedUp.fulfill((() => this.#startup())()); + // `fromScratchIndexPriority` overrides the realm's default priority + // for the from-scratch-index job that `#startup` enqueues when the + // realm has no prior index. Callers that mount-on-demand for a + // user-initiated flow (e.g. realm creation) pass + // `userInitiatedPriority` so the resulting job jumps ahead of any + // backlog of system-priority indexing work. + async start(opts?: { fromScratchIndexPriority?: number }) { + this.#startedUp.fulfill((() => this.#startup(opts))()); if (this.#adapter.fileWatcherEnabled) { await this.startFileWatcher(); @@ -2174,7 +2180,7 @@ export class Realm { await completed; } - async #startup() { + async #startup(opts?: { fromScratchIndexPriority?: number }) { await Promise.resolve(); let startTime = Date.now(); if (this.#copiedFromRealm) { @@ -2188,9 +2194,9 @@ export class Realm { } else { let isNewIndex = await this.#realmIndexUpdater.isNewIndex(); if (isNewIndex || this.#fullIndexOnStartup) { - let promise = this.#realmIndexUpdater.fullIndex( - this.#fromScratchIndexPriority, - ); + let priority = + opts?.fromScratchIndexPriority ?? this.#fromScratchIndexPriority; + let promise = this.#realmIndexUpdater.fullIndex(priority); if (isNewIndex) { // we only await the full indexing at boot if this is a brand new index await promise;