CS-11156: cross-replica clearLocalCaches broadcast via NOTIFY

packages/host/tests/unit/index-query-engine-test.ts

@@ -240,17 +240,17 @@ module('Unit | query', function (hooks) {
         // no-op for tests
         return [];
       },
-      async clearRealmCache(_realmURL: string): Promise<void> {
+      async clearRealmDefinitions(_realmURL: string): Promise<void> {
         // no-op for tests
       },
-      async getModuleCacheEntry(): Promise<undefined> {
+      async getCachedDefinitions(): Promise<undefined> {
         return undefined;
       },
-      async getModuleCacheEntries(): Promise<Record<string, never>> {
+      async getCachedDefinitionsBatch(): Promise<Record<string, never>> {
         return {};
       },
       registerRealm() {},
-      async clearAllModules(): Promise<void> {
+      async clearAllDefinitions(): Promise<void> {
         // no-op for tests
       },
       forRealm() {

packages/realm-server/handlers/handle-delete-realm.ts

@@ -5,6 +5,7 @@ import {
   ensureTrailingSlash,
   fetchRealmPermissions,
   getMatrixUsername,
+  notifyAllFileChanges,
   param,
   PUBLISHED_DIRECTORY_NAME,
   query,
@@ -239,6 +240,17 @@ export default function handleDeleteRealm({
         Sentry.captureException(error);
       }
 
+      // CS-11156. Broadcast a bulk cache-invalidation for the source realm
+      // and each removed published realm so any peer replicas that still
+      // have these realms mounted drop their #sourceCache / #transpiledModuleCache
+      // before the reconciler unmount lands via NOTIFY realm_registry.
+      // Best-effort, fire-and-forget; missed NOTIFY is a bounded
+      // staleness window resolved by the unmount itself.
+      for (let publishedRealm of publishedRealms) {
+        await notifyAllFileChanges(dbAdapter, publishedRealm.url);
+      }
+      await notifyAllFileChanges(dbAdapter, realmURL);
+
       await setContextResponse(
         ctxt,
         new Response(null, {

packages/realm-server/handlers/handle-full-reindex.ts

@@ -15,7 +15,7 @@ export default function handleFullReindex({
   return async function (ctxt: Koa.Context, _next: Koa.Next) {
     let realmUrls = await getFullReindexRealmUrls(dbAdapter);
 
-    await definitionLookup.clearAllModules();
+    await definitionLookup.clearAllDefinitions();
 
     await queue.publish<void>({
       jobType: `full-reindex`,

packages/realm-server/handlers/handle-post-deployment.ts

@@ -27,7 +27,7 @@ export default function handlePostDeployment({
       return;
     }
 
-    await definitionLookup.clearAllModules();
+    await definitionLookup.clearAllDefinitions();
 
     let boxelUiChangeCheckerResult =
       await compareCurrentBoxelUIChecksum(assetsURL);

packages/realm-server/handlers/handle-publish-realm.ts

@@ -223,6 +223,7 @@ function ensureRealmIndexBoilerplateOptIn(publishedRealmPath: string): void {
 
 export default function handlePublishRealm({
   dbAdapter,
+  definitionLookup,
   matrixClient,
   queue,
   realmSecretSeed,
@@ -526,12 +527,20 @@ export default function handlePublishRealm({
           // it up.
           ensureRealmIndexBoilerplateOptIn(publishedRealmPath);
 
-          // Clear stale modules cache for the published realm so that
-          // error entries from a previous publish don't persist
-          await query(dbAdapter, [
-            `DELETE FROM modules WHERE resolved_realm_url =`,
-            param(publishedRealmURL),
-          ]);
+          // Clear stale modules cache for the published realm (including
+          // error entries from a previous publish) before the reindex's
+          // prerender fan-out, so its HTTP module fetches don't hit
+          // cached pre-swap state on this replica or its peers.
+          // `clearRealmDefinitions` bundles the DB DELETE + in-flight prerender
+          // drop + per-realm generation bump + cross-instance NOTIFY on
+          // `module_cache_invalidated` — the modules-cache analog of
+          // `clearLocalSourceCachesAndBroadcast()` below. Without those extra
+          // steps (which a raw `DELETE FROM modules` would miss), an
+          // in-flight prerender that started before the DELETE could
+          // re-insert a stale row at persist time, and peer replicas
+          // would keep their cached rows + generation counters until
+          // their own next invalidation arrived.
+          await definitionLookup.clearRealmDefinitions(publishedRealmURL);
 
           let lastPublishedAt = Date.now().toString();
           try {
@@ -567,12 +576,16 @@ export default function handlePublishRealm({
           //
           // For a new publish, lookupOrMount mounts the realm fresh
           // (registry row was just upserted above); the cache is
-          // empty so clearLocalCaches is a no-op. Either way the
+          // empty so clearLocalSourceCaches is a no-op. Either way the
           // reindex below sees correct source.
           let mountedRealmForCacheClear =
             await reconciler.lookupOrMount(publishedRealmURL);
           if (mountedRealmForCacheClear) {
-            mountedRealmForCacheClear.clearLocalCaches();
+            // Sync local clear + cross-replica NOTIFY in one call. The
+            // local clear is what this replica's reindex fan-out needs;
+            // the broadcast (CS-11156) covers peers that still have the
+            // realm mounted with pre-swap bytes.
+            await mountedRealmForCacheClear.clearLocalSourceCachesAndBroadcast();
           }
 
           // Refresh the index. For a new publish this is redundant

packages/realm-server/handlers/handle-reindex.ts

@@ -80,7 +80,7 @@ export async function reindex({
   definitionLookup: DefinitionLookup;
   priority?: number;
 }) {
-  await definitionLookup.clearRealmCache(realm.url);
+  await definitionLookup.clearRealmDefinitions(realm.url);
   return await enqueueReindexRealmJob(
     realm.url,
     await realm.getRealmOwnerUsername(),

packages/realm-server/handlers/handle-unpublish-realm.ts

@@ -4,6 +4,7 @@ import {
   query,
   SupportedMimeType,
   logger,
+  notifyAllFileChanges,
   param,
   removeRealmPermissions,
   fetchRealmPermissions,
@@ -153,6 +154,15 @@ export default function handleUnpublishRealm({
       // mode where we delete files but the registry row sticks around.
       removeRealmFiles(publishedRealmPath);
 
+      // CS-11156. Broadcast a bulk cache-invalidation to peer replicas so
+      // any that still have this realm mounted drop their #sourceCache /
+      // #transpiledModuleCache before the reconciler unmount lands. The per-file
+      // deleteAll above already emitted per-path NOTIFYs covering bytes
+      // that existed on disk; this bulk emit closes the brief window
+      // between the registry-row delete commit and the peers' reaction.
+      // Best-effort, fire-and-forget.
+      await notifyAllFileChanges(dbAdapter, publishedRealmURL);
+
       // Removing this derivative just changed the source realm's
       // `RealmInfo.lastPublishedAt` map (rows where `source_url =
       // sourceRealmURL`). Without invalidating the source's cached

packages/realm-server/lib/module-cache-coordination.ts

@@ -46,7 +46,7 @@ export function hashCoalesceKeyForAdvisoryLock(key: string): string {
 //
 // The pinned connection ONLY holds the advisory lock and emits the
 // NOTIFY. The `fn` callback issues its DB work (readFromDatabaseCache,
-// persistModuleCacheEntry) through the shared dbAdapter — a separate
+// persistDefinitionCacheEntry) through the shared dbAdapter — a separate
 // pool connection autocommits each query as today. Pool pressure is
 // bounded by N processes (each pins one extra client per concurrent
 // coordinated load) rather than N × M concurrent callers, since the

packages/realm-server/lib/module-cache-invalidation-listener.ts

@@ -9,7 +9,7 @@ const log = logger('realm-server:module-cache-invalidation-listener');
 
 // Cross-instance module-cache invalidation broadcast (CS-10952). Peer
 // realm-server processes emit `NOTIFY module_cache_invalidated, '<payload>'`
-// from CachingDefinitionLookup.invalidate / clearRealmCache / clearAllModules
+// from CachingDefinitionLookup.invalidate / clearRealmDefinitions / clearAllDefinitions
 // after their DELETE commits; this listener parses the payload and replays
 // the appropriate generation bump on the locally-attached
 // CachingDefinitionLookup so its in-flight prerenders observe the

packages/realm-server/lib/realm-file-changes-listener.ts

@@ -1,5 +1,9 @@
 import type { Realm } from '@cardstack/runtime-common';
-import { logger, REALM_FILE_CHANGES_CHANNEL } from '@cardstack/runtime-common';
+import {
+  logger,
+  REALM_FILE_CHANGES_CHANNEL,
+  REALM_FILE_CHANGES_WILDCARD,
+} from '@cardstack/runtime-common';
 import type { PgAdapter, NotificationSubscription } from '@cardstack/postgres';
 
 const log = logger('realm-server:file-changes-listener');
@@ -9,9 +13,15 @@ const log = logger('realm-server:file-changes-listener');
 // runtime-common/realm.ts), every listener subscribed on this channel looks
 // up the URL in its lookup function. If the realm is mounted locally,
 // `realm.invalidateCache(path)` clears the matching #sourceCache /
-// #moduleCache entries. If it's not mounted, the notification is dropped —
+// #transpiledModuleCache entries. If it's not mounted, the notification is dropped —
 // this instance has no stale state to clear.
 //
+// Bulk variant: when the path is the wildcard sentinel `*` (CS-11156),
+// `realm.clearLocalSourceCaches()` drops every cached path for that realm. Emitted
+// by the publish-realm / unpublish-realm / delete-realm handlers after the
+// FS swap or removal so peers (whose file-watcher events do NOT cross
+// replicas) bypass their pre-swap cached bytes on the next read.
+//
 // The LISTEN is backed by `PgAdapter.subscribe` (shared multiplexed
 // notification client). There is no periodic work to run between
 // notifications — the whole dispatch is in the payload — so we don't keep a
@@ -87,7 +97,11 @@ export class RealmFileChangesListener {
       return;
     }
     try {
-      realm.invalidateCache(parsed.path);
+      if (parsed.path === REALM_FILE_CHANGES_WILDCARD) {
+        realm.clearLocalSourceCaches();
+      } else {
+        realm.invalidateCache(parsed.path);
+      }
     } catch (err: unknown) {
       log.warn(
         `invalidateCache failed for ${parsed.url} ${parsed.path}: ${String(err)}`,
@@ -96,11 +110,14 @@ export class RealmFileChangesListener {
   }
 }
 
-// Payload shape: `<realmURL>:<localPath>`. Realm URLs always carry a
-// trailing slash (enforced by `ensureTrailingSlash` throughout the code),
-// so the separator between URL and path is the first `:` that immediately
+// Payload shape: `<realmURL>:<localPath>` or `<realmURL>:*` (bulk
+// invalidation — CS-11156). Realm URLs always carry a trailing slash
+// (enforced by `ensureTrailingSlash` throughout the code), so the
+// separator between URL and path is the first `:` that immediately
 // follows a `/`. That avoids false matches on the scheme colon
-// (`http://...`) and any host:port colon (`localhost:4201`).
+// (`http://...`) and any host:port colon (`localhost:4201`). The same
+// regex handles both shapes; the wildcard payload parses as
+// `path = '*'`.
 const PAYLOAD_SEPARATOR = /\/:/;
 
 export function parsePayload(

packages/realm-server/main.ts

@@ -339,7 +339,7 @@ const getIndexHTML = async () => {
     log.info('Skipping modules cache clear on startup (opted out via env)');
   } else {
     log.info('Clearing modules cache...');
-    await definitionLookup.clearAllModules();
+    await definitionLookup.clearAllDefinitions();
   }
 
   // Backfill realm_registry from CLI args (bootstrap), on-disk source realms,

packages/realm-server/tests/card-source-endpoints-test.ts

@@ -234,14 +234,14 @@ module(basename(__filename), function () {
           );
         });
 
-        // CS-11043. clearLocalCaches() is the public surface the
+        // CS-11043. clearLocalSourceCaches() is the public surface the
         // publish-realm handler invokes after the FS swap so that the
-        // pre-swap bytes living in #sourceCache / #moduleCache don't get
+        // pre-swap bytes living in #sourceCache / #transpiledModuleCache don't get
         // served to the reindex job (which would then write stale
         // isolated_html into boxel_index). Functionally equivalent to
         // __testOnlyClearCaches minus the test-only transpile-counter
         // reset.
-        test('clearLocalCaches drops cached source bytes', async function (assert) {
+        test('clearLocalSourceCaches drops cached source bytes', async function (assert) {
           let cacheTestPath = 'clear-local-caches.gts';
           await testRealm.write(
             cacheTestPath,
@@ -260,15 +260,15 @@ module(basename(__filename), function () {
             'precondition: second fetch hits the source cache',
           );
 
-          testRealm.clearLocalCaches();
+          testRealm.clearLocalSourceCaches();
 
           let afterClear = await request
             .get(`/${cacheTestPath}`)
             .set('Accept', 'application/vnd.card+source');
           assert.strictEqual(
             afterClear.headers['x-boxel-cache'],
             'miss',
-            'fetch after clearLocalCaches is a miss — the #sourceCache entry was dropped',
+            'fetch after clearLocalSourceCaches is a miss — the #sourceCache entry was dropped',
           );
         });
 

packages/realm-server/tests/definition-lookup-test.ts

@@ -1706,7 +1706,7 @@ module(basename(__filename), function () {
       );
     });
 
-    test('in-flight prerender result is dropped when clearRealmCache runs concurrently', async function (assert) {
+    test('in-flight prerender result is dropped when clearRealmDefinitions runs concurrently', async function (assert) {
       await dbAdapter.execute('DELETE FROM modules');
 
       let moduleURL = `${realmURL}stale-persist-clear-realm.gts`;
@@ -1752,14 +1752,14 @@ module(basename(__filename), function () {
       });
       await new Promise((resolve) => setTimeout(resolve, 0));
 
-      await lookup.clearRealmCache(realmURL);
+      await lookup.clearRealmDefinitions(realmURL);
 
       releaseGate();
       let result = await Promise.allSettled([pA]);
       assert.strictEqual(
         result[0].status,
         'rejected',
-        'A rejects after clearRealmCache leaves an empty cache',
+        'A rejects after clearRealmDefinitions leaves an empty cache',
       );
       assert.strictEqual(calls, 1);
 
@@ -1770,14 +1770,14 @@ module(basename(__filename), function () {
       assert.strictEqual(
         rows.length,
         0,
-        'clearRealmCache is honored — A did not re-insert the row',
+        'clearRealmDefinitions is honored — A did not re-insert the row',
       );
     });
 
-    test('in-flight prerender result is dropped when clearAllModules runs concurrently', async function (assert) {
+    test('in-flight prerender result is dropped when clearAllDefinitions runs concurrently', async function (assert) {
       await dbAdapter.execute('DELETE FROM modules');
 
-      // clearAllModules drains state for every realm — including realms
+      // clearAllDefinitions drains state for every realm — including realms
       // that have never been individually invalidated. Use a fresh module
       // URL so the realm has no #generations entry going in; this guards
       // against the per-realm map missing the realm at clear time.
@@ -1824,14 +1824,14 @@ module(basename(__filename), function () {
       });
       await new Promise((resolve) => setTimeout(resolve, 0));
 
-      await lookup.clearAllModules();
+      await lookup.clearAllDefinitions();
 
       releaseGate();
       let result = await Promise.allSettled([pA]);
       assert.strictEqual(
         result[0].status,
         'rejected',
-        'A rejects after clearAllModules leaves an empty cache',
+        'A rejects after clearAllDefinitions leaves an empty cache',
       );
       assert.strictEqual(calls, 1);
 
@@ -1842,7 +1842,7 @@ module(basename(__filename), function () {
       assert.strictEqual(
         rows.length,
         0,
-        'clearAllModules is honored — A did not re-insert the row',
+        'clearAllDefinitions is honored — A did not re-insert the row',
       );
     });
 
@@ -1925,7 +1925,7 @@ module(basename(__filename), function () {
 
     test('a settled in-flight promise does not delete a newer in-flight under the same key', async function (assert) {
       // Identity-check regression guard. Without the identity check in
-      // loadModuleCacheEntry's .finally, A's settle would delete B's
+      // loadDefinitionCacheEntry's .finally, A's settle would delete B's
       // freshly-installed entry and cause D to race a third prerender.
       await dbAdapter.execute('DELETE FROM modules');