diff --git a/packages/boxel-cli/plugin/skills/realm-sync/SKILL.md b/packages/boxel-cli/plugin/skills/realm-sync/SKILL.md index 18de60b92ff..73230824666 100644 --- a/packages/boxel-cli/plugin/skills/realm-sync/SKILL.md +++ b/packages/boxel-cli/plugin/skills/realm-sync/SKILL.md @@ -9,7 +9,7 @@ Wraps the `boxel realm` subcommands that move data between a local directory and - **`push`** — local → remote. Deploy local edits to the realm. - **`pull`** — remote → local. Download a realm into a directory. - **`sync`** — bidirectional. Reconcile both sides; needs a `--prefer-*` flag when there are conflicts. -- **`watch`** — remote → local, continuous. Long-running poller; pulls remote changes into the local directory as they happen. +- **`watch`** — remote → local, continuous. Long-running poller; pulls remote changes into the local directory as they happen. Locally-edited files are *not* overwritten by default — the watcher skips downloads when the local copy diverges from the sync manifest, logs a warning, and keeps polling. Pass `--overwrite-local` to opt back into the unconditional mirror behavior. - **`create`** — provision a new realm on the realm server. - **`remove`** — delete a realm and unlink it from the active profile. - **`list`** — see realms the active profile can access. @@ -43,6 +43,10 @@ A profile must be active. If `boxel profile list` shows none, the user has to ru `push` and `pull` have their own `--delete` and `--dry-run` flags but no `--prefer-*` flags (they're one-directional). When in doubt, dry-run first. +`watch` protects local edits without a flag: by default any file whose local hash differs from the sync manifest is skipped (with a yellow `⚠ skipped …` line) instead of overwritten. The warning re-fires on every poll until the user reconciles via `boxel realm sync …` (e.g. `--prefer-newest`) or reruns watch with `--overwrite-local` to accept the remote. + +If `watch` is starting in a directory that already mirrors the realm but has no `.boxel-sync.json` (e.g. populated by hand, by `git clone`, or by a different tool), run `boxel realm pull` first. Without a manifest every existing file looks "diverged" and the first poll warns about each one until reconciled. + ## Commands @@ -81,6 +85,7 @@ Start watching a Boxel realm for server-side changes and pull them into a local - `-i, --interval ` — Polling interval in seconds - `-d, --debounce ` — Seconds to wait after a burst of changes before applying them - `--realm-secret-seed` — Administrative auth: prompt for a realm secret seed and mint a JWT locally instead of using a Matrix profile (env: BOXEL_REALM_SECRET_SEED) +- `--overwrite-local` — Overwrite local files when the remote changes. Default: skip + warn when the local copy diverges from the sync manifest. ### `boxel realm watch stop` diff --git a/packages/boxel-cli/src/commands/realm/watch/start.ts b/packages/boxel-cli/src/commands/realm/watch/start.ts index a4e802a49b8..c07c80b74df 100644 --- a/packages/boxel-cli/src/commands/realm/watch/start.ts +++ b/packages/boxel-cli/src/commands/realm/watch/start.ts @@ -48,6 +48,12 @@ interface PendingChange { export interface FlushResult { pulled: string[]; deleted: string[]; + /** + * Files whose remote-side change was detected but not applied because the + * local copy diverges from the sync manifest. Cleared by passing + * `overwriteLocal: true`, or by reconciling via `boxel realm sync`. + */ + skipped: string[]; checkpoint: Checkpoint | null; } @@ -59,6 +65,7 @@ export interface FlushResult { export class RealmWatcher extends RealmSyncBase { readonly name: string; private readonly debounceMs: number; + private readonly overwriteLocal: boolean; private readonly checkpointManager: CheckpointManager; private lastKnownMtimes = new Map(); private pendingChanges = new Map(); @@ -68,10 +75,11 @@ export class RealmWatcher extends RealmSyncBase { constructor( spec: WatchRealmSpec, authenticator: RealmAuthenticator, - options: { debounceMs: number }, + options: { debounceMs: number; overwriteLocal?: boolean }, ) { super({ realmUrl: spec.realmUrl, localDir: spec.localDir }, authenticator); this.debounceMs = options.debounceMs; + this.overwriteLocal = options.overwriteLocal ?? false; this.checkpointManager = new CheckpointManager(spec.localDir); this.name = deriveRealmName(this.normalizedRealmUrl); } @@ -186,7 +194,7 @@ export class RealmWatcher extends RealmSyncBase { } if (this.pendingChanges.size === 0) { - return { pulled: [], deleted: [], checkpoint: null }; + return { pulled: [], deleted: [], skipped: [], checkpoint: null }; } // Snapshot then clear before any await — anything an interleaved poll() @@ -197,11 +205,39 @@ export class RealmWatcher extends RealmSyncBase { const pulled: string[] = []; const deleted: string[] = []; + const skipped: string[] = []; const changes: CheckpointChange[] = []; + // Load the manifest once per flush so we hash-compare against a single + // baseline. Skipped when `overwriteLocal` is on — we never look. A + // manifest from a different realm is treated as "no manifest" (same + // policy as `initialize()` and `sync()`), so every local file looks + // unrecorded and is protected by the divergence gate. + let manifest: SyncManifest | null = null; + if (!this.overwriteLocal) { + const loaded = await loadManifest(this.options.localDir); + if (loaded && loaded.realmUrl === this.normalizedRealmUrl) { + manifest = loaded; + } + } + for (const [file, info] of drained) { + const localPath = path.join(this.options.localDir, file); + + if ( + !this.overwriteLocal && + (await this.localDivergesFromManifest( + localPath, + file, + manifest, + info.status, + )) + ) { + skipped.push(file); + continue; + } + if (info.status === 'deleted') { - const localPath = path.join(this.options.localDir, file); try { await fs.unlink(localPath); } catch (err: any) { @@ -210,14 +246,18 @@ export class RealmWatcher extends RealmSyncBase { deleted.push(file); changes.push({ file, status: 'deleted' }); } else { - const localPath = path.join(this.options.localDir, file); await this.downloadFile(file, localPath); pulled.push(file); changes.push({ file, status: info.status }); } } + // Only advance mtimes for files we actually applied. Skipped entries + // keep their old `lastKnownMtimes` value (or absence) so the next poll + // re-detects them — the warning persists until reconciled. + const skippedSet = new Set(skipped); for (const [file, info] of drained) { + if (skippedSet.has(file)) continue; if (info.status === 'deleted') { this.lastKnownMtimes.delete(file); } else { @@ -225,14 +265,45 @@ export class RealmWatcher extends RealmSyncBase { } } - await this.persistManifest(pulled, deleted); + let checkpoint: Checkpoint | null = null; + if (changes.length > 0) { + await this.persistManifest(pulled, deleted); + checkpoint = await this.checkpointManager.createCheckpoint( + 'remote', + changes, + ); + } - const checkpoint = await this.checkpointManager.createCheckpoint( - 'remote', - changes, - ); + return { pulled, deleted, skipped, checkpoint }; + } - return { pulled, deleted, checkpoint }; + /** + * True when the local copy of `relPath` no longer matches the sync + * manifest: hash mismatch, missing manifest record for a present file, + * or — for non-delete operations — the user deleted the file locally + * while the manifest still recorded it (the delete-vs-change conflict + * that `sync-logic.ts` classifies via `'deleted' + 'changed' = conflict`). + */ + private async localDivergesFromManifest( + localPath: string, + relPath: string, + manifest: SyncManifest | null, + operation: 'added' | 'modified' | 'deleted', + ): Promise { + let localHash: string; + try { + localHash = await computeFileHash(localPath); + } catch (err: any) { + if (err.code !== 'ENOENT') throw err; + // Remote also deleting → no local work to lose. Manifest had no + // record → first-time pull, nothing to protect. Manifest had a + // record and remote wants to write → that's the conflict. + if (operation === 'deleted') return false; + return manifest?.files[relPath] !== undefined; + } + const manifestHash = manifest?.files[relPath]; + if (manifestHash === undefined) return true; + return localHash !== manifestHash; } /** @@ -286,10 +357,13 @@ export class RealmWatcher extends RealmSyncBase { pulled: string[], deleted: string[], ): Promise { + // Drop file hashes from a manifest belonging to a different realm — + // otherwise we'd persist cross-realm entries under our `realmUrl`. + // Matches the policy used by `flushPending()` and `initialize()`. const prior = await loadManifest(this.options.localDir); - const files: Record = prior?.files - ? { ...prior.files } - : {}; + const priorFiles = + prior && prior.realmUrl === this.normalizedRealmUrl ? prior.files : null; + const files: Record = priorFiles ? { ...priorFiles } : {}; for (const file of deleted) { delete files[file]; @@ -332,6 +406,12 @@ export interface WatchRealmsOptions { authenticator?: RealmAuthenticator; /** Stops the watch loop when aborted. SIGINT/SIGTERM are wired up when omitted. */ signal?: AbortSignal; + /** + * When true, downloads always overwrite the local file. When false + * (default), files whose local copy diverges from the sync manifest are + * skipped with a warning instead of overwritten. + */ + overwriteLocal?: boolean; } export interface WatchRealmsResult { @@ -358,6 +438,7 @@ export async function watchRealms( const intervalMs = options.intervalMs ?? 30_000; const debounceMs = options.debounceMs ?? 5_000; const quiet = options.quiet ?? false; + const overwriteLocal = options.overwriteLocal ?? false; if (!Number.isFinite(intervalMs) || intervalMs <= 0) { return { watchers: [], error: '`intervalMs` must be a positive number.' }; @@ -408,6 +489,7 @@ export async function watchRealms( for (const spec of specs) { const watcher = new RealmWatcher(spec, authenticator, { debounceMs, + overwriteLocal, }); try { await watcher.initialize(); @@ -546,14 +628,20 @@ function formatLockedError(localDir: string, info: WatchLockInfo): string { function logFlush(name: string, result: FlushResult): void { const total = result.pulled.length + result.deleted.length; - if (total === 0) return; - console.log( - `${DIM}[${timestamp()}]${RESET} [${name}] ${FG_GREEN}applied ${total} change(s)${RESET} (${result.pulled.length} pulled, ${result.deleted.length} deleted)`, - ); - if (result.checkpoint) { - const tag = result.checkpoint.isMajor ? '[MAJOR]' : '[minor]'; + if (total > 0) { + console.log( + `${DIM}[${timestamp()}]${RESET} [${name}] ${FG_GREEN}applied ${total} change(s)${RESET} (${result.pulled.length} pulled, ${result.deleted.length} deleted)`, + ); + if (result.checkpoint) { + const tag = result.checkpoint.isMajor ? '[MAJOR]' : '[minor]'; + console.log( + ` ${DIM}Checkpoint:${RESET} ${result.checkpoint.shortHash} ${tag} ${result.checkpoint.message}`, + ); + } + } + for (const file of result.skipped) { console.log( - ` ${DIM}Checkpoint:${RESET} ${result.checkpoint.shortHash} ${tag} ${result.checkpoint.message}`, + `${DIM}[${timestamp()}]${RESET} [${name}] ${FG_YELLOW}⚠ skipped ${file}: local diverges from sync manifest (rerun with --overwrite-local to discard, or \`boxel realm sync\` to reconcile)${RESET}`, ); } } @@ -614,6 +702,10 @@ export function registerStartCommand(watch: Command): void { '--realm-secret-seed', 'Administrative auth: prompt for a realm secret seed and mint a JWT locally instead of using a Matrix profile (env: BOXEL_REALM_SECRET_SEED)', ) + .option( + '--overwrite-local', + 'Overwrite local files when the remote changes. Default: skip + warn when the local copy diverges from the sync manifest.', + ) .action( async ( realmUrl: string, @@ -622,6 +714,7 @@ export function registerStartCommand(watch: Command): void { interval: number; debounce: number; realmSecretSeed?: boolean; + overwriteLocal?: boolean; }, ) => { const realmSecretSeed = await resolveRealmSecretSeed( @@ -631,6 +724,7 @@ export function registerStartCommand(watch: Command): void { intervalMs: options.interval * 1000, debounceMs: options.debounce * 1000, realmSecretSeed, + overwriteLocal: options.overwriteLocal === true, }); if (result.error) { console.error(`${FG_RED}Error:${RESET} ${result.error}`); diff --git a/packages/boxel-cli/tests/integration/realm-watch.test.ts b/packages/boxel-cli/tests/integration/realm-watch.test.ts index b3e36bb7395..f0565c1f9f5 100644 --- a/packages/boxel-cli/tests/integration/realm-watch.test.ts +++ b/packages/boxel-cli/tests/integration/realm-watch.test.ts @@ -777,4 +777,262 @@ describe('realm watch (integration)', () => { watcher.shutdown(); }); + + // CS-11062: watch must protect locally-edited files from being silently + // overwritten on the next remote-change tick. + it('skips download when the local file diverges from the sync manifest', async () => { + let localDir = makeLocalDir(); + let rel = watchFixture('diverge'); + await writeRemoteFile(realmUrl, rel, 'export const v = 1;\n'); + + let watcher = new RealmWatcher({ realmUrl, localDir }, profileManager, { + debounceMs: 0, + quiet: true, + }); + await watcher.initialize(); + await watcher.poll(); + await watcher.flushPending(); + expect(fs.readFileSync(path.join(localDir, rel), 'utf8')).toContain( + 'v = 1', + ); + + // User edits the local file post-sync. Hash now diverges from manifest. + fs.writeFileSync(path.join(localDir, rel), 'local edit\n', 'utf8'); + + await sleep(1100); + await writeRemoteFile(realmUrl, rel, 'export const v = 2;\n'); + + let hasChanges = await watcher.poll(); + expect(hasChanges).toBe(true); + let result = await watcher.flushPending(); + + expect(result.skipped).toContain(rel); + expect(result.pulled).not.toContain(rel); + expect(fs.readFileSync(path.join(localDir, rel), 'utf8')).toBe( + 'local edit\n', + ); + + watcher.shutdown(); + }); + + it('writes no checkpoint when every change in a flush is skipped', async () => { + let localDir = makeLocalDir(); + let rel = watchFixture('skip-checkpoint'); + await writeRemoteFile(realmUrl, rel, 'export const v = 1;\n'); + + let watcher = new RealmWatcher({ realmUrl, localDir }, profileManager, { + debounceMs: 0, + quiet: true, + }); + await watcher.initialize(); + await watcher.poll(); + let firstFlush = await watcher.flushPending(); + expect(firstFlush.checkpoint).not.toBeNull(); + + let checkpointsAfterFirstFlush = await new CheckpointManager( + localDir, + ).getCheckpoints(); + expect(checkpointsAfterFirstFlush.length).toBe(1); + + // Diverge locally, then change remote → flush will skip the only entry. + fs.writeFileSync(path.join(localDir, rel), 'local edit\n', 'utf8'); + await sleep(1100); + await writeRemoteFile(realmUrl, rel, 'export const v = 2;\n'); + + await watcher.poll(); + let skippedFlush = await watcher.flushPending(); + + expect(skippedFlush.skipped).toContain(rel); + expect(skippedFlush.pulled).toHaveLength(0); + expect(skippedFlush.deleted).toHaveLength(0); + expect(skippedFlush.checkpoint).toBeNull(); + + let checkpointsAfterSkip = await new CheckpointManager( + localDir, + ).getCheckpoints(); + expect(checkpointsAfterSkip.length).toBe(1); + + watcher.shutdown(); + }); + + it('overwrites diverged local files when overwriteLocal is enabled', async () => { + let localDir = makeLocalDir(); + let rel = watchFixture('force'); + await writeRemoteFile(realmUrl, rel, 'export const v = 1;\n'); + + let watcher = new RealmWatcher({ realmUrl, localDir }, profileManager, { + debounceMs: 0, + quiet: true, + overwriteLocal: true, + }); + await watcher.initialize(); + await watcher.poll(); + await watcher.flushPending(); + + fs.writeFileSync(path.join(localDir, rel), 'local edit\n', 'utf8'); + + await sleep(1100); + await writeRemoteFile(realmUrl, rel, 'export const v = 2;\n'); + + await watcher.poll(); + let result = await watcher.flushPending(); + + expect(result.pulled).toContain(rel); + expect(result.skipped ?? []).not.toContain(rel); + expect(fs.readFileSync(path.join(localDir, rel), 'utf8')).toContain( + 'v = 2', + ); + + watcher.shutdown(); + }); + + it('skips first-run downloads when local files exist at remote paths and no manifest', async () => { + let localDir = makeLocalDir(); + let collide = watchFixture('collide'); + let onlyRemote = watchFixture('only-remote'); + + // Local content pre-exists with no sync manifest. + fs.writeFileSync(path.join(localDir, collide), 'precious local\n', 'utf8'); + + await writeRemoteFile(realmUrl, collide, 'export const v = 1;\n'); + await writeRemoteFile(realmUrl, onlyRemote, 'export const r = 1;\n'); + + let watcher = new RealmWatcher({ realmUrl, localDir }, profileManager, { + debounceMs: 0, + quiet: true, + }); + await watcher.initialize(); + await watcher.poll(); + let result = await watcher.flushPending(); + + // Colliding path is left alone, warned about. + expect(result.skipped).toContain(collide); + expect(result.pulled).not.toContain(collide); + expect(fs.readFileSync(path.join(localDir, collide), 'utf8')).toBe( + 'precious local\n', + ); + + // Non-colliding path still pulled normally. + expect(result.pulled).toContain(onlyRemote); + expect(fs.readFileSync(path.join(localDir, onlyRemote), 'utf8')).toContain( + 'r = 1', + ); + + watcher.shutdown(); + }); + + it('downloads when the local file still matches the manifest hash', async () => { + let localDir = makeLocalDir(); + let rel = watchFixture('clean'); + await writeRemoteFile(realmUrl, rel, 'export const v = 1;\n'); + + let watcher = new RealmWatcher({ realmUrl, localDir }, profileManager, { + debounceMs: 0, + quiet: true, + }); + await watcher.initialize(); + await watcher.poll(); + await watcher.flushPending(); + + // No local edit. Bump remote. + await sleep(1100); + await writeRemoteFile(realmUrl, rel, 'export const v = 2;\n'); + + await watcher.poll(); + let result = await watcher.flushPending(); + + expect(result.pulled).toContain(rel); + expect(result.skipped ?? []).not.toContain(rel); + expect(fs.readFileSync(path.join(localDir, rel), 'utf8')).toContain( + 'v = 2', + ); + + watcher.shutdown(); + }); + + it('does not delete a locally-edited file when the remote disappears', async () => { + let localDir = makeLocalDir(); + let rel = watchFixture('rescue'); + await writeRemoteFile(realmUrl, rel, 'export const v = 1;\n'); + + let watcher = new RealmWatcher({ realmUrl, localDir }, profileManager, { + debounceMs: 0, + quiet: true, + }); + await watcher.initialize(); + await watcher.poll(); + await watcher.flushPending(); + + fs.writeFileSync(path.join(localDir, rel), 'local edit\n', 'utf8'); + + await deleteRemoteFile(realmUrl, rel); + await watcher.poll(); + let result = await watcher.flushPending(); + + expect(result.skipped).toContain(rel); + expect(result.deleted).not.toContain(rel); + expect(fs.existsSync(path.join(localDir, rel))).toBe(true); + expect(fs.readFileSync(path.join(localDir, rel), 'utf8')).toBe( + 'local edit\n', + ); + + watcher.shutdown(); + }); + + it('deletes the local file when the remote disappears and the local matches the manifest', async () => { + let localDir = makeLocalDir(); + let rel = watchFixture('clean-delete'); + await writeRemoteFile(realmUrl, rel, 'export const v = 1;\n'); + + let watcher = new RealmWatcher({ realmUrl, localDir }, profileManager, { + debounceMs: 0, + quiet: true, + }); + await watcher.initialize(); + await watcher.poll(); + await watcher.flushPending(); + expect(fs.existsSync(path.join(localDir, rel))).toBe(true); + + // No local edit — local hash still matches the manifest. + await deleteRemoteFile(realmUrl, rel); + await watcher.poll(); + let result = await watcher.flushPending(); + + expect(result.deleted).toContain(rel); + expect(result.skipped ?? []).not.toContain(rel); + expect(fs.existsSync(path.join(localDir, rel))).toBe(false); + + watcher.shutdown(); + }); + + it('keeps re-detecting a skipped divergence on subsequent polls until resolved', async () => { + let localDir = makeLocalDir(); + let rel = watchFixture('nag'); + await writeRemoteFile(realmUrl, rel, 'export const v = 1;\n'); + + let watcher = new RealmWatcher({ realmUrl, localDir }, profileManager, { + debounceMs: 0, + quiet: true, + }); + await watcher.initialize(); + await watcher.poll(); + await watcher.flushPending(); + + fs.writeFileSync(path.join(localDir, rel), 'local edit\n', 'utf8'); + + await sleep(1100); + await writeRemoteFile(realmUrl, rel, 'export const v = 2;\n'); + + await watcher.poll(); + let first = await watcher.flushPending(); + expect(first.skipped).toContain(rel); + + // No further remote change. The previous skip must not have advanced + // lastKnownMtimes, so the same divergence keeps surfacing. + await watcher.poll(); + let second = await watcher.flushPending(); + expect(second.skipped).toContain(rel); + + watcher.shutdown(); + }); });