diff --git a/.changeset/clean-outbound-sync.md b/.changeset/clean-outbound-sync.md new file mode 100644 index 00000000..0b3a321c --- /dev/null +++ b/.changeset/clean-outbound-sync.md @@ -0,0 +1,9 @@ +--- +'@treecrdt/sync': minor +'@treecrdt/sync-protocol': patch +--- + +Add an outbound sync helper for queued local-op upload to remote peer transports. Standard +TreeCRDT operations are deduped by operation id by default; custom op shapes may provide `opKey`. +`SyncPeer` also derives op refs for standard TreeCRDT operations by default, so apps only need +`deriveOpRef` for custom op-ref schemes or nonstandard op shapes. diff --git a/examples/playground/package.json b/examples/playground/package.json index 11dccab4..97884f71 100644 --- a/examples/playground/package.json +++ b/examples/playground/package.json @@ -17,6 +17,7 @@ "@treecrdt/crypto": "workspace:*", "@treecrdt/discovery": "workspace:*", "@treecrdt/interface": "workspace:*", + "@treecrdt/sync": "workspace:*", "@treecrdt/sync-protocol": "workspace:*", "@treecrdt/sync-sqlite": "workspace:*", "@treecrdt/wa-sqlite": "workspace:*", diff --git a/examples/playground/src/App.tsx b/examples/playground/src/App.tsx index 8db5a0a4..95f45a3d 100644 --- a/examples/playground/src/App.tsx +++ b/examples/playground/src/App.tsx @@ -468,7 +468,7 @@ export default function App() { liveAllEnabled, setLiveAllEnabled, toggleLiveChildren, - queueLocalOpsForSync, + queueOps, handleSync, handleScopedSync, postBroadcastMessage, @@ -498,11 +498,11 @@ export default function App() { const handleCommittedLocalOps = React.useCallback( (ops: Operation[]) => { - // The local write already committed. This only feeds playground sync and debug UI state. - queueLocalOpsForSync(ops); + // The local write already committed; publish it to sync and debug UI state. + queueOps(ops); recordOps(ops, { assumeSorted: true }); }, - [queueLocalOpsForSync, recordOps] + [queueOps, recordOps] ); const grantSubtreeToReplicaPubkey = React.useCallback( diff --git a/examples/playground/src/playground/hooks/usePlaygroundSync.ts b/examples/playground/src/playground/hooks/usePlaygroundSync.ts index cf29d297..f4f44d1f 100644 --- a/examples/playground/src/playground/hooks/usePlaygroundSync.ts +++ b/examples/playground/src/playground/hooks/usePlaygroundSync.ts @@ -5,12 +5,8 @@ import { resolveWebSocketAttachment, type ResolveWebSocketAttachmentResult, } from '@treecrdt/discovery'; -import { - SyncPeer, - deriveOpRefV0, - type Filter, - type SyncAuth, -} from '@treecrdt/sync-protocol'; +import { SyncPeer, type Filter, type SyncAuth } from '@treecrdt/sync-protocol'; +import { createOutboundSync, type OutboundSync } from '@treecrdt/sync'; import { createTreecrdtSyncBackendFromClient } from '@treecrdt/sync-sqlite'; import type { BroadcastPresenceAckMessageV1, @@ -21,7 +17,10 @@ import { createBrowserWebSocketTransport, } from '@treecrdt/sync-protocol/browser'; import { treecrdtSyncV0ProtobufCodec } from '@treecrdt/sync-protocol/protobuf'; -import { wrapDuplexTransportWithCodec, type DuplexTransport } from '@treecrdt/sync-protocol/transport'; +import { + wrapDuplexTransportWithCodec, + type DuplexTransport, +} from '@treecrdt/sync-protocol/transport'; import type { TreecrdtClient } from '@treecrdt/wa-sqlite'; import { hexToBytes16, type AuthGrantMessageV1 } from '../../sync-v0'; @@ -48,13 +47,56 @@ import { isDiscoveryBootstrapUrl, isRemotePeerId, isTransientRemoteConnectError, - localOpUploadKey, normalizeSyncServerUrl, previewDiscoveryHost, syncOnceOptionsForPeer, syncTimeoutMsForPeer, withTimeout, } from '../syncHelpers'; + +const RECENT_SYNC_TARGET_MS = 5_000; +const NODE_ID_HEX_RE = /^[0-9a-f]{32}$/i; + +function isNodeIdHex(id: string): boolean { + return NODE_ID_HEX_RE.test(id); +} + +function childrenFilter(parentId: string): Filter { + return { children: { parent: hexToBytes16(parentId) } }; +} + +function syncFilterLabel(filter: Filter, action = 'sync'): string { + return 'all' in filter + ? action + : `${action}(children ${bytesToHex(filter.children.parent).slice(0, 8)}…)`; +} + +async function syncFiltersWithTransport( + peer: SyncPeer, + peerId: string, + transport: DuplexTransport, + filters: readonly Filter[], + opts: { + autoSync?: boolean; + multipleTargets?: boolean; + codewordsPerMessage?: number; + label?: string; + } = {}, +) { + const perPeerTimeoutMs = syncTimeoutMsForPeer(peerId, { + autoSync: opts.autoSync, + multipleTargets: opts.multipleTargets, + }); + const codewordsPerMessage = opts.codewordsPerMessage ?? 2048; + for (const filter of filters) { + await withTimeout( + peer.syncOnce(transport, filter, syncOnceOptionsForPeer(peerId, codewordsPerMessage)), + perPeerTimeoutMs, + `${syncFilterLabel(filter, opts.label)} with ${peerId.slice(0, 8)}… timed out`, + ); + } +} + type PlaygroundSyncApi = { peers: PeerInfo[]; remoteSyncStatus: RemoteSyncStatus; @@ -67,7 +109,7 @@ type PlaygroundSyncApi = { liveAllEnabled: boolean; setLiveAllEnabled: React.Dispatch>; toggleLiveChildren: (parentId: string) => void; - queueLocalOpsForSync: (ops?: Operation[]) => void; + queueOps: (ops: Operation[]) => void; handleSync: (filter: Filter) => Promise; handleScopedSync: () => Promise; postBroadcastMessage: ( @@ -123,6 +165,7 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn } = opts; const [syncBusy, setSyncBusy] = useState(false); + const [outboundBusy, setOutboundBusy] = useState(false); const [syncError, setSyncError] = useState(null); const [remoteSyncStatus, setRemoteSyncStatus] = useState({ state: 'disabled', @@ -133,6 +176,7 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn const onlineRef = useRef(true); useEffect(() => { onlineRef.current = online; + if (online) void outboundSyncRef.current?.flush(); }, [online]); const autoSyncJoinInitial = useRef(autoSyncJoin).current; @@ -144,6 +188,7 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn const syncPeerRef = useRef | null>(null); const syncConnRef = useRef>(new Map()); + const outboundSyncRef = useRef | null>(null); const { liveBusy, liveChildrenParents, @@ -153,8 +198,6 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn toggleLiveChildren, liveChildrenParentsRef, liveAllEnabledRef, - beginLiveWork, - endLiveWork, startLiveAll, stopLiveAllForPeer, stopAllLiveAll, @@ -168,111 +211,13 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn setSyncError, authCanSyncAll, }); - const remoteLivePushScheduledRef = useRef(false); - const remoteLivePushRunningRef = useRef(false); - const remoteLivePushNeedsFullSyncRef = useRef(false); - const remoteLivePushPendingOpsRef = useRef>(new Map()); const autoSyncDoneRef = useRef(false); const autoSyncInFlightRef = useRef(false); const autoSyncAttemptRef = useRef(0); const autoSyncPeerIdRef = useRef(null); - const queueRemoteUploadHints = (ops?: Operation[]) => { - if (!ops || ops.length === 0) { - remoteLivePushNeedsFullSyncRef.current = true; - return; - } - const pendingOps = remoteLivePushPendingOpsRef.current; - for (const op of ops) { - pendingOps.set(localOpUploadKey(op), op); - } - }; - - const queueLocalOpsForSync = (ops?: Operation[]) => { - void syncPeerRef.current?.notifyLocalUpdate(ops); - queueRemoteUploadHints(ops); - if (remoteLivePushRunningRef.current) { - remoteLivePushScheduledRef.current = true; - return; - } - remoteLivePushScheduledRef.current = true; - remoteLivePushRunningRef.current = true; - beginLiveWork(); - void (async () => { - try { - while (remoteLivePushScheduledRef.current) { - remoteLivePushScheduledRef.current = false; - if (!onlineRef.current) continue; - - const peer = syncPeerRef.current; - if (!peer) continue; - - const connections = syncConnRef.current; - const remotePeerIds = Array.from(connections.keys()).filter(isRemotePeerId); - if (remotePeerIds.length === 0) continue; - - const liveChildren = Array.from(liveChildrenParentsRef.current).filter((id) => - /^[0-9a-f]{32}$/i.test(id), - ); - const pendingOps = Array.from(remoteLivePushPendingOpsRef.current.values()); - const needsFullSync = remoteLivePushNeedsFullSyncRef.current; - remoteLivePushPendingOpsRef.current.clear(); - remoteLivePushNeedsFullSyncRef.current = false; - if ( - !needsFullSync && - pendingOps.length === 0 && - !liveAllEnabledRef.current && - liveChildren.length === 0 - ) - continue; - - for (const peerId of remotePeerIds) { - const conn = connections.get(peerId); - if (!conn) continue; - try { - if (pendingOps.length > 0) { - await withTimeout( - peer.pushOps(conn.transport, pendingOps, { - maxOpsPerBatch: PLAYGROUND_SYNC_MAX_OPS_PER_BATCH, - }), - syncTimeoutMsForPeer(peerId, { autoSync: true }), - `live push with ${peerId.slice(0, 8)}… timed out`, - ); - continue; - } - - if (needsFullSync || (liveAllEnabledRef.current && liveChildren.length === 0)) { - await withTimeout( - peer.syncOnce(conn.transport, { all: {} }, syncOnceOptionsForPeer(peerId, 1024)), - syncTimeoutMsForPeer(peerId, { autoSync: true }), - `live sync with ${peerId.slice(0, 8)}… timed out`, - ); - continue; - } - - for (const parentId of liveChildren) { - await withTimeout( - peer.syncOnce( - conn.transport, - { children: { parent: hexToBytes16(parentId) } }, - syncOnceOptionsForPeer(peerId, 1024), - ), - syncTimeoutMsForPeer(peerId, { autoSync: true }), - `live sync(children ${parentId.slice(0, 8)}…) with ${peerId.slice(0, 8)}… timed out`, - ); - } - } catch (err) { - console.error('Remote live sync push failed', err); - setSyncError(formatSyncError(err)); - if (!isCapabilityRevokedError(err)) dropPeerConnection(peerId); - } - } - } - } finally { - remoteLivePushRunningRef.current = false; - endLiveWork(); - } - })(); + const queueOps = (ops: Operation[]) => { + outboundSyncRef.current?.queueOps(ops); }; const dropPeerConnection = (peerId: string) => { @@ -303,7 +248,16 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn else removeMeshPeer(peerId); }; - const handleSync = async (filter: Filter) => { + const selectSyncTargetIds = (connections: ReadonlyMap) => { + const now = Date.now(); + const recentPeerIds = peers + .filter((p) => now - p.lastSeen < RECENT_SYNC_TARGET_MS) + .sort((a, b) => b.lastSeen - a.lastSeen) + .map((p) => p.id); + return recentPeerIds.length > 0 ? recentPeerIds : Array.from(connections.keys()); + }; + + const syncFiltersWithTargets = async (filters: readonly Filter[], label: string) => { if (!onlineRef.current) { setSyncError('Offline: toggle Online to sync.'); return; @@ -322,30 +276,20 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn setSyncBusy(true); setSyncError(null); try { - const now = Date.now(); - const recentPeerIds = peers - .filter((p) => now - p.lastSeen < 5_000) - .sort((a, b) => b.lastSeen - a.lastSeen) - .map((p) => p.id); - const targets = recentPeerIds.length > 0 ? recentPeerIds : Array.from(connections.keys()); + const targets = selectSyncTargetIds(connections); let successes = 0; let lastErr: unknown = null; for (const peerId of targets) { const conn = connections.get(peerId); if (!conn) continue; - const perPeerTimeoutMs = syncTimeoutMsForPeer(peerId, { - multipleTargets: targets.length > 1, - }); try { - await withTimeout( - peer.syncOnce(conn.transport, filter, syncOnceOptionsForPeer(peerId, 2048)), - perPeerTimeoutMs, - `sync with ${peerId.slice(0, 8)}… timed out`, - ); + await syncFiltersWithTransport(peer, peerId, conn.transport, filters, { + multipleTargets: targets.length > 1, + }); successes += 1; } catch (err) { lastErr = err; - console.error('Sync failed for peer', peerId, err); + console.error(`${label} failed for peer`, peerId, err); if (!isCapabilityRevokedError(err)) dropPeerConnection(peerId); } } @@ -355,82 +299,25 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn } await refreshMeta(); } catch (err) { - console.error('Sync failed', err); + console.error(`${label} failed`, err); setSyncError(formatSyncError(err)); } finally { setSyncBusy(false); } }; + const handleSync = async (filter: Filter) => { + await syncFiltersWithTargets([filter], 'Sync'); + }; + const handleScopedSync = async () => { const parents = new Set(getLoadedParentIds()); parents.add(viewRootId); if (viewRootId !== ROOT_ID) parents.delete(ROOT_ID); - const parentIds = Array.from(parents).filter((id) => /^[0-9a-f]{32}$/i.test(id)); + const parentIds = Array.from(parents).filter(isNodeIdHex); parentIds.sort(); - if (!onlineRef.current) { - setSyncError('Offline: toggle Online to sync.'); - return; - } - const peer = syncPeerRef.current; - if (!peer) { - setSyncError('Sync peer is not ready yet.'); - return; - } - const connections = syncConnRef.current; - if (connections.size === 0) { - setSyncError('No peers discovered yet.'); - return; - } - - setSyncBusy(true); - setSyncError(null); - try { - const now = Date.now(); - const recentPeerIds = peers - .filter((p) => now - p.lastSeen < 5_000) - .sort((a, b) => b.lastSeen - a.lastSeen) - .map((p) => p.id); - const targets = recentPeerIds.length > 0 ? recentPeerIds : Array.from(connections.keys()); - let successes = 0; - let lastErr: unknown = null; - for (const peerId of targets) { - const conn = connections.get(peerId); - if (!conn) continue; - const perPeerTimeoutMs = syncTimeoutMsForPeer(peerId, { - multipleTargets: targets.length > 1, - }); - try { - for (const parentId of parentIds) { - await withTimeout( - peer.syncOnce( - conn.transport, - { children: { parent: hexToBytes16(parentId) } }, - syncOnceOptionsForPeer(peerId, 2048), - ), - perPeerTimeoutMs, - `sync(children ${parentId.slice(0, 8)}…) with ${peerId.slice(0, 8)}… timed out`, - ); - } - successes += 1; - } catch (err) { - lastErr = err; - console.error('Scoped sync failed for peer', peerId, err); - if (!isCapabilityRevokedError(err)) dropPeerConnection(peerId); - } - } - if (successes === 0) { - if (lastErr) throw lastErr; - throw new Error('No peers responded to sync.'); - } - await refreshMeta(); - } catch (err) { - console.error('Scoped sync failed', err); - setSyncError(formatSyncError(err)); - } finally { - setSyncBusy(false); - } + await syncFiltersWithTargets(parentIds.map(childrenFilter), 'Scoped sync'); }; const postBroadcastMessage = ( @@ -478,7 +365,7 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn if (!authCanSyncAll) { const clean = viewRootId.toLowerCase(); - if (clean === ROOT_ID || !/^[0-9a-f]{32}$/.test(clean)) return; + if (clean === ROOT_ID || !isNodeIdHex(clean)) return; } if (autoSyncAttemptRef.current >= 3) return; @@ -489,23 +376,11 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn setSyncBusy(true); setSyncError(null); try { - if (authCanSyncAll) { - await withTimeout( - peer.syncOnce(conn.transport, { all: {} }, syncOnceOptionsForPeer(peerId, 2048)), - syncTimeoutMsForPeer(peerId, { autoSync: true }), - `auto sync with ${peerId.slice(0, 8)}… timed out`, - ); - } else { - await withTimeout( - peer.syncOnce( - conn.transport, - { children: { parent: hexToBytes16(viewRootId) } }, - syncOnceOptionsForPeer(peerId, 2048), - ), - syncTimeoutMsForPeer(peerId, { autoSync: true }), - `auto sync(children ${viewRootId.slice(0, 8)}…) with ${peerId.slice(0, 8)}… timed out`, - ); - } + const filter: Filter = authCanSyncAll ? { all: {} } : childrenFilter(viewRootId); + await syncFiltersWithTransport(peer, peerId, conn.transport, [filter], { + autoSync: true, + label: 'auto sync', + }); await refreshMeta(); @@ -661,22 +536,36 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn }, }; - const sharedPeer = new SyncPeer(backend, { + const localPeer = new SyncPeer(backend, { maxCodewords: PLAYGROUND_SYNC_MAX_CODEWORDS, maxOpsPerBatch: PLAYGROUND_SYNC_MAX_OPS_PER_BATCH, - deriveOpRef: (op, ctx) => - deriveOpRefV0(ctx.docId, { replica: op.meta.id.replica, counter: op.meta.id.counter }), ...(syncAuth ? { auth: syncAuth, } : {}), }); - syncPeerRef.current = sharedPeer; + syncPeerRef.current = localPeer; const connections = new Map; detach: () => void }>(); syncConnRef.current = connections; + const outboundSync = createOutboundSync({ + localPeer, + isOnline: () => onlineRef.current, + pushOptions: () => ({ + maxOpsPerBatch: PLAYGROUND_SYNC_MAX_OPS_PER_BATCH, + }), + pushTimeoutMs: (peerId) => syncTimeoutMsForPeer(peerId, { autoSync: true }), + onStatus: (status) => setOutboundBusy(status.running || status.scheduled), + onError: ({ targetId, error }) => { + console.error('Remote op upload failed', error); + setSyncError(formatSyncError(error)); + if (!isCapabilityRevokedError(error)) dropPeerConnection(targetId); + }, + }); + outboundSyncRef.current = outboundSync; + const maybeStartLiveForPeer = (peerId: string) => { if (!isRemotePeerId(peerId)) { const mesh = presenceMeshRef.current; @@ -686,6 +575,28 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn for (const parentId of liveChildrenParentsRef.current) startLiveChildren(peerId, parentId); }; + const queueAutoSyncForPeer = (peerId: string) => { + if (!autoSyncJoinInitial || !joinMode || autoSyncDoneRef.current) return; + autoSyncPeerIdRef.current = peerId; + // Ensure the auto-sync effect runs even if peer readiness toggles without changing `peers.length`. + bumpAutoSyncJoinTick((t) => t + 1); + }; + + const registerPeerConnection = ( + peerId: string, + transport: DuplexTransport, + opts: { outbound?: boolean; markRemoteSeen?: boolean } = {}, + ) => { + const detach = opts.outbound + ? outboundSync.attachTarget(peerId, transport) + : localPeer.attach(transport); + connections.set(peerId, { transport, detach }); + if (opts.markRemoteSeen) setRemotePeer({ id: peerId, lastSeen: Date.now() }); + maybeStartLiveForPeer(peerId); + queueAutoSyncForPeer(peerId); + return detach; + }; + const mesh = channel ? createBroadcastPresenceMesh({ channel, @@ -698,21 +609,10 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn }, onPeerReady: (peerId) => { maybeStartLiveForPeer(peerId); - if (autoSyncJoinInitial && joinMode && !autoSyncDoneRef.current) { - autoSyncPeerIdRef.current = peerId; - // Ensure the auto-sync effect runs even if peer readiness toggles without changing `peers.length`. - bumpAutoSyncJoinTick((t) => t + 1); - } + queueAutoSyncForPeer(peerId); }, onPeerTransport: (peerId, transport) => { - const detach = sharedPeer.attach(transport); - connections.set(peerId, { transport, detach }); - maybeStartLiveForPeer(peerId); - if (autoSyncJoinInitial && joinMode && !autoSyncDoneRef.current) { - autoSyncPeerIdRef.current = peerId; - bumpAutoSyncJoinTick((t) => t + 1); - } - return detach; + return registerPeerConnection(peerId, transport); }, onPeerDisconnected: (peerId) => { connections.delete(peerId); @@ -745,12 +645,15 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn let remoteSocket: WebSocket | null = null; let remotePeerId: string | null = null; - let disposed = false; + let remoteSocketDisposed = false; let remoteOpened = false; let resolvedRemote: ResolveWebSocketAttachmentResult | null = null; - const discoveryRouteCache = getBrowserDiscoveryRouteCache(); if (remoteSyncUrl.length > 0) { + const discoveryRouteCache = getBrowserDiscoveryRouteCache(); + const isRemoteSocketCurrent = () => + !remoteSocketDisposed && syncConnRef.current === connections; + void (async () => { try { setSyncError((prev) => (isTransientRemoteConnectError(prev) ? null : prev)); @@ -766,7 +669,8 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn ? window.fetch.bind(window) : undefined, }); - if (disposed || syncConnRef.current !== connections) return; + if (!isRemoteSocketCurrent()) return; + const remoteUrl = resolvedRemote.url; const connectVerb = resolvedRemote.source === 'network' @@ -783,7 +687,7 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn remoteSocket.binaryType = 'arraybuffer'; remoteSocket.addEventListener('open', () => { - if (disposed || syncConnRef.current !== connections) return; + if (!isRemoteSocketCurrent()) return; if (!remoteSocket || remoteSocket.readyState !== WebSocket.OPEN || !remotePeerId) return; remoteOpened = true; @@ -797,20 +701,14 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn wire, treecrdtSyncV0ProtobufCodec as any, ); - const detach = sharedPeer.attach(transport); - syncConnRef.current.set(remotePeerId, { transport, detach }); - setRemotePeer({ id: remotePeerId, lastSeen: Date.now() }); - maybeStartLiveForPeer(remotePeerId); - - if (autoSyncJoinInitial && joinMode && !autoSyncDoneRef.current) { - autoSyncPeerIdRef.current = remotePeerId; - bumpAutoSyncJoinTick((t) => t + 1); - } + registerPeerConnection(remotePeerId, transport, { + outbound: true, + markRemoteSeen: true, + }); }); remoteSocket.addEventListener('message', () => { - if (disposed || syncConnRef.current !== connections) return; - if (!remotePeerId) return; + if (!isRemoteSocketCurrent() || !remotePeerId) return; setRemotePeer({ id: remotePeerId, lastSeen: Date.now() }); setRemoteSyncStatus((prev) => prev.state === 'connected' @@ -824,7 +722,7 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn remoteSocket.addEventListener('close', () => { if (syncConnRef.current !== connections) return; - if (!disposed) { + if (!remoteSocketDisposed) { setRemoteSyncStatus({ detail: formatRemoteErrorDetail( remoteOpened ? 'disconnected' : 'could_not_connect', @@ -837,12 +735,11 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn if (!remoteOpened && resolvedRemote?.source === 'cache' && resolvedRemote.cacheKey) { void discoveryRouteCache?.delete(resolvedRemote.cacheKey); } - if (!remotePeerId) return; - dropPeerConnection(remotePeerId); + if (remotePeerId) dropPeerConnection(remotePeerId); }); remoteSocket.addEventListener('error', () => { - if (syncConnRef.current !== connections) return; + if (!isRemoteSocketCurrent()) return; setRemoteSyncStatus({ detail: formatRemoteErrorDetail( remoteOpened ? 'connection_error' : 'could_not_reach', @@ -857,7 +754,7 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn setSyncError((prev) => prev ?? `Remote sync socket error (${remoteUrl.host})`); }); } catch (err) { - if (disposed || syncConnRef.current !== connections) return; + if (!isRemoteSocketCurrent()) return; setRemoteSyncStatus({ state: isDiscoveryBootstrapUrl(remoteSyncUrl) ? 'error' : 'invalid', detail: formatSyncError(err), @@ -867,24 +764,29 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn })(); } + const stopRemoteSocket = () => { + remoteSocketDisposed = true; + if (!remoteSocket) return; + try { + remoteSocket.close(); + } catch { + // ignore + } + }; + return () => { - disposed = true; stopAllLiveAll(); stopAllLiveChildren(); if (presenceMeshRef.current === mesh) presenceMeshRef.current = null; mesh?.stop(); - if (remoteSocket) { - try { - remoteSocket.close(); - } catch { - // ignore - } - } + stopRemoteSocket(); if (broadcastChannelRef.current === channel) broadcastChannelRef.current = null; - if (syncPeerRef.current === sharedPeer) syncPeerRef.current = null; + if (syncPeerRef.current === localPeer) syncPeerRef.current = null; + outboundSync.close(); + if (outboundSyncRef.current === outboundSync) { + outboundSyncRef.current = null; + } channel?.close(); - remoteLivePushScheduledRef.current = false; - remoteLivePushRunningRef.current = false; resetLiveWork(); connections.clear(); resetPeers(); @@ -910,7 +812,7 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn peers, remoteSyncStatus, syncBusy, - liveBusy, + liveBusy: liveBusy || outboundBusy, syncError, setSyncError, liveChildrenParents, @@ -918,7 +820,7 @@ export function usePlaygroundSync(opts: UsePlaygroundSyncOptions): PlaygroundSyn liveAllEnabled, setLiveAllEnabled, toggleLiveChildren, - queueLocalOpsForSync, + queueOps, handleSync, handleScopedSync, postBroadcastMessage, diff --git a/examples/playground/src/playground/syncHelpers.ts b/examples/playground/src/playground/syncHelpers.ts index 6b2249e1..a5abf830 100644 --- a/examples/playground/src/playground/syncHelpers.ts +++ b/examples/playground/src/playground/syncHelpers.ts @@ -1,5 +1,3 @@ -import type { Operation } from '@treecrdt/interface'; -import { bytesToHex } from '@treecrdt/interface/ids'; import { createStringStoreRouteCache, isDiscoveryBootstrapUrl, @@ -150,8 +148,4 @@ export function syncTimeoutMsForPeer( return opts.multipleTargets ? 8_000 : 15_000; } -export function localOpUploadKey(op: Operation): string { - return `${bytesToHex(op.meta.id.replica)}:${op.meta.id.counter}`; -} - export { isDiscoveryBootstrapUrl }; diff --git a/examples/playground/tests/playground.spec.ts b/examples/playground/tests/playground.spec.ts index 9b01da4c..0cd8d0a7 100644 --- a/examples/playground/tests/playground.spec.ts +++ b/examples/playground/tests/playground.spec.ts @@ -784,13 +784,12 @@ test("remote live sync all pushes new ops without a manual sync click", async ({ })(), ]); await Promise.race([ - server.waitForServedOps(doc, 1), + expect(treeRowByLabel(pageB, nodeLabel)).toBeVisible({ timeout: 60_000 }), (async () => { - await pageB.getByTestId("sync-error").waitFor({ state: "visible", timeout: 30_000 }); + await pageB.getByTestId("sync-error").waitFor({ state: "visible", timeout: 60_000 }); throw new Error(`sync error (B): ${await pageB.getByTestId("sync-error").textContent()}`); })(), ]); - await expect(treeRowByLabel(pageB, nodeLabel)).toBeVisible({ timeout: 60_000 }); await expect(pageA.getByTestId("sync-error")).toBeHidden(); await expect(pageB.getByTestId("sync-error")).toBeHidden(); diff --git a/packages/sync-protocol/protocol/src/sync.ts b/packages/sync-protocol/protocol/src/sync.ts index 1ec62114..816f50a0 100644 --- a/packages/sync-protocol/protocol/src/sync.ts +++ b/packages/sync-protocol/protocol/src/sync.ts @@ -18,6 +18,7 @@ import { peerSupportsDirectSendEmptyReceiver, peerSupportsDirectSendSmallScope, } from './capabilities.js'; +import { deriveOpRefV0 } from './opref.js'; import { ErrorCode, RibltFailureReason } from './types.js'; import type { Capability, @@ -99,7 +100,10 @@ export type SyncPeerOptions = { requireAuthForFilters?: boolean; /** Auth policy hooks for outgoing filters, incoming ops, and post-verify side effects. */ auth?: SyncAuth; - /** Derive stable op refs from ops so relay fast-forwarding can track delivered entries. */ + /** + * Derive stable op refs from ops so relay fast-forwarding can track delivered entries. + * Defaults to TreeCRDT Operation.meta.id when standard TreeCRDT operations are supplied. + */ deriveOpRef?: (op: Op, ctx: { docId: string }) => OpRef; }; @@ -198,6 +202,13 @@ function bytesToHex(bytes: Uint8Array): string { return out; } +function defaultDeriveOpRef(op: unknown, ctx: { docId: string }): OpRef | undefined { + const id = (op as { meta?: { id?: { replica?: unknown; counter?: unknown } } })?.meta?.id; + if (!(id?.replica instanceof Uint8Array)) return undefined; + if (typeof id.counter !== 'number' && typeof id.counter !== 'bigint') return undefined; + return deriveOpRefV0(ctx.docId, { replica: id.replica, counter: id.counter }); +} + function waitForAbort(signal: AbortSignal): Promise { if (signal.aborted) return Promise.resolve(); return new Promise((resolve) => @@ -234,7 +245,7 @@ export class SyncPeer { private readonly directSendThreshold: number; private readonly requireAuthForFilters: boolean; private readonly auth?: SyncAuth; - private readonly deriveOpRef?: (op: Op, ctx: { docId: string }) => OpRef; + private readonly deriveOpRef: (op: Op, ctx: { docId: string }) => OpRef | undefined; private readonly transportHasAuth = new WeakMap>, boolean>(); private readonly transportPeerCapabilities = new WeakMap< DuplexTransport>, @@ -281,7 +292,7 @@ export class SyncPeer { throw new Error(`invalid directSendThreshold: ${opts.directSendThreshold}`); } this.requireAuthForFilters = opts.requireAuthForFilters ?? Boolean(opts.auth); - this.deriveOpRef = opts.deriveOpRef; + this.deriveOpRef = opts.deriveOpRef ?? defaultDeriveOpRef; } attach( @@ -304,9 +315,13 @@ export class SyncPeer { // feed this push loop until the subscription is removed or the transport fails. notifyLocalUpdate(ops?: readonly Op[]): Promise { if (this.responderSubscriptions.size === 0) return Promise.resolve(); - if (ops && ops.length > 0 && this.deriveOpRef) { + if (ops && ops.length > 0) { for (const op of ops) { const opRef = this.deriveOpRef(op, { docId: this.backend.docId }); + if (!opRef) { + this.pushNeedsFullScan = true; + break; + } const opRefHex = bytesToHex(opRef); this.pendingPushOpsByRefHex.set(opRefHex, { opRef, opRefHex, op }); } @@ -331,7 +346,7 @@ export class SyncPeer { this.pushScheduled = false; // A full scan means "some subscription-relevant state changed, but we // do not have an exact delta set to push from". Delta pushes are only - // safe when notifyLocalUpdate supplied concrete ops and deriveOpRef is available. + // safe when notifyLocalUpdate supplied concrete ops that can be mapped to op refs. const deltaOps = this.pushNeedsFullScan || this.pendingPushOpsByRefHex.size === 0 ? [] diff --git a/packages/treecrdt-sync/README.md b/packages/treecrdt-sync/README.md index 23a449fc..9e9f1d55 100644 --- a/packages/treecrdt-sync/README.md +++ b/packages/treecrdt-sync/README.md @@ -9,10 +9,37 @@ High-level **client** library for TreeCRDT sync v0. It combines **`@treecrdt/dis - You want a single dependency to **open a websocket** to a sync server and run reconciliation against a SQLite-backed client store. - You are fine with the built-in discovery + WebSocket + protobuf wiring. +## Multi-peer outbound upload + +Use `createOutboundSync` with a `localPeer` when one `SyncPeer` owns several transports, such as +local-tab mesh peers plus a remote websocket server. The app still manages transport discovery, but +outbound sync owns the committed-local-op hook: it attaches remote upload targets, wakes live +subscriptions on the `localPeer`, and queues exact local-op upload with dedupe and offline retry. + +```ts +import { createOutboundSync } from '@treecrdt/sync'; + +const outbound = createOutboundSync({ + localPeer: peer, + isOnline: () => navigator.onLine, +}); + +const detachRemote = outbound.attachTarget('remote:server', websocketTransport); + +const op = await client.local.payload(replica, node, payload); +outbound.queueOps([op]); // live subscription wakeup + remote websocket upload/retry +``` + +Use `addTarget` only when the transport was already attached to the same `localPeer`. `queueOps` +dedupes standard TreeCRDT `Operation` values by `meta.id`. Pass `opKey` only for a custom op shape +or custom coalescing behavior. + ## When not to - You only need the protocol types and `SyncPeer` (use **`@treecrdt/sync-protocol`**). - You use a custom transport, no discovery, or an in-memory backend (depend on the protocol and/or **`@treecrdt/discovery`** as needed). +- You want exact low-level control over each `syncOnce`, `startLive`, and direct push call; use + `connectTreecrdtWebSocketSync` directly. ## Repo location diff --git a/packages/treecrdt-sync/src/controller.ts b/packages/treecrdt-sync/src/controller.ts new file mode 100644 index 00000000..39e74892 --- /dev/null +++ b/packages/treecrdt-sync/src/controller.ts @@ -0,0 +1,266 @@ +import type { Operation } from '@treecrdt/interface'; +import { bytesToHex } from '@treecrdt/interface/ids'; +import type { SyncMessage } from '@treecrdt/sync-protocol'; +import type { DuplexTransport } from '@treecrdt/sync-protocol/transport'; +import type { OutboundSync, OutboundSyncOptions, OutboundSyncStatus } from './types.js'; + +function outboundSyncStatusSnapshot( + targets: ReadonlyMap>>, + pendingOps: readonly Op[], + running: boolean, + scheduled: boolean, +): OutboundSyncStatus { + return { + targetCount: targets.size, + pendingOps: pendingOps.length, + running, + scheduled, + }; +} + +function withTimeout(promise: Promise, ms: number | undefined, message: string): Promise { + if (ms === undefined) return promise; + if (!Number.isFinite(ms) || ms <= 0) { + return Promise.reject(new Error(`invalid pushTimeoutMs: ${ms}`)); + } + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(message)), ms); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error) => { + clearTimeout(timer); + reject(error); + }, + ); + }); +} + +function defaultOpKey(op: unknown): string | undefined { + const id = (op as { meta?: { id?: { replica?: unknown; counter?: unknown } } })?.meta?.id; + if (!(id?.replica instanceof Uint8Array)) return undefined; + if (typeof id.counter !== 'number') return undefined; + return `${bytesToHex(id.replica)}:${id.counter}`; +} + +/** + * Queue exact committed local writes for a single {@link SyncPeer} that is attached to one or more + * outbound transports. + * + * `queueOps` also notifies the low-level peer about the local update, so apps can report a local + * write once instead of separately waking live subscriptions and queueing remote upload. + * + * Apps that also use local-tab mesh transports should attach those transports directly to the + * low-level peer, but only register outbound upload targets with this controller. + */ +export function createOutboundSync( + options: OutboundSyncOptions, +): OutboundSync { + const targets = new Map>>(); + const pendingOps: Op[] = []; + const pendingOpKeys = new Set(); + let running = false; + let scheduled = false; + let closed = false; + + const emitStatus = () => { + options.onStatus?.(outboundSyncStatusSnapshot(targets, pendingOps, running, scheduled)); + }; + + const addPendingOps = (ops: readonly Op[]) => { + for (const op of ops) { + const key = options.opKey?.(op) ?? defaultOpKey(op); + if (key !== undefined) { + if (pendingOpKeys.has(key)) continue; + pendingOpKeys.add(key); + } + pendingOps.push(op); + } + }; + + const restorePendingOps = (ops: readonly Op[]) => { + if (ops.length === 0) return; + const existing = pendingOps.splice(0, pendingOps.length); + pendingOpKeys.clear(); + addPendingOps(ops); + addPendingOps(existing); + }; + + const takePendingOps = () => { + const ops = pendingOps.splice(0, pendingOps.length); + pendingOpKeys.clear(); + return ops; + }; + + const pushTimeoutMs = (targetId: string) => + typeof options.pushTimeoutMs === 'function' + ? options.pushTimeoutMs(targetId) + : options.pushTimeoutMs; + + const targetDetaches = new Map< + string, + { transport: DuplexTransport>; detach: () => void } + >(); + + const detachTarget = (targetId: string, expectedTransport?: DuplexTransport>) => { + const attached = targetDetaches.get(targetId); + if (!attached) return; + if (expectedTransport && attached.transport !== expectedTransport) return; + targetDetaches.delete(targetId); + try { + attached.detach(); + } catch { + // ignore + } + }; + + const addTarget = (targetId: string, transport: DuplexTransport>) => { + if (closed) return; + const attached = targetDetaches.get(targetId); + if (attached && attached.transport !== transport) detachTarget(targetId); + targets.set(targetId, transport); + emitStatus(); + if (pendingOps.length > 0) scheduleFlush(); + }; + + const removeTarget = (targetId: string, expectedTransport?: DuplexTransport>) => { + const transport = targets.get(targetId); + if (expectedTransport && transport !== expectedTransport) return; + targets.delete(targetId); + detachTarget(targetId, expectedTransport); + emitStatus(); + }; + + const clearTargets = () => { + targets.clear(); + for (const targetId of Array.from(targetDetaches.keys())) detachTarget(targetId); + emitStatus(); + }; + + const scheduleFlush = () => { + if (closed) return; + if (running) { + scheduled = true; + emitStatus(); + return; + } + if (scheduled) { + emitStatus(); + return; + } + scheduled = true; + emitStatus(); + queueMicrotask(() => { + void controller.flush(); + }); + }; + + const flush = async () => { + if (closed) return; + if (running) { + scheduled = true; + emitStatus(); + return; + } + if (!scheduled && pendingOps.length > 0) scheduled = true; + if (!scheduled) { + emitStatus(); + return; + } + + running = true; + emitStatus(); + try { + while (scheduled && !closed) { + scheduled = false; + if (options.isOnline && !options.isOnline()) { + emitStatus(); + return; + } + + const uploadTargets = Array.from(targets.entries()); + if (uploadTargets.length === 0) { + emitStatus(); + return; + } + + const ops = takePendingOps(); + if (ops.length === 0) { + emitStatus(); + continue; + } + + let failed = false; + for (const [targetId, transport] of uploadTargets) { + try { + await withTimeout( + options.localPeer.pushOps(transport, ops, options.pushOptions?.(targetId)), + pushTimeoutMs(targetId), + `outbound push with ${targetId.slice(0, 8)} timed out`, + ); + } catch (error) { + failed = true; + options.onError?.({ targetId, error }); + } + } + + if (failed) { + restorePendingOps(ops); + emitStatus(); + return; + } + + emitStatus(); + } + } finally { + running = false; + emitStatus(); + } + }; + + const controller: OutboundSync = { + get status() { + return outboundSyncStatusSnapshot(targets, pendingOps, running, scheduled); + }, + get pendingOpCount() { + return pendingOps.length; + }, + get targetCount() { + return targets.size; + }, + attachTarget: (targetId, transport) => { + if (closed) return () => {}; + const detach = options.localPeer.attach(transport); + let detached = false; + addTarget(targetId, transport); + targetDetaches.set(targetId, { transport, detach }); + return () => { + if (detached) return; + detached = true; + removeTarget(targetId, transport); + }; + }, + addTarget, + removeTarget, + clearTargets, + queueOps: (ops) => { + if (closed || ops.length === 0) return; + void options.localPeer.notifyLocalUpdate(ops); + addPendingOps(ops); + if (pendingOps.length > 0) scheduleFlush(); + }, + flush, + close: () => { + closed = true; + scheduled = false; + pendingOps.splice(0, pendingOps.length); + pendingOpKeys.clear(); + clearTargets(); + }, + }; + + emitStatus(); + return controller; +} diff --git a/packages/treecrdt-sync/src/create-sync-from-transport.ts b/packages/treecrdt-sync/src/create-sync-from-transport.ts index cabb8c20..5cc0f896 100644 --- a/packages/treecrdt-sync/src/create-sync-from-transport.ts +++ b/packages/treecrdt-sync/src/create-sync-from-transport.ts @@ -2,7 +2,6 @@ import type { Operation } from '@treecrdt/interface'; import { createTreecrdtSyncBackendFromClient } from '@treecrdt/sync-sqlite/backend'; import { SyncPeer, - deriveOpRefV0, type Filter, type SyncMessage, type SyncOnceOptions, @@ -75,8 +74,6 @@ export function createTreecrdtWebSocketSyncFromTransport( const peerOpts: SyncPeerOptions = { maxCodewords: 2_000_000, maxOpsPerBatch: DEFAULT_SYNC_ONCE.maxOpsPerBatch, - deriveOpRef: (op, ctx) => - deriveOpRefV0(ctx.docId, { replica: op.meta.id.replica, counter: op.meta.id.counter }), ...extraPeerOptions, ...(auth ? { auth } : {}), }; diff --git a/packages/treecrdt-sync/src/index.ts b/packages/treecrdt-sync/src/index.ts index 5bf9c0b7..949f79d7 100644 --- a/packages/treecrdt-sync/src/index.ts +++ b/packages/treecrdt-sync/src/index.ts @@ -1,10 +1,14 @@ export { connectTreecrdtWebSocketSync } from './connect.js'; +export { createOutboundSync } from './controller.js'; export { createTreecrdtWebSocketSyncFromTransport } from './create-sync-from-transport.js'; export type { ConnectTreecrdtWebSocketSyncOptions, CreateTreecrdtWebSocketSyncFromTransportOptions, MaterializationEvent, MaterializationListener, + OutboundSync, + OutboundSyncOptions, + OutboundSyncStatus, TreecrdtWebSocketSync, TreecrdtWebSocketSyncClient, } from './types.js'; diff --git a/packages/treecrdt-sync/src/types.ts b/packages/treecrdt-sync/src/types.ts index 9d7e752a..f42861e4 100644 --- a/packages/treecrdt-sync/src/types.ts +++ b/packages/treecrdt-sync/src/types.ts @@ -8,10 +8,14 @@ import type { TreecrdtSyncBackendClient } from '@treecrdt/sync-sqlite/backend'; import type { Filter, SyncAuth, + SyncMessage, SyncPeerOptions, SyncOnceOptions, + SyncPeer, + SyncPushOptions, SyncSubscribeOptions, } from '@treecrdt/sync-protocol'; +import type { DuplexTransport } from '@treecrdt/sync-protocol/transport'; import type { DiscoveryRouteCache } from '@treecrdt/discovery'; /** @@ -55,8 +59,8 @@ export type TreecrdtWebSocketSync = { startLive: (opts?: SyncSubscribeOptions) => Promise; stopLive: () => void; /** - * Upload local ops to the peer. For local→remote only; pass ops from your edit API. - * No-ops if empty. For full sync, use syncOnce instead. + * Upload local ops to the peer. For local-to-remote only; pass ops from your edit API, not from + * `onChange`. No-ops if empty. For full reconciliation, use `syncOnce`. */ pushLocalOps: (ops?: readonly Operation[]) => Promise; close: () => Promise; @@ -72,4 +76,55 @@ export type CreateTreecrdtWebSocketSyncFromTransportOptions = { onLiveError?: (error: unknown) => void; }; +export type OutboundSyncStatus = { + targetCount: number; + pendingOps: number; + running: boolean; + scheduled: boolean; +}; + +export type OutboundSyncOptions = { + localPeer: SyncPeer; + /** + * Stable key used to coalesce repeated local write hints before upload. + * + * Defaults to TreeCRDT `Operation.meta.id` when the queued op has the standard operation shape. + * Provide this only for custom op shapes or custom coalescing behavior. + */ + opKey?: (op: Op) => string; + /** + * Allows apps to keep queued work while offline instead of turning transient offline state into + * sync errors. + */ + isOnline?: () => boolean; + pushOptions?: (targetId: string) => SyncPushOptions | undefined; + pushTimeoutMs?: number | ((targetId: string) => number | undefined); + onError?: (ctx: { targetId: string; error: unknown }) => void; + onStatus?: (status: OutboundSyncStatus) => void; +}; + +export type OutboundSync = { + readonly status: OutboundSyncStatus; + readonly pendingOpCount: number; + readonly targetCount: number; + /** + * Attach a remote transport to `localPeer` and register it as an outbound upload target. The + * returned cleanup removes the target and detaches the transport. + */ + attachTarget: (targetId: string, transport: DuplexTransport>) => () => void; + /** + * Register an already-attached transport as an outbound upload target. + */ + addTarget: (targetId: string, transport: DuplexTransport>) => void; + removeTarget: (targetId: string) => void; + clearTargets: () => void; + /** + * Report exact committed local ops. Wakes live subscriptions on `localPeer` and queues the same + * ops for registered outbound upload targets. + */ + queueOps: (ops: readonly Op[]) => void; + flush: () => Promise; + close: () => void; +}; + export type { MaterializationEvent, MaterializationListener }; diff --git a/packages/treecrdt-sync/tests/controller.test.ts b/packages/treecrdt-sync/tests/controller.test.ts new file mode 100644 index 00000000..d4ea0e48 --- /dev/null +++ b/packages/treecrdt-sync/tests/controller.test.ts @@ -0,0 +1,191 @@ +import { expect, test, vi } from 'vitest'; +import type { Operation } from '@treecrdt/interface'; +import { makeOp, nodeIdFromInt } from '@treecrdt/benchmark'; +import type { SyncPeer } from '@treecrdt/sync-protocol'; + +import { createOutboundSync } from '../src/controller.js'; +import { ROOT, orderKeyFromPosition } from './test-helpers.js'; + +function replicaFromLabel(label: string): Uint8Array { + const encoded = new TextEncoder().encode(label); + if (encoded.length === 0) throw new Error('replica label must not be empty'); + const out = new Uint8Array(32); + for (let i = 0; i < out.length; i += 1) out[i] = encoded[i % encoded.length]!; + return out; +} + +const replicas = { a: replicaFromLabel('a') }; + +function makeInsertOp(counter = 1): Operation { + return makeOp(replicas.a, counter, counter, { + type: 'insert', + parent: ROOT, + node: nodeIdFromInt(counter), + orderKey: orderKeyFromPosition(counter - 1), + }); +} + +function createFakePeer(opts: { failPushes?: number } = {}) { + let failPushes = opts.failPushes ?? 0; + const pushed: Op[][] = []; + const detachTransport = vi.fn(); + const peer = { + attach: vi.fn(() => detachTransport), + notifyLocalUpdate: vi.fn(async () => {}), + pushOps: vi.fn(async (_transport: unknown, ops: readonly Op[]) => { + if (failPushes > 0) { + failPushes -= 1; + throw new Error('direct push failed'); + } + pushed.push([...ops]); + }), + } as unknown as SyncPeer; + return { detachTransport, peer, pushed }; +} + +test('outbound sync queues local ops until an outbound target is available', async () => { + const op = makeInsertOp(); + const { peer, pushed } = createFakePeer(); + const controller = createOutboundSync({ + localPeer: peer, + }); + + controller.queueOps([op, op]); + await controller.flush(); + + expect(peer.notifyLocalUpdate).toHaveBeenCalledWith([op, op]); + expect(controller.pendingOpCount).toBe(1); + expect(pushed).toHaveLength(0); + + controller.addTarget('remote:server', {} as any); + await controller.flush(); + + expect(controller.pendingOpCount).toBe(0); + expect(pushed).toEqual([[op]]); +}); + +test('outbound sync can attach and clean up a remote target', async () => { + const op = makeInsertOp(); + const transport = {} as any; + const { detachTransport, peer, pushed } = createFakePeer(); + const controller = createOutboundSync({ + localPeer: peer, + }); + + const detachTarget = controller.attachTarget('remote:server', transport); + + expect(peer.attach).toHaveBeenCalledWith(transport); + expect(controller.targetCount).toBe(1); + + controller.queueOps([op]); + await controller.flush(); + + expect(pushed).toEqual([[op]]); + + detachTarget(); + detachTarget(); + + expect(detachTransport).toHaveBeenCalledTimes(1); + expect(controller.targetCount).toBe(0); +}); + +test('outbound sync close detaches attached remote targets', () => { + const { detachTransport, peer } = createFakePeer(); + const controller = createOutboundSync({ + localPeer: peer, + }); + + controller.attachTarget('remote:server', {} as any); + controller.close(); + controller.close(); + + expect(detachTransport).toHaveBeenCalledTimes(1); + expect(controller.targetCount).toBe(0); +}); + +test('outbound sync keeps queued ops while offline', async () => { + const op = makeInsertOp(); + let online = false; + const { peer, pushed } = createFakePeer(); + const controller = createOutboundSync({ + localPeer: peer, + isOnline: () => online, + }); + controller.addTarget('remote:server', {} as any); + + controller.queueOps([op]); + await controller.flush(); + + expect(controller.pendingOpCount).toBe(1); + expect(pushed).toEqual([]); + + online = true; + await controller.flush(); + + expect(controller.pendingOpCount).toBe(0); + expect(pushed).toEqual([[op]]); +}); + +test('outbound sync keeps failed direct pushes queued', async () => { + const op = makeInsertOp(); + const { peer, pushed } = createFakePeer({ failPushes: 1 }); + const errors: unknown[] = []; + const controller = createOutboundSync({ + localPeer: peer, + onError: ({ error }) => errors.push(error), + }); + controller.addTarget('remote:server', {} as any); + + controller.queueOps([op]); + await controller.flush(); + + expect(controller.pendingOpCount).toBe(1); + expect(errors).toHaveLength(1); + expect(pushed).toHaveLength(0); + + await controller.flush(); + + expect(controller.pendingOpCount).toBe(0); + expect(pushed).toEqual([[op]]); +}); + +test('outbound sync accepts custom op keys for non-TreeCRDT op shapes', async () => { + type CustomOp = { id: string }; + const op: CustomOp = { id: 'local-write-1' }; + const { peer, pushed } = createFakePeer(); + const controller = createOutboundSync({ + localPeer: peer, + opKey: (next) => next.id, + }); + + controller.queueOps([op, { id: op.id }]); + await controller.flush(); + + expect(peer.notifyLocalUpdate).toHaveBeenCalledWith([op, { id: op.id }]); + expect(controller.pendingOpCount).toBe(1); + expect(pushed).toHaveLength(0); + + controller.addTarget('remote:server', {} as any); + await controller.flush(); + + expect(controller.pendingOpCount).toBe(0); + expect(pushed).toEqual([[op]]); +}); + +test('outbound sync ignores empty op batches', async () => { + const { peer, pushed } = createFakePeer(); + const statuses: number[] = []; + const controller = createOutboundSync({ + localPeer: peer, + onStatus: (status) => statuses.push(status.pendingOps), + }); + controller.addTarget('remote:server', {} as any); + + controller.queueOps([]); + await controller.flush(); + + expect(peer.notifyLocalUpdate).not.toHaveBeenCalled(); + expect(controller.pendingOpCount).toBe(0); + expect(pushed).toEqual([]); + expect(statuses.at(-1)).toBe(0); +}); diff --git a/packages/treecrdt-sync/tests/in-memory-sync.test.ts b/packages/treecrdt-sync/tests/in-memory-sync.test.ts index 94322300..f3ae2c92 100644 --- a/packages/treecrdt-sync/tests/in-memory-sync.test.ts +++ b/packages/treecrdt-sync/tests/in-memory-sync.test.ts @@ -3,7 +3,7 @@ import { bytesToHex, nodeIdToBytes16 } from '@treecrdt/interface/ids'; import { makeOp, maxLamport, nodeIdFromInt } from '@treecrdt/benchmark'; import { createTreecrdtSyncBackendFromClient } from '@treecrdt/sync-sqlite/backend'; import { treecrdtSyncV0ProtobufCodec } from '@treecrdt/sync-protocol/protobuf'; -import { SyncPeer, deriveOpRefV0 } from '@treecrdt/sync-protocol'; +import { SyncPeer } from '@treecrdt/sync-protocol'; import { createInMemoryDuplex, wrapDuplexTransportWithCodec, @@ -44,8 +44,6 @@ async function runSyncOnceInMemory( const peerB = new SyncPeer(backendB, { maxCodewords: 100_000, maxOpsPerBatch: 2_000, - deriveOpRef: (op, ctx) => - deriveOpRefV0(ctx.docId, { replica: op.meta.id.replica, counter: op.meta.id.counter }), }); const detachB = peerB.attach(transportB); @@ -126,8 +124,6 @@ test('syncOnce defaults split inbound applies into modest batches', async () => }); const peerB = new SyncPeer(backendB, { maxCodewords: 100_000, - deriveOpRef: (op, ctx) => - deriveOpRefV0(ctx.docId, { replica: op.meta.id.replica, counter: op.meta.id.counter }), }); const detachB = peerB.attach(transportB); const sync = createTreecrdtWebSocketSyncFromTransport(aClient, transportA, detachB); @@ -248,8 +244,6 @@ test('pushLocalOps uploads an insert to the remote peer (in-memory transport)', const peerB = new SyncPeer(backendB, { maxCodewords: 100_000, maxOpsPerBatch: 2_000, - deriveOpRef: (op, ctx) => - deriveOpRefV0(ctx.docId, { replica: op.meta.id.replica, counter: op.meta.id.counter }), }); const detachB = peerB.attach(transportB); const onCloseB = () => { @@ -301,8 +295,6 @@ test('pushLocalOps with no ops is a no-op (no syncOnce)', async () => { const peerB = new SyncPeer(backendB, { maxCodewords: 100_000, maxOpsPerBatch: 2_000, - deriveOpRef: (op, ctx) => - deriveOpRefV0(ctx.docId, { replica: op.meta.id.replica, counter: op.meta.id.counter }), }); const detachB = peerB.attach(transportB); const onCloseB = () => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b78fc2ba..9fa00ae9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -41,6 +41,9 @@ importers: '@treecrdt/interface': specifier: workspace:* version: link:../../packages/treecrdt-ts + '@treecrdt/sync': + specifier: workspace:* + version: link:../../packages/treecrdt-sync '@treecrdt/sync-protocol': specifier: workspace:* version: link:../../packages/sync-protocol/protocol