diff --git a/.changeset/fix-expired-shapes-self-healing.md b/.changeset/fix-expired-shapes-self-healing.md new file mode 100644 index 0000000000..f0367cdf9c --- /dev/null +++ b/.changeset/fix-expired-shapes-self-healing.md @@ -0,0 +1,5 @@ +--- +'@electric-sql/client': patch +--- + +Fix permanently stuck expired shape handles in localStorage by adding self-healing retry. When stale cache retries are exhausted (3 attempts with cache busters), the client now clears the expired entry from localStorage and retries once without the `expired_handle` parameter. Since the server never reuses handles (documented as SPEC.md S0), the fresh response will have a new handle and bypass stale detection. This prevents shapes from being permanently unloadable when a proxy strips cache-buster query parameters. diff --git a/examples/burn/assets/package.json b/examples/burn/assets/package.json index ec23e2a5e7..fd6a42dbd2 100644 --- a/examples/burn/assets/package.json +++ b/examples/burn/assets/package.json @@ -50,7 +50,7 @@ "eslint-plugin-prettier": "^5.4.0", "eslint-plugin-react-hooks": "^4.6.0", "eslint-plugin-react-refresh": "^0.4.6", - "prettier": "^3.2.4", + "prettier": "^3.6.2", "typescript": "^5.2.2", "vite": "^6.2.3" } diff --git a/examples/redis/package.json b/examples/redis/package.json index 223b5e1d96..67e96cda3e 100644 --- a/examples/redis/package.json +++ b/examples/redis/package.json @@ -40,7 +40,7 @@ "eslint-config-prettier": "^9.1.0", "eslint-plugin-prettier": "^5.1.3", "glob": "^10.3.10", - "prettier": "^3.3.2", + "prettier": "^3.6.2", "shx": "^0.3.4", "tsup": "^8.0.1", "tsx": "^4.19.1", diff --git a/package.json b/package.json index 54a2afaa2c..4ad47aa603 100644 --- a/package.json +++ b/package.json @@ -33,9 +33,9 @@ "lint-staged": { "*.{js,jsx,ts,tsx}": [ "eslint --fix", - "prettier --write" + "node_modules/.bin/prettier --write" ], - "*.{json,css,md,yml,yaml}": "prettier --write" + "*.{json,css,md,yml,yaml}": "node_modules/.bin/prettier --write" }, "pnpm": { "patchedDependencies": { diff --git a/packages/experimental/package.json b/packages/experimental/package.json index 20efc0dbdb..059ee8f874 100644 --- a/packages/experimental/package.json +++ b/packages/experimental/package.json @@ -19,7 +19,7 @@ "eslint-plugin-prettier": "^5.1.3", "glob": "^10.3.10", "pg": "^8.12.0", - "prettier": "^3.3.2", + "prettier": "^3.6.2", "shx": "^0.3.4", "tsup": "^8.0.1", "typescript": "^5.5.2", diff --git a/packages/react-hooks/package.json b/packages/react-hooks/package.json index 5c403a95d8..b92c8e4102 100644 --- a/packages/react-hooks/package.json +++ b/packages/react-hooks/package.json @@ -26,7 +26,7 @@ "glob": "^10.3.10", "jsdom": "^25.0.0", "pg": "^8.12.0", - "prettier": "^3.3.2", + "prettier": "^3.6.2", "react": "^18.3.1", "react-dom": "^18.3.1", "shx": "^0.3.4", diff --git a/packages/start/package.json b/packages/start/package.json index 105742241e..0eb1f2320e 100644 --- a/packages/start/package.json +++ b/packages/start/package.json @@ -35,7 +35,7 @@ "eslint": "^8.57.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-prettier": "^5.1.3", - "prettier": "^3.3.2", + "prettier": "^3.6.2", "shx": "^0.3.4", "tsup": "^8.0.1", "typescript": "^5.5.2", diff --git a/packages/typescript-client/SPEC.md b/packages/typescript-client/SPEC.md index 83dc2b83c8..346187b26a 100644 --- a/packages/typescript-client/SPEC.md +++ b/packages/typescript-client/SPEC.md @@ -66,6 +66,26 @@ Any ──markMustRefetch─► Initial (offset = -1) - `response` on Paused delegates to `previousState`, preserving the Paused wrapper for `accepted` and `stale-retry` transitions; `ignored` returns `this` - `response`/`messages`/`sseClose` on Error return `this` (ignored) +## Server Assumptions + +Properties of the sync service that the client state machine depends on. + +### S0: Shape handles are unique and never reused + +The server generates handles as `{phash2_hash}-{microsecond_timestamp}`. Uniqueness +is enforced by monotonic timestamps, a SQLite `UNIQUE INDEX` on the handle column, +and ETS `insert_new` checks. Even after server restarts, old handles persist in +SQLite and new ones receive fresh timestamps, so collisions cannot occur. + +**Implication for expired shapes cache**: Once a handle is marked expired (after a +409 response), the server will never issue that handle again. If a response contains +an expired handle, it must be coming from a caching layer (browser HTTP cache, +CDN, or proxy) — not from the server itself. + +**Source**: `packages/sync-service/lib/electric/shapes/shape.ex` (`generate_id/1`), +`packages/sync-service/lib/electric/shape_cache/shape_status/shape_db/connection.ex` +(`shapes_handle_idx`). + ## Invariants Properties that must hold after every state transition. Checked automatically by @@ -346,25 +366,26 @@ This is enforced by the path-specific guards listed below. Live requests Six sites in `client.ts` recurse or loop to issue a new fetch: -| # | Site | Line | Trigger | URL changes because | Guard | -| --- | --------------------------------------- | ---- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------------------- | -| L1 | `#requestShape` → `#requestShape` | 940 | Normal completion after `#fetchShape()` | Offset advances from response headers | `#checkFastLoop` (non-live) | -| L2 | `#requestShape` catch → `#requestShape` | 874 | Abort with `FORCE_DISCONNECT_AND_REFRESH` or `SYSTEM_WAKE` | `isRefreshing` flag changes `canLongPoll`, affecting `live` param | Abort signals are discrete events | -| L3 | `#requestShape` catch → `#requestShape` | 886 | `StaleCacheError` thrown by `#onInitialResponse` | `StaleRetryState` adds `cache_buster` param | `maxStaleCacheRetries` counter in state machine | -| L4 | `#requestShape` catch → `#requestShape` | 924 | HTTP 409 (shape rotation) | `#reset()` sets offset=-1 + new handle; or request-scoped cache buster if no handle | New handle from 409 response or unique retry URL | -| L5 | `#start` catch → `#start` | 782 | Exception + `onError` returns retry opts | Params/headers merged from `retryOpts` | `#maxConsecutiveErrorRetries` (50) | -| L6 | `fetchSnapshot` catch → `fetchSnapshot` | 1975 | HTTP 409 on snapshot fetch | New handle via `withHandle()`; or local retry cache buster if same/no handle | `#maxSnapshotRetries` (5) + cache buster on same handle | +| # | Site | Line | Trigger | URL changes because | Guard | +| --- | --------------------------------------- | ---- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | +| L1 | `#requestShape` → `#requestShape` | 940 | Normal completion after `#fetchShape()` | Offset advances from response headers | `#checkFastLoop` (non-live) | +| L2 | `#requestShape` catch → `#requestShape` | 874 | Abort with `FORCE_DISCONNECT_AND_REFRESH` or `SYSTEM_WAKE` | `isRefreshing` flag changes `canLongPoll`, affecting `live` param | Abort signals are discrete events | +| L3 | `#requestShape` catch → `#requestShape` | 886 | `StaleCacheError` thrown by `#onInitialResponse` | `StaleRetryState` adds `cache_buster` param; after max retries, self-healing clears expired entry + resets stream | `maxStaleCacheRetries` counter + `#expiredShapeRecoveryKey` (once per shape) | +| L4 | `#requestShape` catch → `#requestShape` | 924 | HTTP 409 (shape rotation) | `#reset()` sets offset=-1 + new handle; or request-scoped cache buster if no handle | New handle from 409 response or unique retry URL | +| L5 | `#start` catch → `#start` | 782 | Exception + `onError` returns retry opts | Params/headers merged from `retryOpts` | `#maxConsecutiveErrorRetries` (50) | +| L6 | `fetchSnapshot` catch → `fetchSnapshot` | 1975 | HTTP 409 on snapshot fetch | New handle via `withHandle()`; or local retry cache buster if same/no handle | `#maxSnapshotRetries` (5) + cache buster on same handle | ### Guard mechanisms -| Guard | Scope | How it works | -| ----------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | -| `#checkFastLoop` | Non-live `#requestShape` only | Detects N requests at same offset within a time window. First: clears caches + resets. Persistent: exponential backoff → throws FetchError(502). | -| `maxStaleCacheRetries` | Stale response path (L3) | State machine counts stale retries. Throws FetchError(502) after 3 consecutive stale responses. | -| `#maxSnapshotRetries` | Snapshot 409 path (L6) | Counts consecutive snapshot 409s. Adds cache buster when handle unchanged. Throws FetchError(502) after 5. | -| `#maxConsecutiveErrorRetries` | `#start` onError retry (L5) | Counts consecutive error retries. Sends error to subscribers and tears down after 50. Reset on successful message batch. | -| Pause lock | `#requestShape` entry | Returns immediately if paused. Prevents fetches during snapshots. | -| Up-to-date exit | `#requestShape` entry | Returns if `!subscribe` and `isUpToDate`. Breaks loop for one-shot syncs. | +| Guard | Scope | How it works | +| ----------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `#checkFastLoop` | Non-live `#requestShape` only | Detects N requests at same offset within a time window. First: clears caches + resets. Persistent: exponential backoff → throws FetchError(502). | +| `maxStaleCacheRetries` | Stale response path (L3) | State machine counts stale retries. After 3 consecutive stale responses, clears expired entry and attempts one self-healing retry. Throws FetchError(502) if self-healing also fails. | +| `#expiredShapeRecoveryKey` | Self-healing (L3 extension) | Records shape key after first self-healing attempt. Second exhaustion on same key skips self-healing → FetchError(502). Cleared on up-to-date. | +| `#maxSnapshotRetries` | Snapshot 409 path (L6) | Counts consecutive snapshot 409s. Adds cache buster when handle unchanged. Throws FetchError(502) after 5. | +| `#maxConsecutiveErrorRetries` | `#start` onError retry (L5) | Counts consecutive error retries. Sends error to subscribers and tears down after 50. Reset on successful message batch. | +| Pause lock | `#requestShape` entry | Returns immediately if paused. Prevents fetches during snapshots. | +| Up-to-date exit | `#requestShape` entry | Returns if `!subscribe` and `isUpToDate`. Breaks loop for one-shot syncs. | ### Coverage gaps diff --git a/packages/typescript-client/package.json b/packages/typescript-client/package.json index e8f2a56083..6ba2ddee30 100644 --- a/packages/typescript-client/package.json +++ b/packages/typescript-client/package.json @@ -25,7 +25,7 @@ "glob": "^10.3.10", "jsdom": "^26.1.0", "pg": "^8.12.0", - "prettier": "^3.3.2", + "prettier": "^3.6.2", "shx": "^0.3.4", "tsup": "^8.0.1", "typescript": "^5.5.2", diff --git a/packages/typescript-client/src/client.ts b/packages/typescript-client/src/client.ts index fcc7590eb6..1112a95f2a 100644 --- a/packages/typescript-client/src/client.ts +++ b/packages/typescript-client/src/client.ts @@ -623,6 +623,8 @@ export class ShapeStream = Row> #fastLoopMaxCount = 5 #pendingRequestShapeCacheBuster?: string #maxSnapshotRetries = 5 + #expiredShapeRecoveryKey: string | null = null + #pendingSelfHealCheck: { shapeKey: string; staleHandle: string } | null = null #consecutiveErrorRetries = 0 #maxConsecutiveErrorRetries = 50 @@ -914,10 +916,11 @@ export class ShapeStream = Row> } if (e instanceof StaleCacheError) { - // Received a stale cached response from CDN with an expired handle. - // The #staleCacheBuster has been set in #onInitialResponse, so retry - // the request which will include a random cache buster to bypass the - // misconfigured CDN cache. + // Two paths throw StaleCacheError: + // 1. Normal stale-retry: response handle matched expired handle, + // #staleCacheBuster set to bypass CDN cache on next request. + // 2. Self-healing: stale retries exhausted, expired entry cleared, + // stream reset — retry without expired_handle param. return this.#requestShape() } @@ -1248,6 +1251,25 @@ export class ShapeStream = Row> ? expiredShapesCache.getExpiredHandle(shapeKey) : null + // If this response is the first one after a self-healing retry, check + // whether the proxy/CDN returned the exact handle we just marked expired. + // If so, the client is about to accept stale data silently — loudly warn + // so operators can detect and fix the proxy misconfiguration. + if (this.#pendingSelfHealCheck) { + const { shapeKey: healedKey, staleHandle } = this.#pendingSelfHealCheck + this.#pendingSelfHealCheck = null + if (shapeKey === healedKey && shapeHandle === staleHandle) { + console.warn( + `[Electric] Self-healing retry received the same handle "${staleHandle}" that was just marked expired. ` + + `This means your proxy/CDN is serving a stale cached response and ignoring cache-buster query params. ` + + `The client will proceed with this stale data to avoid a permanent failure, but it may be out of date until the cache refreshes. ` + + `Fix: configure your proxy/CDN to include all query parameters (especially 'handle' and 'offset') in its cache key. ` + + `For more information visit the troubleshooting guide: ${TROUBLESHOOTING_URL}`, + new Error(`stack trace`) + ) + } + } + const transition = this.#syncState.handleResponseMetadata({ status, responseHandle: shapeHandle, @@ -1262,6 +1284,12 @@ export class ShapeStream = Row> this.#syncState = transition.state + // Clear recovery guard on 204 (no-content), since the empty body means + // #onMessages won't run to clear it via the up-to-date path. + if (status === 204) { + this.#expiredShapeRecoveryKey = null + } + if (transition.action === `accepted` && status === 204) { this.#consecutiveErrorRetries = 0 } @@ -1270,6 +1298,38 @@ export class ShapeStream = Row> // Cancel the response body to release the connection before retrying. await response.body?.cancel() if (transition.exceededMaxRetries) { + if (shapeKey) { + // Clear the expired entry — keeping it only poisons future sessions. + expiredShapesCache.delete(shapeKey) + + // Try one self-healing retry per shape: reset the stream and + // retry without the expired_handle param. Since handles are never + // reused (see SPEC.md S0), the fresh response will have a new + // handle and won't trigger stale detection. + if (this.#expiredShapeRecoveryKey !== shapeKey) { + console.warn( + `[Electric] Stale cache retries exhausted (${this.#maxStaleCacheRetries} attempts). ` + + `Clearing expired handle entry and attempting self-healing retry without the expired_handle parameter. ` + + `For more information visit the troubleshooting guide: ${TROUBLESHOOTING_URL}`, + new Error(`stack trace`) + ) + this.#expiredShapeRecoveryKey = shapeKey + // Arm a post-self-heal check: if the next response comes back + // with the same handle we just marked expired, the proxy/CDN is + // still serving stale data and we'll warn loudly instead of + // accepting it silently. + if (shapeHandle) { + this.#pendingSelfHealCheck = { + shapeKey, + staleHandle: shapeHandle, + } + } + this.#reset() + throw new StaleCacheError( + `Expired handle entry evicted for self-healing retry` + ) + } + } throw new FetchError( 502, undefined, @@ -1351,6 +1411,7 @@ export class ShapeStream = Row> shapeKey, this.#syncState.liveCacheBuster ) + this.#expiredShapeRecoveryKey = null } } @@ -1770,9 +1831,10 @@ export class ShapeStream = Row> #reset(handle?: string) { this.#syncState = this.#syncState.markMustRefetch(handle) this.#connected = false - // releaseAllMatching intentionally doesn't fire onReleased — it's called - // from within the running stream loop (#requestShape's 409 handler), so - // the stream is already active and doesn't need a resume signal. + // releaseAllMatching intentionally doesn't fire onReleased — every caller + // (#requestShape's 409 handler, #checkFastLoop, and stale-retry + // self-healing in #onInitialResponse) runs inside the active stream loop, + // so the stream is already active and doesn't need a resume signal. this.#pauseLock.releaseAllMatching(`snapshot`) } diff --git a/packages/typescript-client/test/expired-shapes-cache.test.ts b/packages/typescript-client/test/expired-shapes-cache.test.ts index 5510556652..44a32c4edb 100644 --- a/packages/typescript-client/test/expired-shapes-cache.test.ts +++ b/packages/typescript-client/test/expired-shapes-cache.test.ts @@ -1,4 +1,12 @@ -import { beforeEach, afterEach, describe, expect, it, vi } from 'vitest' +import { + beforeEach, + afterEach, + describe, + expect, + it, + vi, + type MockInstance, +} from 'vitest' import { ShapeStream } from '../src' import { ExpiredShapesCache, @@ -7,6 +15,8 @@ import { import { CACHE_BUSTER_QUERY_PARAM, EXPIRED_HANDLE_QUERY_PARAM, + OFFSET_QUERY_PARAM, + SHAPE_HANDLE_QUERY_PARAM, } from '../src/constants' function waitForFetch(stream: ShapeStream): Promise { @@ -28,6 +38,10 @@ describe(`ExpiredShapesCache`, () => { (input: RequestInfo | URL, init?: RequestInit) => Promise > > + // Many tests in this suite intentionally simulate stale-cache scenarios + // that produce expected console.warn output. Silence by default; tests + // that need to assert on warnings can read warnSpy.mock.calls directly. + let warnSpy: MockInstance beforeEach(() => { localStorage.clear() @@ -39,9 +53,13 @@ describe(`ExpiredShapesCache`, () => { (input: RequestInfo | URL, init?: RequestInit) => Promise >() vi.clearAllMocks() + warnSpy = vi.spyOn(console, `warn`).mockImplementation(() => {}) }) - afterEach(() => aborter.abort()) + afterEach(() => { + aborter.abort() + warnSpy.mockRestore() + }) it(`should mark shapes as expired and check expiration status`, () => { const shapeUrl1 = `https://example.com/v1/shape?table=test1` @@ -515,9 +533,11 @@ describe(`ExpiredShapesCache`, () => { expect(fetchCount).toBeGreaterThanOrEqual(3) }) - it(`should throw error after max stale cache retries exceeded`, async () => { - // This test verifies that the client doesn't retry forever when CDN - // continues serving stale responses despite cache buster + it(`should self-heal after stale cache retries by clearing expired entry and retrying`, async () => { + // This test verifies the full stale-retry + self-healing flow: + // 1. CDN serves stale response with expired handle (3 retries with cache busters) + // 2. After retries exhaust, expired entry is cleared and self-healing retry fires + // 3. Self-healing request has no expired_handle param, gets fresh response const expectedShapeUrl = `${shapeUrl}?table=test` const staleHandle = `persistent-stale-handle` @@ -529,9 +549,29 @@ describe(`ExpiredShapesCache`, () => { fetchMock.mockImplementation((input: RequestInfo | URL) => { requestCount++ - capturedUrls.push(input.toString()) + const urlStr = input.toString() + capturedUrls.push(urlStr) - // Always return stale response - simulating severely misconfigured CDN + const url = new URL(urlStr) + if (!url.searchParams.has(EXPIRED_HANDLE_QUERY_PARAM)) { + // Self-healing retry: no expired_handle param, return fresh response + return Promise.resolve( + new Response( + JSON.stringify([{ headers: { control: `up-to-date` } }]), + { + status: 200, + headers: { + 'electric-handle': `fresh-handle`, + 'electric-offset': `0_0`, + 'electric-schema': `{"id":"int4"}`, + 'electric-up-to-date': ``, + }, + } + ) + ) + } + + // Stale response while expired_handle param is present (CDN serving old data) return Promise.resolve( new Response(JSON.stringify([{ value: { id: 1 } }]), { status: 200, @@ -561,22 +601,302 @@ describe(`ExpiredShapesCache`, () => { // Subscribe to trigger fetching stream.subscribe(() => {}) - // Wait for retries to exhaust (should be fast since no real network) - await new Promise((resolve) => setTimeout(resolve, 100)) + // Wait for retries + self-healing + await new Promise((resolve) => setTimeout(resolve, 200)) + + // Should have made initial request + 3 stale retries + 1 self-healing = 5 + expect(requestCount).toBe(5) - // Should have made initial request + 3 retries = 4 total requests - expect(requestCount).toBe(4) + // First 4 requests should have expired_handle param + for (let i = 0; i < 4; i++) { + const url = new URL(capturedUrls[i]) + expect(url.searchParams.get(EXPIRED_HANDLE_QUERY_PARAM)).toBe(staleHandle) + } - // Verify each retry after the first includes cache buster - for (let i = 1; i < capturedUrls.length; i++) { + // Retries (requests 2-4) should include cache buster + for (let i = 1; i < 4; i++) { const url = new URL(capturedUrls[i]) expect(url.searchParams.has(CACHE_BUSTER_QUERY_PARAM)).toBe(true) } - // Should have thrown an error after max retries - expect(caughtError).not.toBe(null) - expect(caughtError!.message).toContain(`stale cached responses`) - expect(caughtError!.message).toContain(`3 retry attempts`) + // Self-healing request (5th) should be a fresh start: no expired_handle, + // no handle, and offset reset to -1 + const selfHealingUrl = new URL(capturedUrls[4]) + expect(selfHealingUrl.searchParams.has(EXPIRED_HANDLE_QUERY_PARAM)).toBe( + false + ) + expect(selfHealingUrl.searchParams.has(SHAPE_HANDLE_QUERY_PARAM)).toBe( + false + ) + expect(selfHealingUrl.searchParams.get(OFFSET_QUERY_PARAM)).toBe(`-1`) + + // Expired entry should have been cleared + expect(expiredShapesCache.getExpiredHandle(expectedShapeUrl)).toBeNull() + + // No error — self-healing succeeded + expect(caughtError).toBe(null) + }) + + it(`should clear expired entry and attempt self-healing even when CDN always returns stale handle`, async () => { + // When CDN caches by path only (ignoring all query params), even the + // self-healing retry gets the expired handle back. Verify that + // self-healing still fires and the expired entry is cleared. + // (The eventual fast-loop error is tested separately in stream.test.ts) + const expectedShapeUrl = `${shapeUrl}?table=test` + const staleHandle = `persistent-stale-handle` + + expiredShapesCache.markExpired(expectedShapeUrl, staleHandle) + + const capturedUrls: string[] = [] + + fetchMock.mockImplementation((input: RequestInfo | URL) => { + capturedUrls.push(input.toString()) + + // CDN always returns stale handle regardless of query params + return Promise.resolve( + new Response(JSON.stringify([{ value: { id: 1 } }]), { + status: 200, + headers: { + 'electric-handle': staleHandle, + 'electric-offset': `0_0`, + 'electric-schema': `{"id":"int4"}`, + 'electric-cursor': `cursor-1`, + }, + }) + ) + }) + + const stream = new ShapeStream({ + url: shapeUrl, + params: { table: `test` }, + signal: aborter.signal, + fetchClient: fetchMock, + subscribe: false, + // Responses never reach up-to-date, so the stream loops forever until + // the fast-loop detector or abort fires. Swallow those terminal errors + // so they don't surface as unhandled rejections in CI. + onError: () => {}, + }) + + stream.subscribe(() => {}) + + // Poll until self-healing fires (a request without expired_handle), then + // abort immediately so the stream doesn't keep looping in the background. + const deadline = Date.now() + 2000 + while ( + !capturedUrls.some( + (url) => !new URL(url).searchParams.has(EXPIRED_HANDLE_QUERY_PARAM) + ) && + Date.now() < deadline + ) { + await new Promise((resolve) => setTimeout(resolve, 10)) + } + aborter.abort() + + // Self-healing should have been attempted (a request without expired_handle) + const selfHealingFired = capturedUrls.some( + (url) => !new URL(url).searchParams.has(EXPIRED_HANDLE_QUERY_PARAM) + ) + expect(selfHealingFired).toBe(true) + + // Expired entry should be cleared + expect(expiredShapesCache.getExpiredHandle(expectedShapeUrl)).toBeNull() + }) + + it(`self-healing accepts stale data when proxy always serves expired handle (by design) but warns loudly`, async () => { + // Finding 1 from external review: after self-healing clears the expired + // entry, if the proxy serves the same stale response with up-to-date, + // the client accepts it. This is by design — stale data is better than + // permanent 502 for one-shot streams — but the client MUST warn loudly + // so operators can detect and fix the proxy misconfiguration. + const expectedShapeUrl = `${shapeUrl}?table=test` + const staleHandle = `expired-handle` + + expiredShapesCache.markExpired(expectedShapeUrl, staleHandle) + + let requestCount = 0 + + fetchMock.mockImplementation((input: RequestInfo | URL) => { + requestCount++ + const url = new URL(input.toString()) + + if (!url.searchParams.has(EXPIRED_HANDLE_QUERY_PARAM)) { + // Self-healing request: proxy returns same stale handle + up-to-date + return Promise.resolve( + new Response( + JSON.stringify([ + { value: { id: 1 } }, + { headers: { control: `up-to-date` } }, + ]), + { + status: 200, + headers: { + 'electric-handle': staleHandle, // same expired handle! + 'electric-offset': `0_0`, + 'electric-schema': `{"id":"int4"}`, + 'electric-up-to-date': ``, + }, + } + ) + ) + } + + // Stale response during retry phase + return Promise.resolve( + new Response(JSON.stringify([{ value: { id: 1 } }]), { + status: 200, + headers: { + 'electric-handle': staleHandle, + 'electric-offset': `0_0`, + 'electric-schema': `{"id":"int4"}`, + 'electric-cursor': `cursor-1`, + }, + }) + ) + }) + + let caughtError: Error | null = null + const stream = new ShapeStream({ + url: shapeUrl, + params: { table: `test` }, + signal: aborter.signal, + fetchClient: fetchMock, + subscribe: false, + onError: (error) => { + caughtError = error + }, + }) + + stream.subscribe(() => {}) + await new Promise((resolve) => setTimeout(resolve, 300)) + + // Self-healing accepts stale data — no error, stream reached up-to-date + expect(caughtError).toBe(null) + expect(stream.isUpToDate).toBe(true) + // 4 stale retries + 1 self-healing = 5 + expect(requestCount).toBe(5) + + // CRITICAL: silent acceptance of stale data would be a bug. The client + // MUST emit a warning naming the stale handle so operators can detect + // and fix the proxy misconfiguration. Without this signal, apps would + // sit on stale data with no way to know. + const staleAcceptWarning = warnSpy.mock.calls.find( + (args) => + typeof args[0] === `string` && + args[0].includes( + `Self-healing retry received the same handle "${staleHandle}"` + ) + ) + expect(staleAcceptWarning).toBeTruthy() + }) + + it(`should clear recovery guard after 204 so self-healing works again`, async () => { + // Finding 2 from external review: 204 response transitions directly to + // LiveState without going through #onMessages, so #expiredShapeRecoveryKey + // is never cleared. With subscribe:true, the stream continues after 204. + // When the live long-poll hits a new stale cache entry, self-healing must + // be able to fire again (recovery guard was cleared by the 204 path). + const expectedShapeUrl = `${shapeUrl}?table=test` + + expiredShapesCache.markExpired(expectedShapeUrl, `stale-handle-1`) + + let selfHealCount = 0 + let healingDone = false + + fetchMock.mockImplementation((input: RequestInfo | URL) => { + const url = new URL(input.toString()) + const hasExpiredHandle = url.searchParams.has(EXPIRED_HANDLE_QUERY_PARAM) + + // Self-healing requests: no expired_handle param, healing not yet done + if (!hasExpiredHandle && !healingDone) { + selfHealCount++ + + if (selfHealCount === 1) { + // First self-healing: return 204 (deprecated path) + // Mark a new handle as expired for phase 2 + expiredShapesCache.markExpired(expectedShapeUrl, `stale-handle-2`) + return Promise.resolve( + new Response(null, { + status: 204, + headers: { + 'electric-handle': `fresh-handle-1`, + 'electric-offset': `0_0`, + 'electric-schema': `{"id":"int4"}`, + }, + }) + ) + } + + // Second self-healing: return 200 with up-to-date + healingDone = true + return Promise.resolve( + new Response( + JSON.stringify([{ headers: { control: `up-to-date` } }]), + { + status: 200, + headers: { + 'electric-handle': `fresh-handle-2`, + 'electric-offset': `0_0`, + 'electric-schema': `{"id":"int4"}`, + 'electric-cursor': `cursor-2`, + 'electric-up-to-date': ``, + }, + } + ) + ) + } + + // Post-healing live requests or stale responses with expired_handle + if (hasExpiredHandle) { + // Stale response: echo back the expired handle to trigger stale detection + const staleHandle = url.searchParams.get(EXPIRED_HANDLE_QUERY_PARAM)! + return Promise.resolve( + new Response(JSON.stringify([{ value: { id: 1 } }]), { + status: 200, + headers: { + 'electric-handle': staleHandle, + 'electric-offset': `0_0`, + 'electric-schema': `{"id":"int4"}`, + 'electric-cursor': `cursor-1`, + }, + }) + ) + } + + // Post-healing live long-poll: abort to stop the stream + aborter.abort() + return Promise.resolve( + new Response(`[]`, { + status: 200, + headers: { + 'electric-handle': `fresh-handle-2`, + 'electric-offset': `0_0`, + 'electric-schema': `{"id":"int4"}`, + 'electric-cursor': `cursor-2`, + }, + }) + ) + }) + + const stream = new ShapeStream({ + url: shapeUrl, + params: { table: `test` }, + signal: aborter.signal, + fetchClient: fetchMock, + subscribe: true, + // On slower CI runners the fast-loop detector can fire before the + // test's abort propagates. Swallow it so it doesn't surface as an + // unhandled rejection — we only care about the selfHealCount signal. + onError: () => {}, + }) + + stream.subscribe(() => {}) + await new Promise((resolve) => setTimeout(resolve, 1000)) + + // The recovery guard was cleared after the 204, so the second + // self-healing attempt fires. This is the precise signal — if the + // guard were stuck, the code throws 502 before incrementing. + expect(selfHealCount).toBe(2) }) it(`client should retry with cache buster when local handle matches expired handle`, async () => { @@ -655,22 +975,39 @@ describe(`ExpiredShapesCache`, () => { // 3. First fetch returns a stale response with data messages // 4. checkStaleResponse enters stale-retry (adds cache buster) // 5. Client retries with cache buster to bypass CDN - // 6. After max retries, client errors (CDN keeps serving stale) + // 6. After max retries, self-healing clears the entry and retries + // 7. Self-healing request gets fresh response and succeeds const expectedShapeUrl = `${shapeUrl}?table=test` expiredShapesCache.markExpired(expectedShapeUrl, `stale-handle`) - let fetchCount = 0 const capturedUrls: string[] = [] - const errors: Error[] = [] fetchMock.mockImplementation( (input: RequestInfo | URL, _init?: RequestInit) => { - fetchCount++ - capturedUrls.push(input.toString()) + const urlStr = input.toString() + capturedUrls.push(urlStr) - if (fetchCount >= 10) { - aborter.abort() + const url = new URL(urlStr) + if (!url.searchParams.has(EXPIRED_HANDLE_QUERY_PARAM)) { + // Self-healing retry: return fresh response + return Promise.resolve( + new Response( + JSON.stringify([{ headers: { control: `up-to-date` } }]), + { + status: 200, + headers: { + 'electric-handle': `fresh-handle`, + 'electric-offset': `0_0`, + 'electric-schema': JSON.stringify({ + id: { type: `text` }, + name: { type: `text` }, + }), + 'electric-up-to-date': ``, + }, + } + ) + ) } return Promise.resolve( @@ -698,15 +1035,11 @@ describe(`ExpiredShapesCache`, () => { signal: aborter.signal, fetchClient: fetchMock, subscribe: false, - onError: (error) => { - errors.push(error) - return - }, }) stream.subscribe(() => {}) - await new Promise((resolve) => setTimeout(resolve, 500)) + await new Promise((resolve) => setTimeout(resolve, 200)) // Should have used cache busters to try to bypass stale CDN const usedCacheBuster = capturedUrls.some((url) => @@ -714,8 +1047,8 @@ describe(`ExpiredShapesCache`, () => { ) expect(usedCacheBuster).toBe(true) - // Should eventually error after max stale retries - expect(errors.length).toBeGreaterThan(0) + // Expired entry should have been cleared by self-healing + expect(expiredShapesCache.getExpiredHandle(expectedShapeUrl)).toBeNull() }) it(`should use cache buster instead of handle mutation on 409 without handle header`, async () => { diff --git a/packages/y-electric/package.json b/packages/y-electric/package.json index d22f69dc88..3400bc1a6c 100644 --- a/packages/y-electric/package.json +++ b/packages/y-electric/package.json @@ -22,7 +22,7 @@ "eslint": "^8.57.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-prettier": "^5.1.3", - "prettier": "^3.3.2", + "prettier": "^3.6.2", "shx": "^0.3.4", "tsup": "^8.0.1", "typescript": "^5.5.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5c20b3462e..c9770cd354 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -80,6 +80,8 @@ importers: examples/bash: {} + examples/basic-example: {} + examples/burn: dependencies: camelcase: @@ -674,6 +676,8 @@ importers: specifier: ^5.3.4 version: 5.4.10(@types/node@22.19.1)(lightningcss@1.30.1)(terser@5.44.0) + examples/nextjs-example: {} + examples/phoenix-liveview: dependencies: camelcase: @@ -835,13 +839,13 @@ importers: version: 9.1.0(eslint@8.57.1) eslint-plugin-prettier: specifier: ^5.1.3 - version: 5.2.1(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.3.3) + version: 5.2.1(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.6.2) glob: specifier: ^10.3.10 version: 10.4.5 prettier: - specifier: ^3.3.2 - version: 3.3.3 + specifier: ^3.6.2 + version: 3.6.2 shx: specifier: ^0.3.4 version: 0.3.4 @@ -859,6 +863,8 @@ importers: specifier: ^4.18.1 version: 4.24.4 + examples/redis-sync: {} + examples/remix: dependencies: '@electric-sql/client': @@ -938,6 +944,8 @@ importers: specifier: ^5.3.4 version: 5.4.10(@types/node@22.19.1)(lightningcss@1.30.1)(terser@5.44.0) + examples/remix-basic: {} + examples/tanstack: dependencies: '@electric-sql/client': @@ -1253,6 +1261,8 @@ importers: specifier: ^5.1.0 version: 5.1.0 + examples/tanstack-example: {} + examples/todo-app: dependencies: '@electric-sql/client': @@ -1558,7 +1568,7 @@ importers: version: 9.1.0(eslint@8.57.1) eslint-plugin-prettier: specifier: ^5.1.3 - version: 5.2.1(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.3.3) + version: 5.2.1(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.6.2) glob: specifier: ^10.3.10 version: 10.4.5 @@ -1566,8 +1576,8 @@ importers: specifier: ^8.12.0 version: 8.13.1 prettier: - specifier: ^3.3.2 - version: 3.3.3 + specifier: ^3.6.2 + version: 3.6.2 shx: specifier: ^0.3.4 version: 0.3.4 @@ -1632,7 +1642,7 @@ importers: version: 9.1.0(eslint@8.57.1) eslint-plugin-prettier: specifier: ^5.1.3 - version: 5.2.1(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.3.3) + version: 5.2.1(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.6.2) glob: specifier: ^10.3.10 version: 10.4.5 @@ -1643,8 +1653,8 @@ importers: specifier: ^8.12.0 version: 8.13.1 prettier: - specifier: ^3.3.2 - version: 3.3.3 + specifier: ^3.6.2 + version: 3.6.2 react: specifier: ^18.3.1 version: 18.3.1 @@ -1691,7 +1701,7 @@ importers: specifier: ^5.1.3 version: 5.5.3(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.6.2) prettier: - specifier: ^3.3.2 + specifier: ^3.6.2 version: 3.6.2 shx: specifier: ^0.3.4 @@ -1746,7 +1756,7 @@ importers: version: 9.1.0(eslint@8.57.1) eslint-plugin-prettier: specifier: ^5.1.3 - version: 5.2.1(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.3.3) + version: 5.2.1(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.6.2) fast-check: specifier: ^4.6.0 version: 4.6.0 @@ -1760,8 +1770,8 @@ importers: specifier: ^8.12.0 version: 8.13.1 prettier: - specifier: ^3.3.2 - version: 3.3.3 + specifier: ^3.6.2 + version: 3.6.2 shx: specifier: ^0.3.4 version: 0.3.4 @@ -1823,10 +1833,10 @@ importers: version: 9.1.0(eslint@8.57.1) eslint-plugin-prettier: specifier: ^5.1.3 - version: 5.2.1(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.3.3) + version: 5.2.1(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.6.2) prettier: - specifier: ^3.3.2 - version: 3.3.3 + specifier: ^3.6.2 + version: 3.6.2 shx: specifier: ^0.3.4 version: 0.3.4 @@ -12774,11 +12784,6 @@ packages: engines: {node: '>=10.13.0'} hasBin: true - prettier@3.3.3: - resolution: {integrity: sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==} - engines: {node: '>=14'} - hasBin: true - prettier@3.6.2: resolution: {integrity: sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==} engines: {node: '>=14'} @@ -24832,15 +24837,6 @@ snapshots: dependencies: eslint: 8.57.1 - eslint-plugin-prettier@5.2.1(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.3.3): - dependencies: - eslint: 8.57.1 - prettier: 3.3.3 - prettier-linter-helpers: 1.0.0 - synckit: 0.9.2 - optionalDependencies: - eslint-config-prettier: 9.1.0(eslint@8.57.1) - eslint-plugin-prettier@5.2.1(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.6.2): dependencies: eslint: 8.57.1 @@ -28912,8 +28908,6 @@ snapshots: prettier@2.8.8: {} - prettier@3.3.3: {} - prettier@3.6.2: {} pretty-bytes@5.6.0: {}