From 68b71601a4a10103df76013b9aac2cf29e794257 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 12 Jul 2026 15:57:58 +0200 Subject: [PATCH] feat(content): add versioned payload codec --- .changeset/calm-images-arrive.md | 5 + README.md | 1 + package.json | 4 +- packages/treecrdt-content/README.md | 21 +++ packages/treecrdt-content/package.json | 38 +++++ packages/treecrdt-content/src/index.ts | 159 ++++++++++++++++++ .../treecrdt-content/tests/content.test.ts | 102 +++++++++++ packages/treecrdt-content/tsconfig.json | 9 + pnpm-lock.yaml | 9 + 9 files changed, 346 insertions(+), 2 deletions(-) create mode 100644 .changeset/calm-images-arrive.md create mode 100644 packages/treecrdt-content/README.md create mode 100644 packages/treecrdt-content/package.json create mode 100644 packages/treecrdt-content/src/index.ts create mode 100644 packages/treecrdt-content/tests/content.test.ts create mode 100644 packages/treecrdt-content/tsconfig.json diff --git a/.changeset/calm-images-arrive.md b/.changeset/calm-images-arrive.md new file mode 100644 index 00000000..97939bad --- /dev/null +++ b/.changeset/calm-images-arrive.md @@ -0,0 +1,5 @@ +--- +'@treecrdt/content': minor +--- + +Add an app-layer payload codec for raw UTF-8 text and versioned inline images. diff --git a/README.md b/README.md index 6bfc9425..7facbdf0 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ pnpm test - `@treecrdt/wa-sqlite`: browser SQLite client adapter (in-memory WASM on Node). - `@treecrdt/sync`: client sync over discovery + WebSocket + SQLite backends. - `@treecrdt/interface`: shared TypeScript interfaces. +- `@treecrdt/content`: versioned app-layer codecs for payload text and inline images. - `@treecrdt/sync-protocol`: transport-agnostic sync protocol runtime. - `@treecrdt/discovery`: bootstrap contract for resolving docs to sync attachments. - `@treecrdt/sync-server-postgres-node`: Postgres-backed WebSocket sync server. diff --git a/package.json b/package.json index 2dce4f18..e3741214 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "build:ext:native": "cargo rustc -p treecrdt-sqlite-ext --release --no-default-features --features \"ext-sqlite rusqlite-storage\" -- --crate-type=cdylib", "build": "pnpm -r --if-present run build", "test:browser": "pnpm -C packages/treecrdt-wa-sqlite/e2e test:e2e", - "test:native-node": "pnpm -C packages/sync-protocol/material/sqlite run test && pnpm -C packages/treecrdt-sqlite-node run test && pnpm -C packages/treecrdt-wa-sqlite run test", + "test:native-node": "pnpm -C packages/treecrdt-content run test && pnpm -C packages/sync-protocol/material/sqlite run test && pnpm -C packages/treecrdt-sqlite-node run test && pnpm -C packages/treecrdt-wa-sqlite run test", "benchmark": "rm -rf benchmarks && mkdir -p benchmarks && pnpm run benchmark:web && pnpm run benchmark:sqlite-node:ops && pnpm run benchmark:sqlite-node:note-paths && pnpm run benchmark:sync:direct && pnpm run benchmark:wasm && pnpm run benchmark:postgres && pnpm run benchmark:core && pnpm run benchmark:aggregate", "benchmark:sqlite-node": "pnpm run benchmark:sqlite-node:ops && pnpm run benchmark:sqlite-node:note-paths && pnpm run benchmark:sqlite-node:sync", "benchmark:core": "cargo bench -p treecrdt-core --bench core --features bench", @@ -55,7 +55,7 @@ "sync-server:postgres:db:stop": "node scripts/run-sync-server-postgres-db-stop.mjs", "changeset": "changeset", "version-packages": "changeset version && pnpm install --lockfile-only", - "build:publish": "pnpm --filter @treecrdt/wa-sqlite-vendor run build && pnpm --filter @treecrdt/wa-sqlite... --filter @treecrdt/sync... --filter @treecrdt/sync-server-postgres-node... --filter @treecrdt/discovery-server-node... run build", + "build:publish": "pnpm --filter @treecrdt/wa-sqlite-vendor run build && pnpm --filter @treecrdt/content... --filter @treecrdt/wa-sqlite... --filter @treecrdt/sync... --filter @treecrdt/sync-server-postgres-node... --filter @treecrdt/discovery-server-node... run build", "release": "pnpm run build:publish && changeset publish && node scripts/set-npm-public-access.mjs" }, "devDependencies": { diff --git a/packages/treecrdt-content/README.md b/packages/treecrdt-content/README.md new file mode 100644 index 00000000..53b0179c --- /dev/null +++ b/packages/treecrdt-content/README.md @@ -0,0 +1,21 @@ +# @treecrdt/content + +A small app-layer codec for interpreting TreeCRDT payload bytes as text or inline images. +TreeCRDT storage remains media-agnostic and continues to store ordinary `Uint8Array` payloads. + +```ts +import { decodeContent, encodeImageContent, encodeTextContent } from '@treecrdt/content'; + +const textPayload = encodeTextContent('hello'); +const imagePayload = encodeImageContent({ + mime: 'image/png', + bytes: imageBytes, +}); + +const content = decodeContent(imagePayload); +``` + +Text uses raw UTF-8 as the canonical zero-envelope fast path. Images use a versioned binary envelope containing small +JSON metadata (including the exact image byte length) followed by the original image bytes. +Recognized envelopes fail closed when corrupt or unsupported; byte-size limits and browser +object-URL lifecycles remain application policy. diff --git a/packages/treecrdt-content/package.json b/packages/treecrdt-content/package.json new file mode 100644 index 00000000..086ee388 --- /dev/null +++ b/packages/treecrdt-content/package.json @@ -0,0 +1,38 @@ +{ + "name": "@treecrdt/content", + "version": "0.0.1", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "files": [ + "dist", + "src" + ], + "scripts": { + "build": "tsc -p tsconfig.json", + "test": "pnpm run build && vitest run" + }, + "devDependencies": { + "typescript": "^5.9.3", + "vitest": "^1.6.0" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/cybersemics/treecrdt.git", + "directory": "packages/treecrdt-content" + }, + "bugs": { + "url": "https://github.com/cybersemics/treecrdt/issues" + }, + "homepage": "https://github.com/cybersemics/treecrdt#readme", + "publishConfig": { + "access": "public" + } +} diff --git a/packages/treecrdt-content/src/index.ts b/packages/treecrdt-content/src/index.ts new file mode 100644 index 00000000..e870c2de --- /dev/null +++ b/packages/treecrdt-content/src/index.ts @@ -0,0 +1,159 @@ +export const SUPPORTED_IMAGE_MIME_TYPES = [ + 'image/png', + 'image/jpeg', + 'image/webp', + 'image/gif', +] as const; + +export type SupportedImageMime = (typeof SUPPORTED_IMAGE_MIME_TYPES)[number]; + +export type TreecrdtContent = + | { kind: 'empty' } + | { kind: 'text'; text: string; bytes: Uint8Array } + | { + kind: 'image'; + mime: SupportedImageMime; + name?: string; + size: number; + bytes: Uint8Array; + }; + +export type TreecrdtImageContentInput = { + mime: string; + name?: string; + bytes: Uint8Array; +}; + +const CONTENT_MAGIC = new Uint8Array([0x89, 0x54, 0x43, 0x52, 0x43, 0x4e, 0x54]); +const CONTENT_VERSION = 1; +const METADATA_LENGTH_BYTES = 4; +const MAX_METADATA_BYTES = 16 * 1024; +const HEADER_BYTES = CONTENT_MAGIC.byteLength + 1 + METADATA_LENGTH_BYTES; + +const textEncoder = new TextEncoder(); +const textDecoder = new TextDecoder(); +const metadataDecoder = new TextDecoder('utf-8', { fatal: true }); + +export function isSupportedImageMime(mime: string): mime is SupportedImageMime { + return (SUPPORTED_IMAGE_MIME_TYPES as readonly string[]).includes(mime); +} + +/** Text uses raw UTF-8 as the canonical zero-envelope representation. */ +export function encodeTextContent(text: string): Uint8Array { + return textEncoder.encode(text); +} + +export function encodeImageContent(input: TreecrdtImageContentInput): Uint8Array { + if (!isSupportedImageMime(input.mime)) { + throw new Error(`Unsupported image content MIME type: ${input.mime || '(empty)'}`); + } + + const name = input.name?.trim(); + const metadata = { + kind: 'image', + mime: input.mime, + size: input.bytes.byteLength, + ...(name ? { name } : {}), + }; + const metadataBytes = textEncoder.encode(JSON.stringify(metadata)); + if (metadataBytes.byteLength > MAX_METADATA_BYTES) { + throw new Error('Image content metadata is too large'); + } + + const output = new Uint8Array(HEADER_BYTES + metadataBytes.byteLength + input.bytes.byteLength); + output.set(CONTENT_MAGIC); + output[CONTENT_MAGIC.byteLength] = CONTENT_VERSION; + new DataView( + output.buffer, + output.byteOffset + CONTENT_MAGIC.byteLength + 1, + METADATA_LENGTH_BYTES, + ).setUint32(0, metadataBytes.byteLength, false); + output.set(metadataBytes, HEADER_BYTES); + output.set(input.bytes, HEADER_BYTES + metadataBytes.byteLength); + return output; +} + +/** + * Decodes the TreeCRDT app-layer content protocol. + * + * Bytes without the binary envelope are interpreted as canonical UTF-8 text. Once + * the envelope magic is present, malformed or unsupported data throws instead + * of being silently reclassified as text. + */ +export function decodeContent(bytes: Uint8Array | null): TreecrdtContent { + if (bytes === null) return { kind: 'empty' }; + if (!hasContentMagic(bytes)) { + return { kind: 'text', text: textDecoder.decode(bytes), bytes }; + } + return decodeEnvelope(bytes); +} + +function decodeEnvelope(bytes: Uint8Array): TreecrdtContent { + if (bytes.byteLength < HEADER_BYTES) throw new Error('Truncated TreeCRDT content envelope'); + + const version = bytes[CONTENT_MAGIC.byteLength]; + if (version !== CONTENT_VERSION) { + throw new Error(`Unsupported TreeCRDT content envelope version: ${String(version)}`); + } + + const metadataLength = new DataView( + bytes.buffer, + bytes.byteOffset + CONTENT_MAGIC.byteLength + 1, + METADATA_LENGTH_BYTES, + ).getUint32(0, false); + if (metadataLength === 0 || metadataLength > MAX_METADATA_BYTES) { + throw new Error('Invalid TreeCRDT content metadata length'); + } + + const payloadOffset = HEADER_BYTES + metadataLength; + if (payloadOffset > bytes.byteLength) throw new Error('Truncated TreeCRDT content metadata'); + + let decoded: unknown; + try { + decoded = JSON.parse(metadataDecoder.decode(bytes.subarray(HEADER_BYTES, payloadOffset))); + } catch { + throw new Error('Invalid TreeCRDT content metadata'); + } + if (typeof decoded !== 'object' || decoded === null || Array.isArray(decoded)) { + throw new Error('Invalid TreeCRDT content metadata'); + } + + const metadata = decoded as Record; + if (metadata.kind !== 'image') throw new Error('Unsupported TreeCRDT content kind'); + if (typeof metadata.mime !== 'string' || !isSupportedImageMime(metadata.mime)) { + throw new Error('Unsupported image content MIME type'); + } + if (metadata.name !== undefined && typeof metadata.name !== 'string') { + throw new Error('Invalid image content name'); + } + + if ( + typeof metadata.size !== 'number' || + !Number.isSafeInteger(metadata.size) || + metadata.size < 0 + ) { + throw new Error('Invalid image content size'); + } + const payloadLength = bytes.byteLength - payloadOffset; + if (metadata.size !== payloadLength) { + throw new Error('TreeCRDT image content size mismatch'); + } + + const imageBytes = bytes.subarray(payloadOffset); + const name = metadata.name?.trim(); + return { + kind: 'image', + mime: metadata.mime, + ...(name ? { name } : {}), + size: metadata.size, + bytes: imageBytes, + }; +} + +function hasContentMagic(bytes: Uint8Array): boolean { + if (bytes.byteLength < CONTENT_MAGIC.byteLength) return false; + for (let i = 0; i < CONTENT_MAGIC.byteLength; i += 1) { + if (bytes[i] !== CONTENT_MAGIC[i]) return false; + } + return true; +} diff --git a/packages/treecrdt-content/tests/content.test.ts b/packages/treecrdt-content/tests/content.test.ts new file mode 100644 index 00000000..b81922d0 --- /dev/null +++ b/packages/treecrdt-content/tests/content.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, test } from 'vitest'; + +import { decodeContent, encodeImageContent, encodeTextContent } from '../src/index.js'; + +function hexToBytes(hex: string): Uint8Array { + if (!/^(?:[0-9a-f]{2})+$/i.test(hex)) throw new Error('invalid test fixture hex'); + return Uint8Array.from(hex.match(/.{2}/g)!.map((byte) => Number.parseInt(byte, 16))); +} + +describe('@treecrdt/content', () => { + test('keeps text as raw UTF-8 and treats null as empty content', () => { + const bytes = encodeTextContent('hello image world'); + + expect(decodeContent(bytes)).toMatchObject({ + kind: 'text', + text: 'hello image world', + }); + expect(decodeContent(null)).toEqual({ kind: 'empty' }); + }); + + test('roundtrips an image envelope', () => { + const encoded = encodeImageContent({ + mime: 'image/png', + name: ' pixel.png ', + bytes: new Uint8Array([1, 2, 3, 4]), + }); + + const decoded = decodeContent(encoded); + expect(decoded).toMatchObject({ + kind: 'image', + mime: 'image/png', + name: 'pixel.png', + size: 4, + }); + expect(decoded.kind === 'image' ? Array.from(decoded.bytes) : []).toEqual([1, 2, 3, 4]); + }); + + test('locks the v1 wire format with a static fixture', () => { + const fixture = hexToBytes( + '89544352434e54010000003f7b226b696e64223a22696d616765222c226d696d65223a22696d6167652f706e67222c2273697a65223a332c226e616d65223a22706978656c2e706e67227d010203', + ); + + expect(decodeContent(fixture)).toMatchObject({ + kind: 'image', + mime: 'image/png', + name: 'pixel.png', + size: 3, + }); + expect( + encodeImageContent({ + mime: 'image/png', + name: 'pixel.png', + bytes: new Uint8Array([1, 2, 3]), + }), + ).toEqual(fixture); + }); + + test('handles Uint8Array views with non-zero offsets', () => { + const encoded = encodeImageContent({ + mime: 'image/webp', + bytes: new Uint8Array([9, 8, 7]), + }); + const padded = new Uint8Array(encoded.byteLength + 4); + padded.set(encoded, 2); + + expect(decodeContent(padded.subarray(2, -2))).toMatchObject({ + kind: 'image', + mime: 'image/webp', + size: 3, + }); + }); + + test('rejects unsupported image MIME types', () => { + expect(() => + encodeImageContent({ + mime: 'image/svg+xml', + bytes: new Uint8Array([1]), + }), + ).toThrow(/Unsupported image content MIME type/); + }); + + test('fails closed for malformed or unsupported recognized envelopes', () => { + const encoded = encodeImageContent({ + mime: 'image/png', + bytes: new Uint8Array([1]), + }); + + const unsupportedVersion = encoded.slice(); + unsupportedVersion[7] = 2; + expect(() => decodeContent(unsupportedVersion)).toThrow(/unsupported.*version/i); + expect(() => decodeContent(encoded.subarray(0, 10))).toThrow(/truncated/i); + + const invalidMetadataLength = encoded.slice(); + invalidMetadataLength.fill(0, 8, 12); + expect(() => decodeContent(invalidMetadataLength)).toThrow(/metadata length/i); + + expect(() => decodeContent(encoded.subarray(0, -1))).toThrow(/size mismatch/i); + const appended = new Uint8Array(encoded.byteLength + 1); + appended.set(encoded); + expect(() => decodeContent(appended)).toThrow(/size mismatch/i); + }); +}); diff --git a/packages/treecrdt-content/tsconfig.json b/packages/treecrdt-content/tsconfig.json new file mode 100644 index 00000000..92d2502b --- /dev/null +++ b/packages/treecrdt-content/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "declaration": true + }, + "include": ["src/**/*"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 947bd490..3285ef93 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -306,6 +306,15 @@ importers: specifier: ^5.9.3 version: 5.9.3 + packages/treecrdt-content: + devDependencies: + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^1.6.0 + version: 1.6.1(@types/node@20.19.25) + packages/treecrdt-crypto: dependencies: '@noble/ed25519':