feat(content): add a versioned payload codec

.changeset/calm-images-arrive.md

@@ -0,0 +1,5 @@
+---
+'@treecrdt/content': minor
+---
+
+Add an app-layer payload codec for raw UTF-8 text and versioned inline images.

README.md

@@ -15,6 +15,7 @@ pnpm test
 - `@treecrdt/wa-sqlite`: browser SQLite client adapter (in-memory WASM on Node).
 - `@treecrdt/sync`: client sync over discovery + WebSocket + SQLite backends.
 - `@treecrdt/interface`: shared TypeScript interfaces.
+- `@treecrdt/content`: versioned app-layer codecs for payload text and inline images.
 - `@treecrdt/sync-protocol`: transport-agnostic sync protocol runtime.
 - `@treecrdt/discovery`: bootstrap contract for resolving docs to sync attachments.
 - `@treecrdt/sync-server-postgres-node`: Postgres-backed WebSocket sync server.

package.json

@@ -22,7 +22,7 @@
     "build:ext:native": "cargo rustc -p treecrdt-sqlite-ext --release --no-default-features --features \"ext-sqlite rusqlite-storage\" -- --crate-type=cdylib",
     "build": "pnpm -r --if-present run build",
     "test:browser": "pnpm -C packages/treecrdt-wa-sqlite/e2e test:e2e",
-    "test:native-node": "pnpm -C packages/sync-protocol/material/sqlite run test && pnpm -C packages/treecrdt-sqlite-node run test && pnpm -C packages/treecrdt-wa-sqlite run test",
+    "test:native-node": "pnpm -C packages/treecrdt-content run test && pnpm -C packages/sync-protocol/material/sqlite run test && pnpm -C packages/treecrdt-sqlite-node run test && pnpm -C packages/treecrdt-wa-sqlite run test",
     "benchmark": "rm -rf benchmarks && mkdir -p benchmarks && pnpm run benchmark:web && pnpm run benchmark:sqlite-node:ops && pnpm run benchmark:sqlite-node:note-paths && pnpm run benchmark:sync:direct && pnpm run benchmark:wasm && pnpm run benchmark:postgres && pnpm run benchmark:core && pnpm run benchmark:aggregate",
     "benchmark:sqlite-node": "pnpm run benchmark:sqlite-node:ops && pnpm run benchmark:sqlite-node:note-paths && pnpm run benchmark:sqlite-node:sync",
     "benchmark:core": "cargo bench -p treecrdt-core --bench core --features bench",
@@ -55,7 +55,7 @@
     "sync-server:postgres:db:stop": "node scripts/run-sync-server-postgres-db-stop.mjs",
     "changeset": "changeset",
     "version-packages": "changeset version && pnpm install --lockfile-only",
-    "build:publish": "pnpm --filter @treecrdt/wa-sqlite-vendor run build && pnpm --filter @treecrdt/wa-sqlite... --filter @treecrdt/sync... --filter @treecrdt/sync-server-postgres-node... --filter @treecrdt/discovery-server-node... run build",
+    "build:publish": "pnpm --filter @treecrdt/wa-sqlite-vendor run build && pnpm --filter @treecrdt/content... --filter @treecrdt/wa-sqlite... --filter @treecrdt/sync... --filter @treecrdt/sync-server-postgres-node... --filter @treecrdt/discovery-server-node... run build",
     "release": "pnpm run build:publish && changeset publish && node scripts/set-npm-public-access.mjs"
   },
   "devDependencies": {

packages/treecrdt-content/README.md

@@ -0,0 +1,21 @@
+# @treecrdt/content
+
+A small app-layer codec for interpreting TreeCRDT payload bytes as text or inline images.
+TreeCRDT storage remains media-agnostic and continues to store ordinary `Uint8Array` payloads.
+
+```ts
+import { decodeContent, encodeImageContent, encodeTextContent } from '@treecrdt/content';
+
+const textPayload = encodeTextContent('hello');
+const imagePayload = encodeImageContent({
+  mime: 'image/png',
+  bytes: imageBytes,
+});
+
+const content = decodeContent(imagePayload);
+```
+
+Text uses raw UTF-8 as the canonical zero-envelope fast path. Images use a versioned binary envelope containing small
+JSON metadata (including the exact image byte length) followed by the original image bytes.
+Recognized envelopes fail closed when corrupt or unsupported; byte-size limits and browser
+object-URL lifecycles remain application policy.

packages/treecrdt-content/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@treecrdt/content",
+  "version": "0.0.1",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "test": "pnpm run build && vitest run"
+  },
+  "devDependencies": {
+    "typescript": "^5.9.3",
+    "vitest": "^1.6.0"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/cybersemics/treecrdt.git",
+    "directory": "packages/treecrdt-content"
+  },
+  "bugs": {
+    "url": "https://github.com/cybersemics/treecrdt/issues"
+  },
+  "homepage": "https://github.com/cybersemics/treecrdt#readme",
+  "publishConfig": {
+    "access": "public"
+  }
+}

packages/treecrdt-content/src/index.ts

@@ -0,0 +1,159 @@
+export const SUPPORTED_IMAGE_MIME_TYPES = [
+  'image/png',
+  'image/jpeg',
+  'image/webp',
+  'image/gif',
+] as const;
+
+export type SupportedImageMime = (typeof SUPPORTED_IMAGE_MIME_TYPES)[number];
+
+export type TreecrdtContent =
+  | { kind: 'empty' }
+  | { kind: 'text'; text: string; bytes: Uint8Array }
+  | {
+      kind: 'image';
+      mime: SupportedImageMime;
+      name?: string;
+      size: number;
+      bytes: Uint8Array;
+    };
+
+export type TreecrdtImageContentInput = {
+  mime: string;
+  name?: string;
+  bytes: Uint8Array;
+};
+
+const CONTENT_MAGIC = new Uint8Array([0x89, 0x54, 0x43, 0x52, 0x43, 0x4e, 0x54]);
+const CONTENT_VERSION = 1;
+const METADATA_LENGTH_BYTES = 4;
+const MAX_METADATA_BYTES = 16 * 1024;
+const HEADER_BYTES = CONTENT_MAGIC.byteLength + 1 + METADATA_LENGTH_BYTES;
+
+const textEncoder = new TextEncoder();
+const textDecoder = new TextDecoder();
+const metadataDecoder = new TextDecoder('utf-8', { fatal: true });
+
+export function isSupportedImageMime(mime: string): mime is SupportedImageMime {
+  return (SUPPORTED_IMAGE_MIME_TYPES as readonly string[]).includes(mime);
+}
+
+/** Text uses raw UTF-8 as the canonical zero-envelope representation. */
+export function encodeTextContent(text: string): Uint8Array {
+  return textEncoder.encode(text);
+}
+
+export function encodeImageContent(input: TreecrdtImageContentInput): Uint8Array {
+  if (!isSupportedImageMime(input.mime)) {
+    throw new Error(`Unsupported image content MIME type: ${input.mime || '(empty)'}`);
+  }
+
+  const name = input.name?.trim();
+  const metadata = {
+    kind: 'image',
+    mime: input.mime,
+    size: input.bytes.byteLength,
+    ...(name ? { name } : {}),
+  };
+  const metadataBytes = textEncoder.encode(JSON.stringify(metadata));
+  if (metadataBytes.byteLength > MAX_METADATA_BYTES) {
+    throw new Error('Image content metadata is too large');
+  }
+
+  const output = new Uint8Array(HEADER_BYTES + metadataBytes.byteLength + input.bytes.byteLength);
+  output.set(CONTENT_MAGIC);
+  output[CONTENT_MAGIC.byteLength] = CONTENT_VERSION;
+  new DataView(
+    output.buffer,
+    output.byteOffset + CONTENT_MAGIC.byteLength + 1,
+    METADATA_LENGTH_BYTES,
+  ).setUint32(0, metadataBytes.byteLength, false);
+  output.set(metadataBytes, HEADER_BYTES);
+  output.set(input.bytes, HEADER_BYTES + metadataBytes.byteLength);
+  return output;
+}
+
+/**
+ * Decodes the TreeCRDT app-layer content protocol.
+ *
+ * Bytes without the binary envelope are interpreted as canonical UTF-8 text. Once
+ * the envelope magic is present, malformed or unsupported data throws instead
+ * of being silently reclassified as text.
+ */
+export function decodeContent(bytes: Uint8Array | null): TreecrdtContent {
+  if (bytes === null) return { kind: 'empty' };
+  if (!hasContentMagic(bytes)) {
+    return { kind: 'text', text: textDecoder.decode(bytes), bytes };
+  }
+  return decodeEnvelope(bytes);
+}
+
+function decodeEnvelope(bytes: Uint8Array): TreecrdtContent {
+  if (bytes.byteLength < HEADER_BYTES) throw new Error('Truncated TreeCRDT content envelope');
+
+  const version = bytes[CONTENT_MAGIC.byteLength];
+  if (version !== CONTENT_VERSION) {
+    throw new Error(`Unsupported TreeCRDT content envelope version: ${String(version)}`);
+  }
+
+  const metadataLength = new DataView(
+    bytes.buffer,
+    bytes.byteOffset + CONTENT_MAGIC.byteLength + 1,
+    METADATA_LENGTH_BYTES,
+  ).getUint32(0, false);
+  if (metadataLength === 0 || metadataLength > MAX_METADATA_BYTES) {
+    throw new Error('Invalid TreeCRDT content metadata length');
+  }
+
+  const payloadOffset = HEADER_BYTES + metadataLength;
+  if (payloadOffset > bytes.byteLength) throw new Error('Truncated TreeCRDT content metadata');
+
+  let decoded: unknown;
+  try {
+    decoded = JSON.parse(metadataDecoder.decode(bytes.subarray(HEADER_BYTES, payloadOffset)));
+  } catch {
+    throw new Error('Invalid TreeCRDT content metadata');
+  }
+  if (typeof decoded !== 'object' || decoded === null || Array.isArray(decoded)) {
+    throw new Error('Invalid TreeCRDT content metadata');
+  }
+
+  const metadata = decoded as Record<string, unknown>;
+  if (metadata.kind !== 'image') throw new Error('Unsupported TreeCRDT content kind');
+  if (typeof metadata.mime !== 'string' || !isSupportedImageMime(metadata.mime)) {
+    throw new Error('Unsupported image content MIME type');
+  }
+  if (metadata.name !== undefined && typeof metadata.name !== 'string') {
+    throw new Error('Invalid image content name');
+  }
+
+  if (
+    typeof metadata.size !== 'number' ||
+    !Number.isSafeInteger(metadata.size) ||
+    metadata.size < 0
+  ) {
+    throw new Error('Invalid image content size');
+  }
+  const payloadLength = bytes.byteLength - payloadOffset;
+  if (metadata.size !== payloadLength) {
+    throw new Error('TreeCRDT image content size mismatch');
+  }
+
+  const imageBytes = bytes.subarray(payloadOffset);
+  const name = metadata.name?.trim();
+  return {
+    kind: 'image',
+    mime: metadata.mime,
+    ...(name ? { name } : {}),
+    size: metadata.size,
+    bytes: imageBytes,
+  };
+}
+
+function hasContentMagic(bytes: Uint8Array): boolean {
+  if (bytes.byteLength < CONTENT_MAGIC.byteLength) return false;
+  for (let i = 0; i < CONTENT_MAGIC.byteLength; i += 1) {
+    if (bytes[i] !== CONTENT_MAGIC[i]) return false;
+  }
+  return true;
+}

packages/treecrdt-content/tests/content.test.ts

@@ -0,0 +1,102 @@
+import { describe, expect, test } from 'vitest';
+
+import { decodeContent, encodeImageContent, encodeTextContent } from '../src/index.js';
+
+function hexToBytes(hex: string): Uint8Array {
+  if (!/^(?:[0-9a-f]{2})+$/i.test(hex)) throw new Error('invalid test fixture hex');
+  return Uint8Array.from(hex.match(/.{2}/g)!.map((byte) => Number.parseInt(byte, 16)));
+}
+
+describe('@treecrdt/content', () => {
+  test('keeps text as raw UTF-8 and treats null as empty content', () => {
+    const bytes = encodeTextContent('hello image world');
+
+    expect(decodeContent(bytes)).toMatchObject({
+      kind: 'text',
+      text: 'hello image world',
+    });
+    expect(decodeContent(null)).toEqual({ kind: 'empty' });
+  });
+
+  test('roundtrips an image envelope', () => {
+    const encoded = encodeImageContent({
+      mime: 'image/png',
+      name: ' pixel.png ',
+      bytes: new Uint8Array([1, 2, 3, 4]),
+    });
+
+    const decoded = decodeContent(encoded);
+    expect(decoded).toMatchObject({
+      kind: 'image',
+      mime: 'image/png',
+      name: 'pixel.png',
+      size: 4,
+    });
+    expect(decoded.kind === 'image' ? Array.from(decoded.bytes) : []).toEqual([1, 2, 3, 4]);
+  });
+
+  test('locks the v1 wire format with a static fixture', () => {
+    const fixture = hexToBytes(
+      '89544352434e54010000003f7b226b696e64223a22696d616765222c226d696d65223a22696d6167652f706e67222c2273697a65223a332c226e616d65223a22706978656c2e706e67227d010203',
+    );
+
+    expect(decodeContent(fixture)).toMatchObject({
+      kind: 'image',
+      mime: 'image/png',
+      name: 'pixel.png',
+      size: 3,
+    });
+    expect(
+      encodeImageContent({
+        mime: 'image/png',
+        name: 'pixel.png',
+        bytes: new Uint8Array([1, 2, 3]),
+      }),
+    ).toEqual(fixture);
+  });
+
+  test('handles Uint8Array views with non-zero offsets', () => {
+    const encoded = encodeImageContent({
+      mime: 'image/webp',
+      bytes: new Uint8Array([9, 8, 7]),
+    });
+    const padded = new Uint8Array(encoded.byteLength + 4);
+    padded.set(encoded, 2);
+
+    expect(decodeContent(padded.subarray(2, -2))).toMatchObject({
+      kind: 'image',
+      mime: 'image/webp',
+      size: 3,
+    });
+  });
+
+  test('rejects unsupported image MIME types', () => {
+    expect(() =>
+      encodeImageContent({
+        mime: 'image/svg+xml',
+        bytes: new Uint8Array([1]),
+      }),
+    ).toThrow(/Unsupported image content MIME type/);
+  });
+
+  test('fails closed for malformed or unsupported recognized envelopes', () => {
+    const encoded = encodeImageContent({
+      mime: 'image/png',
+      bytes: new Uint8Array([1]),
+    });
+
+    const unsupportedVersion = encoded.slice();
+    unsupportedVersion[7] = 2;
+    expect(() => decodeContent(unsupportedVersion)).toThrow(/unsupported.*version/i);
+    expect(() => decodeContent(encoded.subarray(0, 10))).toThrow(/truncated/i);
+
+    const invalidMetadataLength = encoded.slice();
+    invalidMetadataLength.fill(0, 8, 12);
+    expect(() => decodeContent(invalidMetadataLength)).toThrow(/metadata length/i);
+
+    expect(() => decodeContent(encoded.subarray(0, -1))).toThrow(/size mismatch/i);
+    const appended = new Uint8Array(encoded.byteLength + 1);
+    appended.set(encoded);
+    expect(() => decodeContent(appended)).toThrow(/size mismatch/i);
+  });
+});

packages/treecrdt-content/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist",
+    "declaration": true
+  },
+  "include": ["src/**/*"]
+}