feat(yjs): collaborative editing framework

.woodpecker.star

@@ -175,6 +175,13 @@ config = {
                 "OCM_OCM_PROVIDER_AUTHORIZER_PROVIDERS_FILE": "%s" % dir["ocmProviders"],
             },
         },
+        "collab": {
+            "earlyFail": True,
+            "skip": False,
+            "suites": [
+                "collaboration",
+            ],
+        },
         "mobile-view": {
             "skip": False,
             "suites": [
@@ -627,6 +634,12 @@ def e2eTests(ctx):
             elif "ocm" in suite:
                 steps += openCloudService(params["extraServerEnvironment"]) + \
                          (openCloudService(params["extraServerEnvironment"], "federation") if params["federationServer"] else [])
+            elif "collab" in suite:
+                # Realtime collab: start hocuspocus alongside OC. The OC
+                # reverse proxy forwards /realtime to it via the proxy.yaml
+                # mounted in openCloudService.
+                steps += hocuspocusService() + \
+                         openCloudService(params["extraServerEnvironment"])
             else:
                 # OpenCloud specific steps
                 steps += (tikaService() if params["tikaNeeded"] else []) + \
@@ -946,6 +959,7 @@ def openCloudService(extra_env_config = {}, deploy_type = "opencloud"):
                 "mkdir -p /srv/app/tmp/opencloud/storage/users/",
                 "./opencloud init",
                 "cp %s/tests/woodpecker/app-registry.yaml /root/.opencloud/config/app-registry.yaml" % dir["web"],
+                "cp %s/tests/woodpecker/proxy.yaml /root/.opencloud/config/proxy.yaml" % dir["web"],
                 "./opencloud server",
             ],
         },
@@ -1409,6 +1423,34 @@ def tikaService():
         "detach": True,
     }] + waitForService("tika", "9998")
 
+def hocuspocusService():
+    # Build + start the hocuspocus realtime collab sidecar in-place from the
+    # checked-out web tree. Plain HTTP on :1234; OC's reverse proxy
+    # WebSocket-upgrades and forwards via the `additional_policies` entry in
+    # tests/woodpecker/proxy.yaml. Mirrors the dev/docker/hocuspocus image
+    # setup but skips Docker — we install + patch the same way the Dockerfile
+    # does, then run server.js with node.
+    sidecar_dir = "%s/dev/docker/hocuspocus" % dir["web"]
+    return [{
+        "name": "hocuspocus",
+        "image": OC_CI_NODEJS,
+        "detach": True,
+        "environment": {
+            "PORT": "1234",
+            "DB_PATH": "/tmp/hocuspocus-state.db",
+            "OPENCLOUD_URL": "https://opencloud:9200",
+            "NODE_TLS_REJECT_UNAUTHORIZED": "0",
+        },
+        "commands": [
+            "cd %s" % sidecar_dir,
+            "npm install --omit=dev --no-audit --no-fund --loglevel=error",
+            # The Dockerfile applies a pre-built patch on top of
+            # @hocuspocus/server@4.0.0. `patch` is in nodejs-ci:24.
+            "cd node_modules/@hocuspocus/server && patch -p1 < %s/patches/hocuspocus-server-4.0.0.patch && cd -" % sidecar_dir,
+            "node server.js",
+        ],
+    }] + waitForService("hocuspocus", "1234")
+
 def collaboraService():
     return [
         {

cucumber.mjs

@@ -8,7 +8,8 @@ if (!fs.existsSync(config.reportDir)) {
 
 const e2e = `
   --loader ts-node/esm
-  --import ./tests/e2e/**/*.ts
+  --import ./tests/e2e/cucumber/**/*.ts
+  --import ./tests/e2e/support/**/*.ts
   --retry ${config.retry}
   --format @cucumber/pretty-formatter
   --format pretty

dev/docker/hocuspocus/Dockerfile

@@ -0,0 +1,10 @@
+FROM node:22-alpine
+
+WORKDIR /app
+
+COPY package.json ./
+RUN npm install --omit=dev --no-audit --no-fund --loglevel=error
+
+COPY server.js ./
+
+CMD ["node", "server.js"]

dev/docker/hocuspocus/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "opencloud-hocuspocus-dev",
+  "version": "0.0.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "start": "node server.js"
+  },
+  "dependencies": {
+    "@hocuspocus/server": "^4.1.0"
+  }
+}

dev/docker/hocuspocus/server.js

@@ -0,0 +1,265 @@
+import { Server } from '@hocuspocus/server'
+
+const port = parseInt(process.env.PORT ?? '1234', 10)
+const opencloudUrl = (process.env.OPENCLOUD_URL ?? 'https://host.docker.internal:9200').replace(
+  /\/$/,
+  ''
+)
+const devFakeToken = process.env.DEV_FAKE_TOKEN ?? ''
+
+// Per-document first-seen app version. Acts as the authoritative gate for
+// "everybody in this room must run the same client version". First connect
+// for a documentName sets the baseline; subsequent connects with a different
+// appVersion are rejected at authenticate-time. In-memory only; on restart
+// the next connecter becomes the new baseline (acceptable for a stateless
+// sidecar). Empty appVersion is tolerated for legacy/test clients.
+const appVersionByDocument = new Map()
+
+function deterministicColor(seed) {
+  let hash = 0
+  for (let i = 0; i < seed.length; i++) hash = seed.charCodeAt(i) + ((hash << 5) - hash)
+  return `hsl(${Math.abs(hash) % 360}, 70%, 50%)`
+}
+
+async function validateTokenAgainstOpenCloud(token) {
+  const res = await fetch(`${opencloudUrl}/graph/v1.0/me`, {
+    headers: { Authorization: `Bearer ${token}` }
+  })
+  if (!res.ok) {
+    const detail = await res.text().catch(() => '')
+    throw new Error(`graph /me returned ${res.status}: ${detail.slice(0, 200)}`)
+  }
+  return res.json()
+}
+
+// Heuristic: a libregraph permission action implies write access when its
+// trailing verb is create/update/delete/allTasks on driveItem properties.
+const WRITE_ACTION = /\/(update|create|delete|allTasks)$/
+
+// Splits OC's canonical composite id `<storageid>$<spaceid>!<opaqueid>` into
+// the (driveId, itemId) pair the Graph endpoint expects: driveID =
+// `<storageid>$<spaceid>`, itemID = the FULL composite.
+//
+// The wrapper now namespaces room names by app id to avoid schema
+// collisions between different editors opening the same file
+// (e.g. `text-editor::<composite>` vs `codemirror::<composite>`). Strip
+// any `<scope>::` prefix before parsing so the Graph probe targets the
+// raw file id.
+function parseDocumentId(documentName) {
+  const scopeSep = documentName.indexOf('::')
+  const fileId = scopeSep >= 0 ? documentName.slice(scopeSep + 2) : documentName
+  const sep = fileId.indexOf('!')
+  if (sep <= 0 || sep === fileId.length - 1) {
+    throw new Error(`malformed documentName="${documentName}"`)
+  }
+  return { driveId: fileId.slice(0, sep), itemId: fileId }
+}
+
+// Probes OC's Graph API for the user's effective access AND the file's
+// current native etag. Returns `{ canWrite, etag }` on success; `null` when
+// OC denies access entirely (401/403/404).
+//
+// Two parallel calls:
+// - Graph /permissions for the effective action set (top-level
+//   @libre.graph.permissions.actions.allowedValues, which is the merged
+//   PermissionSet that backs WebDAV's oc:permissions).
+// - WebDAV HEAD for the native eTag (Graph's /items endpoint is share-jail-
+//   only and 400s on personal drives; WebDAV works uniformly).
+async function probeFileAccess(token, documentName) {
+  const { driveId, itemId } = parseDocumentId(documentName)
+  const permsUrl =
+    `${opencloudUrl}/graph/v1beta1/drives/${encodeURIComponent(driveId)}` +
+    `/items/${encodeURIComponent(itemId)}/permissions`
+  const davUrl = `${opencloudUrl}/remote.php/dav/spaces/${encodeURIComponent(itemId)}`
+  const headers = { Authorization: `Bearer ${token}` }
+
+  const [permsRes, headRes] = await Promise.all([
+    fetch(permsUrl, { headers }),
+    fetch(davUrl, { method: 'HEAD', headers })
+  ])
+
+  if ([permsRes.status, headRes.status].some((s) => s === 401 || s === 403 || s === 404)) {
+    return null
+  }
+  if (!permsRes.ok) {
+    const detail = await permsRes.text().catch(() => '')
+    throw new Error(`graph permissions returned ${permsRes.status}: ${detail.slice(0, 200)}`)
+  }
+  if (!headRes.ok) {
+    const detail = await headRes.text().catch(() => '')
+    throw new Error(`webdav HEAD returned ${headRes.status}: ${detail.slice(0, 200)}`)
+  }
+
+  const permsBody = await permsRes.json()
+  const allowed = Array.isArray(permsBody?.['@libre.graph.permissions.actions.allowedValues'])
+    ? permsBody['@libre.graph.permissions.actions.allowedValues']
+    : []
+  const canWrite = allowed.some((a) => WRITE_ACTION.test(a))
+
+  // WebDAV emits the strong validator under `ETag` (and sometimes `OC-ETag`
+  // for OC-specific extensions). Strip surrounding quotes for consistency
+  // with the etag the wrapper sees from `props.resource.etag`.
+  const rawEtag = headRes.headers.get('etag') || headRes.headers.get('oc-etag') || ''
+  const etag = rawEtag.replace(/^"(.*)"$/, '$1')
+
+  return { canWrite, etag }
+}
+
+const META_KEY = '_oc_meta'
+
+const server = new Server({
+  port,
+  address: '0.0.0.0',
+  // No server-side persistence: every doc is file-backed via WebDAV.
+  // Cold-start for a fresh peer = hydrate from `currentContent` in the
+  // wrapper. The persisted SQLite snapshot would get discarded on stale-
+  // state recovery anyway (etag drift triggers rehydrate); keeping it
+  // here is "mostly ceremony" per the migration plan. Stale detection
+  // moved to the client (see CollaborativeWrapper.onProviderSynced).
+
+  async onAuthenticate({ token, documentName, requestParameters }) {
+    if (!token) {
+      throw new Error('missing token')
+    }
+
+    // App-version gate. First connect to a documentName sets the baseline,
+    // subsequent connects with a different appVersion are rejected so old
+    // clients can't poison the room. Empty client appVersion is permitted
+    // (back-compat for the integration test harness using a raw provider).
+    const clientAppVersion = requestParameters.get('appVersion') ?? ''
+    const baselineAppVersion = appVersionByDocument.get(documentName)
+    if (clientAppVersion && baselineAppVersion && clientAppVersion !== baselineAppVersion) {
+      throw new Error(
+        `app version mismatch for document="${documentName}": ` +
+          `client=${clientAppVersion} room=${baselineAppVersion}, please reload`
+      )
+    }
+    if (clientAppVersion && !baselineAppVersion) {
+      appVersionByDocument.set(documentName, clientAppVersion)
+    }
+
+    // Dev shortcut for integration tests: any token matching DEV_FAKE_TOKEN
+    // returns a synthetic identity. ACL check is skipped (tests use random
+    // documentNames that don't exist in OC). Disabled when DEV_FAKE_TOKEN is
+    // unset. Tests can pass `devEtag` to drive the stale-state detection
+    // path without touching real OC.
+    if (devFakeToken && token === devFakeToken) {
+      const id = 'dev-fake-user'
+      const nativeEtag = requestParameters.get('devEtag') ?? ''
+      console.log(`[onAuthenticate] dev-fake document="${documentName}" nativeEtag="${nativeEtag}"`)
+      return {
+        nativeEtag,
+        user: {
+          id,
+          displayName: 'Dev Fake User',
+          color: deterministicColor(id)
+        }
+      }
+    }
+
+    const me = await validateTokenAgainstOpenCloud(token)
+    const id = me.id ?? me.userPrincipalName ?? 'unknown'
+
+    // ACL + native etag probe via Graph: enforces access AND captures the
+    // current native etag so onLoadDocument can detect a stale persisted
+    // Y.Doc snapshot (Hocuspocus persistence vs external file write).
+    const access = await probeFileAccess(token, documentName)
+    if (access === null) {
+      throw new Error(`access denied for document="${documentName}"`)
+    }
+    const readOnly = !access.canWrite
+
+    console.log(
+      `[onAuthenticate] document="${documentName}" user="${me.displayName ?? id}" ` +
+        `id="${id}" readOnly=${readOnly} nativeEtag="${access.etag}"`
+    )
+    return {
+      readOnly,
+      nativeEtag: access.etag,
+      clientAppVersion,
+      user: {
+        id,
+        displayName: me.displayName ?? me.userPrincipalName ?? id,
+        color: deterministicColor(id)
+      }
+    }
+  },
+
+  // Stale-state detection — DISABLED.
+  //
+  // This hook fires once when a doc is loaded into memory. It used to do
+  // useful work when we shipped the SQLite extension: at cold load it
+  // compared the persisted `_oc_meta.etag` against the live native etag
+  // and flagged drift so the wrapper would rehydrate. Without persistence
+  // the doc is always freshly created at load time, `_oc_meta` is empty,
+  // and the wrapper's etag mirror runs strictly AFTER this hook — so the
+  // comparison can never fire. The equivalent check now lives in
+  // CollaborativeWrapper.onProviderSynced (runs on the client, sees the
+  // CRDT-synced `_oc_meta.etag` from whichever peer joined first).
+  //
+  // Kept commented out as a reference: if persistence is reintroduced
+  // (extension-sqlite, redis, etc.), uncomment to get the server-side
+  // cold-load probe back.
+  //
+  // async onLoadDocument({ document, context }) {
+  //   const meta = document.getMap(META_KEY)
+  //   const persistedEtag = meta.get('etag')
+  //   const nativeEtag = context?.nativeEtag
+  //   const persistedAppVersion = meta.get('appVersion')
+  //   const clientAppVersion = context?.clientAppVersion
+  //
+  //   const etagDrift = !!persistedEtag && !!nativeEtag && persistedEtag !== nativeEtag
+  //   const versionDrift =
+  //     !!persistedAppVersion && !!clientAppVersion && persistedAppVersion !== clientAppVersion
+  //
+  //   if (!etagDrift && !versionDrift) return
+  //
+  //   const reasons = []
+  //   if (etagDrift) reasons.push(`etag(${persistedEtag}→${nativeEtag})`)
+  //   if (versionDrift) reasons.push(`appVersion(${persistedAppVersion}→${clientAppVersion})`)
+  //   console.log(
+  //     `[onLoadDocument] stale state document="${document.name}" ` +
+  //       `${reasons.join(' ')} → marked for rehydrate`
+  //   )
+  //   document.transact(() => {
+  //     meta.set('isStale', true)
+  //     if (nativeEtag) meta.set('nativeEtag', nativeEtag)
+  //   })
+  // },
+
+  async onConnect({ documentName, requestHeaders }) {
+    console.log(`[onConnect] document="${documentName}" origin=${requestHeaders.origin ?? '-'}`)
+  },
+
+  async onDisconnect({ documentName, clientsCount }) {
+    console.log(`[onDisconnect] document="${documentName}" remaining=${clientsCount}`)
+    if (clientsCount === 0) {
+      // Forget the version baseline once the room empties out so a new
+      // deploy can start fresh without manual restart.
+      appVersionByDocument.delete(documentName)
+    }
+  },
+
+  // Anti-spoof identity stamp: before each inbound awareness update is
+  // applied, overwrite the `user` field on every state in the update with
+  // the authenticated identity from the connection's context. This used
+  // to require a patch on hocuspocus 4.0.0; 4.1.0 ships the
+  // `beforeHandleAwareness` callback natively with positional args.
+  async beforeHandleAwareness(_document, states, origin) {
+    const connection = origin?.source === 'connection' ? origin.connection : null
+    const user = connection?.context?.user
+    if (!user) return
+    const canonical = {
+      id: user.id,
+      name: user.displayName,
+      color: user.color
+    }
+    for (const state of states.values()) {
+      state.user = canonical
+    }
+  }
+})
+
+server.listen().then(() => {
+  console.log(`hocuspocus v4 listening on :${port}, oc=${opencloudUrl}`)
+})

dev/docker/opencloud.web.config.json

@@ -18,6 +18,9 @@
     "activities",
     "preview",
     "mail",
-    "contacts"
+    "contacts",
+    "codemirror",
+    "tiptap",
+    "excalidraw"
   ]
 }

dev/docker/opencloud/proxy.yaml

@@ -4,6 +4,18 @@
 additional_policies:
   - name: default
     routes:
+      # Realtime collab sidecar. The hocuspocus container terminates plain
+      # HTTP on :1234; OC's reverse proxy upgrades the incoming WebSocket
+      # request and forwards it on. Pattern is borrowed from the
+      # opencloud-music sidecar setup. `unprotected: true` because
+      # hocuspocus does its own bearer-token validation against OC's
+      # Graph API (see dev/docker/hocuspocus/server.js
+      # `validateTokenAgainstOpenCloud`) and must see the Authorization
+      # header the client sent verbatim, not whatever OC's proxy would
+      # substitute.
+      - endpoint: /realtime
+        backend: http://hocuspocus:1234
+        unprotected: true
       - endpoint: /caldav/
         backend: http://host.docker.internal:5232
         remote_user_header: X-Remote-User