Import and querying stability improvements

.changeset/two-oranges-march.md

@@ -0,0 +1,10 @@
+---
+'rushdb-core': patch
+'rushdb-docs': patch
+'@rushdb/javascript-sdk': patch
+'@rushdb/mcp-server': patch
+'@rushdb/skills': patch
+'rushdb-dashboard': patch
+---
+
+Import and querying stability improvements

.github/scripts/notify-content-release.mjs

@@ -0,0 +1,35 @@
+import { createHmac } from 'node:crypto'
+
+const required = (name) => {
+  const value = process.env[name]
+  if (!value) throw new Error(`Missing required environment variable: ${name}`)
+  return value
+}
+
+const version = required('RELEASE_VERSION')
+const body = JSON.stringify({
+  event: 'release.completed',
+  repository: process.env.GITHUB_REPOSITORY ?? 'rush-db/rushdb',
+  version,
+  tag: `v${version}`,
+  sha: required('RELEASE_SHA'),
+  workflowUrl: required('RELEASE_RUN_URL'),
+  packages: [
+    { name: '@rushdb/javascript-sdk', version },
+    { name: '@rushdb/mcp-server', version }
+  ],
+  occurredAt: new Date().toISOString()
+})
+const timestamp = String(Math.floor(Date.now() / 1000))
+const signature = `sha256=${createHmac('sha256', required('CONTENT_WEBHOOK_SECRET')).update(`${timestamp}.${body}`).digest('hex')}`
+const response = await fetch(required('CONTENT_WEBHOOK_URL'), {
+  method: 'POST',
+  headers: {
+    'content-type': 'application/json',
+    'x-content-timestamp': timestamp,
+    'x-content-signature': signature
+  },
+  body
+})
+if (!response.ok) throw new Error(`Content webhook failed (${response.status}): ${await response.text()}`)
+console.log(await response.text())

.github/workflows/release.yml

@@ -65,7 +65,8 @@ jobs:
           STATUS=$(pnpm changeset status --no-color --verbose || true)
           VERSION=$(echo "$STATUS" | grep -E '@rushdb/(javascript-sdk|mcp-server)' | awk '{print $NF}' | sed 's/[^[:print:]]//g' | head -n1)
           if [ -z "$VERSION" ]; then
-            echo "No upcoming version detected for javascript-sdk or mcp-server; leaving empty"
+            VERSION=$(node -p "require('./packages/javascript-sdk/package.json').version")
+            echo "No upcoming changeset version; using checked-in SDK version ${VERSION}"
           fi
           echo "version=${VERSION}" >> $GITHUB_OUTPUT
 
@@ -298,3 +299,21 @@ jobs:
           service: ${{ env.ECS_SERVICE_MCP }}
           cluster: ${{ env.ECS_CLUSTER }}
           wait-for-service-stability: false
+
+  notify-content:
+    name: Notify content control plane
+    needs: [release, docker, deploy]
+    if: needs.release.outputs.is_version_bump == 'true' && needs.deploy.result == 'success'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Send signed release event
+        env:
+          CONTENT_WEBHOOK_URL: ${{ secrets.CONTENT_WEBHOOK_URL }}
+          CONTENT_WEBHOOK_SECRET: ${{ secrets.CONTENT_WEBHOOK_SECRET }}
+          RELEASE_VERSION: ${{ needs.release.outputs.version }}
+          RELEASE_SHA: ${{ github.sha }}
+          RELEASE_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: node .github/scripts/notify-content-release.mjs

e2e/import-write-path.e2e-spec.ts

@@ -104,7 +104,9 @@ describe('record import write path (e2e)', () => {
       key: `r-${i}`,
       name: `Section ${i}`
     }))
+    const firstStartedAt = Date.now()
     await db.records.createMany({ label, data: first, options })
+    const firstElapsed = Date.now() - firstStartedAt
 
     const afterFirst = await db.records.find({ labels: [label], limit: 1 })
     expect(afterFirst.total).toBe(100)
@@ -114,7 +116,14 @@ describe('record import write path (e2e)', () => {
       key: `r-${i + 50}`,
       grade: 'III'
     }))
+    const secondStartedAt = Date.now()
     await db.records.createMany({ label, data: second, options })
+    const secondElapsed = Date.now() - secondStartedAt
+    // The reported bug was ~0.45s/merged row driven by an unindexed, project-wide (not
+    // label-scoped) match — a 100-row batch took 40+ seconds. Scoping the match to this
+    // label's node set must keep a 100-row merge on an idle local stack far below that.
+    expect(firstElapsed).toBeLessThan(20_000)
+    expect(secondElapsed).toBeLessThan(20_000)
 
     const afterSecond = await db.records.find({ labels: [label], limit: 1 })
     expect(afterSecond.total).toBe(150)

e2e/sdk/records.find.where-nested-pagination.e2e.test.ts

@@ -0,0 +1,127 @@
+/**
+ * E2E regression test for a bug where a related-label `where` traversal filter
+ * combined with `skip`/`limit` returned incomplete `data` while `total` stayed
+ * correct.
+ *
+ * Root cause (fixed in EntityQueryService.findRecords): SKIP/LIMIT/ORDER BY were
+ * glued onto the root MATCH clause, which ran *before* the OPTIONAL MATCH
+ * traversal to the related label and before the WITH...WHERE barrier that
+ * applies the traversal predicate. So pagination sliced the raw, unfiltered
+ * root scan first, and only afterwards was the traversal filter applied to
+ * whatever records happened to survive that slice — silently dropping matches
+ * that lived outside the pagination window.
+ *
+ * This test pins down an explicit, deterministic ordering (via a `seq`
+ * property) so the reproduction doesn't depend on incidental id ordering:
+ * parent "A" owns children seq 0-149, parent "B" owns children seq 150-299.
+ * Querying children scoped to "B", ordered by seq ascending, means the first
+ * `limit` window of the *unfiltered* scan is entirely "A"'s children — exactly
+ * the failure mode from the bug report.
+ *
+ * Prerequisites
+ * ─────────────
+ * RUSHDB_API_KEY and RUSHDB_API_URL must be set in packages/javascript-sdk/.env
+ *
+ * If RUSHDB_API_KEY is absent the whole suite is skipped gracefully.
+ */
+
+import RushDB from '../../packages/javascript-sdk/src/index.node'
+
+jest.setTimeout(60_000)
+
+const apiKey = process.env.RUSHDB_API_KEY
+const apiUrl = process.env.RUSHDB_API_URL || 'http://localhost:3000'
+
+if (!apiKey) {
+  describe('records.find() – traversal filter + pagination (e2e)', () => {
+    it('skips because RUSHDB_API_KEY is not set', () => expect(true).toBe(true))
+  })
+} else {
+  const db = new RushDB(apiKey, { url: apiUrl })
+
+  const tenantId = `where-nested-pg-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`
+
+  const L_PARENT = 'PGParent'
+  const L_CHILD = 'PGChild'
+
+  const CHILDREN_PER_PARENT = 150
+
+  beforeAll(async () => {
+    await db.records.delete({ where: { tenantId } }).catch(() => {})
+
+    await db.records.importJson({
+      label: L_PARENT,
+      data: [
+        {
+          name: 'A',
+          tenantId,
+          [L_CHILD]: Array.from({ length: CHILDREN_PER_PARENT }, (_, i) => ({ seq: i, tenantId }))
+        },
+        {
+          name: 'B',
+          tenantId,
+          [L_CHILD]: Array.from({ length: CHILDREN_PER_PARENT }, (_, i) => ({
+            seq: i + CHILDREN_PER_PARENT,
+            tenantId
+          }))
+        }
+      ]
+    })
+  })
+
+  afterAll(async () => {
+    await db.records.delete({ where: { tenantId } }).catch(() => {})
+  })
+
+  describe('records.find() – traversal filter + pagination (e2e)', () => {
+    it('reports the correct total for a related-label filter beyond the first page', async () => {
+      const res = await db.records.find({
+        labels: [L_CHILD],
+        where: { tenantId, [L_PARENT]: { name: 'B' } },
+        orderBy: { seq: 'asc' },
+        limit: 100
+      })
+      expect(res.total).toBe(CHILDREN_PER_PARENT)
+    })
+
+    it('returns a full first page of matches for a parent whose children sit outside the raw scan window', async () => {
+      const res = await db.records.find({
+        labels: [L_CHILD],
+        where: { tenantId, [L_PARENT]: { name: 'B' } },
+        orderBy: { seq: 'asc' },
+        limit: 100
+      })
+
+      expect(res.data).toHaveLength(100)
+      // Every returned record must actually belong to parent B (seq >= 150),
+      // never a record from A that leaked in from the unfiltered window.
+      expect((res.data as any[]).every((r) => r.data.seq >= CHILDREN_PER_PARENT)).toBe(true)
+    })
+
+    it('returns the remaining matches on the second page', async () => {
+      const res = await db.records.find({
+        labels: [L_CHILD],
+        where: { tenantId, [L_PARENT]: { name: 'B' } },
+        orderBy: { seq: 'asc' },
+        skip: 100,
+        limit: 100
+      })
+
+      expect(res.data).toHaveLength(50)
+      expect((res.data as any[]).every((r) => r.data.seq >= CHILDREN_PER_PARENT)).toBe(true)
+    })
+
+    it('still paginates correctly for a parent whose children DO sit inside the first window (regression sanity check)', async () => {
+      const res = await db.records.find({
+        labels: [L_CHILD],
+        where: { tenantId, [L_PARENT]: { name: 'A' } },
+        orderBy: { seq: 'asc' },
+        limit: 100
+      })
+
+      expect(res.total).toBe(CHILDREN_PER_PARENT)
+      expect(res.data).toHaveLength(100)
+      expect((res.data as any[]).every((r) => r.data.seq < CHILDREN_PER_PARENT)).toBe(true)
+    })
+  })
+}

platform/core/src/core/entity/entity-query.service.ts

@@ -187,8 +187,9 @@ export class EntityQueryService {
   importUpsertRecords({
     withResults = false,
     rewrite = false,
-    mergeBy
-  }: { withResults?: boolean; rewrite?: boolean; mergeBy?: string[] } = {}) {
+    mergeBy,
+    label: recordLabel
+  }: { withResults?: boolean; rewrite?: boolean; mergeBy?: string[]; label?: string } = {}) {
     const queryBuilder = new QueryBuilder()
 
     // When mergeBy is provided, inline the keys as static property accesses
@@ -200,6 +201,15 @@ export class EntityQueryService {
     const staticMergeKeys = (mergeBy ?? []).filter((k) => typeof k === 'string' && k.trim().length > 0)
     const useStaticMergeKeys = staticMergeKeys.length > 0
 
+    // Callers group an upsert batch by business label before calling this method, so every
+    // row in $records shares `label` — inline it as a static pattern label (record:Record:`Label`)
+    // instead of matching every :Record in the project and filtering by r.label afterward. This
+    // scopes the match to one business label's node set rather than the whole project. Rows with
+    // no label (rare: an unlabeled root import) are called without `label` and fall back to the
+    // original untyped match + runtime label check.
+    const labelPart = recordLabel ? `:\`${recordLabel.replace(/`/g, '``')}\`` : ''
+    const labelGuard = labelPart ? '' : '(r.label IS NULL OR ANY(l IN labels(record) WHERE l = r.label)) AND '
+
     queryBuilder
       .append(`WITH $records as recordsToUpsert, datetime() as time, coalesce($mergeBy, []) as mergeBy`)
       .append(`UNWIND recordsToUpsert as r`)
@@ -217,17 +227,13 @@ export class EntityQueryService {
         .join(' AND ')
 
       queryBuilder
-        .append(`OPTIONAL MATCH (record:${RUSHDB_LABEL_RECORD} { ${projectIdInline()} })`)
-        .append(
-          `WHERE (r.label IS NULL OR ANY(l IN labels(record) WHERE l = r.label)) AND ${mergePredicates}`
-        )
+        .append(`OPTIONAL MATCH (record:${RUSHDB_LABEL_RECORD}${labelPart} { ${projectIdInline()} })`)
+        .append(`WHERE ${labelGuard}${mergePredicates}`)
     } else {
       queryBuilder
         .append(`WITH *, CASE WHEN size(mergeBy)=0 THEN keys(valuesMap) ELSE mergeBy END as keysToMatch`)
-        .append(`OPTIONAL MATCH (record:${RUSHDB_LABEL_RECORD} { ${projectIdInline()} })`)
-        .append(
-          `WHERE (r.label IS NULL OR ANY(l IN labels(record) WHERE l = r.label)) AND ALL(k IN keysToMatch WHERE record[k] = valuesMap[k])`
-        )
+        .append(`OPTIONAL MATCH (record:${RUSHDB_LABEL_RECORD}${labelPart} { ${projectIdInline()} })`)
+        .append(`WHERE ${labelGuard}ALL(k IN keysToMatch WHERE record[k] = valuesMap[k])`)
     }
 
     queryBuilder
@@ -290,17 +296,7 @@ export class EntityQueryService {
       : { ...buildAggregation(searchQuery?.aggregate, aliasesMap, searchQuery?.groupBy ?? []), matchPart: '' }
 
     // convert a clause array to string
-    const normalizedQueryClauses = queryClauses
-      .map((clause, index) => {
-        if (!orderByAggregatedField && index === 0) {
-          return `${clause} ${sortParams} ${pagination}`.trim()
-        } else {
-          return clause
-        }
-      })
-      // @FYI: Keep it in this order
-      .filter(toBoolean)
-      .join(`\n`)
+    const normalizedQueryClauses = queryClauses.filter(toBoolean).join(`\n`)
 
     const queryBuilder = new QueryBuilder()
 
@@ -314,14 +310,23 @@ export class EntityQueryService {
       queryBuilder.append(`WITH ${parsedWhere.nodeAliases.join(', ')} ${wherePart}`.trim())
     }
 
+    // @FYI: Pagination/sort must be applied AFTER the traversal WITH...WHERE barrier
+    // above — otherwise SKIP/LIMIT truncates the root record scan before related-label
+    // matches are even evaluated, silently dropping rows that only appear once the
+    // traversal filter is applied. For the aggregated-order-by case, ORDER BY must
+    // additionally follow aggregateProjections since it sorts by the computed field.
+    if (!orderByAggregatedField) {
+      queryBuilder.append(`${sortParams} ${pagination}`.trim())
+    }
+
     if (matchPart) {
       queryBuilder.append(matchPart)
     }
 
     queryBuilder.append(aggregateProjections)
 
     if (orderByAggregatedField) {
-      queryBuilder.append(`${sortParams} ${pagination}`)
+      queryBuilder.append(`${sortParams} ${pagination}`.trim())
     }
 
     queryBuilder.append(`RETURN ${returnPart}`)