Add shape (character-class skeleton) index layer

src/constants.js

@@ -29,3 +29,16 @@ export const defaultNgramLength = 5
 // identically; an index with no such key (built before this) is read as the
 // empty set, i.e. plain alphanumeric, and keeps working unchanged.
 export const defaultNgramChars = '"{}:'
+
+// Length of shape (character-class skeleton) n-grams. Shape n-grams collapse
+// letters to L and digits to D over a tiny alphabet, so a short window stays
+// selective for structured patterns (a phone shape DDD-DDDD is rare even though
+// every individual character class is common). 4 captures the digit-group
+// structure of phones, SSNs, IPs and dates.
+export const defaultShapeNgramLength = 4
+
+// Structural punctuation kept verbatim inside shape runs (everything else that
+// is not a letter or digit is a shape boundary). These are the separators that
+// give numeric patterns their selectivity: dots, dashes, slashes, colons,
+// underscores and the email '@'.
+export const defaultShapeChars = '@.-_/:'

src/createIndex.js

@@ -1,7 +1,8 @@
 import { parquetMetadataAsync, parquetReadObjects } from 'hyparquet'
 import { ParquetWriter, schemaFromColumnData } from 'hyparquet-writer'
-import { defaultBlockSize, defaultIndexRowGroupSize, defaultNgramChars, defaultNgramLength, hypGrepVersion } from './constants.js'
+import { defaultBlockSize, defaultIndexRowGroupSize, defaultNgramChars, defaultNgramLength, defaultShapeChars, defaultShapeNgramLength, hypGrepVersion } from './constants.js'
 import { extractNgrams } from './ngrams.js'
+import { extractShapeNgrams } from './shape.js'
 import { assertNonNegativeSafeInteger, assertPositiveSafeInteger, getTextColumnsFromSchema } from './utils.js'
 
 /**
@@ -32,6 +33,12 @@ export async function createIndex({
   }
   assertNonNegativeSafeInteger(sourceFile.byteLength, 'sourceFile.byteLength')
 
+  // The shape layer is always on and not configurable; it is a second, coarser
+  // tokenization of the same text. Its settings are recorded in kv metadata so
+  // queries tokenize identically and so an index built without it is detected.
+  const shapeNgramLength = defaultShapeNgramLength
+  const shapeChars = defaultShapeChars
+
   const metadata = sourceMetadata ?? await parquetMetadataAsync(sourceFile)
   const numRows = Number(metadata.num_rows)
   assertNonNegativeSafeInteger(numRows, 'source row count')
@@ -103,7 +110,7 @@ export async function createIndex({
       columns: textColumns,
     })
     for (const row of rows) {
-      collectRowNgrams(row, textColumns, ngramLength, ngramChars, blockNgrams)
+      collectRowNgrams(row, textColumns, ngramLength, ngramChars, shapeNgramLength, shapeChars, blockNgrams)
       if (++rowsInBlock === blockSize) flushBlock()
     }
     groupStart += groupRows
@@ -118,6 +125,10 @@ export async function createIndex({
     // missing this key (built before structural tokenization) is read as '',
     // i.e. plain alphanumeric, and keeps working unchanged.
     { key: 'hypgrep.ngram_chars', value: ngramChars },
+    // Shape (character-class skeleton) layer. Absence of these keys means the
+    // index has no shape tokens, so queryIndex must not require any.
+    { key: 'hypgrep.shape_ngram_length', value: String(shapeNgramLength) },
+    { key: 'hypgrep.shape_chars', value: shapeChars },
     { key: 'hypgrep.text_columns', value: textColumns.join(',') },
     { key: 'hypgrep.source_rows', value: String(numRows) },
     // Can save network requests on the source file
@@ -183,21 +194,27 @@ export async function createIndex({
 }
 
 /**
- * Collect the distinct n-grams of one row into a block's accumulator set.
+ * Collect the distinct n-grams of one row into a block's accumulator set:
+ * ordinary character n-grams plus shape (character-class skeleton) n-grams,
+ * which share the column but are namespaced by SHAPE_PREFIX.
  *
  * @param {Record<string, any>} row
  * @param {string[]} textColumns
  * @param {number} n
  * @param {string} chars extra characters kept inside n-gram runs
+ * @param {number} shapeN shape n-gram length
+ * @param {string} shapeChars structural punctuation kept verbatim in shapes
  * @param {Set<string>} ngrams block accumulator to add into
  */
-function collectRowNgrams(row, textColumns, n, chars, ngrams) {
+function collectRowNgrams(row, textColumns, n, chars, shapeN, shapeChars, ngrams) {
   if (!row) return
   for (const columnName of textColumns) {
     const value = row[columnName]
-    if (typeof value !== 'string' || value.length < n) continue
-    for (const g of extractNgrams(value, n, chars)) {
-      ngrams.add(g)
+    if (typeof value !== 'string') continue
+    if (value.length >= n) {
+      for (const g of extractNgrams(value, n, chars)) ngrams.add(g)
     }
+    // Shape grams stream straight into the block accumulator (no throwaway set).
+    extractShapeNgrams(value, shapeN, shapeChars, ngrams)
   }
 }

src/queryIndex.js

@@ -1,7 +1,7 @@
 import { parquetMetadataAsync, parquetQuery } from 'hyparquet'
 import { hypGrepVersion } from './constants.js'
 import { literalsToNgrams, queryNgrams } from './ngrams.js'
-import { extractRegexLiterals } from './regex.js'
+import { extractRegexLiterals, extractRegexShapes } from './regex.js'
 import { assertNonNegativeSafeInteger, assertPositiveSafeInteger } from './utils.js'
 
 /**
@@ -32,19 +32,31 @@ export async function queryIndex({ query, indexFile, indexMetadata }) {
   // Read index kv metadata
   indexMetadata ??= await parquetMetadataAsync(indexFile)
   const kvMetadata = indexMetadata.key_value_metadata || []
-  const { blockSize, ngramLength, ngramChars, textColumns, sourceByteLength, sourceRows } = parseKvMetadata(kvMetadata)
+  const { blockSize, ngramLength, ngramChars, shapeNgramLength, shapeChars, textColumns, sourceByteLength, sourceRows } = parseKvMetadata(kvMetadata)
 
   // A "branch" is a conjunction of n-grams that ALL must appear in a block.
   // A query matches a block if ANY branch is fully satisfied (DNF).
   //
   // - String query: one branch, the n-grams of the string itself.
-  // - RegExp query: one branch per top-level alternation arm.
-  // The query MUST be tokenized with the same kept-character set the index was
-  // built with, or its n-grams won't line up with the postings.
+  // - RegExp query: one branch per top-level alternation arm, plus any mandatory
+  //   shape n-grams (character-class skeletons) ANDed in. Shape tokens let a
+  //   literal-free pattern (a phone number, SSN, IP) prune by its structure.
+  //   Gated on shapeNgramLength so an index with no shape layer is never asked
+  //   for shape tokens (which would be a false negative).
   /** @type {string[][]} */
-  const branches = query instanceof RegExp
-    ? extractRegexLiterals(query).map(lits => literalsToNgrams(lits, ngramLength, ngramChars))
-    : [queryNgrams(query, ngramLength, ngramChars)]
+  let branches
+  if (query instanceof RegExp) {
+    const shapeGrams = shapeNgramLength ? extractRegexShapes(query, shapeNgramLength, shapeChars) : []
+    branches = extractRegexLiterals(query).map(lits => {
+      const grams = literalsToNgrams(lits, ngramLength, ngramChars)
+      // Shape grams are near-universal (any 4-digit run, any hyphenated word),
+      // so their posting lists cost bytes but prune little. Only spend them on a
+      // branch with no literal n-gram to prune on; otherwise the literal wins.
+      return grams.length ? grams : grams.concat(shapeGrams)
+    })
+  } else {
+    branches = [queryNgrams(query, ngramLength, ngramChars)]
+  }
 
   // If any branch is empty, that branch matches anything — falling back to a
   // full scan is correct and bounded.
@@ -124,6 +136,11 @@ export function parseKvMetadata(kvMetadata) {
   // Extra characters kept inside n-gram runs. Absent in pre-structural indexes,
   // which were tokenized as plain alphanumeric, so '' reproduces that exactly.
   let ngramChars = ''
+  // Shape layer settings. shapeNgramLength stays undefined for indexes built
+  // without a shape layer, which signals queryIndex not to require shape tokens.
+  /** @type {number | undefined} */
+  let shapeNgramLength
+  let shapeChars = ''
   /** @type {string[]} */
   let textColumns = []
   /** @type {number | undefined} */
@@ -141,6 +158,16 @@ export function parseKvMetadata(kvMetadata) {
     if (key === 'hypgrep.ngram_chars' && typeof value === 'string') {
       ngramChars = value
     }
+    if (key === 'hypgrep.shape_ngram_length') {
+      // The shape layer is optional: a malformed value degrades to "no shape
+      // layer" (like a missing key) instead of failing the whole index — the
+      // text n-gram layer is still intact and queryable.
+      const n = Number(value)
+      if (Number.isSafeInteger(n) && n > 0) shapeNgramLength = n
+    }
+    if (key === 'hypgrep.shape_chars' && typeof value === 'string') {
+      shapeChars = value
+    }
     if (key === 'hypgrep.version') {
       version = Number(value)
       if (version !== hypGrepVersion) {
@@ -180,5 +207,5 @@ export function parseKvMetadata(kvMetadata) {
   assertNonNegativeSafeInteger(sourceRows, 'hypgrep.source_rows')
   assertNonNegativeSafeInteger(sourceByteLength, 'hypgrep.source_bytelength')
 
-  return { blockSize, ngramLength, ngramChars, textColumns, sourceByteLength, sourceRows }
+  return { blockSize, ngramLength, ngramChars, shapeNgramLength, shapeChars, textColumns, sourceByteLength, sourceRows }
 }