Prakhar/ivfplus benchmarking

README.md

@@ -710,4 +710,4 @@ If `numa_miss` / `numa_foreign` counters are high relative to `numa_hit`, querie
 
 ## License
 
-See LICENSE file for details.
+See LICENSE file for details.

config/cohere-1m-cos/edb_vectorplus-ivfplus-1k.yaml

@@ -0,0 +1,19 @@
+edb_vectorplus-ivfplus-cohere-1m-1k:
+  indexType: ivfplus
+  dataset: cohere-1m-cos
+  datasetType: parquet
+  metric: cos
+  lists: 1000 #sqrt(1m)
+  maintenance_work_mem: 4GB
+  pg_parallel_workers: 32
+  top: 10
+  # Fixed, suite-level GUCs (optional -- omit to use ivfplus defaults).
+  # iterative_scan: relaxed_order       # off | relaxed_order | strict_order
+  # max_probes: 2000
+  # hierarchy_threshold: 4
+  benchmarks:
+    "20":  { probes: 20 }
+    "40":  { probes: 40 }
+    "80":  { probes: 80 }
+    "150": { probes: 150 }
+    "300": { probes: 300 }

config/cohere-2m-cos/edb_vectorplus-ivfplus-1414.yaml

@@ -0,0 +1,19 @@
+edb_vectorplus-ivfplus-cohere-2m-1414:
+  indexType: ivfplus
+  dataset: cohere-2m-cos
+  datasetType: parquet
+  metric: cos
+  lists: 1414 #sqrt(2m)
+  maintenance_work_mem: 8GB
+  pg_parallel_workers: 32
+  top: 10
+  # Fixed, suite-level GUCs (optional -- omit to use ivfplus defaults).
+  # iterative_scan: relaxed_order       # off | relaxed_order | strict_order
+  # max_probes: 2000
+  # hierarchy_threshold: 4
+  benchmarks:
+    "20":  { probes: 20 }
+    "40":  { probes: 40 }
+    "80":  { probes: 80 }
+    "150": { probes: 150 }
+    "300": { probes: 300 }

config/cohere-3m-cos/edb_vectorplus-ivfplus-1800.yaml

@@ -0,0 +1,19 @@
+edb_vectorplus-ivfplus-cohere-3m-1800:
+  indexType: ivfplus
+  dataset: cohere-2m-cos
+  datasetType: parquet
+  metric: cos
+  lists: 1800 #sqrt(3m)
+  maintenance_work_mem: 12GB
+  pg_parallel_workers: 32
+  top: 10
+  # Fixed, suite-level GUCs (optional -- omit to use ivfplus defaults).
+  # iterative_scan: relaxed_order       # off | relaxed_order | strict_order
+  # max_probes: 2000
+  # hierarchy_threshold: 4
+  benchmarks:
+    "20":  { probes: 20 }
+    "40":  { probes: 40 }
+    "80":  { probes: 80 }
+    "150": { probes: 150 }
+    "300": { probes: 300 }

config/cohere-50k-cos/edb_vectorplus-ivfplus-224.yaml

@@ -0,0 +1,12 @@
+edb_vectorplus-ivfplus-cohere-50k-224:
+  indexType: ivfplus
+  dataset: cohere-50k-cos
+  datasetType: parquet
+  metric: cos
+  lists: 224  # ~sqrt(50k)
+  maintenance_work_mem: 1GB
+  pg_parallel_workers: 1
+  top: 10
+  benchmarks:
+    "40":  { probes: 40 }
+    "224": { probes: 224 }

config/cohere-5k-cos/edb_vectorplus-ivfplus-71.yaml

@@ -0,0 +1,12 @@
+edb_vectorplus-ivfplus-cohere-5k-71:
+  indexType: ivfplus
+  dataset: cohere-5k-cos
+  datasetType: parquet
+  metric: cos
+  lists: 71  # ~sqrt(5k)
+  maintenance_work_mem: 1GB
+  pg_parallel_workers: 1
+  top: 10
+  benchmarks:
+    "16": { probes: 16 }
+    "71": { probes: 71 }

config/laion-50k-ip/edb_vectorplus-ivfplus-224.yaml

@@ -0,0 +1,12 @@
+edb_vectorplus-ivfplus-laion-50k-224:
+  indexType: ivfplus
+  dataset: laion-50k-ip
+  datasetType: parquet
+  metric: ip
+  lists: 224  # ~sqrt(50k)
+  maintenance_work_mem: 1GB
+  pg_parallel_workers: 1
+  top: 10
+  benchmarks:
+    "40":  { probes: 40 }
+    "224": { probes: 224 }

config/laion-5k-ip/edb_vectorplus-ivfplus-71.yaml

@@ -0,0 +1,12 @@
+edb_vectorplus-ivfplus-laion-5k-71:
+  indexType: ivfplus
+  dataset: laion-5k-ip
+  datasetType: parquet
+  metric: ip
+  lists: 71  # ~sqrt(5k)
+  maintenance_work_mem: 1GB
+  pg_parallel_workers: 1
+  top: 10
+  benchmarks:
+    "16": { probes: 16 }
+    "71": { probes: 71 }

config/laion-5m-test-ip/edb_vectorplus-ivfplus-2236.yaml

@@ -0,0 +1,16 @@
+edb_vectorplus-ivfplus-laion-5m-2236:
+  indexType: ivfplus
+  dataset: laion-5m-test-ip
+  datasetType: hdf5
+  metric: cos
+  lists: 2236  # ~sqrt(5M)
+  rotation: true
+  maintenance_work_mem: 4GB
+  pg_parallel_workers: 32
+  top: 10
+  benchmarks:
+    "20":  { probes: 20 }
+    "40":  { probes: 40 }
+    "80":  { probes: 80 }
+    "150": { probes: 150 }
+    "300": { probes: 300 }

config/openai-1m-cos/edb_vectorplus_ivfplus-1k.yaml

@@ -0,0 +1,19 @@
+edb_vectorplus-ivfplus-openai-1m-cos-1k:
+  indexType: ivfplus
+  dataset: openai-1m-cos
+  datasetType: parquet
+  metric: cos
+  lists: 1000 #sqrt(1m)
+  maintenance_work_mem: 4GB
+  pg_parallel_workers: 32
+  top: 10
+  # Fixed, suite-level GUCs (optional -- omit to use ivfplus defaults).
+  # iterative_scan: relaxed_order       # off | relaxed_order | strict_order
+  # max_probes: 2000
+  # hierarchy_threshold: 4
+  benchmarks:
+    "20":  { probes: 20 }
+    "40":  { probes: 40 }
+    "80":  { probes: 80 }
+    "150": { probes: 150 }
+    "300": { probes: 300 }

config/openai-2m-cos/edb_vectorplus_ivfplus-1414.yaml

@@ -0,0 +1,19 @@
+edb_vectorplus-ivfplus-cohere-2m-1414:
+  indexType: ivfplus
+  dataset: openai-2m-cos
+  datasetType: parquet
+  metric: cos
+  lists: 1414 #sqrt(2m)
+  maintenance_work_mem: 8GB
+  pg_parallel_workers: 32
+  top: 10
+  # Fixed, suite-level GUCs (optional -- omit to use ivfplus defaults).
+  # iterative_scan: relaxed_order       # off | relaxed_order | strict_order
+  # max_probes: 2000
+  # hierarchy_threshold: 4
+  benchmarks:
+    "20":  { probes: 20 }
+    "40":  { probes: 40 }
+    "80":  { probes: 80 }
+    "150": { probes: 150 }
+    "300": { probes: 300 }

config/openai-500k-cos/edb_vectorplus-ivfplus-707.yaml

@@ -0,0 +1,15 @@
+edb_vectorplus-ivfplus-openai-500k-707:
+  indexType: ivfplus
+  dataset: openai-500k-cos
+  datasetType: parquet
+  metric: cos
+  lists: 707  # ~sqrt(500k)
+  maintenance_work_mem: 1GB
+  pg_parallel_workers: 32
+  top: 10
+  benchmarks:
+    "10":  { probes: 10 }
+    "30":  { probes: 30 }
+    "60":  { probes: 60 }
+    "100": { probes: 100 }
+    "200": { probes: 200 }

config/openai-50k-cos/edb_vectorplus-ivfplus-224.yaml

@@ -0,0 +1,12 @@
+edb_vectorplus-ivfplus-openai-50k-224:
+  indexType: ivfplus
+  dataset: openai-50k-cos
+  datasetType: parquet
+  metric: cos
+  lists: 224  # ~sqrt(50k)
+  maintenance_work_mem: 1GB
+  pg_parallel_workers: 1
+  top: 10
+  benchmarks:
+    "40":  { probes: 40 }
+    "224": { probes: 224 }

config/openai-5k-cos/edb_vectorplus-ivfplus-71.yaml

@@ -0,0 +1,19 @@
+edb_vectorplus-ivfplus-openai-5k-71:
+  indexType: ivfplus
+  dataset: openai-5k-cos
+  datasetType: parquet
+  metric: cos
+  lists: 71  # ~sqrt(5k)
+  maintenance_work_mem: 1GB
+  pg_parallel_workers: 1
+  top: 10
+  benchmarks:
+    "16": { probes: 16 }
+    "71": { probes: 71 }
+
+# Also consumed by edb_vectorplus CI (the benchmark-smoke job in build.yml),
+# which asserts a recall floor against it -- so lists/probes/top here are that
+# gate's specification, not just a local choice.
+#
+# probes 71 == lists, i.e. a full scan, which is what the recall gate asserts
+# on -- k-means variation cancels out there. Measured recall 0.9995.

config/openai-5m-cos/edb_vectorplus-ivfplus-2236.yaml

@@ -0,0 +1,20 @@
+edb_vectorplus-ivfplus-openai-5m-2236:
+  indexType: ivfplus
+  dataset: openai-5m-cos
+  datasetType: parquet
+  metric: cos
+  lists: 2236  # ~sqrt(5M)
+  rotation: true
+  maintenance_work_mem: 4GB
+  pg_parallel_workers: 32
+  top: 10
+  # Fixed, suite-level GUCs (optional -- omit to use ivfplus defaults).
+  # iterative_scan: relaxed_order       # off | relaxed_order | strict_order
+  # max_probes: 2000
+  # hierarchy_threshold: 4
+  benchmarks:
+    "20":  { probes: 20 }
+    "40":  { probes: 40 }
+    "80":  { probes: 80 }
+    "150": { probes: 150 }
+    "300": { probes: 300 }

config/toy-5-cos/edb_vectorplus-ivfplus-toy-5-2.yaml

@@ -0,0 +1,11 @@
+edb_vectorplus-ivfplus-toy-5-2:
+  indexType: ivfplus
+  dataset: toy-5-cos
+  datasetType: parquet
+  metric: cos
+  lists: 2 #sqrt(1m)
+  maintenance_work_mem: 1GB
+  pg_parallel_workers: 1
+  top: 3
+  benchmarks:
+    "2":  { probes: 2 }

datasets.py

@@ -9,6 +9,7 @@
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from tqdm import tqdm
 
+
 # --- CONFIGURATION ---
 DATA_DIR = os.environ.get("DATASET_LOCAL_DIR", "./datasets")
 DATASETS = {
@@ -109,6 +110,10 @@
         "s3_prefix": "s3://enterprisedb-vector-datasets/openai/5m",
         "base_dir": os.path.join(DATA_DIR, "openai/5m"),
     },
+    # --- CI/CD subsets (real slices of each family's smallest dataset, ground
+    #     truth recomputed against the subset). Tiny; downloaded from S3 like
+    #     the other parquet datasets. Generated offline via
+    #     utils/derive_datasets.py. ---
     "openai-5k-cos": {
         "type": "parquet",
         "metric": "cos",