diff --git a/README.md b/README.md index 4f9419f..f3ae42e 100644 --- a/README.md +++ b/README.md @@ -710,4 +710,4 @@ If `numa_miss` / `numa_foreign` counters are high relative to `numa_hit`, querie ## License -See LICENSE file for details. +See LICENSE file for details. \ No newline at end of file diff --git a/config/cohere-1m-cos/edb_vectorplus-ivfplus-1k.yaml b/config/cohere-1m-cos/edb_vectorplus-ivfplus-1k.yaml new file mode 100644 index 0000000..f44139a --- /dev/null +++ b/config/cohere-1m-cos/edb_vectorplus-ivfplus-1k.yaml @@ -0,0 +1,19 @@ +edb_vectorplus-ivfplus-cohere-1m-1k: + indexType: ivfplus + dataset: cohere-1m-cos + datasetType: parquet + metric: cos + lists: 1000 #sqrt(1m) + maintenance_work_mem: 4GB + pg_parallel_workers: 32 + top: 10 + # Fixed, suite-level GUCs (optional -- omit to use ivfplus defaults). + # iterative_scan: relaxed_order # off | relaxed_order | strict_order + # max_probes: 2000 + # hierarchy_threshold: 4 + benchmarks: + "20": { probes: 20 } + "40": { probes: 40 } + "80": { probes: 80 } + "150": { probes: 150 } + "300": { probes: 300 } diff --git a/config/cohere-2m-cos/edb_vectorplus-ivfplus-1414.yaml b/config/cohere-2m-cos/edb_vectorplus-ivfplus-1414.yaml new file mode 100644 index 0000000..ccff9a1 --- /dev/null +++ b/config/cohere-2m-cos/edb_vectorplus-ivfplus-1414.yaml @@ -0,0 +1,19 @@ +edb_vectorplus-ivfplus-cohere-2m-1414: + indexType: ivfplus + dataset: cohere-2m-cos + datasetType: parquet + metric: cos + lists: 1414 #sqrt(2m) + maintenance_work_mem: 8GB + pg_parallel_workers: 32 + top: 10 + # Fixed, suite-level GUCs (optional -- omit to use ivfplus defaults). + # iterative_scan: relaxed_order # off | relaxed_order | strict_order + # max_probes: 2000 + # hierarchy_threshold: 4 + benchmarks: + "20": { probes: 20 } + "40": { probes: 40 } + "80": { probes: 80 } + "150": { probes: 150 } + "300": { probes: 300 } diff --git a/config/cohere-3m-cos/edb_vectorplus-ivfplus-1800.yaml b/config/cohere-3m-cos/edb_vectorplus-ivfplus-1800.yaml new file mode 100644 index 0000000..b94bfe6 --- /dev/null +++ b/config/cohere-3m-cos/edb_vectorplus-ivfplus-1800.yaml @@ -0,0 +1,19 @@ +edb_vectorplus-ivfplus-cohere-3m-1800: + indexType: ivfplus + dataset: cohere-2m-cos + datasetType: parquet + metric: cos + lists: 1800 #sqrt(3m) + maintenance_work_mem: 12GB + pg_parallel_workers: 32 + top: 10 + # Fixed, suite-level GUCs (optional -- omit to use ivfplus defaults). + # iterative_scan: relaxed_order # off | relaxed_order | strict_order + # max_probes: 2000 + # hierarchy_threshold: 4 + benchmarks: + "20": { probes: 20 } + "40": { probes: 40 } + "80": { probes: 80 } + "150": { probes: 150 } + "300": { probes: 300 } diff --git a/config/cohere-50k-cos/edb_vectorplus-ivfplus-224.yaml b/config/cohere-50k-cos/edb_vectorplus-ivfplus-224.yaml new file mode 100644 index 0000000..84b460a --- /dev/null +++ b/config/cohere-50k-cos/edb_vectorplus-ivfplus-224.yaml @@ -0,0 +1,12 @@ +edb_vectorplus-ivfplus-cohere-50k-224: + indexType: ivfplus + dataset: cohere-50k-cos + datasetType: parquet + metric: cos + lists: 224 # ~sqrt(50k) + maintenance_work_mem: 1GB + pg_parallel_workers: 1 + top: 10 + benchmarks: + "40": { probes: 40 } + "224": { probes: 224 } diff --git a/config/cohere-5k-cos/edb_vectorplus-ivfplus-71.yaml b/config/cohere-5k-cos/edb_vectorplus-ivfplus-71.yaml new file mode 100644 index 0000000..291a963 --- /dev/null +++ b/config/cohere-5k-cos/edb_vectorplus-ivfplus-71.yaml @@ -0,0 +1,12 @@ +edb_vectorplus-ivfplus-cohere-5k-71: + indexType: ivfplus + dataset: cohere-5k-cos + datasetType: parquet + metric: cos + lists: 71 # ~sqrt(5k) + maintenance_work_mem: 1GB + pg_parallel_workers: 1 + top: 10 + benchmarks: + "16": { probes: 16 } + "71": { probes: 71 } diff --git a/config/laion-50k-ip/edb_vectorplus-ivfplus-224.yaml b/config/laion-50k-ip/edb_vectorplus-ivfplus-224.yaml new file mode 100644 index 0000000..9bd2a5b --- /dev/null +++ b/config/laion-50k-ip/edb_vectorplus-ivfplus-224.yaml @@ -0,0 +1,12 @@ +edb_vectorplus-ivfplus-laion-50k-224: + indexType: ivfplus + dataset: laion-50k-ip + datasetType: parquet + metric: ip + lists: 224 # ~sqrt(50k) + maintenance_work_mem: 1GB + pg_parallel_workers: 1 + top: 10 + benchmarks: + "40": { probes: 40 } + "224": { probes: 224 } diff --git a/config/laion-5k-ip/edb_vectorplus-ivfplus-71.yaml b/config/laion-5k-ip/edb_vectorplus-ivfplus-71.yaml new file mode 100644 index 0000000..46044b0 --- /dev/null +++ b/config/laion-5k-ip/edb_vectorplus-ivfplus-71.yaml @@ -0,0 +1,12 @@ +edb_vectorplus-ivfplus-laion-5k-71: + indexType: ivfplus + dataset: laion-5k-ip + datasetType: parquet + metric: ip + lists: 71 # ~sqrt(5k) + maintenance_work_mem: 1GB + pg_parallel_workers: 1 + top: 10 + benchmarks: + "16": { probes: 16 } + "71": { probes: 71 } diff --git a/config/laion-5m-test-ip/edb_vectorplus-ivfplus-2236.yaml b/config/laion-5m-test-ip/edb_vectorplus-ivfplus-2236.yaml new file mode 100644 index 0000000..94fc6ff --- /dev/null +++ b/config/laion-5m-test-ip/edb_vectorplus-ivfplus-2236.yaml @@ -0,0 +1,16 @@ +edb_vectorplus-ivfplus-laion-5m-2236: + indexType: ivfplus + dataset: laion-5m-test-ip + datasetType: hdf5 + metric: cos + lists: 2236 # ~sqrt(5M) + rotation: true + maintenance_work_mem: 4GB + pg_parallel_workers: 32 + top: 10 + benchmarks: + "20": { probes: 20 } + "40": { probes: 40 } + "80": { probes: 80 } + "150": { probes: 150 } + "300": { probes: 300 } diff --git a/config/openai-1m-cos/edb_vectorplus_ivfplus-1k.yaml b/config/openai-1m-cos/edb_vectorplus_ivfplus-1k.yaml new file mode 100644 index 0000000..b54f4f1 --- /dev/null +++ b/config/openai-1m-cos/edb_vectorplus_ivfplus-1k.yaml @@ -0,0 +1,19 @@ +edb_vectorplus-ivfplus-openai-1m-cos-1k: + indexType: ivfplus + dataset: openai-1m-cos + datasetType: parquet + metric: cos + lists: 1000 #sqrt(1m) + maintenance_work_mem: 4GB + pg_parallel_workers: 32 + top: 10 + # Fixed, suite-level GUCs (optional -- omit to use ivfplus defaults). + # iterative_scan: relaxed_order # off | relaxed_order | strict_order + # max_probes: 2000 + # hierarchy_threshold: 4 + benchmarks: + "20": { probes: 20 } + "40": { probes: 40 } + "80": { probes: 80 } + "150": { probes: 150 } + "300": { probes: 300 } diff --git a/config/openai-2m-cos/edb_vectorplus_ivfplus-1414.yaml b/config/openai-2m-cos/edb_vectorplus_ivfplus-1414.yaml new file mode 100644 index 0000000..47542c2 --- /dev/null +++ b/config/openai-2m-cos/edb_vectorplus_ivfplus-1414.yaml @@ -0,0 +1,19 @@ +edb_vectorplus-ivfplus-cohere-2m-1414: + indexType: ivfplus + dataset: openai-2m-cos + datasetType: parquet + metric: cos + lists: 1414 #sqrt(2m) + maintenance_work_mem: 8GB + pg_parallel_workers: 32 + top: 10 + # Fixed, suite-level GUCs (optional -- omit to use ivfplus defaults). + # iterative_scan: relaxed_order # off | relaxed_order | strict_order + # max_probes: 2000 + # hierarchy_threshold: 4 + benchmarks: + "20": { probes: 20 } + "40": { probes: 40 } + "80": { probes: 80 } + "150": { probes: 150 } + "300": { probes: 300 } diff --git a/config/openai-500k-cos/edb_vectorplus-ivfplus-707.yaml b/config/openai-500k-cos/edb_vectorplus-ivfplus-707.yaml new file mode 100644 index 0000000..758b7e3 --- /dev/null +++ b/config/openai-500k-cos/edb_vectorplus-ivfplus-707.yaml @@ -0,0 +1,15 @@ +edb_vectorplus-ivfplus-openai-500k-707: + indexType: ivfplus + dataset: openai-500k-cos + datasetType: parquet + metric: cos + lists: 707 # ~sqrt(500k) + maintenance_work_mem: 1GB + pg_parallel_workers: 32 + top: 10 + benchmarks: + "10": { probes: 10 } + "30": { probes: 30 } + "60": { probes: 60 } + "100": { probes: 100 } + "200": { probes: 200 } diff --git a/config/openai-50k-cos/edb_vectorplus-ivfplus-224.yaml b/config/openai-50k-cos/edb_vectorplus-ivfplus-224.yaml new file mode 100644 index 0000000..fb82dd0 --- /dev/null +++ b/config/openai-50k-cos/edb_vectorplus-ivfplus-224.yaml @@ -0,0 +1,12 @@ +edb_vectorplus-ivfplus-openai-50k-224: + indexType: ivfplus + dataset: openai-50k-cos + datasetType: parquet + metric: cos + lists: 224 # ~sqrt(50k) + maintenance_work_mem: 1GB + pg_parallel_workers: 1 + top: 10 + benchmarks: + "40": { probes: 40 } + "224": { probes: 224 } diff --git a/config/openai-5k-cos/edb_vectorplus-ivfplus-71.yaml b/config/openai-5k-cos/edb_vectorplus-ivfplus-71.yaml new file mode 100644 index 0000000..ebc718f --- /dev/null +++ b/config/openai-5k-cos/edb_vectorplus-ivfplus-71.yaml @@ -0,0 +1,19 @@ +edb_vectorplus-ivfplus-openai-5k-71: + indexType: ivfplus + dataset: openai-5k-cos + datasetType: parquet + metric: cos + lists: 71 # ~sqrt(5k) + maintenance_work_mem: 1GB + pg_parallel_workers: 1 + top: 10 + benchmarks: + "16": { probes: 16 } + "71": { probes: 71 } + +# Also consumed by edb_vectorplus CI (the benchmark-smoke job in build.yml), +# which asserts a recall floor against it -- so lists/probes/top here are that +# gate's specification, not just a local choice. +# +# probes 71 == lists, i.e. a full scan, which is what the recall gate asserts +# on -- k-means variation cancels out there. Measured recall 0.9995. diff --git a/config/openai-5m-cos/edb_vectorplus-ivfplus-2236.yaml b/config/openai-5m-cos/edb_vectorplus-ivfplus-2236.yaml new file mode 100644 index 0000000..d638295 --- /dev/null +++ b/config/openai-5m-cos/edb_vectorplus-ivfplus-2236.yaml @@ -0,0 +1,20 @@ +edb_vectorplus-ivfplus-openai-5m-2236: + indexType: ivfplus + dataset: openai-5m-cos + datasetType: parquet + metric: cos + lists: 2236 # ~sqrt(5M) + rotation: true + maintenance_work_mem: 4GB + pg_parallel_workers: 32 + top: 10 + # Fixed, suite-level GUCs (optional -- omit to use ivfplus defaults). + # iterative_scan: relaxed_order # off | relaxed_order | strict_order + # max_probes: 2000 + # hierarchy_threshold: 4 + benchmarks: + "20": { probes: 20 } + "40": { probes: 40 } + "80": { probes: 80 } + "150": { probes: 150 } + "300": { probes: 300 } diff --git a/config/toy-5-cos/edb_vectorplus-ivfplus-toy-5-2.yaml b/config/toy-5-cos/edb_vectorplus-ivfplus-toy-5-2.yaml new file mode 100644 index 0000000..db8adf7 --- /dev/null +++ b/config/toy-5-cos/edb_vectorplus-ivfplus-toy-5-2.yaml @@ -0,0 +1,11 @@ +edb_vectorplus-ivfplus-toy-5-2: + indexType: ivfplus + dataset: toy-5-cos + datasetType: parquet + metric: cos + lists: 2 #sqrt(1m) + maintenance_work_mem: 1GB + pg_parallel_workers: 1 + top: 3 + benchmarks: + "2": { probes: 2 } diff --git a/datasets.py b/datasets.py index e882e25..65498c1 100644 --- a/datasets.py +++ b/datasets.py @@ -9,6 +9,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from tqdm import tqdm + # --- CONFIGURATION --- DATA_DIR = os.environ.get("DATASET_LOCAL_DIR", "./datasets") DATASETS = { @@ -109,6 +110,10 @@ "s3_prefix": "s3://enterprisedb-vector-datasets/openai/5m", "base_dir": os.path.join(DATA_DIR, "openai/5m"), }, + # --- CI/CD subsets (real slices of each family's smallest dataset, ground + # truth recomputed against the subset). Tiny; downloaded from S3 like + # the other parquet datasets. Generated offline via + # utils/derive_datasets.py. --- "openai-5k-cos": { "type": "parquet", "metric": "cos", diff --git a/pgvector_suite.py b/pgvector_suite.py index 9ae5be1..bef7975 100644 --- a/pgvector_suite.py +++ b/pgvector_suite.py @@ -2,9 +2,10 @@ pgvector Benchmark Suite Benchmarks vector search using the pgvector extension with HNSW or -IVFFlat indexes (vanilla and BQ + rerank variants) for PostgreSQL. +IVFFlat indexes (vanilla and BQ + rerank variants), plus the +edb_vectorplus extension's ivfplus access method, for PostgreSQL. -A single TestSuite handles all three index types. The per-index +A single TestSuite handles all four index types. The per-index variation lives in INDEX_SPECS — small functions that build the CREATE INDEX statement, the per-benchmark session GUCs, and the per-benchmark query template. Everything else (warmup, sequential / parallel @@ -231,10 +232,57 @@ def _bq_rerank_debug_print(config, dataset): print() +def _ivfplus_session_gucs(benchmark): + return [ + f"SET ivfplus.probes = {benchmark['probes']}", + "SET enable_seqscan = off", + ] + + +def _ivfplus_fixed_gucs(config: dict) -> list[str]: + """Optional, suite-level fixed GUCs for ivfplus — set once per suite + (not swept per benchmark point) and applied on top of the per-benchmark + `probes` GUC above.""" + stmts = [] + iterative_scan = config.get("iterative_scan") + if iterative_scan is not None: + stmts.append(f"SET ivfplus.iterative_scan = '{iterative_scan}'") + max_probes = config.get("max_probes") + if max_probes is not None: + stmts.append(f"SET ivfplus.max_probes = {max_probes}") + hierarchy_threshold = config.get("hierarchy_threshold") + if hierarchy_threshold is not None: + stmts.append(f"SET ivfplus.hierarchy_threshold = {hierarchy_threshold}") + return stmts + + +def _ivfplus_create_index_sql(table_name, config, dataset): + lists = _ivfflat_resolve_lists(config, dataset) + metric_func = _metric_func(dataset["metric"]) + rotation_sql = "true" + return ( + f"CREATE INDEX {table_name}_embedding_idx ON {table_name} " + f"USING ivfplus (embedding {metric_func}) " + f"WITH (lists = {lists}, rotation = {rotation_sql})" + ) + + +def _ivfplus_debug_print(config, dataset): + lists = _ivfflat_resolve_lists(config, dataset) + print(f"\nšŸ”§ Index Configuration (ivfplus):") + print(f" • Lists: {lists}") + print(f" • Metric Function: {_metric_func(dataset['metric'])}") + print() + + _IVFFLAT_CONFIG_COLUMNS = ( ("Lists", lambda c, r: str(c.get("lists", r.get("lists", "N/A")))), ) +_IVFPLUS_CONFIG_COLUMNS = ( + ("Lists", lambda c, r: str(c.get("lists", r.get("lists", "N/A")))), +) + INDEX_SPECS = { "hnsw": IndexSpec( @@ -275,6 +323,18 @@ def _bq_rerank_debug_print(config, dataset): ), config_columns=_IVFFLAT_CONFIG_COLUMNS, ), + "ivfplus": IndexSpec( + index_type="ivfplus", + suite_type="edb_vectorplus", + # Identical single-stage query shape to IVFFlat — reused as-is. + query_template=_ivfflat_query_template, + bind_kind="single", + session_gucs=_ivfplus_session_gucs, + create_index_sql=_ivfplus_create_index_sql, + debug_print=_ivfplus_debug_print, + bench_param_columns=(("probes", "Probes"),), + config_columns=_IVFPLUS_CONFIG_COLUMNS, + ), } @@ -340,9 +400,10 @@ def build_arg_parse(): class TestSuite(common.TestSuite): - """Single suite for HNSW / IVFFlat / IVFFlat-BQ-Rerank, dispatched by - the YAML `indexType` field via INDEX_SPECS. The HNSW path produces - byte-identical SQL/GUCs/reports to upstream.""" + """Single suite for HNSW / IVFFlat / IVFFlat-BQ-Rerank / ivfplus + (edb_vectorplus), dispatched by the YAML `indexType` field via + INDEX_SPECS. The HNSW path produces byte-identical SQL/GUCs/reports + to upstream.""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -363,6 +424,11 @@ def __init__(self, *args, **kwargs): ) self.spec = INDEX_SPECS[index_type] self._batch_dataset = None + # ivfplus-only: optional, suite-level fixed GUCs (iterative_scan / + # max_probes / hierarchy_threshold), populated in run_benchmarks() + # since apply_session_guc/make_batch_args don't receive suite_name. + # Stays empty (harmless no-op) for every other index type. + self._ivfplus_fixed_gucs_cache: list[str] = [] def create_connection(self): """Create a database connection with pgvector support.""" @@ -375,7 +441,11 @@ def init_ext(self, suite_name: Optional[str] = None): conn = super().create_connection() # Surface server NOTICEs in vsbt output conn.add_notice_handler(common.psql_log_handler) - conn.execute("CREATE EXTENSION IF NOT EXISTS vector") + if self.spec.index_type == "ivfplus": + # CASCADE pulls in pgvector's `vector` extension automatically. + conn.execute("CREATE EXTENSION IF NOT EXISTS edb_vectorplus CASCADE") + else: + conn.execute("CREATE EXTENSION IF NOT EXISTS vector") conn.execute("CREATE EXTENSION IF NOT EXISTS pg_prewarm") conn.close() self.debug_log("Extensions initialized successfully.") @@ -540,14 +610,24 @@ def make_batch_args(self, test, answer, top, metric, table_name, benchmark, query_sql, self.spec.bind_kind, rerank_limit, - self.spec.session_gucs(benchmark), + self.spec.session_gucs(benchmark) + self._ivfplus_fixed_gucs_cache, self.url, warmup_n, ) + def run_benchmarks(self, suite_name, table_name, dataset, query_clients): + """Stash ivfplus's optional, suite-level fixed GUCs before the + benchmark loop starts. No-op (empty list) for every other index + type.""" + if self.spec.index_type == "ivfplus": + self._ivfplus_fixed_gucs_cache = _ivfplus_fixed_gucs(self.config[suite_name]) + return super().run_benchmarks(suite_name, table_name, dataset, query_clients) + def apply_session_guc(self, conn, benchmark): for stmt in self.spec.session_gucs(benchmark): conn.execute(stmt) + for stmt in self._ivfplus_fixed_gucs_cache: + conn.execute(stmt) def warmup_query(self, table_name, dataset, metric_ops, top, benchmark): return self.spec.query_template( @@ -556,7 +636,7 @@ def warmup_query(self, table_name, dataset, metric_ops, top, benchmark): def create_index(self, suite_name: str, table_name: str, dataset: dict) -> None: """Create the pgvector index for this suite's IndexSpec - (HNSW, IVFFlat, or IVFFlat-BQ-Rerank).""" + (HNSW, IVFFlat, IVFFlat-BQ-Rerank, or ivfplus).""" event, index_monitor_thread = super().create_index( suite_name, table_name, dataset ) @@ -583,6 +663,8 @@ def create_index(self, suite_name: str, table_name: str, dataset: dict) -> None: self.results[suite_name]["lists"] = _ivfflat_resolve_lists( config, dataset ) + if self.spec.index_type == "ivfplus": + self.results[suite_name]["rotation"] = True if self.debug: self.spec.debug_print(config, dataset) diff --git a/results.py b/results.py index 3b11cc9..7064298 100644 --- a/results.py +++ b/results.py @@ -161,6 +161,7 @@ def append_to_consolidated( "sampling_factor": config.get("samplingFactor", "N/A"), "nprob": benchmark_config.get("nprob", "N/A"), "epsilon": benchmark_config.get("epsilon", "N/A"), + "probes": benchmark_config.get("probes", "N/A"), "residual_quantization": config.get("residual_quantization", "N/A"), "build_threads": results.get("build_threads", "N/A"), "load_time_s": results.get("load_time", "N/A"), @@ -412,9 +413,10 @@ def _generate_run_report( ["Build Threads", str(results.get("build_threads", "N/A"))], ["K-means Hierarchical", str(config.get("kmeans_hierarchical", "N/A"))], ]) - elif suite_type in ("pgvector-ivfflat", "pgvector-ivfflat-bq-rerank"): - # New pgvector index types ship their own column specs; the - # caller passes config_columns from the suite. Each entry is + elif suite_type in ("pgvector-ivfflat", "pgvector-ivfflat-bq-rerank", "edb_vectorplus"): + # New pgvector index types (and other new suite_types) ship + # their own column specs; the caller passes config_columns + # from the suite. Each entry is # (label, extractor(config_dict, results_dict) -> str). for label, extractor in (config_columns or []): config_rows.append([label, extractor(config, results)]) @@ -438,7 +440,7 @@ def _generate_run_report( # --- Benchmark Results --- benchmarks = config.get("benchmarks", {}) - is_ivfflat = suite_type in ("pgvector-ivfflat", "pgvector-ivfflat-bq-rerank") + is_ivfflat = suite_type in ("pgvector-ivfflat", "pgvector-ivfflat-bq-rerank", "edb_vectorplus") bench_cols = bench_columns if is_ivfflat else None bench_rows = [] for bench_name, bench_config in benchmarks.items(): @@ -587,7 +589,7 @@ def generate_markdown_report( # --- Benchmark Results (unified table across all runs, ordered by timestamp) --- lines.extend(["", "---", "", "## Benchmark Results", ""]) - is_ivfflat = suite_type in ("pgvector-ivfflat", "pgvector-ivfflat-bq-rerank") + is_ivfflat = suite_type in ("pgvector-ivfflat", "pgvector-ivfflat-bq-rerank", "edb_vectorplus") bench_cols = bench_columns if is_ivfflat else None bench_rows = [] diff --git a/utils/derive_datasets.py b/utils/derive_datasets.py new file mode 100644 index 0000000..a3d1164 --- /dev/null +++ b/utils/derive_datasets.py @@ -0,0 +1,289 @@ +"""Generate small CI/CD subsets of the openai / cohere / laion datasets. + +Slices the first N base vectors (5k / 50k by default) out of each family's +smallest locally-available source dataset, reuses a slice of its query set, +and **recomputes** ground-truth neighbours against each subset (correctness +requires this regardless of whether the parent's GT happens to overlap). + +Source per family (see FAMILIES below): + openai <- openai-500k (parquet, dim 1536, metric cos) + cohere <- cohere-1m (parquet, dim 768, metric cos) + laion <- laion-5m-test-ip (HDF5, dim 768, metric ip) + +Whatever the source format, output is always the three-file parquet shape +`_load_parquet` in datasets.py expects. `//