From 1bbfedc893a7bf34dbde0e9c0338caf4e27556b7 Mon Sep 17 00:00:00 2001 From: jc543239 Date: Wed, 29 Jul 2026 09:37:41 +0000 Subject: [PATCH 1/8] feat: add AutoTune V1 orchestration Signed-off-by: jc543239 Assisted-by: Codex:gpt-5 --- .github/workflows/pr-ci.yml | 5 +- CMakeLists.txt | 6 +- docs/docs/en/src/SUMMARY.md | 2 + docs/docs/en/src/resources/autotune.md | 248 ++++ docs/docs/en/src/resources/autotune_api_v1.md | 494 ++++++++ docs/docs/zh/src/SUMMARY.md | 2 + docs/docs/zh/src/resources/autotune.md | 235 ++++ docs/docs/zh/src/resources/autotune_api_v1.md | 473 ++++++++ ..._feature_create_index_with_constraints.cpp | 85 ++ .../327_feature_autotune_existing_index.cpp | 119 ++ .../328_feature_autotune_existing_pyramid.cpp | 204 ++++ examples/cpp/CMakeLists.txt | 14 + tools/CMakeLists.txt | 1 + tools/README.md | 1 + tools/autotune/CMakeLists.txt | 54 + tools/autotune/README.md | 25 + tools/autotune/README_zh.md | 25 + tools/autotune/autotune.cpp | 1071 +++++++++++++++++ tools/autotune/autotune.h | 174 +++ tools/autotune/autotune_candidate.cpp | 424 +++++++ tools/autotune/autotune_evaluation.cpp | 374 ++++++ tools/autotune/autotune_internal.h | 129 ++ tools/autotune/autotune_test.cpp | 1001 +++++++++++++++ tools/autotune/examples/README.md | 19 + .../sift_hgraph_autotune_request.json | 48 + tools/autotune/main.cpp | 54 + tools/eval/CMakeLists.txt | 1 + tools/eval/case/build_eval_case.cpp | 28 +- tools/eval/case/build_eval_case.h | 6 +- tools/eval/case/build_search_eval_case.h | 9 +- tools/eval/case/eval_case.cpp | 24 +- tools/eval/case/eval_case.h | 9 +- tools/eval/case/search_eval_case.cpp | 122 +- tools/eval/case/search_eval_case.h | 6 +- tools/eval/eval_dataset.cpp | 201 +++- tools/eval/eval_dataset.h | 96 +- tools/eval/eval_dataset_test.cpp | 377 ++++++ tools/eval/evaluator.cpp | 79 ++ tools/eval/evaluator.h | 32 + tools/eval/monitor/monitor.h | 12 + tools/eval/monitor/recall_monitor.cpp | 53 +- 41 files changed, 6246 insertions(+), 96 deletions(-) create mode 100644 docs/docs/en/src/resources/autotune.md create mode 100644 docs/docs/en/src/resources/autotune_api_v1.md create mode 100644 docs/docs/zh/src/resources/autotune.md create mode 100644 docs/docs/zh/src/resources/autotune_api_v1.md create mode 100644 examples/cpp/326_feature_create_index_with_constraints.cpp create mode 100644 examples/cpp/327_feature_autotune_existing_index.cpp create mode 100644 examples/cpp/328_feature_autotune_existing_pyramid.cpp create mode 100644 tools/autotune/CMakeLists.txt create mode 100644 tools/autotune/README.md create mode 100644 tools/autotune/README_zh.md create mode 100644 tools/autotune/autotune.cpp create mode 100644 tools/autotune/autotune.h create mode 100644 tools/autotune/autotune_candidate.cpp create mode 100644 tools/autotune/autotune_evaluation.cpp create mode 100644 tools/autotune/autotune_internal.h create mode 100644 tools/autotune/autotune_test.cpp create mode 100644 tools/autotune/examples/README.md create mode 100644 tools/autotune/examples/sift_hgraph_autotune_request.json create mode 100644 tools/autotune/main.cpp create mode 100644 tools/eval/evaluator.cpp create mode 100644 tools/eval/evaluator.h diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 98c8ae10bb..4b63daeaec 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -250,8 +250,11 @@ jobs: echo leak:libomp.so > omp.supp echo leak:libgomp.so >> omp.supp export LSAN_OPTIONS=suppressions=omp.supp - chmod +x ./build/tests/unittests + chmod +x ./build/tests/unittests ./build/tools/eval/eval_dataset_test \ + ./build/tools/autotune/autotune_test ./scripts/testing/test_parallel_by_name.sh unittests + ./build/tools/eval/eval_dataset_test -d yes + ./build/tools/autotune/autotune_test -d yes - name: Upload Log uses: actions/upload-artifact@v5 with: diff --git a/CMakeLists.txt b/CMakeLists.txt index b4e2a200a3..86553755fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,12 +57,12 @@ add_subdirectory (src) if (ENABLE_MOCKIMPL) add_subdirectory (mockimpl) endif () -if (ENABLE_EXAMPLES) - add_subdirectory (examples/cpp) -endif () if (ENABLE_TOOLS AND ENABLE_CXX11_ABI) add_subdirectory (tools) endif () +if (ENABLE_EXAMPLES) + add_subdirectory (examples/cpp) +endif () if (ENABLE_TESTS) add_subdirectory (tests) target_compile_definitions (vsag PRIVATE ENABLE_TESTS) diff --git a/docs/docs/en/src/SUMMARY.md b/docs/docs/en/src/SUMMARY.md index f57c6a7ce6..f534fffefc 100644 --- a/docs/docs/en/src/SUMMARY.md +++ b/docs/docs/en/src/SUMMARY.md @@ -83,6 +83,8 @@ - [Optimizer (Tune)](advanced/optimizer.md) - [Benchmarks](resources/performance.md) - [Evaluation Tool](resources/eval.md) +- [AutoTune Tool](resources/autotune.md) + - [AutoTune V1 CLI JSON Contract](resources/autotune_api_v1.md) - [HDF5 Dataset Format](resources/dataset_format.md) - [Index Analysis Tool](resources/analyze_index.md) - [Compatibility Check Tool](resources/check_compatibility.md) diff --git a/docs/docs/en/src/resources/autotune.md b/docs/docs/en/src/resources/autotune.md new file mode 100644 index 0000000000..f1aa3ecad5 --- /dev/null +++ b/docs/docs/en/src/resources/autotune.md @@ -0,0 +1,248 @@ +# AutoTune Tool (`autotune`) + +`autotune` automates the VSAG build-and-search parameter sweep. AutoTune V1 evaluates real VSAG +indexes through the shared eval core, filters trials by constraints, and selects the best feasible +trial for one KNN workload. + +The CLI JSON request and shared report schema are defined in the +[AutoTune V1 CLI JSON contract](autotune_api_v1.md). + +V1 builds and tunes HGraph and IVF on dense float32 data. It also tunes search parameters for an +existing HGraph, IVF, or Pyramid index. The CLI reads HDF5; the C++ API accepts in-memory datasets +and indexes. The C++ API is experimental and build-tree-only; it is not installed with the VSAG +library. + +## Building + +```bash +cmake -S . -B build-release \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TOOLS=ON +cmake --build build-release --target autotune -j +``` + +The executable is `build-release/tools/autotune/autotune`. + +A runnable request template is available at [the SIFT request template][sift-example]. +It expects the dataset at `/tmp/sift-128-euclidean.hdf5`; edit `data_path` when necessary. + +[sift-example]: https://github.com/antgroup/vsag/blob/main/tools/autotune/examples/sift_hgraph_autotune_request.json + +## C++ API + +The CLI and C++ entry points share candidate generation, trial planning, evaluation, constraint +filtering, and result selection. Only their input adapters differ: + +- the CLI loads JSON and HDF5; +- `TuneIndex` accepts in-memory datasets and jointly tunes build and search parameters; +- `TuneSearch` accepts an already built or loaded `IndexPtr` and tunes search parameters only. + +Consumers that link the build-tree `vsag::autotune` target can run the complete tune, select, and +load loop directly: + +```cpp +vsag::autotune::IndexRequest request; +request.base = base; +request.metric_type = vsag::METRIC_L2; +request.workload = {queries, ground_truth, 10, 48}; +request.index_spaces = { + {"hgraph", create_candidate_space, search_candidate_space}, +}; +request.constraints = { + {vsag::autotune::Metric::RECALL_AT_K, 0.95}, + {vsag::autotune::Metric::INDEX_MEMORY_MB, 8192.0}, +}; +request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + +auto result = vsag::autotune::TuneIndex(request).value(); +auto neighbors = + result.index->KnnSearch(query, 10, result.search_parameters).value(); +``` + +`base` and `queries` are dense float32 datasets. `base` IDs must be present and unique. The +ground-truth dataset stores `query_count` rows, uses `Dim()` as ground-truth K, and contains +flattened IDs from the base ID space. It is required when `RECALL_AT_K` is a constraint or the +objective. Dataset buffers must remain valid until the synchronous call returns. + +`IndexSpace::create_parameter_space` and `search_parameter_space` remain JSON strings because each +index owns its native parameter schema. An empty `index_spaces` vector selects the built-in HGraph +and IVF spaces; otherwise arrays and `$range` expressions use the same semantics as the CLI +contract. + +The result contains the loaded `index`, `index_name`, complete concrete `create_parameters`, +validated `search_parameters`, metrics, artifact path, and the complete report. The recommended +artifact is retained, so another process can recreate an empty index from +`index_name + create_parameters` and deserialize that file. Unselected intermediate artifacts are +removed by default. + +If no candidate satisfies every constraint, `TuneIndex` returns an `INVALID_ARGUMENT` error +instead of loading `best_effort`. + +See +[`examples/cpp/326_feature_create_index_with_constraints.cpp`][factory-example] for a complete +example. Enable both tools and examples to build it: + +```bash +cmake -S . -B build-release \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TOOLS=ON \ + -DENABLE_EXAMPLES=ON +cmake --build build-release --target 326_feature_create_index_with_constraints -j +``` + +[factory-example]: https://github.com/antgroup/vsag/blob/main/examples/cpp/326_feature_create_index_with_constraints.cpp + +## Request + +Save a request such as the following as `request.json`. `dim`, `dtype`, and `metric_type` are read +from the dataset and need not be repeated. If provided, they must match the dataset. + +```json +{ + "version": 1, + "data_path": "/tmp/sift-128-euclidean.hdf5", + "indexes": [ + { + "name": "hgraph", + "create_params": { + "index_param": { + "base_quantization_type": ["fp32", "sq8_uniform"], + "max_degree": [16, 32], + "ef_construction": 200 + } + }, + "search_params": { + "hgraph": { + "ef_search": { + "$range": {"start": 40, "stop": 1000} + } + } + } + } + ], + "workload": { + "top_k": 10, + "concurrency": 1 + }, + "constraints": { + "recall_at_k": 0.95, + "index_memory_mb": 8192 + }, + "objective": { + "metric": "latency_avg_ms" + }, + "tuning_config": { + "workspace_path": "/tmp/vsag_autotune", + "keep_intermediate": false, + "max_trials": 1000 + } +} +``` + +Run it with: + +```bash +./build-release/tools/autotune/autotune request.json +``` + +Every parameter leaf may be a scalar, an array, or a `$range` with `start`, `stop`, and `step`. +Explicit values are preserved. HGraph `ef_search` additionally accepts an integer `$range` without +`step`; AutoTune then doubles from `start` until it brackets a passing value and binary-searches +that interval for the smallest passing value. This assumes that recall does not decrease as +`ef_search` increases for the fixed configuration. Every evaluated point uses all queries. For +omitted tuning fields, V1 supplies a small built-in candidate set for HGraph or IVF. If `indexes` +is omitted, both indexes are evaluated. + +Candidates with identical concrete create parameters share one build. Each concrete search +parameter set remains a separate trial. An adaptive `ef_search` range records only the concrete +points that were actually evaluated. + +## Existing Index + +Set `index_path` and provide exactly one index specification to tune search parameters without +rebuilding. Its `create_params` must be the concrete parameters used to create the serialized +index. AutoTune does not expand `create_params` in this mode, so native index fields that happen to +be arrays remain unchanged. + +Programmatic callers use a separate `SearchRequest` and call `TuneSearch`: + +```cpp +vsag::autotune::SearchRequest request; +request.index = existing_index; +request.index_name = "hgraph"; +request.base = base; +request.metric_type = vsag::METRIC_L2; +request.workload = {queries, ground_truth, 10, 48}; +request.parameter_space = R"({"hgraph":{"ef_search":[40,80,120]}})"; +request.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.95}}; +request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + +auto result = vsag::autotune::TuneSearch(request).value(); +auto neighbors = existing_index->KnnSearch(query, 10, result.parameters).value(); +``` + +This avoids serialization and file I/O and reuses the caller's index for all search trials. +`base` is currently required by the shared recall evaluator. The CLI's `index_path` field remains +an offline adapter: it creates and deserializes an index, then enters the same search-only flow. + +Pyramid is supported in this search-only mode. The HDF5 CLI adapter does not provide query paths, +so it can evaluate only Pyramid's native default/root search. Typed requests take paths directly +from `workload.queries->GetPaths()`: when paths are present, each query is evaluated against its +path; when absent, Pyramid applies its native root-search behavior. V1 supports only the default +unnamed hierarchy. To select different `ef_search` values for different paths, run one typed +AutoTune request per representative path workload, with the matching ground truth. V1 does not +aggregate path-specific recommendations into one result. + +A complete example is available at +[`examples/cpp/327_feature_autotune_existing_index.cpp`][existing-index-example]. +For Pyramid path tuning, see +[`examples/cpp/328_feature_autotune_existing_pyramid.cpp`][pyramid-example]. +It tunes the same Pyramid index separately for 512-vector and 4096-vector leaf subgraphs under the +same recall target, illustrating why different paths may need different `ef_search` values. + +[existing-index-example]: https://github.com/antgroup/vsag/blob/main/examples/cpp/327_feature_autotune_existing_index.cpp +[pyramid-example]: https://github.com/antgroup/vsag/blob/main/examples/cpp/328_feature_autotune_existing_pyramid.cpp + +## Metrics + +The following metrics can be used as constraints or as the objective: + +| Metric | Constraint comparison | +| --- | --- | +| `recall_at_k`, `qps` | At least the requested value | +| `latency_avg_ms`, `latency_p99_ms` | At most the requested value | +| `index_memory_mb`, `index_size_mb` | At most the requested value | +| `build_seconds`, `search_seconds`, `build_and_search_seconds` | At most the requested value | + +`build_seconds`, `index_size_mb`, and `build_and_search_seconds` are unavailable in +existing-index mode. The objective direction is inferred from the metric, so no `direction` field +is needed. + +AutoTune uses a dedicated latency/QPS pass and a separate recall/statistics pass, so a logical +query runs more than once per trial. `search_seconds` covers both in-memory passes and metric +collection, but excludes index deserialization. Candidates run in a fixed order without warm-up +or randomization. Measure latency and QPS on a representative, controlled machine and repeat runs +when these metrics determine the recommendation. + +## Reading the Output + +Standard output is a short summary ordered for copying: + +- `recommendation`: the selected concrete parameters, metrics, and trial evidence. Build-and-search + results also include create parameters and artifact/build evidence; search-only results do not. +- `best_effort`: present only when no trial satisfies every constraint. +- `failure`: present when the request or all evaluations fail. +- `report_path`: the full reproducible report. + +The full report also contains every build and trial, constraint violations, the normalized request, +and structured per-stage failures. In search-only mode `builds` is empty and trials contain no +artifact or build fields. For the CLI and `RunAutoTune`, `keep_intermediate=false` removes every +generated index after evaluation. Typed `TuneIndex` always retains the recommended artifact so it +can return a reproducible result, while removing unselected artifacts when the option is false. +The report itself is retained in both cases. + +## V1 Boundaries + +V1 evaluates one KNN workload and performs a full sweep except for the supported HGraph +`ef_search` adaptive search. It does not provide filtered or range-search workloads, adaptive query +sampling, cross-request build cache, or model-based candidate generation. diff --git a/docs/docs/en/src/resources/autotune_api_v1.md b/docs/docs/en/src/resources/autotune_api_v1.md new file mode 100644 index 0000000000..0bcc84824b --- /dev/null +++ b/docs/docs/en/src/resources/autotune_api_v1.md @@ -0,0 +1,494 @@ +# AutoTune V1 CLI JSON Input and Output Contract + +This page defines the offline JSON/HDF5 adapter used by the `autotune` command-line tool. The +experimental, build-tree-only C++ API instead accepts typed `IndexRequest` and `SearchRequest` +objects. It is not installed with the VSAG library. All entry points use the same normalized +request and tuning engine after input conversion. AutoTune V1 supports one full-dataset KNN +workload on dense float32 data, with HGraph and IVF as the supported build-and-search index types. + +Unknown fields are rejected in the request, index specification, workload, objective, +`tuning_config`, and `output` objects. Native fields inside `create_params` and `search_params` are +passed to the concrete VSAG index and are validated when that index is created or searched. + +## Request Object + +| Field | Type | Required | Meaning | +| --- | --- | --- | --- | +| `version` | integer | Yes | Must be `1`. | +| `data_path` | string | Yes | Readable HDF5 evaluation dataset. | +| `index_path` | string | No | Existing serialized index for search-only tuning. | +| `indexes` | array | No | Index specifications. Defaults to HGraph and IVF in normal mode. | +| `workload` | object | Yes | The KNN workload being optimized. | +| `constraints` | object | Yes | Non-empty metric-to-threshold map. | +| `objective` | object | Yes | Metric used to rank feasible candidates. | +| `tuning_config` | object | No | Workspace, artifact retention, and trial limit. | +| `output` | object | No | Report path and raw-eval control. | + +### Dataset Rules + +`data_path` must identify a readable regular file in the HDF5 format accepted by the VSAG +evaluation tool. V1 requires: + +- dense vectors; +- float32 base and query vectors; +- at least one base vector, query vector, and ground-truth neighbor; +- `workload.top_k` no greater than the ground-truth width when recall is requested; +- a distance attribute that maps to `l2`, `ip`, or `cosine`. + +AutoTune reads `dim`, `dtype`, and `metric_type` from the dataset and injects them into every +concrete `create_params`. A request may repeat one of these fields only when it exactly matches the +dataset. + +## Index Specifications + +Each `indexes[]` item has the following shape: + +```json +{ + "name": "hgraph", + "create_params": { + "index_param": { + "max_degree": [16, 32] + } + }, + "search_params": { + "hgraph": { + "ef_search": {"$range": {"start": 40, "stop": 1000}} + } + } +} +``` + +| Field | Type | Required | Meaning | +| --- | --- | --- | --- | +| `name` | string | Yes | `hgraph`, `ivf`, or search-only `pyramid`, case-insensitive. | +| `create_params` | object | No | VSAG create parameters or candidate expressions. | +| `search_params` | object | No | Search parameters, keyed by the same index name. | + +When `indexes` is absent in normal build-and-search mode, AutoTune evaluates both HGraph and IVF. +When `index_path` is present, `indexes` is required and must contain exactly one item. +Pyramid is accepted only with an existing in-memory or serialized index. + +### Candidate Expressions + +Every leaf in `create_params` and `search_params` can be written as: + +- a scalar, which fixes that value; +- a non-empty array, which provides discrete candidate values; +- an inclusive `$range` containing exactly `start`, `stop`, and non-zero `step`. + +Examples: + +```json +{ + "fixed": 32, + "choices": [16, 32, 48], + "integer_range": {"$range": {"start": 40, "stop": 120, "step": 40}}, + "float_range": {"$range": {"start": 0.1, "stop": 0.3, "step": 0.1}} +} +``` + +HGraph `ef_search` has one additional form: + +```json +{"$range": {"start": 40, "stop": 1000}} +``` + +The bounds must be positive integers with `start <= stop`. This form requires a `recall_at_k` +constraint and an objective of `latency_avg_ms`, `latency_p99_ms`, `qps`, `search_seconds`, or +`build_and_search_seconds`. It cannot be combined with `timeout_ms` or `hops_limit`. For every +fixed combination of the other parameters, AutoTune evaluates concrete `ef_search` values and +doubles from `start`, capped at `stop`, until recall satisfies the constraint. It then +binary-searches the last failing/passing interval for the smallest passing value. Branching uses +recall only; latency and QPS remain measured metrics used by final constraint evaluation and +selection. This strategy assumes that recall does not decrease as `ef_search` increases while all +other parameters and the workload remain fixed. + +Every probe evaluates the full query workload. If a probe fails to execute or does not produce +recall, AutoTune stops that adaptive range because it cannot choose the next interval safely. + +V1 treats every JSON array and stepped range in these parameter objects as candidate choices. It +computes their full Cartesian product and removes identical complete candidates. A no-step range +is not accepted for other parameters. + +The request is rejected when the planned worst-case evaluation count exceeds +`tuning_config.max_trials`. A normal concrete candidate costs one trial. An adaptive `ef_search` +range costs one when `start == stop`. Otherwise, the planner reserves the largest probe count +through any possible passing bound plus the binary-search cost of that bound's preceding interval. +For example, `40..1000` reserves 15 trials in the worst case. + +### Built-in Missing-Field Candidates + +Explicit user fields always win. The following values are supplied only when the corresponding +field is absent: + +| Index | Missing field | V1 candidates | +| --- | --- | --- | +| HGraph | `base_quantization_type` | `fp32`, `sq8_uniform` | +| HGraph | `max_degree` | `16`, `32` | +| HGraph | `ef_construction` | `100`, `200` | +| HGraph | `ef_search` | Three values derived from `top_k`, with floors `40`, `80`, `120` | +| Pyramid, search-only | `ef_search` | Three values derived from `top_k`, with floors `40`, `80`, `120` | +| IVF | `base_quantization_type` | `fp32`, `sq8_uniform` | +| IVF | `buckets_count` | Up to `1024` and `2048`, capped by base-vector count | +| IVF, build-and-search | `scan_buckets_count` | Up to four values derived from concrete `buckets_count` | +| IVF, search-only | `scan_buckets_count` | `1`, `4`, `16`, `64` | + +The exact HGraph and Pyramid search values are `max(40, top_k)`, `max(80, 2 * top_k)`, and +`max(120, 4 * top_k)`, with duplicates removed. Build-and-search IVF candidates never exceed the +concrete bucket count. AutoTune cannot infer an existing IVF index's bucket count, so search-only +mode uses the conservative generic values above. Values rejected by the loaded index become +failed trials; callers can avoid them by providing an explicit `scan_buckets_count` space. + +Candidates with the same normalized index name and identical concrete `create_params` belong to +one build group. AutoTune builds that group once, serializes it once as evidence, and evaluates +every associated search candidate against the same in-memory index instance. + +### Existing-Index Mode + +With `index_path`: + +- the path must identify a readable regular file; +- exactly one index specification is required; +- `create_params.index_param` is required; +- `create_params` is treated as one concrete native index configuration and is not expanded; +- AutoTune does not generate missing create candidates or invoke build; +- missing search fields may still receive built-in search candidates; +- `build_seconds`, `index_size_mb`, and `build_and_search_seconds` cannot be constraints or + objectives; +- the caller-owned index is never deleted. + +The concrete create parameters are needed to instantiate the correct VSAG index before +deserialization. AutoTune does not infer them from the serialized file in V1. + +For a typed `SearchRequest`, the index is already instantiated and loaded. It therefore needs an +index name and search `parameter_space`, but no create parameters. + +The HDF5 adapter does not provide Pyramid query paths, so the CLI can evaluate only Pyramid's +native default/root search. Path-specific Pyramid tuning requires a typed `SearchRequest` whose +query dataset supplies `Dataset::Paths()`. V1 forwards only the default unnamed hierarchy; named +Pyramid hierarchies are not supported by AutoTune. + +## Workload + +```json +{ + "workload": { + "top_k": 10, + "concurrency": 48 + } +} +``` + +| Field | Type | Required | Default | Validation | +| --- | --- | --- | --- | --- | +| `top_k` | positive integer | Yes | — | No greater than base count, and no greater than ground-truth width when recall is requested; representable by native `int`. | +| `concurrency` | positive integer | No | `1` | Maximum `200`. | + +V1 always evaluates the complete query set in the HDF5 file and supports only KNN search. The +benchmark machine, system load, and runtime environment are part of the measured workload. +Latency and QPS are not portable across different machine specifications or load conditions. + +## Constraints and Objective + +`constraints` maps metric names directly to thresholds. `objective.metric` names one metric. The +comparison and objective direction are inferred from the metric and cannot be overridden. + +```json +{ + "constraints": { + "recall_at_k": 0.95, + "latency_p99_ms": 2.0, + "index_memory_mb": 8192 + }, + "objective": { + "metric": "latency_avg_ms" + } +} +``` + +| Metric | Constraint | Objective direction | Existing index | +| --- | --- | --- | --- | +| `recall_at_k` | actual ≥ threshold | Maximize | Yes | +| `qps` | actual ≥ threshold | Maximize | Yes | +| `latency_avg_ms` | actual ≤ threshold | Minimize | Yes | +| `latency_p99_ms` | actual ≤ threshold | Minimize | Yes | +| `index_memory_mb` | actual ≤ threshold | Minimize | Yes | +| `index_size_mb` | actual ≤ threshold | Minimize | No | +| `build_seconds` | actual ≤ threshold | Minimize | No | +| `search_seconds` | actual ≤ threshold | Minimize | Yes | +| `build_and_search_seconds` | actual ≤ threshold | Minimize | No | + +Thresholds must be finite and non-negative. `recall_at_k` must also be at most `1.0`. + +Metric meanings in V1: + +- `build_seconds` measures the index `Build` operation reported by the evaluation tool. +- It excludes index serialization, dataset loading, and candidate orchestration. +- `search_seconds` measures the wall time of a complete in-memory search evaluation trial. It + excludes index deserialization, but includes all search passes and metric collection. Use + latency or QPS for serving-performance goals. +- `build_and_search_seconds` is the sum of those two metrics for a build-and-search trial. +- `index_memory_mb` comes from the loaded index's positive `GetMemoryUsage()` result. When an index + reports zero, AutoTune treats the metric as unavailable instead of as zero usage; a corresponding + constraint is recorded as a missing-metric violation. +- `index_size_mb` is the generated serialized artifact's file size and is unavailable in + search-only mode. +- AutoTune uses a dedicated latency/QPS pass and a separate recall/statistics pass. Latency wraps + each `KnnSearch` call with `steady_clock`; QPS is successful queries divided by the wall time of + the concurrent performance pass. The same logical query is therefore executed more than once + per trial. + +`search_seconds` remains evaluation cost because it includes both passes and monitor work; it is +not the time for one logical production query batch. Candidates run in a fixed order against a +reused index, without a warm-up or randomized ordering. Cache state and unrelated system load can +therefore affect performance metrics; benchmark in a representative, controlled environment and +repeat runs when latency or QPS drives selection. + +## Tuning and Output Configuration + +```json +{ + "tuning_config": { + "workspace_path": "/tmp/vsag_autotune", + "keep_intermediate": false, + "max_trials": 1000 + }, + "output": { + "result_path": "/tmp/vsag_autotune/report.json", + "include_raw_eval": false + } +} +``` + +| Field | Default | Meaning | +| --- | --- | --- | +| `tuning_config.workspace_path` | `/tmp/vsag_autotune` | Run artifacts and default reports. | +| `tuning_config.keep_intermediate` | `false` | Retain generated intermediate index artifacts. | +| `tuning_config.max_trials` | `1000` | Maximum planned worst-case trials; hard maximum `100000`. | +| `output.result_path` | `/run-.json` | Full report path. | +| `output.include_raw_eval` | `false` | Include native eval JSON in build and trial records. | + +`output.result_path` cannot alias `data_path` or `index_path`. For the offline JSON/CLI entry, +generated indexes are removed after evaluation when `keep_intermediate` is false. Typed +`TuneIndex` always retains the recommended artifact so it can return a reproducible result; the +option controls retention of unselected artifacts. The report is retained independently. + +## Full Report + +A completed evaluation writes and returns this top-level shape: + +```json +{ + "version": 1, + "status": "success", + "recommendation": {}, + "best_effort": null, + "builds": [], + "trials": [], + "request": {}, + "elapsed_seconds": 84.2, + "report_path": "/tmp/vsag_autotune/run-....json" +} +``` + +| Field | Meaning | +| --- | --- | +| `version` | Report contract version, always `1`. | +| `status` | `success`, `no_feasible_candidate`, or `failed`. | +| `recommendation` | Best feasible trial, otherwise `null`. | +| `best_effort` | Closest successful trial when constraints are infeasible, otherwise `null`. | +| `builds` | One record per concrete generated build group; empty in search-only mode. | +| `trials` | One record per executed concrete search candidate. | +| `request` | Effective normalized request used by the tuning engine. | +| `elapsed_seconds` | AutoTune wall time through selection, excluding report writing and cleanup. | +| `report_path` | Persisted full report path. | +| `failure` | Present when the overall run failed. | + +Early validation failures do not have `builds`, `trials`, or `report_path`, and no report file is +written. A run in which all candidate evaluations fail does contain the attempted build and trial +records and is written when the report path itself is usable. + +The normalized `request` contains derived dataset metadata, workload defaults, constraints, +objective, config, and output fields. Build-and-search mode adds `index_spaces`; search-only mode +adds `index_name` and `parameter_space`. CLI reports also retain the offline `data_path` and, when +present, `index_path` as top-level fields inside this normalized object. + +### Status Semantics + +- `success`: at least one successful trial satisfies every constraint; `recommendation` is set. +- `no_feasible_candidate`: successful trials exist, but none satisfies every constraint; + `best_effort` is set for explanation and is not a valid recommendation. +- `failed`: the request is invalid, execution/reporting fails, or no trial can be evaluated + successfully with the objective metric. + +### Recommendation and Best Effort + +In build-and-search mode, `recommendation` contains: + +```json +{ + "index_name": "hgraph", + "create_params": {}, + "search_params": {}, + "workload": {"top_k": 10, "concurrency": 1}, + "metrics": {}, + "artifacts": {}, + "evidence": {"build_id": "build-0", "trial_id": "trial-0"} +} +``` + +In search-only mode, `create_params`, `artifacts`, and `evidence.build_id` are omitted; the +recommendation contains the concrete search parameters, workload, metrics, and +`evidence.trial_id`. `best_effort` has the same mode-specific fields plus +`constraint_evaluation`. It is selected first by the number of violated or missing constraints, +then by normalized violation magnitude. It is only explanatory. + +### Build Records + +`builds` is empty in search-only mode. In build-and-search mode, each `builds[]` item contains: + +| Field | Meaning | +| --- | --- | +| `build_id` | Stable ID referenced by trials. | +| `index_name` | Concrete index type. | +| `create_params` | Concrete create parameters. | +| `status` | `success` or `failed`. | +| `metrics` | Available build-shared metrics. | +| `artifacts` | `source`, `index_path`, `use_existing_index`, and `retained`. | +| `failure` | Structured failure or `null`. | +| `elapsed_seconds` | Build-group preparation time. | +| `raw_eval_result` | Present only when requested and a native build eval ran. | + +### Trial Records + +Each `trials[]` item contains: + +| Field | Meaning | +| --- | --- | +| `trial_id` | Stable trial ID. | +| `build_id` | Associated build group; build-and-search mode only. | +| `index_name` | Concrete index type. | +| `create_params` | Concrete create parameters; build-and-search mode only. | +| `search_params` | Concrete search parameters. | +| `status` | `success` or `failed`. | +| `metrics` | Available metrics used by constraint evaluation and selection. | +| `constraint_evaluation` | `satisfied` plus a `violations` array. | +| `artifacts` | Artifact evidence copied from the build group; build-and-search mode only. | +| `failure` | Structured failure or `null`. | +| `elapsed_seconds` | Search trial wall time. | +| `raw_eval_result` | Present only when requested and native search eval succeeded. | + +For an adaptive HGraph `ef_search` range, the report contains one trial for each point actually +evaluated. Every recorded `search_params` value is concrete; the `$range` expression remains only +in the normalized request. Every recorded trial evaluates the full query workload. + +A constraint violation contains `metric`, `comparison`, `expected`, and `actual`. `actual` is +`null` when the required metric is missing or non-finite. + +Search trials in one build group reuse the same loaded index instance. A generated index is built +once and serialized once. In search-only mode the caller's index, or the index deserialized by the +CLI adapter, is reused directly for every trial. + +### Artifact Semantics + +Artifact fields appear only in build-and-search records in V1. `artifacts.source` is `generated`, +and `artifacts.index_path` is evidence of where the evaluated index was stored, not a promise that +the path still exists. Check `artifacts.retained`: + +- `true`: this is the selected artifact returned by typed `TuneIndex`, or retention was requested; +- `false`: AutoTune planned to remove or already removed the generated artifact. + +### Structured Failures + +Failures use one common object: + +```json +{ + "stage": "validation", + "code": "invalid_request", + "message": "request.workload.top_k is required" +} +``` + +Typical stages are `cli`, `validation`, `candidate_generation`, `build`, `search`, `evaluation`, +`selection`, and `report`. Per-build and per-trial failures remain in their corresponding records +so other candidates can continue. + +| Stage | Code | Meaning | +| --- | --- | --- | +| `cli` | `request_file_error` | The CLI could not read or parse its request file. | +| `validation` | `invalid_request` | Request validation failed. | +| `candidate_generation` | `invalid_request` | A candidate expression or count was invalid. | +| `build` | `build_evaluation_failed` | One build group failed. | +| `search` | `build_failed` | Search was skipped because its build failed. | +| `search` | `search_evaluation_failed` | One search trial failed. | +| `evaluation` | `all_trials_failed` | Every candidate trial failed. | +| `selection` | `objective_metric_unavailable` | Trials succeeded, but none produced the objective metric. | +| `evaluation`, `report` | `execution_failed` | Top-level execution or report writing failed. | + +## CLI Summary + +The CLI prints a compact subset of the full report in this order: + +1. `recommendation`, when present; +2. `best_effort`, when present; +3. `failure`, when present; +4. `status`; +5. `elapsed_seconds`; +6. `report_path`, when present; +7. `version`. + +Null result branches and the detailed build/trial arrays are omitted from standard output. Read +`report_path` for complete evidence. + +The CLI exits with code `1` for `status=failed` and command-line/request-file errors. It currently +exits with code `0` for both `success` and `no_feasible_candidate`; callers must inspect `status` +rather than using the exit code to decide whether a recommendation exists. + +## Build-Tree C++ Entry Point + +The experimental optional `vsag::autotune` CMake target exposes the declarations in +`tools/autotune/autotune.h`. The target and header are build-tree-only and are not installed: + +```cpp +tl::expected TuneIndex(const IndexRequest& request); +tl::expected TuneSearch(const SearchRequest& request); +JsonType RunAutoTune(const JsonType& request); // CLI adapter +``` + +The JSON entry point is an offline adapter: it loads `data_path`, creates or deserializes an index +when `index_path` is present, and constructs the same internal request used by the typed entry +points. The target and header are intentionally not installed in V1. Calls inside one process are +serialized because the underlying evaluation path configures process-global OpenMP state. + +`TuneIndex` jointly tunes build and search parameters: + +```cpp +IndexRequest request; +request.base = base; +request.metric_type = METRIC_L2; +request.workload = {queries, ground_truth, 10, 48}; +request.index_spaces = {{"hgraph", create_candidate_space, search_candidate_space}}; +request.constraints = {{Metric::RECALL_AT_K, 0.95}}; +request.objective = Metric::LATENCY_AVG_MS; +auto result = TuneIndex(request); +``` + +It returns a loaded, query-ready selected index together with concrete create and search +parameters. `TuneSearch` tunes an already built or loaded index: + +```cpp +SearchRequest request; +request.index = existing_index; +request.index_name = "hgraph"; +request.base = base; +request.metric_type = METRIC_L2; +request.workload = {queries, ground_truth, 10, 48}; +request.parameter_space = search_candidate_space; +request.constraints = {{Metric::RECALL_AT_K, 0.95}}; +request.objective = Metric::LATENCY_AVG_MS; +auto result = TuneSearch(request); +``` + +Build-time metrics and `index_size_mb` are unavailable to `TuneSearch`. diff --git a/docs/docs/zh/src/SUMMARY.md b/docs/docs/zh/src/SUMMARY.md index 8c3910f7ab..58639ff396 100644 --- a/docs/docs/zh/src/SUMMARY.md +++ b/docs/docs/zh/src/SUMMARY.md @@ -83,6 +83,8 @@ - [优化器](advanced/optimizer.md) - [标准环境性能参考](resources/performance.md) - [性能评估工具](resources/eval.md) +- [AutoTune 工具](resources/autotune.md) + - [AutoTune V1 CLI JSON 契约](resources/autotune_api_v1.md) - [HDF5 数据集格式](resources/dataset_format.md) - [索引分析工具](resources/analyze_index.md) - [兼容性检查工具](resources/check_compatibility.md) diff --git a/docs/docs/zh/src/resources/autotune.md b/docs/docs/zh/src/resources/autotune.md new file mode 100644 index 0000000000..87edc22093 --- /dev/null +++ b/docs/docs/zh/src/resources/autotune.md @@ -0,0 +1,235 @@ +# AutoTune 工具（`autotune`） + +`autotune` 用于自动完成 VSAG 构建参数和查询参数的枚举评测。AutoTune V1 通过共享的 eval +核心运行真实 VSAG 索引，按约束过滤 trial，并为一个 KNN workload 选择最优 +可行候选。 + +CLI JSON 请求和共用报告结构见 [AutoTune V1 CLI JSON 契约](autotune_api_v1.md)。 + +V1 支持在 dense float32 数据上构建和调优 HGraph、IVF，也支持对已有 HGraph、IVF 或 +Pyramid 索引调 search 参数。CLI 读取 HDF5；C++ API 接收内存 Dataset 和 Index。C++ API +是实验性接口，只在构建目录中提供，不会随 VSAG 库安装。 + +## 编译 + +```bash +cmake -S . -B build-release \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TOOLS=ON +cmake --build build-release --target autotune -j +``` + +可执行文件位于 `build-release/tools/autotune/autotune`。 + +仓库提供了[可运行的 SIFT 请求模板][sift-example]。 +它默认读取 `/tmp/sift-128-euclidean.hdf5`；数据集位于其他目录时请修改 `data_path`。 + +[sift-example]: https://github.com/antgroup/vsag/blob/main/tools/autotune/examples/sift_hgraph_autotune_request.json + +## C++ API + +CLI 和 C++ 入口复用同一套候选生成、trial 规划、评测、约束过滤和结果选择逻辑，区别仅在 +输入适配： + +- CLI 解析 JSON 并加载 HDF5； +- `TuneIndex` 接收内存 Dataset，同时调优构建参数和查询参数； +- `TuneSearch` 接收已经构建或加载的 `IndexPtr`，只调查询参数。 + +链接构建目录中的 `vsag::autotune` target 后，可以直接完成“调优、选择、加载”闭环： + +```cpp +vsag::autotune::IndexRequest request; +request.base = base; +request.metric_type = vsag::METRIC_L2; +request.workload = {queries, ground_truth, 10, 48}; +request.index_spaces = { + {"hgraph", create_candidate_space, search_candidate_space}, +}; +request.constraints = { + {vsag::autotune::Metric::RECALL_AT_K, 0.95}, + {vsag::autotune::Metric::INDEX_MEMORY_MB, 8192.0}, +}; +request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + +auto result = vsag::autotune::TuneIndex(request).value(); +auto neighbors = + result.index->KnnSearch(query, 10, result.search_parameters).value(); +``` + +`base` 和 `queries` 必须是 dense float32 Dataset，base ID 必须存在且唯一。ground truth 的 +`NumElements()` 等于 query 数量，`Dim()` 表示 ground-truth K，`Ids()` 是来自 base ID 空间的 +展平数组。当 `RECALL_AT_K` 是约束或目标时必须提供 ground truth。Dataset 的 buffer 需保持有效， +直到这个同步调用返回。 + +每个索引拥有不同的原生参数模式，因此 `IndexSpace::create_parameter_space` 和 +`search_parameter_space` 仍使用 JSON 字符串。`index_spaces` 为空时使用内置 HGraph 和 +IVF 候选；显式参数中的数组和 `$range` 与 CLI 契约语义相同。 + +返回值包括已加载的 `index`、`index_name`、完整具体 `create_parameters`、经过验证的 +`search_parameters`、metrics、artifact 路径和完整报告。最终推荐 artifact 会保留，因此 +可以用 `index_name + create_parameters` 重新创建空索引并反序列化该文件；未选中的中间 +artifact 默认删除。 + +如果没有候选满足全部约束，`TuneIndex` 返回 `INVALID_ARGUMENT`，不会加载 `best_effort`。 + +完整示例见 +[`examples/cpp/326_feature_create_index_with_constraints.cpp`][factory-example]。同时启用工具和 +示例即可编译： + +```bash +cmake -S . -B build-release \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TOOLS=ON \ + -DENABLE_EXAMPLES=ON +cmake --build build-release --target 326_feature_create_index_with_constraints -j +``` + +[factory-example]: https://github.com/antgroup/vsag/blob/main/examples/cpp/326_feature_create_index_with_constraints.cpp + +## 请求 + +将下面的请求保存为 `request.json`。`dim`、`dtype` 和 `metric_type` 会从数据集读取，用户 +不需要重复填写。 + +```json +{ + "version": 1, + "data_path": "/tmp/sift-128-euclidean.hdf5", + "indexes": [ + { + "name": "hgraph", + "create_params": { + "index_param": { + "base_quantization_type": ["fp32", "sq8_uniform"], + "max_degree": [16, 32], + "ef_construction": 200 + } + }, + "search_params": { + "hgraph": { + "ef_search": { + "$range": {"start": 40, "stop": 1000} + } + } + } + } + ], + "workload": { + "top_k": 10, + "concurrency": 1 + }, + "constraints": { + "recall_at_k": 0.95, + "index_memory_mb": 8192 + }, + "objective": { + "metric": "latency_avg_ms" + }, + "tuning_config": { + "workspace_path": "/tmp/vsag_autotune", + "keep_intermediate": false, + "max_trials": 1000 + } +} +``` + +运行方式： + +```bash +./build-release/tools/autotune/autotune request.json +``` + +参数叶子可以是标量、数组，或带有 `start`、`stop`、`step` 的 `$range`。HGraph +`ef_search` 还可以省略 `step`；AutoTune 会从 `start` 倍增，找到首个满足 `recall_at_k` +约束的区间，再在该区间内二分查找最小通过值。该策略假设固定其他配置时 recall 不会随 +`ef_search` 增大而降低。每个实际评测点都使用全部 query。用户显式提供的值保持不变； +对于缺失的调优字段，V1 会补充一小组 HGraph 或 IVF 内置候选。省略 `indexes` 时会同时 +评测这两种索引。 + +具体 create 参数相同的候选只构建一次；每组具体 search 参数仍然是独立 trial。自适应 +`ef_search` 范围只记录实际评测过的具体参数点。 + +## 已有索引 + +设置 `index_path` 并且只提供一个索引规格，可以跳过构建、只调查询参数。其中 +`create_params` 必须是序列化该索引时使用的具体参数。该模式不会展开 `create_params`， +因此索引原生的数组字段会保持不变。 + +程序调用方使用独立的 `SearchRequest` 和 `TuneSearch`： + +```cpp +vsag::autotune::SearchRequest request; +request.index = existing_index; +request.index_name = "hgraph"; +request.base = base; +request.metric_type = vsag::METRIC_L2; +request.workload = {queries, ground_truth, 10, 48}; +request.parameter_space = R"({"hgraph":{"ef_search":[40,80,120]}})"; +request.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.95}}; +request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + +auto result = vsag::autotune::TuneSearch(request).value(); +auto neighbors = existing_index->KnnSearch(query, 10, result.parameters).value(); +``` + +该接口避免序列化和文件 I/O，并让全部 search trial 复用调用方的索引。共享 recall evaluator +目前仍要求提供 `base`。CLI 的 `index_path` 只是离线适配：先创建并反序列化 Index，再进入 +同一条 search-only 流程。 + +Pyramid 通过这个 search-only 模式接入。HDF5 CLI 适配器不提供 query path，因此只能评测 +Pyramid 原生的默认/root 搜索。typed 请求直接从 `workload.queries->GetPaths()` 取得 path： +提供 path 时，每条 query 按自己的 path 评测；不提供时沿用 root 搜索语义。V1 只支持默认 +未命名 hierarchy。若不同 path 需要不同 `ef_search`，应为每个代表性 path workload 分别 +发起一次 typed AutoTune 请求，并提供与该 path 对应的 ground truth。V1 不把多个 path 的 +推荐聚合成一条结果。 + +完整示例见 +[`examples/cpp/327_feature_autotune_existing_index.cpp`][existing-index-example]。 +Pyramid path 调优示例见 +[`examples/cpp/328_feature_autotune_existing_pyramid.cpp`][pyramid-example]。 +它对同一个 Pyramid 索引中的 512-vector 和 4096-vector 叶子子图使用相同 recall 目标分别 +调优，用于展示不同 path 可能需要不同的 `ef_search`。 + +[existing-index-example]: https://github.com/antgroup/vsag/blob/main/examples/cpp/327_feature_autotune_existing_index.cpp +[pyramid-example]: https://github.com/antgroup/vsag/blob/main/examples/cpp/328_feature_autotune_existing_pyramid.cpp + +## 指标 + +以下指标可以作为约束或优化目标： + +| 指标 | 约束比较方式 | +| --- | --- | +| `recall_at_k`、`qps` | 不低于请求值 | +| `latency_avg_ms`、`latency_p99_ms` | 不高于请求值 | +| `index_memory_mb`、`index_size_mb` | 不高于请求值 | +| `build_seconds`、`search_seconds`、`build_and_search_seconds` | 不高于请求值 | + +已有索引模式不提供 `build_seconds`、`index_size_mb` 和 `build_and_search_seconds`。 +目标方向由指标本身决定，因此不需要 `direction` 字段。 + +AutoTune 使用独立的 latency/QPS pass 和 recall/statistics pass，因此同一条逻辑 query 在 +每个 trial 中会执行多次。`search_seconds` 包含两个内存 pass 和指标采集，但不包含索引 +反序列化。候选按固定顺序运行，不做 warm-up 或随机排序。如果 latency 或 QPS 决定最终 +推荐，应在有代表性且受控的机器上测量，并通过重复实验确认。 + +## 阅读输出 + +标准输出是便于阅读和复制的短 summary，字段顺序如下： + +- `recommendation`：最终选中的具体参数、指标和 trial 证据。build-and-search 结果还包含 + create 参数和 artifact/build 证据；search-only 结果不包含这些字段。 +- `best_effort`：只有在没有 trial 满足全部约束时出现。 +- `failure`：请求失败或所有评测失败时出现。 +- `report_path`：完整可复现报告的位置。 + +完整报告还包含每个 build 和 trial、约束 violation、补齐后的请求，以及分阶段的结构化 +失败。search-only 模式的 `builds` 为空，trial 不包含 artifact 或 build 字段。当 +CLI 或 `RunAutoTune` 设置 `keep_intermediate=false` 时，评测后会删除全部生成索引。typed +`TuneIndex` 为返回可复现结果始终保留推荐 artifact；该选项为 false 时只删除未选中的 +artifact。两种入口都会保留报告文件。 + +## V1 边界 + +V1 评测一个 KNN workload；除已支持的 HGraph `ef_search` 自适应搜索外，其他候选仍完整遍历。 +它暂不支持过滤或范围查询 workload、query sampling、跨请求 build cache，以及基于模型的 +候选生成。 diff --git a/docs/docs/zh/src/resources/autotune_api_v1.md b/docs/docs/zh/src/resources/autotune_api_v1.md new file mode 100644 index 0000000000..41a9dbe3c0 --- /dev/null +++ b/docs/docs/zh/src/resources/autotune_api_v1.md @@ -0,0 +1,473 @@ +# AutoTune V1 CLI JSON 输入输出契约 + +本文档定义 `autotune` 命令行工具使用的离线 JSON/HDF5 适配契约。实验性的 C++ API 只在 +构建目录中提供，接收 typed `IndexRequest` 和 `SearchRequest`，不会随 VSAG 库安装。所有 +入口完成输入转换后使用同一个规范化请求和调优引擎。AutoTune V1 支持 dense float32 数据、 +HGraph/IVF 的构建和查询调优，以及一个使用全量 query 的 KNN workload。 + +请求、索引规格、workload、objective、`tuning_config` 和 `output` 中的未知字段会被拒绝。 +`create_params` 和 `search_params` 内部的原生参数会传给具体 VSAG 索引，在创建或查询时由 +索引校验。 + +## 请求对象 + +| 字段 | 类型 | 必填 | 含义 | +| --- | --- | --- | --- | +| `version` | integer | 是 | 必须为 `1`。 | +| `data_path` | string | 是 | 可读的 HDF5 评测数据集。 | +| `index_path` | string | 否 | 用于 search-only 调优的已有序列化索引。 | +| `indexes` | array | 否 | 索引规格；普通模式默认使用 HGraph 和 IVF。 | +| `workload` | object | 是 | 要优化的 KNN workload。 | +| `constraints` | object | 是 | 非空的“指标到阈值”映射。 | +| `objective` | object | 是 | 对可行候选排序的目标指标。 | +| `tuning_config` | object | 否 | workspace、产物保留和 trial 上限。 | +| `output` | object | 否 | 报告路径和原始 eval 输出开关。 | + +### 数据集规则 + +`data_path` 必须指向 VSAG eval 工具可读取的普通 HDF5 文件。V1 要求： + +- dense vector； +- base 和 query 均为 float32； +- 至少包含一个 base、一个 query 和一个 ground-truth neighbor； +- 使用 recall 指标时，`workload.top_k` 不超过 ground truth 的宽度； +- distance 属性能够映射为 `l2`、`ip` 或 `cosine`。 + +AutoTune 从数据集读取 `dim`、`dtype` 和 `metric_type`，并注入每组具体 +`create_params`。请求可以重复提供这些字段，但值必须与数据集完全一致。 + +## 索引规格 + +每个 `indexes[]` 元素的形态如下： + +```json +{ + "name": "hgraph", + "create_params": { + "index_param": { + "max_degree": [16, 32] + } + }, + "search_params": { + "hgraph": { + "ef_search": {"$range": {"start": 40, "stop": 1000}} + } + } +} +``` + +| 字段 | 类型 | 必填 | 含义 | +| --- | --- | --- | --- | +| `name` | string | 是 | `hgraph`、`ivf`，或 search-only 的 `pyramid`；不区分大小写。 | +| `create_params` | object | 否 | VSAG 创建参数或候选表达式。 | +| `search_params` | object | 否 | 查询参数，key 必须是同一个索引名。 | + +普通 build-and-search 模式省略 `indexes` 时，AutoTune 会同时评测 HGraph 和 IVF。设置 +`index_path` 时，必须提供 `indexes`，并且只能包含一个元素。 +Pyramid 只接受已有的内存索引或序列化索引。 + +### 候选表达式 + +`create_params` 和 `search_params` 中的每个叶子都可以写成： + +- 标量：固定该值； +- 非空数组：提供离散候选值； +- `$range`：必须且只能包含 `start`、`stop` 和非零 `step`，范围包含端点。 + +示例： + +```json +{ + "fixed": 32, + "choices": [16, 32, 48], + "integer_range": {"$range": {"start": 40, "stop": 120, "step": 40}}, + "float_range": {"$range": {"start": 0.1, "stop": 0.3, "step": 0.1}} +} +``` + +HGraph `ef_search` 还支持一种特殊写法： + +```json +{"$range": {"start": 40, "stop": 1000}} +``` + +上下界必须是正整数并满足 `start <= stop`。这种写法要求存在 `recall_at_k` 约束，且目标为 +`latency_avg_ms`、`latency_p99_ms`、`qps`、`search_seconds` 或 +`build_and_search_seconds`；不能同时设置 `timeout_ms` 或 `hops_limit`。对于其他参数的每个 +固定组合，AutoTune 从 `start` 开始倍增 `ef_search`（不超过 `stop`），直到实测 recall 满足 +约束；随后在最后一个失败/通过区间内二分，查找最小通过值。分支方向只由 recall 决定； +latency 和 QPS 仍作为实测指标参与最终约束过滤和结果选择。该策略假设其他参数和 workload +固定时，recall 不会随 `ef_search` 增大而降低。 + +每个探测点都会评测完整 query workload。如果某个评测点执行失败或没有 recall，AutoTune +会停止该自适应范围，因为此时无法安全选择下一个区间。 + +V1 会把参数对象中的所有 JSON 数组和带 `step` 的范围解释成候选集合，计算完整笛卡尔积 +并去除完全相同的候选。其他参数不接受省略 `step` 的范围。 + +计划的最坏评测次数超过 `tuning_config.max_trials` 时，请求失败。普通具体候选占一个 +trial；自适应 `ef_search` 在 `start == stop` 时占一个。否则，planner 会为每个可能的首次 +通过探测点计算“到达该点的探测次数 + 前一区间的二分次数”，并取其中最大值。例如 +`40..1000` 最坏预留 15 个 trial。 + +### 缺失字段的内置候选 + +用户显式字段始终优先。只有字段缺失时才补充以下候选： + +| 索引 | 缺失字段 | V1 候选 | +| --- | --- | --- | +| HGraph | `base_quantization_type` | `fp32`、`sq8_uniform` | +| HGraph | `max_degree` | `16`、`32` | +| HGraph | `ef_construction` | `100`、`200` | +| HGraph | `ef_search` | 根据 `top_k` 生成三个值，下限为 `40`、`80`、`120` | +| Pyramid，search-only | `ef_search` | 根据 `top_k` 生成三个值，下限为 `40`、`80`、`120` | +| IVF | `base_quantization_type` | `fp32`、`sq8_uniform` | +| IVF | `buckets_count` | 不超过 base 数量的 `1024`、`2048` | +| IVF，build-and-search | `scan_buckets_count` | 根据具体 `buckets_count` 生成至多四个值 | +| IVF，search-only | `scan_buckets_count` | `1`、`4`、`16`、`64` | + +HGraph 和 Pyramid 查询候选具体为 `max(40, top_k)`、`max(80, 2 * top_k)` 和 +`max(120, 4 * top_k)`，重复值会删除。build-and-search 模式的 IVF 查询候选不会超过具体 +bucket 数量。AutoTune 无法推断已有 IVF 的 bucket 数量，因此 search-only 模式使用上面的 +保守通用值；被已加载索引拒绝的值会记录为 failed trial，调用方可以显式提供 +`scan_buckets_count` 候选以避免这些 trial。 + +规范化索引名和具体 `create_params` 相同的候选属于同一个 build group。AutoTune 对该组 +只构建一次、序列化一次作为证据，再使用同一个内存索引实例执行所有关联的 search 候选。 + +### 已有索引模式 + +设置 `index_path` 后： + +- 路径必须指向可读普通文件； +- 必须且只能提供一个索引规格； +- 必须提供 `create_params.index_param`； +- `create_params` 整体视为一份具体的索引原生配置，不再做候选展开； +- AutoTune 不补充 create 候选，也不会执行 build； +- 缺失的 search 字段仍可使用内置查询候选； +- `build_seconds`、`index_size_mb` 和 `build_and_search_seconds` 不能作为约束或目标； +- 不会删除调用方提供的索引。 + +V1 反序列化前需要使用具体创建参数实例化正确的 VSAG 索引，目前不会从序列化文件推导 +这些参数。 + +对于 typed `SearchRequest`，索引已经完成实例化和加载，因此只需索引名和 search +`parameter_space`，不需要 create 参数。 + +HDF5 适配器不提供 Pyramid query path，因此 CLI 只能评测 Pyramid 原生的默认/root 查询。 +按 path 调优 Pyramid 需要使用 typed `SearchRequest`，并通过 query Dataset 的 +`Dataset::Paths()` 提供 path。V1 只转发默认的未命名 hierarchy，不支持 Pyramid named +hierarchy。 + +## Workload + +```json +{ + "workload": { + "top_k": 10, + "concurrency": 48 + } +} +``` + +| 字段 | 类型 | 必填 | 默认值 | 校验 | +| --- | --- | --- | --- | --- | +| `top_k` | positive integer | 是 | — | 不超过 base 数量；使用 recall 指标时不超过 ground truth 宽度；并能用原生 `int` 表示。 | +| `concurrency` | positive integer | 否 | `1` | 最大为 `200`。 | + +V1 始终评测 HDF5 文件中的全量 query，并且只支持 KNN。benchmark 机器、系统负载和运行 +环境都属于 workload 的一部分；延迟和 QPS 不能直接移植到不同机器规格或负载条件。 + +## 约束和优化目标 + +`constraints` 直接把指标名映射到阈值；`objective.metric` 指定一个目标指标。比较方式和 +目标方向由指标决定，用户不能覆盖。 + +```json +{ + "constraints": { + "recall_at_k": 0.95, + "latency_p99_ms": 2.0, + "index_memory_mb": 8192 + }, + "objective": { + "metric": "latency_avg_ms" + } +} +``` + +| 指标 | 约束 | 目标方向 | 已有索引可用 | +| --- | --- | --- | --- | +| `recall_at_k` | actual ≥ threshold | 最大化 | 是 | +| `qps` | actual ≥ threshold | 最大化 | 是 | +| `latency_avg_ms` | actual ≤ threshold | 最小化 | 是 | +| `latency_p99_ms` | actual ≤ threshold | 最小化 | 是 | +| `index_memory_mb` | actual ≤ threshold | 最小化 | 是 | +| `index_size_mb` | actual ≤ threshold | 最小化 | 否 | +| `build_seconds` | actual ≤ threshold | 最小化 | 否 | +| `search_seconds` | actual ≤ threshold | 最小化 | 是 | +| `build_and_search_seconds` | actual ≤ threshold | 最小化 | 否 | + +阈值必须是有限非负数，`recall_at_k` 还必须不大于 `1.0`。 + +V1 指标口径： + +- `build_seconds` 是 eval 工具报告的索引 `Build` 操作耗时。 +- 它不包含索引序列化、数据集加载和候选编排。 +- `search_seconds` 是完整内存 search eval trial 的墙钟时间，不包含索引反序列化，但包含 + 所有 search pass 和指标采集；线上查询性能目标应使用 latency 或 QPS。 +- `build_and_search_seconds` 是一次 build-and-search trial 中前两项之和。 +- `index_memory_mb` 来自已加载索引大于零的 `GetMemoryUsage()` 结果。索引报告零时， + AutoTune 会把该指标视为不可用，而不是零内存；相关约束会记录 missing-metric violation。 +- `index_size_mb` 是生成的序列化索引文件大小，在 search-only 模式不可用。 +- AutoTune 使用独立的 latency/QPS pass 和 recall/statistics pass。latency 用 + `steady_clock` 包围每次 `KnnSearch`；QPS 是成功查询数除以并发性能 pass 的墙钟时间。 + 因此，同一条逻辑 query 在每个 trial 中会执行多次。 + +`search_seconds` 仍表示评测成本，因为它包含两个 pass 和 monitor 处理；它不是一次线上 +逻辑 query batch 的执行时间。候选会按固定顺序在复用的索引上运行，不做 warm-up 或随机 +排序，因此 cache 状态和无关系统负载会影响性能指标。如果 latency 或 QPS 决定结果，应在 +有代表性且受控的环境中运行，并通过重复实验确认。 + +## Tuning 和输出配置 + +```json +{ + "tuning_config": { + "workspace_path": "/tmp/vsag_autotune", + "keep_intermediate": false, + "max_trials": 1000 + }, + "output": { + "result_path": "/tmp/vsag_autotune/report.json", + "include_raw_eval": false + } +} +``` + +| 字段 | 默认值 | 含义 | +| --- | --- | --- | +| `tuning_config.workspace_path` | `/tmp/vsag_autotune` | run 产物和默认报告目录。 | +| `tuning_config.keep_intermediate` | `false` | 是否保留生成的中间索引产物。 | +| `tuning_config.max_trials` | `1000` | 计划最坏 trial 数上限；硬上限为 `100000`。 | +| `output.result_path` | `/run-.json` | 完整报告路径。 | +| `output.include_raw_eval` | `false` | 在 build/trial 中包含原生 eval JSON。 | + +`output.result_path` 不能与 `data_path` 或 `index_path` 指向同一文件。离线 JSON/CLI 入口 +设置 `keep_intermediate=false` 时，评测后删除所有生成索引。typed `TuneIndex` 为返回可复现 +结果始终保留推荐 artifact；该选项只控制未选中 artifact 的保留。报告单独保留。 + +## 完整报告 + +完成评测后会写入并返回以下顶层结构： + +```json +{ + "version": 1, + "status": "success", + "recommendation": {}, + "best_effort": null, + "builds": [], + "trials": [], + "request": {}, + "elapsed_seconds": 84.2, + "report_path": "/tmp/vsag_autotune/run-....json" +} +``` + +| 字段 | 含义 | +| --- | --- | +| `version` | 报告契约版本，固定为 `1`。 | +| `status` | `success`、`no_feasible_candidate` 或 `failed`。 | +| `recommendation` | 最优可行 trial；不存在时为 `null`。 | +| `best_effort` | 约束不可行时最接近的成功 trial，否则为 `null`。 | +| `builds` | 每组具体生成 build 一条记录；search-only 模式为空数组。 | +| `trials` | 每组已执行的具体 search 候选一条记录。 | +| `request` | 调优引擎实际使用的有效规范化请求。 | +| `elapsed_seconds` | 截止结果选择的 AutoTune 墙钟时间，不含报告写入和清理。 | +| `report_path` | 持久化完整报告路径。 | +| `failure` | 整体执行失败时出现。 | + +早期校验失败不包含 `builds`、`trials` 或 `report_path`，也不会写报告。如果所有候选评测 +失败，只要报告路径仍然可用，就会保留尝试过的 build/trial 记录并写报告。 + +规范化 `request` 包含推导出的 dataset 元数据、workload 默认值、constraints、objective、 +config 和 output。build-and-search 模式还包含 `index_spaces`；search-only 模式包含 +`index_name` 和 `parameter_space`。CLI 报告还会在这个规范化对象顶层保留离线 +`data_path`，存在已有索引输入时也保留 `index_path`。 + +### Status 语义 + +- `success`：至少一个成功 trial 满足全部约束，设置 `recommendation`。 +- `no_feasible_candidate`：存在成功 trial，但没有任何一个满足全部约束；设置 + `best_effort` 用于解释，它不是有效推荐。 +- `failed`：请求非法、执行或报告阶段失败，或者没有成功且带目标指标的 trial。 + +### Recommendation 和 Best Effort + +build-and-search 模式下，`recommendation` 包含： + +```json +{ + "index_name": "hgraph", + "create_params": {}, + "search_params": {}, + "workload": {"top_k": 10, "concurrency": 1}, + "metrics": {}, + "artifacts": {}, + "evidence": {"build_id": "build-0", "trial_id": "trial-0"} +} +``` + +search-only 模式会省略 `create_params`、`artifacts` 和 `evidence.build_id`，只包含具体 +查询参数、workload、metrics 和 `evidence.trial_id`。`best_effort` 使用相同的模式相关 +字段，并额外包含 `constraint_evaluation`。它首先按违反或缺失约束数量选择，再比较归一化 +violation 大小，只用于解释。 + +### Build 记录 + +search-only 模式的 `builds` 为空。build-and-search 模式下，每个 `builds[]` 元素包含： + +| 字段 | 含义 | +| --- | --- | +| `build_id` | 被 trial 引用的稳定 ID。 | +| `index_name` | 具体索引类型。 | +| `create_params` | 具体创建参数。 | +| `status` | `success` 或 `failed`。 | +| `metrics` | 可用的 build 共享指标。 | +| `artifacts` | `source`、`index_path`、`use_existing_index` 和 `retained`。 | +| `failure` | 结构化失败或 `null`。 | +| `elapsed_seconds` | build group 准备耗时。 | +| `raw_eval_result` | 请求原始输出且真实 build eval 成功时出现。 | + +### Trial 记录 + +每个 `trials[]` 元素包含： + +| 字段 | 含义 | +| --- | --- | +| `trial_id` | 稳定 trial ID。 | +| `build_id` | 关联的 build group；仅 build-and-search 模式存在。 | +| `index_name` | 具体索引类型。 | +| `create_params` | 具体创建参数；仅 build-and-search 模式存在。 | +| `search_params` | 具体查询参数。 | +| `status` | `success` 或 `failed`。 | +| `metrics` | 用于约束评估和选择的可用指标。 | +| `constraint_evaluation` | `satisfied` 和 `violations` 数组。 | +| `artifacts` | 从 build group 复制的产物证据；仅 build-and-search 模式存在。 | +| `failure` | 结构化失败或 `null`。 | +| `elapsed_seconds` | search trial 墙钟时间。 | +| `raw_eval_result` | 请求原始输出且原生 search eval 成功时出现。 | + +对于自适应 HGraph `ef_search` 范围，报告会为每个实际评测点记录一条 trial。每条记录中的 +`search_params` 都是具体值；`$range` 表达式只保留在规范化请求中。每条记录都会评测完整 +query workload。 + +每条 constraint violation 包含 `metric`、`comparison`、`expected` 和 `actual`。指标缺失或 +非有限数时，`actual` 为 `null`。 + +同一 build group 的 search trial 复用同一个已加载索引实例。新生成的索引只构建和序列化 +一次。search-only 模式直接为所有 trial 复用调用方索引，或复用 CLI 适配器反序列化得到的 +索引。 + +### Artifact 语义 + +V1 只在 build-and-search 记录中提供 artifact 字段。`artifacts.source` 为 `generated`； +`artifacts.index_path` 用于说明被评测索引曾存放在哪里，不保证响应返回时路径仍然存在。 +需要检查 `artifacts.retained`： + +- `true`：这是 typed `TuneIndex` 返回的推荐产物，或者请求保留； +- `false`：AutoTune 计划删除或已经删除生成产物。 + +### 结构化失败 + +所有失败使用统一结构： + +```json +{ + "stage": "validation", + "code": "invalid_request", + "message": "request.workload.top_k is required" +} +``` + +常见 stage 包括 `cli`、`validation`、`candidate_generation`、`build`、`search`、 +`evaluation`、`selection` 和 `report`。build/trial 失败保留在对应记录中，其他候选可以 +继续执行。 + +| Stage | Code | 含义 | +| --- | --- | --- | +| `cli` | `request_file_error` | CLI 无法读取或解析请求文件。 | +| `validation` | `invalid_request` | 请求校验失败。 | +| `candidate_generation` | `invalid_request` | 候选表达式或数量非法。 | +| `build` | `build_evaluation_failed` | 一组 build 失败。 | +| `search` | `build_failed` | 关联 build 失败，search 被跳过。 | +| `search` | `search_evaluation_failed` | 一组 search trial 失败。 | +| `evaluation` | `all_trials_failed` | 所有候选 trial 都执行失败。 | +| `selection` | `objective_metric_unavailable` | 存在成功 trial，但都没有产生目标指标。 | +| `evaluation`、`report` | `execution_failed` | 顶层执行或报告写入失败。 | + +## CLI Summary + +CLI 按以下顺序输出完整报告的紧凑子集： + +1. `recommendation`（存在时）； +2. `best_effort`（存在时）； +3. `failure`（存在时）； +4. `status`； +5. `elapsed_seconds`； +6. `report_path`（存在时）； +7. `version`。 + +标准输出会省略值为 null 的结果分支和详细 build/trial 数组。完整证据请读取 +`report_path`。 + +CLI 在 `status=failed` 或命令行/请求文件出错时返回 `1`。目前 `success` 和 +`no_feasible_candidate` 都返回 `0`；调用方必须检查 `status`，不能只用退出码判断是否 +存在 recommendation。 + +## Build-tree C++ 入口 + +实验性的可选 CMake target `vsag::autotune` 暴露 `tools/autotune/autotune.h` 中的接口。 +该 target 和头文件只在构建目录中提供，不会安装： + +```cpp +tl::expected TuneIndex(const IndexRequest& request); +tl::expected TuneSearch(const SearchRequest& request); +JsonType RunAutoTune(const JsonType& request); // CLI adapter +``` + +JSON 入口只负责离线适配：加载 `data_path`；存在 `index_path` 时创建并反序列化索引；随后 +构造与 typed 入口相同的内部请求。V1 不安装该 target 和头文件。由于底层 eval 路径会配置 +进程级 OpenMP 状态，同一进程内的多次调用会串行执行。 + +`TuneIndex` 同时调优构建参数和查询参数： + +```cpp +IndexRequest request; +request.base = base; +request.metric_type = METRIC_L2; +request.workload = {queries, ground_truth, 10, 48}; +request.index_spaces = {{"hgraph", create_candidate_space, search_candidate_space}}; +request.constraints = {{Metric::RECALL_AT_K, 0.95}}; +request.objective = Metric::LATENCY_AVG_MS; +auto result = TuneIndex(request); +``` + +返回值包含已加载、可查询的推荐索引，以及具体 create 和 search 参数。`TuneSearch` 对已经 +构建或加载的索引调查询参数： + +```cpp +SearchRequest request; +request.index = existing_index; +request.index_name = "hgraph"; +request.base = base; +request.metric_type = METRIC_L2; +request.workload = {queries, ground_truth, 10, 48}; +request.parameter_space = search_candidate_space; +request.constraints = {{Metric::RECALL_AT_K, 0.95}}; +request.objective = Metric::LATENCY_AVG_MS; +auto result = TuneSearch(request); +``` + +`TuneSearch` 不提供构建阶段指标和 `index_size_mb`。 diff --git a/examples/cpp/326_feature_create_index_with_constraints.cpp b/examples/cpp/326_feature_create_index_with_constraints.cpp new file mode 100644 index 0000000000..f480496f81 --- /dev/null +++ b/examples/cpp/326_feature_create_index_with_constraints.cpp @@ -0,0 +1,85 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include + +#include "autotune.h" + +int +main() { + constexpr int64_t DIM = 2; + vsag::Options::Instance().set_block_size_limit(2UL * 1024 * 1024); + std::vector base_ids{101, 205, 309, 413}; + std::vector base_vectors{0.0F, 0.0F, 1.0F, 0.0F, 0.0F, 1.0F, 1.0F, 1.0F}; + std::vector query_vectors{0.0F, 0.0F, 1.0F, 1.0F}; + std::vector ground_truth_ids{101, 413}; + + auto base = vsag::Dataset::Make() + ->NumElements(4) + ->Dim(DIM) + ->Ids(base_ids.data()) + ->Float32Vectors(base_vectors.data()) + ->Owner(false); + auto queries = vsag::Dataset::Make() + ->NumElements(2) + ->Dim(DIM) + ->Float32Vectors(query_vectors.data()) + ->Owner(false); + auto ground_truth = + vsag::Dataset::Make()->NumElements(2)->Dim(1)->Ids(ground_truth_ids.data())->Owner(false); + + vsag::autotune::IndexRequest request; + request.base = base; + request.metric_type = vsag::METRIC_L2; + request.workload = {queries, ground_truth, 1, 1}; + request.index_spaces = { + {"hgraph", + R"({"index_param":{"base_quantization_type":"fp32","max_degree":8,"ef_construction":40}})", + R"({"hgraph":{"ef_search":[10,20]}})"}}; + request.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 1.0}, + {vsag::autotune::Metric::INDEX_MEMORY_MB, 1024.0}}; + request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + request.config.max_trials = 2; + + auto tuned = vsag::autotune::TuneIndex(request); + if (!tuned.has_value()) { + std::cerr << "Failed to tune index: " << tuned.error().message << std::endl; + return 1; + } + + const auto& result = tuned.value(); + std::cout << "index_name: " << result.index_name << '\n' + << "create_params: " << result.create_parameters << '\n' + << "search_params: " << result.search_parameters << '\n' + << "validated_metrics: " << result.metrics << '\n' + << "index_artifact: " << result.artifact_path << '\n' + << "report: " << result.report_path << std::endl; + + auto query = vsag::Dataset::Make(); + query->NumElements(1)->Dim(DIM)->Float32Vectors(query_vectors.data())->Owner(false); + + auto neighbors = result.index->KnnSearch(query, 1, result.search_parameters); + if (!neighbors.has_value()) { + std::cerr << "The selected index could not be queried: " << neighbors.error().message + << std::endl; + return 1; + } + std::cout << "first neighbor id: " << neighbors.value()->GetIds()[0] << std::endl; + return 0; +} diff --git a/examples/cpp/327_feature_autotune_existing_index.cpp b/examples/cpp/327_feature_autotune_existing_index.cpp new file mode 100644 index 0000000000..7bf5eb8e2d --- /dev/null +++ b/examples/cpp/327_feature_autotune_existing_index.cpp @@ -0,0 +1,119 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include + +#include "autotune.h" + +int +main() { + constexpr int64_t DIM = 4; + constexpr int64_t BASE_COUNT = 16; + constexpr int64_t QUERY_COUNT = 4; + vsag::Options::Instance().set_block_size_limit(2UL * 1024 * 1024); + + std::vector base_ids(BASE_COUNT); + std::vector base_vectors(BASE_COUNT * DIM); + for (int64_t i = 0; i < BASE_COUNT; ++i) { + base_ids[i] = 1000 + i; + for (int64_t j = 0; j < DIM; ++j) { + base_vectors[i * DIM + j] = static_cast(i * DIM + j); + } + } + + std::vector query_vectors(base_vectors.begin(), + base_vectors.begin() + QUERY_COUNT * DIM); + std::vector ground_truth_ids(base_ids.begin(), base_ids.begin() + QUERY_COUNT); + + auto base = vsag::Dataset::Make() + ->NumElements(BASE_COUNT) + ->Dim(DIM) + ->Ids(base_ids.data()) + ->Float32Vectors(base_vectors.data()) + ->Owner(false); + auto queries = vsag::Dataset::Make() + ->NumElements(QUERY_COUNT) + ->Dim(DIM) + ->Float32Vectors(query_vectors.data()) + ->Owner(false); + auto ground_truth = vsag::Dataset::Make() + ->NumElements(QUERY_COUNT) + ->Dim(1) + ->Ids(ground_truth_ids.data()) + ->Owner(false); + + const std::string create_params = R"( + { + "dim": 4, + "dtype": "float32", + "metric_type": "l2", + "index_param": { + "base_quantization_type": "fp32", + "max_degree": 8, + "ef_construction": 40 + } + })"; + auto created = vsag::Factory::CreateIndex("hgraph", create_params); + if (!created.has_value()) { + std::cerr << "Failed to create index: " << created.error().message << std::endl; + return 1; + } + auto index = created.value(); + auto built = index->Build(base); + if (!built.has_value()) { + std::cerr << "Failed to build index: " << built.error().message << std::endl; + return 1; + } + + vsag::autotune::SearchRequest request; + request.index = index; + request.base = base; + request.metric_type = vsag::METRIC_L2; + request.workload = {queries, ground_truth, 1, 1}; + request.index_name = "hgraph"; + request.parameter_space = R"({"hgraph":{"ef_search":[4,8,16]}})"; + request.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 1.0}}; + request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + request.config.workspace_path = "/tmp/vsag_autotune_existing_index_example"; + request.config.max_trials = 3; + + const auto tuned = vsag::autotune::TuneSearch(request); + if (!tuned.has_value()) { + std::cerr << "AutoTune failed: " << tuned.error().message << std::endl; + return 1; + } + + const auto& result = tuned.value(); + std::cout << "recommended search_params: " << result.parameters << '\n' + << "validated metrics: " << result.metrics << '\n' + << "report: " << result.report_path << std::endl; + + auto query = vsag::Dataset::Make() + ->NumElements(1) + ->Dim(DIM) + ->Float32Vectors(query_vectors.data()) + ->Owner(false); + auto neighbors = index->KnnSearch(query, 1, result.parameters); + if (!neighbors.has_value()) { + std::cerr << "Search failed: " << neighbors.error().message << std::endl; + return 1; + } + std::cout << "first neighbor id: " << neighbors.value()->GetIds()[0] << std::endl; + return 0; +} diff --git a/examples/cpp/328_feature_autotune_existing_pyramid.cpp b/examples/cpp/328_feature_autotune_existing_pyramid.cpp new file mode 100644 index 0000000000..3dcae015b1 --- /dev/null +++ b/examples/cpp/328_feature_autotune_existing_pyramid.cpp @@ -0,0 +1,204 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "autotune.h" + +namespace { + +constexpr int64_t DIM = 32; +constexpr int64_t EASY_COUNT = 512; +constexpr int64_t HARD_COUNT = 4096; +constexpr int64_t QUERY_COUNT = 512; +constexpr int64_t TOP_K = 10; + +struct PathWorkload { + std::vector query_vectors; + std::vector ground_truth_ids; + std::vector query_paths; + vsag::DatasetPtr queries; + vsag::DatasetPtr ground_truth; +}; + +float +L2(const float* left, const float* right) { + float result = 0.0F; + for (int64_t i = 0; i < DIM; ++i) { + const auto difference = left[i] - right[i]; + result += difference * difference; + } + return result; +} + +void +PrepareWorkload(PathWorkload& workload, + const std::vector& base_ids, + const std::vector& base_vectors, + int64_t begin, + int64_t count, + const std::string& path, + std::mt19937& random) { + std::uniform_real_distribution unit(0.0F, 1.0F); + workload.query_vectors.resize(QUERY_COUNT * DIM); + workload.ground_truth_ids.resize(QUERY_COUNT * TOP_K); + workload.query_paths.assign(QUERY_COUNT, path); + + for (int64_t query = 0; query < QUERY_COUNT; ++query) { + auto* query_vector = workload.query_vectors.data() + query * DIM; + for (int64_t column = 0; column < DIM; ++column) { + query_vector[column] = unit(random); + } + + std::vector> ranked; + ranked.reserve(count); + for (int64_t row = begin; row < begin + count; ++row) { + ranked.emplace_back(L2(query_vector, base_vectors.data() + row * DIM), base_ids[row]); + } + std::partial_sort(ranked.begin(), ranked.begin() + TOP_K, ranked.end()); + for (int64_t k = 0; k < TOP_K; ++k) { + workload.ground_truth_ids[query * TOP_K + k] = ranked[k].second; + } + } + + workload.queries = vsag::Dataset::Make() + ->NumElements(QUERY_COUNT) + ->Dim(DIM) + ->Float32Vectors(workload.query_vectors.data()) + ->Paths(workload.query_paths.data()) + ->Owner(false); + workload.ground_truth = vsag::Dataset::Make() + ->NumElements(QUERY_COUNT) + ->Dim(TOP_K) + ->Ids(workload.ground_truth_ids.data()) + ->Owner(false); +} + +tl::expected +TunePath(const vsag::IndexPtr& index, + const vsag::DatasetPtr& base, + const PathWorkload& workload, + const std::string& workspace) { + vsag::autotune::SearchRequest request; + request.index = index; + request.index_name = "pyramid"; + request.base = base; + request.metric_type = vsag::METRIC_L2; + request.workload = {workload.queries, workload.ground_truth, TOP_K, 1}; + request.parameter_space = R"({"pyramid":{"ef_search":[10,20,40,80,160]}})"; + request.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.80}}; + request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + request.config.workspace_path = workspace; + request.config.max_trials = 5; + return vsag::autotune::TuneSearch(request); +} + +void +PrintResult(const std::string& path, const vsag::autotune::SearchResult& result) { + std::cout << "\npath: " << path << '\n' + << "recommended search_params: " << result.parameters << '\n' + << "validated metrics: " << result.metrics << '\n' + << "report: " << result.report_path << std::endl; +} + +} // namespace + +int +main() { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + vsag::Options::Instance().set_block_size_limit(2UL * 1024 * 1024); + std::mt19937 random(47); + std::uniform_real_distribution unit(0.0F, 1.0F); + + constexpr int64_t BASE_COUNT = EASY_COUNT + HARD_COUNT; + std::vector base_ids(BASE_COUNT); + std::vector base_vectors(BASE_COUNT * DIM); + std::vector base_paths(BASE_COUNT); + for (int64_t row = 0; row < BASE_COUNT; ++row) { + base_ids[row] = 1000 + row; + auto* vector = base_vectors.data() + row * DIM; + base_paths[row] = row < EASY_COUNT ? "catalog/easy" : "catalog/hard"; + for (int64_t column = 0; column < DIM; ++column) { + vector[column] = unit(random); + } + } + + auto base = vsag::Dataset::Make() + ->NumElements(BASE_COUNT) + ->Dim(DIM) + ->Ids(base_ids.data()) + ->Float32Vectors(base_vectors.data()) + ->Paths(base_paths.data()) + ->Owner(false); + + const std::string create_params = + R"({ + "dim": 32, + "dtype": "float32", + "metric_type": "l2", + "index_param": { + "base_quantization_type": "fp32", + "max_degree": 16, + "alpha": 1.2, + "graph_iter_turn": 5, + "neighbor_sample_rate": 0.2, + "no_build_levels": [0, 1], + "use_reorder": true, + "graph_type": "odescent", + "build_thread_count": 8 + } + })"; + auto created = vsag::Factory::CreateIndex("pyramid", create_params); + if (!created.has_value()) { + std::cerr << "Failed to create Pyramid: " << created.error().message << std::endl; + return 1; + } + auto index = created.value(); + auto built = index->Build(base); + if (!built.has_value()) { + std::cerr << "Failed to build Pyramid: " << built.error().message << std::endl; + return 1; + } + + PathWorkload easy_workload; + PathWorkload hard_workload; + PrepareWorkload(easy_workload, base_ids, base_vectors, 0, EASY_COUNT, "catalog/easy", random); + PrepareWorkload( + hard_workload, base_ids, base_vectors, EASY_COUNT, HARD_COUNT, "catalog/hard", random); + + auto easy_result = + TunePath(index, base, easy_workload, "/tmp/vsag_autotune_existing_pyramid_example/easy"); + auto hard_result = + TunePath(index, base, hard_workload, "/tmp/vsag_autotune_existing_pyramid_example/hard"); + if (!easy_result.has_value() || !hard_result.has_value()) { + const auto message = + !easy_result.has_value() ? easy_result.error().message : hard_result.error().message; + std::cerr << "AutoTune failed: " << message << std::endl; + return 1; + } + + std::cout << "The same Pyramid index is tuned once per representative path workload." + << std::endl; + PrintResult("catalog/easy (512 random vectors)", easy_result.value()); + PrintResult("catalog/hard (4096 random vectors)", hard_result.value()); + return 0; +} diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 3793625a87..d71f93af3a 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -157,3 +157,17 @@ target_link_libraries(325_feature_uring_io vsag) add_executable(324_feature_hgraph_mci_companion 324_feature_hgraph_mci_companion.cpp) target_link_libraries(324_feature_hgraph_mci_companion vsag) + +if (TARGET vsag::autotune) + add_executable (326_feature_create_index_with_constraints + 326_feature_create_index_with_constraints.cpp) + target_link_libraries (326_feature_create_index_with_constraints vsag::autotune) + + add_executable (327_feature_autotune_existing_index + 327_feature_autotune_existing_index.cpp) + target_link_libraries (327_feature_autotune_existing_index vsag::autotune) + + add_executable (328_feature_autotune_existing_pyramid + 328_feature_autotune_existing_pyramid.cpp) + target_link_libraries (328_feature_autotune_existing_pyramid vsag::autotune) +endif () diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index afcad94544..30408efaee 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -15,6 +15,7 @@ add_subdirectory (eval) +add_subdirectory (autotune) add_subdirectory (check_compatibility) add_subdirectory (analyze_index) add_subdirectory (visualize_index) diff --git a/tools/README.md b/tools/README.md index 5d60142128..d98103d7e8 100644 --- a/tools/README.md +++ b/tools/README.md @@ -8,6 +8,7 @@ nearby entry points for local development and AI coding agents. | `analyze_index` | [analyze_index/README.md](analyze_index/README.md) | [docs/docs/en/src/resources/analyze_index.md](../docs/docs/en/src/resources/analyze_index.md) | | | `check_compatibility` | [check_compatibility/README.md](check_compatibility/README.md) | [docs/docs/en/src/resources/check_compatibility.md](../docs/docs/en/src/resources/check_compatibility.md) | | | `eval_performance` | [eval/README.md](eval/README.md) | [docs/docs/en/src/resources/eval.md](../docs/docs/en/src/resources/eval.md) | | +| `autotune` | [autotune/README.md](autotune/README.md) | [docs/docs/en/src/resources/autotune.md](../docs/docs/en/src/resources/autotune.md) | | Chinese documentation is available under `docs/docs/zh/src/resources/` and on the Chinese website paths under . diff --git a/tools/autotune/CMakeLists.txt b/tools/autotune/CMakeLists.txt new file mode 100644 index 0000000000..1d2f277d38 --- /dev/null +++ b/tools/autotune/CMakeLists.txt @@ -0,0 +1,54 @@ +# Copyright 2024-present the vsag project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_library (vsag_autotune SHARED + autotune.cpp + autotune_candidate.cpp + autotune_evaluation.cpp + $) +add_library (vsag::autotune ALIAS vsag_autotune) +target_include_directories (vsag_autotune PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories (vsag_autotune PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../eval) +target_link_libraries (vsag_autotune PUBLIC nlohmann_json::nlohmann_json vsag) +target_link_libraries (vsag_autotune PRIVATE + simd + vsag_eval_common + ${HDF5_CPP_STATIC_LIBRARY} + ${HDF5_C_STATIC_LIBRARY} + z + ${CMAKE_DL_LIBS}) +add_dependencies (vsag_autotune hdf5) + +add_executable (autotune main.cpp) +target_include_directories (autotune PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries (autotune PRIVATE vsag_autotune vsag) + +if (ENABLE_TESTS) + add_executable (autotune_test autotune_test.cpp) + target_include_directories (autotune_test PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../eval) + target_link_libraries (autotune_test PRIVATE + Catch2::Catch2WithMain + vsag_autotune + vsag + vsag_eval_common + ${HDF5_CPP_STATIC_LIBRARY} + ${HDF5_C_STATIC_LIBRARY} + z + ${CMAKE_DL_LIBS}) + add_dependencies (autotune_test hdf5 Catch2) +endif () diff --git a/tools/autotune/README.md b/tools/autotune/README.md new file mode 100644 index 0000000000..8eabdcf64b --- /dev/null +++ b/tools/autotune/README.md @@ -0,0 +1,25 @@ +# VSAG AutoTune Tool + +The canonical documentation for `autotune` lives in the website source tree. +The typed C++ API is experimental, build-tree-only, and is not installed with VSAG. + +- Local English docs: + [docs/docs/en/src/resources/autotune.md](../../docs/docs/en/src/resources/autotune.md) +- Local Chinese docs: + [docs/docs/zh/src/resources/autotune.md](../../docs/docs/zh/src/resources/autotune.md) +- V1 CLI JSON input/output contract: + [AutoTune V1 API](../../docs/docs/en/src/resources/autotune_api_v1.md) +- Website: +- Chinese website: +- Example request: + [examples/sift_hgraph_autotune_request.json](examples/sift_hgraph_autotune_request.json) +- Build-and-search tuning example: + [examples/cpp/326_feature_create_index_with_constraints.cpp][factory-example] +- Existing-index search tuning example: + [examples/cpp/327_feature_autotune_existing_index.cpp][existing-index-example] +- Existing-Pyramid path tuning example: + [examples/cpp/328_feature_autotune_existing_pyramid.cpp][pyramid-example] + +[factory-example]: ../../examples/cpp/326_feature_create_index_with_constraints.cpp +[existing-index-example]: ../../examples/cpp/327_feature_autotune_existing_index.cpp +[pyramid-example]: ../../examples/cpp/328_feature_autotune_existing_pyramid.cpp diff --git a/tools/autotune/README_zh.md b/tools/autotune/README_zh.md new file mode 100644 index 0000000000..b1b5f1732a --- /dev/null +++ b/tools/autotune/README_zh.md @@ -0,0 +1,25 @@ +# VSAG AutoTune 工具 + +`autotune` 的规范文档位于网站源码目录。 +typed C++ API 是实验性接口，只在构建目录中提供，不会随 VSAG 安装。 + +- 本地英文文档： + [docs/docs/en/src/resources/autotune.md](../../docs/docs/en/src/resources/autotune.md) +- 本地中文文档： + [docs/docs/zh/src/resources/autotune.md](../../docs/docs/zh/src/resources/autotune.md) +- V1 CLI JSON 输入输出契约： + [AutoTune V1 API](../../docs/docs/zh/src/resources/autotune_api_v1.md) +- 网站： +- 中文网站： +- 示例请求： + [examples/sift_hgraph_autotune_request.json](examples/sift_hgraph_autotune_request.json) +- 构建和查询联合调优示例： + [examples/cpp/326_feature_create_index_with_constraints.cpp][factory-example] +- 已有索引 search 调优示例： + [examples/cpp/327_feature_autotune_existing_index.cpp][existing-index-example] +- 已有 Pyramid path 调优示例： + [examples/cpp/328_feature_autotune_existing_pyramid.cpp][pyramid-example] + +[factory-example]: ../../examples/cpp/326_feature_create_index_with_constraints.cpp +[existing-index-example]: ../../examples/cpp/327_feature_autotune_existing_index.cpp +[pyramid-example]: ../../examples/cpp/328_feature_autotune_existing_pyramid.cpp diff --git a/tools/autotune/autotune.cpp b/tools/autotune/autotune.cpp new file mode 100644 index 0000000000..630d88ba57 --- /dev/null +++ b/tools/autotune/autotune.cpp @@ -0,0 +1,1071 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "autotune.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "autotune_internal.h" +#include "eval_dataset.h" +#include "vsag/constants.h" +#include "vsag/factory.h" + +namespace vsag::autotune::internal { + +namespace { + +double +elapsed(const std::chrono::steady_clock::time_point& start) { + return std::chrono::duration(std::chrono::steady_clock::now() - start).count(); +} + +void +require(bool condition, const std::string& message) { + if (!condition) { + throw std::invalid_argument(message); + } +} + +void +known_keys(const JsonType& object, + std::initializer_list keys, + const std::string& path) { + require(object.is_object(), path + " must be an object"); + std::set known(keys.begin(), keys.end()); + for (const auto& item : object.items()) { + require(known.find(item.key()) != known.end(), path + "." + item.key() + " is unsupported"); + } +} + +std::string +required_string(const JsonType& object, const std::string& key, const std::string& path) { + require(object.contains(key) && object[key].is_string(), path + "." + key + " is required"); + auto value = object[key].get(); + require(!value.empty(), path + "." + key + " must not be empty"); + return value; +} + +uint64_t +positive_integer(const JsonType& object, const std::string& key, const std::string& path) { + require(object.contains(key) && object[key].is_number_integer(), + path + "." + key + " must be a positive integer"); + if (object[key].is_number_unsigned()) { + const auto value = object[key].get(); + require(value > 0, path + "." + key + " must be positive"); + return value; + } + const auto value = object[key].get(); + require(value > 0, path + "." + key + " must be positive"); + return static_cast(value); +} + +std::string +normalize(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), [](unsigned char character) { + return static_cast(std::tolower(character)); + }); + return value; +} + +std::string +metric_name(Metric metric) { + switch (metric) { + case Metric::RECALL_AT_K: + return "recall_at_k"; + case Metric::LATENCY_AVG_MS: + return "latency_avg_ms"; + case Metric::LATENCY_P99_MS: + return "latency_p99_ms"; + case Metric::QPS: + return "qps"; + case Metric::INDEX_MEMORY_MB: + return "index_memory_mb"; + case Metric::INDEX_SIZE_MB: + return "index_size_mb"; + case Metric::BUILD_SECONDS: + return "build_seconds"; + case Metric::SEARCH_SECONDS: + return "search_seconds"; + case Metric::BUILD_AND_SEARCH_SECONDS: + return "build_and_search_seconds"; + case Metric::UNSPECIFIED: + break; + } + throw std::invalid_argument("AutoTune metric must be specified"); +} + +Metric +parse_metric(const std::string& metric) { + if (metric == "recall_at_k") { + return Metric::RECALL_AT_K; + } + if (metric == "latency_avg_ms") { + return Metric::LATENCY_AVG_MS; + } + if (metric == "latency_p99_ms") { + return Metric::LATENCY_P99_MS; + } + if (metric == "qps") { + return Metric::QPS; + } + if (metric == "index_memory_mb") { + return Metric::INDEX_MEMORY_MB; + } + if (metric == "index_size_mb") { + return Metric::INDEX_SIZE_MB; + } + if (metric == "build_seconds") { + return Metric::BUILD_SECONDS; + } + if (metric == "search_seconds") { + return Metric::SEARCH_SECONDS; + } + if (metric == "build_and_search_seconds") { + return Metric::BUILD_AND_SEARCH_SECONDS; + } + throw std::invalid_argument("unsupported metric: " + metric); +} + +bool +supported_metric(const std::string& metric) { + static const std::set metrics{"build_seconds", + "index_size_mb", + "index_memory_mb", + "recall_at_k", + "latency_avg_ms", + "latency_p99_ms", + "qps", + "search_seconds", + "build_and_search_seconds"}; + return metrics.find(metric) != metrics.end(); +} + +bool +higher_is_better(const std::string& metric) { + return metric == "recall_at_k" || metric == "qps"; +} + +bool +available_for_existing_index(const std::string& metric) { + return metric != "build_seconds" && metric != "build_and_search_seconds"; +} + +bool +same_path(const std::string& left, const std::string& right) { + if (left.empty() || right.empty()) { + return false; + } + + std::error_code equivalent_error; + if (std::filesystem::equivalent(left, right, equivalent_error)) { + return true; + } + + const auto normalized = [](const std::string& value) { + std::error_code error; + const auto canonical = std::filesystem::weakly_canonical(value, error); + return error ? std::filesystem::absolute(value).lexically_normal() : canonical; + }; + return normalized(left) == normalized(right); +} + +std::string +metric_type(const std::string& dataset_metric) { + if (dataset_metric == "euclidean") { + return "l2"; + } + if (dataset_metric == "ip") { + return "ip"; + } + if (dataset_metric == "angular") { + return "cosine"; + } + throw std::invalid_argument("unsupported dataset distance: " + dataset_metric); +} + +struct offline_dataset_owner { + eval::EvalDatasetPtr source; + DatasetPtr base; + DatasetPtr queries; + DatasetPtr ground_truth; +}; + +template +void +attach_offline_dataset(Request& request, const eval::EvalDatasetPtr& dataset) { + require(dataset->GetVectorType() == "dense_vectors", + "AutoTune V1 supports only dense vector datasets"); + require(dataset->GetTrainDataType() == vsag::DATATYPE_FLOAT32 && + dataset->GetTestDataType() == vsag::DATATYPE_FLOAT32, + "AutoTune V1 supports only float32 datasets"); + + auto owner = std::make_shared(); + owner->source = dataset; + const auto base_count = dataset->GetNumberOfBase(); + const auto query_count = dataset->GetNumberOfQuery(); + const auto ground_truth_k = static_cast(dataset->GetGroundTruthK()); + owner->base = Dataset::Make() + ->NumElements(base_count) + ->Dim(dataset->GetDim()) + ->Ids(dataset->GetTrainIds()) + ->Float32Vectors(static_cast(dataset->GetTrain())) + ->Owner(false); + owner->queries = Dataset::Make() + ->NumElements(query_count) + ->Dim(dataset->GetDim()) + ->Float32Vectors(static_cast(dataset->GetTest())) + ->Owner(false); + if (ground_truth_k > 0) { + owner->ground_truth = Dataset::Make() + ->NumElements(query_count) + ->Dim(ground_truth_k) + ->Ids(dataset->GetNeighbors(0)) + ->Distances(dataset->GetDistances(0)) + ->Owner(false); + } + + request.base = DatasetPtr(owner, owner->base.get()); + request.workload.queries = DatasetPtr(owner, owner->queries.get()); + if (owner->ground_truth != nullptr) { + request.workload.ground_truth = DatasetPtr(owner, owner->ground_truth.get()); + } + request.metric_type = metric_type(dataset->GetMetric()); +} + +void +check_file(const std::string& path, const std::string& field) { + std::error_code error; + require(std::filesystem::is_regular_file(path, error) && !error, + field + " must name a readable regular file: " + path); + std::ifstream input(path, std::ios::binary); + require(input.good(), field + " must name a readable regular file: " + path); +} + +void +merge_dataset_field(JsonType& create_params, + const std::string& name, + const JsonType& value, + const std::string& index_name) { + if (create_params.contains(name)) { + require(create_params[name] == value, + index_name + " create_params." + name + " must match the dataset"); + } + create_params[name] = value; +} + +MetricMap +read_metrics(const JsonType& value) { + MetricMap metrics; + if (!value.is_object()) { + return metrics; + } + for (const auto& item : value.items()) { + if (item.value().is_number()) { + metrics[item.key()] = item.value().get(); + } + } + return metrics; +} + +JsonType +constraint_evaluation(const RequestContext& request, const JsonType& trial) { + JsonType violations = JsonType::array(); + const auto metrics = read_metrics(trial.value("metrics", JsonType::object())); + for (const auto& [name, expected] : request.constraints) { + const auto actual = metrics.find(name); + const bool present = actual != metrics.end() && std::isfinite(actual->second); + const bool satisfied = present && (higher_is_better(name) ? actual->second >= expected + : actual->second <= expected); + if (!satisfied) { + violations.push_back( + {{"metric", name}, + {"comparison", higher_is_better(name) ? "at_least" : "at_most"}, + {"expected", expected}, + {"actual", present ? JsonType(actual->second) : JsonType(nullptr)}}); + } + } + return {{"satisfied", violations.empty()}, {"violations", std::move(violations)}}; +} + +double +objective_value(const RequestContext& request, const JsonType& trial) { + const auto metrics = read_metrics(trial.value("metrics", JsonType::object())); + const auto value = metrics.find(request.objective); + if (value == metrics.end() || !std::isfinite(value->second)) { + return higher_is_better(request.objective) ? -std::numeric_limits::infinity() + : std::numeric_limits::infinity(); + } + return value->second; +} + +bool +has_objective(const RequestContext& request, const JsonType& trial) { + const auto metrics = read_metrics(trial.value("metrics", JsonType::object())); + const auto value = metrics.find(request.objective); + return value != metrics.end() && std::isfinite(value->second); +} + +JsonType +recommendation(const RequestContext& request, const JsonType& trial) { + JsonType result{{"index_name", trial["index_name"]}, + {"search_params", trial["search_params"]}, + {"workload", {{"top_k", request.top_k}, {"concurrency", request.concurrency}}}, + {"metrics", trial["metrics"]}, + {"evidence", {{"trial_id", trial["trial_id"]}}}}; + if (trial.contains("create_params")) { + result["create_params"] = trial["create_params"]; + } + if (trial.contains("artifacts")) { + result["artifacts"] = trial["artifacts"]; + } + if (trial.contains("build_id")) { + result["evidence"]["build_id"] = trial["build_id"]; + } + return result; +} + +std::mutex& +run_mutex() { + static std::mutex mutex; + return mutex; +} + +std::string +new_run_name() { + static std::atomic serial{0}; + const auto now = std::chrono::steady_clock::now().time_since_epoch().count(); + return "run-" + std::to_string(now) + "-" + std::to_string(serial++); +} + +} // namespace + +namespace { + +JsonType +parse_parameters(const std::string& value, const std::string& path) { + try { + auto parsed = JsonType::parse(value); + require(parsed.is_object(), path + " must encode a JSON object"); + return parsed; + } catch (const nlohmann::json::exception& error) { + throw std::invalid_argument(path + " is invalid JSON: " + error.what()); + } +} + +RequestContext +make_context(const DatasetPtr& base, + const std::string& metric_type, + const Workload& workload, + const std::vector& constraints, + Metric objective, + const Config& config, + bool search_only) { + RequestContext request; + request.workspace_path = + config.workspace_path.empty() ? "/tmp/vsag_autotune" : config.workspace_path; + request.result_path = config.report_path; + request.keep_intermediate = config.keep_intermediate; + request.include_raw_eval = config.include_raw_evaluation; + request.max_trials = config.max_trials; + request.top_k = workload.top_k; + request.concurrency = workload.concurrency; + + require(base != nullptr, "request.base is required"); + require(base->GetIds() != nullptr, "request.base IDs are required"); + require(workload.queries != nullptr, "request.workload.queries is required"); + require(request.top_k > 0, "request.workload.top_k must be positive"); + require(request.top_k <= static_cast(std::numeric_limits::max()), + "request.workload.top_k is too large"); + require(request.concurrency > 0 && request.concurrency <= 200, + "request.workload.concurrency must be in [1, 200]"); + require(request.max_trials > 0 && request.max_trials <= 100000, + "request.config.max_trials must be in [1, 100000]"); + + request.objective = metric_name(objective); + request.enable_recall = objective == Metric::RECALL_AT_K; + require(!constraints.empty(), "request.constraints must not be empty"); + for (const auto& constraint : constraints) { + const auto name = metric_name(constraint.metric); + require(std::isfinite(constraint.value) && constraint.value >= 0.0, + "request.constraints." + name + " must be finite and non-negative"); + require(name != "recall_at_k" || constraint.value <= 1.0, + "request.constraints.recall_at_k must be in [0, 1]"); + require(!search_only || available_for_existing_index(name), + "request.constraints." + name + " is unavailable for search tuning"); + require(!search_only || name != "index_size_mb", + "request.constraints.index_size_mb is unavailable for an in-memory index"); + require(request.constraints.emplace(name, constraint.value).second, + "request.constraints contains duplicate metric: " + name); + request.enable_recall = request.enable_recall || constraint.metric == Metric::RECALL_AT_K; + } + require(!search_only || available_for_existing_index(request.objective), + "request.objective is unavailable for search tuning"); + require(!search_only || request.objective != "index_size_mb", + "request.objective index_size_mb is unavailable for an in-memory index"); + + const auto metric = normalize(metric_type); + require(metric == "l2" || metric == "ip" || metric == "cosine", + "request.metric_type must be l2, ip, or cosine"); + request.dataset = + eval::EvalDataset::FromDatasets(base, workload.queries, workload.ground_truth, metric); + request.base_count = static_cast(request.dataset->GetNumberOfBase()); + request.query_count = static_cast(request.dataset->GetNumberOfQuery()); + request.ground_truth_k = request.dataset->GetGroundTruthK(); + require(request.top_k <= request.base_count, + "request.workload.top_k exceeds the base dataset size"); + if (request.enable_recall) { + require(workload.ground_truth != nullptr, + "request.workload.ground_truth is required for recall_at_k"); + require(request.ground_truth_k >= request.top_k, + "request.workload.top_k exceeds ground truth k"); + } + + request.effective_request = { + {"version", 1}, + {"dataset", + {{"base_count", request.base_count}, + {"query_count", request.query_count}, + {"ground_truth_k", request.ground_truth_k}, + {"dim", request.dataset->GetDim()}, + {"dtype", vsag::DATATYPE_FLOAT32}, + {"metric_type", metric}}}, + {"workload", {{"top_k", request.top_k}, {"concurrency", request.concurrency}}}, + {"constraints", JsonType::object()}, + {"objective", {{"metric", request.objective}}}, + {"config", + {{"workspace_path", request.workspace_path}, + {"keep_intermediate", request.keep_intermediate}, + {"max_trials", request.max_trials}}}, + {"output", + {{"report_path", request.result_path}, + {"include_raw_evaluation", request.include_raw_eval}}}}; + for (const auto& [name, value] : request.constraints) { + request.effective_request["constraints"][name] = value; + } + return request; +} + +IndexInput +parse_index_space(const IndexSpace& value, uint64_t position, bool search_only) { + const auto path = "request.index_spaces[" + std::to_string(position) + "]"; + IndexInput index; + index.name = normalize(value.name); + require( + index.name == "hgraph" || index.name == "ivf" || (search_only && index.name == "pyramid"), + "unsupported index: " + index.name); + index.create_params = search_only ? JsonType::object() + : parse_parameters(value.create_parameter_space, + path + ".create_parameter_space"); + const auto search = + parse_parameters(value.search_parameter_space, path + ".search_parameter_space"); + for (const auto& item : search.items()) { + require(normalize(item.key()) == index.name, + path + ".search_parameter_space." + item.key() + " is unsupported"); + index.search_params[index.name] = item.value(); + } + return index; +} + +} // namespace + +IndexTuningRequest +ParseRequest(const IndexRequest& input) { + IndexTuningRequest request; + request.context = make_context(input.base, + input.metric_type, + input.workload, + input.constraints, + input.objective, + input.config, + false); + if (input.index_spaces.empty()) { + request.indexes = {{"hgraph"}, {"ivf"}}; + } else { + for (uint64_t i = 0; i < input.index_spaces.size(); ++i) { + request.indexes.emplace_back(parse_index_space(input.index_spaces[i], i, false)); + } + } + const auto dim = static_cast(request.context.dataset->GetDim()); + const auto metric = normalize(input.metric_type); + for (auto& index : request.indexes) { + merge_dataset_field(index.create_params, "dim", dim, index.name); + merge_dataset_field(index.create_params, "dtype", vsag::DATATYPE_FLOAT32, index.name); + merge_dataset_field(index.create_params, "metric_type", metric, index.name); + request.context.effective_request["index_spaces"].push_back( + {{"name", index.name}, + {"create_parameter_space", index.create_params}, + {"search_parameter_space", index.search_params}}); + } + return request; +} + +SearchTuningRequest +ParseRequest(const SearchRequest& input) { + require(input.index != nullptr, "request.index is required"); + SearchTuningRequest request; + request.context = make_context(input.base, + input.metric_type, + input.workload, + input.constraints, + input.objective, + input.config, + true); + IndexSpace space; + space.name = input.index_name; + space.search_parameter_space = input.parameter_space; + request.index_input = parse_index_space(space, 0, true); + request.index = input.index; + request.context.effective_request["index_name"] = request.index_input.name; + request.context.effective_request["parameter_space"] = request.index_input.search_params; + return request; +} + +ParsedRequest +ParseRequest(const JsonType& input) { + known_keys(input, + {"version", + "data_path", + "index_path", + "indexes", + "workload", + "constraints", + "objective", + "tuning_config", + "output"}, + "request"); + require(input.contains("version") && input["version"] == 1, "request.version must be 1"); + const auto data_path = required_string(input, "data_path", "request"); + check_file(data_path, "data_path"); + + IndexRequest typed; + attach_offline_dataset(typed, eval::EvalDataset::Load(data_path)); + if (input.contains("indexes")) { + require(input["indexes"].is_array() && !input["indexes"].empty(), + "request.indexes must be a non-empty array"); + for (uint64_t i = 0; i < input["indexes"].size(); ++i) { + const auto& value = input["indexes"][i]; + const auto path = "request.indexes[" + std::to_string(i) + "]"; + known_keys(value, {"name", "create_params", "search_params"}, path); + IndexSpace index; + index.name = required_string(value, "name", path); + if (value.contains("create_params")) { + require(value["create_params"].is_object(), + path + ".create_params must be an object"); + index.create_parameter_space = value["create_params"].dump(); + } + if (value.contains("search_params")) { + require(value["search_params"].is_object(), + path + ".search_params must be an object"); + index.search_parameter_space = value["search_params"].dump(); + } + typed.index_spaces.emplace_back(std::move(index)); + } + } + require(input.contains("workload"), "request.workload is required"); + known_keys(input["workload"], {"top_k", "concurrency"}, "request.workload"); + typed.workload.top_k = positive_integer(input["workload"], "top_k", "request.workload"); + if (input["workload"].contains("concurrency")) { + typed.workload.concurrency = + positive_integer(input["workload"], "concurrency", "request.workload"); + } + require(input.contains("constraints") && input["constraints"].is_object() && + !input["constraints"].empty(), + "request.constraints must be a non-empty object"); + for (const auto& item : input["constraints"].items()) { + require(supported_metric(item.key()), "unsupported metric: " + item.key()); + require(item.value().is_number(), + "request.constraints." + item.key() + " must be a number"); + typed.constraints.push_back({parse_metric(item.key()), item.value().get()}); + } + require(input.contains("objective"), "request.objective is required"); + known_keys(input["objective"], {"metric"}, "request.objective"); + typed.objective = + parse_metric(required_string(input["objective"], "metric", "request.objective")); + if (input.contains("tuning_config")) { + const auto& config = input["tuning_config"]; + known_keys( + config, {"workspace_path", "keep_intermediate", "max_trials"}, "request.tuning_config"); + if (config.contains("workspace_path")) { + typed.config.workspace_path = + required_string(config, "workspace_path", "request.tuning_config"); + } + if (config.contains("keep_intermediate")) { + require(config["keep_intermediate"].is_boolean(), + "request.tuning_config.keep_intermediate must be a boolean"); + typed.config.keep_intermediate = config["keep_intermediate"].get(); + } + if (config.contains("max_trials")) { + typed.config.max_trials = + positive_integer(config, "max_trials", "request.tuning_config"); + } + } + if (input.contains("output")) { + const auto& output = input["output"]; + known_keys(output, {"result_path", "include_raw_eval"}, "request.output"); + if (output.contains("result_path")) { + typed.config.report_path = required_string(output, "result_path", "request.output"); + require(!same_path(typed.config.report_path, data_path), + "request.output.result_path must not alias data_path"); + } + if (output.contains("include_raw_eval")) { + require(output["include_raw_eval"].is_boolean(), + "request.output.include_raw_eval must be a boolean"); + typed.config.include_raw_evaluation = output["include_raw_eval"].get(); + } + } + + if (!input.contains("index_path")) { + auto parsed = ParseRequest(typed); + parsed.context.effective_request["data_path"] = data_path; + return parsed; + } + + const auto index_path = required_string(input, "index_path", "request"); + check_file(index_path, "index_path"); + require(!same_path(typed.config.report_path, index_path), + "request.output.result_path must not alias index_path"); + require(typed.index_spaces.size() == 1, + "index_path requires exactly one indexes[] specification"); + auto& space = typed.index_spaces.front(); + auto create_params = + parse_parameters(space.create_parameter_space, "request.indexes[0].create_params"); + require(create_params.contains("index_param") && create_params["index_param"].is_object(), + "request.indexes[0].create_params.index_param is required"); + merge_dataset_field( + create_params, "dim", static_cast(typed.base->GetDim()), space.name); + merge_dataset_field(create_params, "dtype", vsag::DATATYPE_FLOAT32, space.name); + merge_dataset_field(create_params, "metric_type", normalize(typed.metric_type), space.name); + auto created = Factory::CreateIndex(normalize(space.name), create_params.dump()); + if (!created.has_value()) { + throw std::invalid_argument(created.error().message); + } + std::ifstream serialized(index_path, std::ios::binary); + auto loaded = created.value()->Deserialize(serialized); + if (!loaded.has_value()) { + throw std::invalid_argument(loaded.error().message); + } + + SearchRequest search; + search.index = created.value(); + search.index_name = space.name; + search.base = typed.base; + search.metric_type = typed.metric_type; + search.workload = typed.workload; + search.parameter_space = space.search_parameter_space; + search.constraints = typed.constraints; + search.objective = typed.objective; + search.config = typed.config; + auto parsed = ParseRequest(search); + parsed.context.effective_request["data_path"] = data_path; + parsed.context.effective_request["index_path"] = index_path; + parsed.context.effective_request["create_params"] = create_params; + return parsed; +} + +JsonType +SelectResult(const RequestContext& request, const Evaluation& evaluation) { + JsonType trials = JsonType::array(); + int64_t best = -1; + int64_t best_effort = -1; + uint64_t best_violations = std::numeric_limits::max(); + double best_violation_score = std::numeric_limits::infinity(); + bool has_successful_trial = false; + bool has_successful_objective = false; + + for (const auto& source : evaluation.trials) { + auto trial = source; + trial["constraint_evaluation"] = constraint_evaluation(request, trial); + trials.push_back(std::move(trial)); + const auto stored_index = static_cast(trials.size() - 1); + const auto& stored = trials.back(); + if (stored["status"] != "success") { + continue; + } + has_successful_trial = true; + if (!has_objective(request, stored)) { + continue; + } + has_successful_objective = true; + + const auto violation_count = + static_cast(stored["constraint_evaluation"]["violations"].size()); + if (violation_count == 0) { + if (best < 0 || + (higher_is_better(request.objective) + ? objective_value(request, stored) > objective_value(request, trials[best]) + : objective_value(request, stored) < objective_value(request, trials[best]))) { + best = stored_index; + } + continue; + } + + double score = 0.0; + for (const auto& violation : stored["constraint_evaluation"]["violations"]) { + if (violation["actual"].is_null()) { + score += 1.0; + } else { + score += std::abs(violation["actual"].get() - + violation["expected"].get()) / + std::max(violation["expected"].get(), 1e-12); + } + } + if (best_effort < 0 || violation_count < best_violations || + (violation_count == best_violations && score < best_violation_score)) { + best_effort = stored_index; + best_violations = violation_count; + best_violation_score = score; + } + } + + JsonType result{{"status", best >= 0 ? "success" : "no_feasible_candidate"}, + {"recommendation", nullptr}, + {"best_effort", nullptr}, + {"builds", evaluation.builds}, + {"trials", std::move(trials)}}; + if (best >= 0) { + result["recommendation"] = recommendation(request, result["trials"][best]); + } else if (best_effort >= 0) { + result["best_effort"] = recommendation(request, result["trials"][best_effort]); + result["best_effort"]["constraint_evaluation"] = + result["trials"][best_effort]["constraint_evaluation"]; + } else { + result["status"] = "failed"; + result["failure"] = + has_successful_trial && !has_successful_objective + ? Failure("selection", + "objective_metric_unavailable", + "objective metric is unavailable: " + request.objective) + : Failure("evaluation", "all_trials_failed", "all candidate evaluations failed"); + } + return result; +} + +JsonType +Failure(const std::string& stage, const std::string& code, const std::string& message) { + return {{"stage", stage}, {"code", code}, {"message", message}}; +} + +void +WriteJson(const std::string& path, const JsonType& value) { + const auto parent = std::filesystem::path(path).parent_path(); + if (!parent.empty()) { + std::filesystem::create_directories(parent); + } + std::ofstream output(path); + if (!output.good()) { + throw std::runtime_error("failed to open report path: " + path); + } + output << value.dump(2) << std::endl; + if (!output.good()) { + throw std::runtime_error("failed to write report: " + path); + } +} + +} // namespace vsag::autotune::internal + +namespace vsag::autotune { + +namespace { + +template +JsonType +run_tuning_locked(Parser parser, + bool cleanup_run_on_failure, + std::chrono::steady_clock::time_point start) { + std::string stage = "validation"; + std::string run_path; + bool keep_intermediate = false; + const auto failure = [&](const std::string& message) { + if ((cleanup_run_on_failure || !keep_intermediate) && !run_path.empty()) { + std::error_code cleanup_error; + std::filesystem::remove_all(run_path, cleanup_error); + } + const char* const code = stage == "validation" || stage == "candidate_generation" + ? "invalid_request" + : "execution_failed"; + return JsonType{{"version", 1}, + {"status", "failed"}, + {"recommendation", nullptr}, + {"best_effort", nullptr}, + {"elapsed_seconds", internal::elapsed(start)}, + {"failure", internal::Failure(stage, code, message)}}; + }; + try { + auto request = parser(); + auto& context = request.context; + keep_intermediate = context.keep_intermediate; + stage = "candidate_generation"; + const auto candidates = internal::GenerateCandidates(request); + const auto run_name = internal::new_run_name(); + stage = "evaluation"; + internal::Evaluation evaluation; + if constexpr (std::is_same_v) { + run_path = context.workspace_path + "/runs/" + run_name; + std::filesystem::create_directories(run_path + "/artifacts"); + evaluation = internal::EvaluateCandidates(request, candidates, run_path); + } else { + evaluation = internal::EvaluateCandidates(request, candidates); + } + auto report = internal::SelectResult(context, evaluation); + report["version"] = 1; + report["request"] = context.effective_request; + report["elapsed_seconds"] = internal::elapsed(start); + report["report_path"] = context.result_path.empty() + ? context.workspace_path + "/" + run_name + ".json" + : context.result_path; + stage = "report"; + internal::WriteJson(report["report_path"].template get(), report); + if (!context.keep_intermediate && !run_path.empty()) { + std::error_code error; + std::filesystem::remove_all(run_path, error); + } + return report; + } catch (const std::exception& error) { + return failure(error.what()); + } catch (...) { + return failure("unknown AutoTune error"); + } +} + +template +JsonType +run_tuning(Parser parser, + bool cleanup_run_on_failure = false, + std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now()) { + std::lock_guard lock(internal::run_mutex()); + return run_tuning_locked(std::move(parser), cleanup_run_on_failure, start); +} + +Error +report_error(const JsonType& report) { + const auto status = report.value("status", std::string("failed")); + auto type = + status == "no_feasible_candidate" ? ErrorType::INVALID_ARGUMENT : ErrorType::INTERNAL_ERROR; + auto message = status == "no_feasible_candidate" + ? std::string("no candidate satisfies all constraints") + : std::string("AutoTune failed"); + if (report.contains("failure") && report["failure"].is_object()) { + const auto& failure = report["failure"]; + if (failure.value("stage", std::string()) == "validation" || + failure.value("stage", std::string()) == "candidate_generation") { + type = ErrorType::INVALID_ARGUMENT; + } + message = failure.value("message", message); + } + if (report.contains("report_path") && report["report_path"].is_string()) { + message += " (report: " + report["report_path"].get() + ")"; + } + return {type, std::move(message)}; +} + +void +update_artifact(JsonType& value, + bool keep_all, + const std::optional& selected, + std::unordered_set& removed) { + if (!value.is_object() || !value.contains("artifacts") || !value["artifacts"].is_object() || + !value["artifacts"].contains("index_path") || + !value["artifacts"]["index_path"].is_string()) { + return; + } + + auto& artifacts = value["artifacts"]; + const auto path = artifacts["index_path"].get(); + std::error_code exists_error; + const auto exists = std::filesystem::is_regular_file(path, exists_error) && !exists_error; + const auto should_retain = keep_all || (selected.has_value() && path == *selected); + if (exists && !should_retain && removed.emplace(path).second) { + std::error_code remove_error; + if (std::filesystem::remove(path, remove_error)) { + std::filesystem::remove(std::filesystem::path(path).parent_path(), remove_error); + std::filesystem::remove(std::filesystem::path(path).parent_path().parent_path(), + remove_error); + } + } + std::error_code retained_error; + artifacts["retained"] = + std::filesystem::is_regular_file(path, retained_error) && !retained_error; +} + +void +finalize_index_report(JsonType& report, + const Config& config, + const std::optional& selected) { + if (report.contains("request") && report["request"].is_object() && + report["request"].contains("config") && report["request"]["config"].is_object()) { + report["request"]["config"]["keep_intermediate"] = config.keep_intermediate; + } + + std::unordered_set removed; + if (report.contains("builds") && report["builds"].is_array()) { + for (auto& build : report["builds"]) { + update_artifact(build, config.keep_intermediate, selected, removed); + } + } + if (report.contains("trials") && report["trials"].is_array()) { + for (auto& trial : report["trials"]) { + update_artifact(trial, config.keep_intermediate, selected, removed); + } + } + for (const auto* key : {"recommendation", "best_effort"}) { + if (report.contains(key)) { + update_artifact(report[key], config.keep_intermediate, selected, removed); + } + } + + if (report.contains("report_path") && report["report_path"].is_string()) { + internal::WriteJson(report["report_path"].get(), report); + } +} + +void +finalize_index_report_noexcept(JsonType& report, const Config& config) noexcept { + try { + finalize_index_report(report, config, std::nullopt); + } catch (...) { + } +} + +JsonType +validation_failure(const std::chrono::steady_clock::time_point& start, const std::string& message) { + return {{"version", 1}, + {"status", "failed"}, + {"recommendation", nullptr}, + {"best_effort", nullptr}, + {"elapsed_seconds", internal::elapsed(start)}, + {"failure", internal::Failure("validation", "invalid_request", message)}}; +} + +} // namespace + +tl::expected +TuneIndex(const IndexRequest& request) { + auto retained_request = request; + retained_request.config.keep_intermediate = true; + auto report = + run_tuning([&retained_request]() { return internal::ParseRequest(retained_request); }, + !request.config.keep_intermediate); + if (report.value("status", std::string("failed")) != "success") { + try { + finalize_index_report(report, request.config, std::nullopt); + } catch (const std::exception& error) { + finalize_index_report_noexcept(report, request.config); + return tl::unexpected(Error(ErrorType::INTERNAL_ERROR, error.what())); + } + return tl::unexpected(report_error(report)); + } + try { + const auto& recommendation = report.at("recommendation"); + const auto index_name = recommendation.at("index_name").get(); + const auto create_parameters = recommendation.at("create_params").dump(); + const auto search_parameters = recommendation.at("search_params").dump(); + const auto artifact_path = + recommendation.at("artifacts").at("index_path").get(); + auto created = Factory::CreateIndex(index_name, create_parameters); + if (!created.has_value()) { + finalize_index_report(report, request.config, std::nullopt); + return tl::unexpected(created.error()); + } + std::ifstream input(artifact_path, std::ios::binary); + if (!input.good()) { + finalize_index_report(report, request.config, std::nullopt); + return tl::unexpected( + Error(ErrorType::MISSING_FILE, "failed to open index artifact: " + artifact_path)); + } + auto loaded = created.value()->Deserialize(input); + if (!loaded.has_value()) { + finalize_index_report(report, request.config, std::nullopt); + return tl::unexpected(loaded.error()); + } + finalize_index_report(report, request.config, artifact_path); + return IndexResult{created.value(), + index_name, + create_parameters, + search_parameters, + recommendation.at("metrics").dump(), + artifact_path, + report.at("report_path").get(), + report.dump()}; + } catch (const std::exception& error) { + finalize_index_report_noexcept(report, request.config); + return tl::unexpected(Error(ErrorType::INTERNAL_ERROR, error.what())); + } +} + +tl::expected +TuneSearch(const SearchRequest& request) { + auto report = run_tuning([&request]() { return internal::ParseRequest(request); }); + if (report.value("status", std::string("failed")) != "success") { + return tl::unexpected(report_error(report)); + } + const auto& recommendation = report["recommendation"]; + return SearchResult{recommendation["search_params"].dump(), + recommendation["metrics"].dump(), + report["report_path"].get(), + report.dump()}; +} + +JsonType +RunAutoTune(const JsonType& request) { + const auto start = std::chrono::steady_clock::now(); + try { + std::lock_guard lock(internal::run_mutex()); + return std::visit( + [start](auto parsed) { + return run_tuning_locked( + [parsed = std::move(parsed)]() { return parsed; }, false, start); + }, + internal::ParseRequest(request)); + } catch (const H5::Exception& error) { + return validation_failure(start, + "failed to load evaluation dataset: " + error.getDetailMsg()); + } catch (const std::exception& error) { + return validation_failure(start, error.what()); + } catch (...) { + return validation_failure(start, "unknown request validation error"); + } +} + +std::string +FormatResultSummaryForCli(const JsonType& report) { + nlohmann::ordered_json result; + for (const auto* key : {"recommendation", + "best_effort", + "failure", + "status", + "elapsed_seconds", + "report_path", + "version"}) { + if (report.contains(key) && !report[key].is_null()) { + result[key] = report[key]; + } + } + return result.dump(2); +} + +} // namespace vsag::autotune diff --git a/tools/autotune/autotune.h b/tools/autotune/autotune.h new file mode 100644 index 0000000000..8157bf0456 --- /dev/null +++ b/tools/autotune/autotune.h @@ -0,0 +1,174 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#include "nlohmann/json.hpp" +#include "vsag/dataset.h" +#include "vsag/errors.h" +#include "vsag/expected.hpp" +#include "vsag/index.h" + +namespace vsag::autotune { + +using JsonType = nlohmann::json; + +/// Metrics exposed by the experimental AutoTune evaluator. +enum class Metric { + UNSPECIFIED = 0, + RECALL_AT_K, + LATENCY_AVG_MS, + LATENCY_P99_MS, + QPS, + INDEX_MEMORY_MB, + INDEX_SIZE_MB, + BUILD_SECONDS, + SEARCH_SECONDS, + BUILD_AND_SEARCH_SECONDS, +}; + +/// An inclusive upper or lower bound, according to the metric's direction. +struct Constraint { + Metric metric{Metric::UNSPECIFIED}; + /// Minimum for recall/QPS and maximum for all other metrics. + double value{0.0}; +}; + +/// Query workload evaluated for every search candidate. +struct Workload { + /// Query vectors. Buffers referenced by a non-owning Dataset must outlive the tuning call. + DatasetPtr queries; + /// Ground truth required when recall is a constraint or objective. + DatasetPtr ground_truth; + /// Number of neighbors requested from every query. + uint64_t top_k{0}; + /// Number of evaluator search threads. + uint64_t concurrency{1}; +}; + +/// Execution and report options shared by index and search tuning. +struct Config { + /// Directory for generated reports and index artifacts. + std::string workspace_path{"/tmp/vsag_autotune"}; + /// Optional report destination; an empty value generates a path under workspace_path. + std::string report_path; + /// Maximum number of concrete search trials planned for the request. + uint64_t max_trials{1000}; + /// Keep every generated index artifact instead of only the selected artifact. + bool keep_intermediate{false}; + bool include_raw_evaluation{false}; +}; + +/// Candidate space for one concrete index type. +struct IndexSpace { + /// Concrete index type, currently hgraph or ivf. + std::string name; + /// JSON object whose scalar, array, and range leaves define build candidates. + std::string create_parameter_space{"{}"}; + /// JSON object whose scalar, array, and range leaves define search candidates. + std::string search_parameter_space{"{}"}; +}; + +/// Input for jointly tuning index construction and search. +struct IndexRequest { + /// Base vectors and IDs. Buffers referenced by a non-owning Dataset must outlive TuneIndex. + DatasetPtr base; + /// l2, ip, or cosine. + std::string metric_type; + Workload workload; + std::vector index_spaces; + std::vector constraints; + Metric objective{Metric::UNSPECIFIED}; + Config config; +}; + +/// Input for tuning search parameters on an already built index. +struct SearchRequest { + /// Existing index reused in place; TuneSearch neither rebuilds nor takes exclusive ownership. + IndexPtr index; + /// Concrete type of index, including pyramid for search-only tuning. + std::string index_name; + /// Base vectors and IDs used to evaluate recall and validate ground truth. + DatasetPtr base; + /// l2, ip, or cosine; it must match the existing index and datasets. + std::string metric_type; + Workload workload; + /// JSON search candidate space; missing supported fields receive built-in proposals. + std::string parameter_space{"{}"}; + std::vector constraints; + Metric objective{Metric::UNSPECIFIED}; + Config config; +}; + +/// Successful index-tuning result. +struct IndexResult { + /// Loaded selected index, ready for queries. + IndexPtr index; + std::string index_name; + /// Concrete JSON parameters required to recreate and deserialize the selected index. + std::string create_parameters; + /// Concrete JSON parameters recommended for queries. + std::string search_parameters; + /// Validated metric values as a JSON object. + std::string metrics; + /// Selected serialized artifact; the caller removes it when it is no longer needed. + std::string artifact_path; + std::string report_path; + /// Complete report as JSON. + std::string report; +}; + +/// Successful search-only tuning result. +struct SearchResult { + /// Concrete JSON parameters recommended for the existing index. + std::string parameters; + /// Validated metric values as a JSON object. + std::string metrics; + std::string report_path; + /// Complete report as JSON. + std::string report; +}; + +/** + * Experimental, synchronous build-tree tool API. It is not installed as part of the VSAG SDK. + * + * Returns INVALID_ARGUMENT for invalid requests or no feasible candidate, and an execution error + * for evaluation, artifact, or report failures. On success, only the selected artifact is retained + * unless Config::keep_intermediate is true. + */ +tl::expected +TuneIndex(const IndexRequest& request); + +/** + * Synchronously evaluates search candidates against SearchRequest::index without rebuilding it. + * + * Returns INVALID_ARGUMENT for invalid requests or no feasible candidate, and an execution error + * for evaluation or report failures. + */ +tl::expected +TuneSearch(const SearchRequest& request); + +/// Offline JSON adapter used by the CLI. Validation failures are returned as structured JSON. +JsonType +RunAutoTune(const JsonType& request); + +/// Returns the compact, recommendation-first JSON shown by the CLI. +std::string +FormatResultSummaryForCli(const JsonType& report); + +} // namespace vsag::autotune diff --git a/tools/autotune/autotune_candidate.cpp b/tools/autotune/autotune_candidate.cpp new file mode 100644 index 0000000000..cdc1e3c4f7 --- /dev/null +++ b/tools/autotune/autotune_candidate.cpp @@ -0,0 +1,424 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include + +#include "autotune_internal.h" + +namespace vsag::autotune::internal { + +namespace { + +using Emit = std::function; + +void +expand(const JsonType& value, const Emit& emit); + +void +expand_object(const JsonType& object, + const JsonType::const_iterator& field, + JsonType partial, + const Emit& emit) { + if (field == object.end()) { + emit(partial); + return; + } + + const auto& key = field.key(); + const auto next = std::next(field); + expand(field.value(), [&](const JsonType& expanded) { + auto next_partial = partial; + next_partial[key] = expanded; + expand_object(object, next, std::move(next_partial), emit); + }); +} + +void +expand_range(const JsonType& range, const Emit& emit) { + if (!range.is_object() || range.size() != 3 || !range.contains("start") || + !range.contains("stop") || !range.contains("step")) { + throw std::invalid_argument("$range requires start, stop and step"); + } + if (!range["start"].is_number() || !range["stop"].is_number() || !range["step"].is_number()) { + throw std::invalid_argument("$range start, stop and step must be numbers"); + } + + if (range["start"].is_number_integer() && range["stop"].is_number_integer() && + range["step"].is_number_integer()) { + const auto start = range["start"].get(); + const auto stop = range["stop"].get(); + const auto step = range["step"].get(); + if (step == 0 || (start < stop && step < 0) || (start > stop && step > 0)) { + throw std::invalid_argument("$range step does not reach stop"); + } + auto current = start; + for (;;) { + emit(current); + if (current == stop || + (step > 0 && current > std::numeric_limits::max() - step) || + (step < 0 && current < std::numeric_limits::min() - step)) { + break; + } + const auto next = current + step; + if ((step > 0 && next > stop) || (step < 0 && next < stop)) { + break; + } + current = next; + } + return; + } + + const auto start = range["start"].get(); + const auto stop = range["stop"].get(); + const auto step = range["step"].get(); + if (!std::isfinite(start) || !std::isfinite(stop) || !std::isfinite(step) || step == 0.0 || + (start < stop && step < 0.0) || (start > stop && step > 0.0)) { + throw std::invalid_argument("$range step does not reach stop"); + } + const auto tolerance = std::numeric_limits::epsilon() * + std::max({1.0, std::abs(start), std::abs(stop)}) * 8.0; + double previous = 0.0; + bool emitted = false; + for (uint64_t i = 0;; ++i) { + auto current = start + static_cast(i) * step; + if (!std::isfinite(current) || + (step > 0.0 && current > stop && current - stop > tolerance) || + (step < 0.0 && current < stop && stop - current > tolerance)) { + return; + } + if (std::abs(current - stop) <= tolerance) { + current = stop; + } + if (emitted && current == previous) { + throw std::invalid_argument("$range step is too small to advance"); + } + emit(current); + if (current == stop) { + return; + } + previous = current; + emitted = true; + } +} + +void +expand(const JsonType& value, const Emit& emit) { + if (value.is_array()) { + if (value.empty()) { + throw std::invalid_argument("candidate array must not be empty"); + } + std::set seen; + for (const auto& item : value) { + expand(item, [&](const JsonType& expanded) { + if (seen.emplace(expanded.dump()).second) { + emit(expanded); + } + }); + } + return; + } + if (value.is_object()) { + if (value.contains("$range")) { + if (value.size() != 1) { + throw std::invalid_argument("$range cannot be mixed with other keys"); + } + expand_range(value["$range"], emit); + return; + } + expand_object(value, value.begin(), JsonType::object(), emit); + return; + } + emit(value); +} + +void +fill_hgraph_create(JsonType& create_params) { + if (!create_params.contains("index_param")) { + create_params["index_param"] = JsonType::object(); + } + auto& params = create_params["index_param"]; + if (!params.is_object()) { + throw std::invalid_argument("hgraph create_params.index_param must be an object"); + } + if (!params.contains("base_quantization_type")) { + params["base_quantization_type"] = JsonType::array({"fp32", "sq8_uniform"}); + } + if (!params.contains("max_degree")) { + params["max_degree"] = JsonType::array({16, 32}); + } + if (!params.contains("ef_construction")) { + params["ef_construction"] = JsonType::array({100, 200}); + } +} + +void +fill_ivf_create(JsonType& create_params, uint64_t base_count) { + if (!create_params.contains("index_param")) { + create_params["index_param"] = JsonType::object(); + } + auto& params = create_params["index_param"]; + if (!params.is_object()) { + throw std::invalid_argument("ivf create_params.index_param must be an object"); + } + if (!params.contains("base_quantization_type")) { + params["base_quantization_type"] = JsonType::array({"fp32", "sq8_uniform"}); + } + if (!params.contains("buckets_count")) { + const auto first = std::min(1024, base_count); + const auto second = std::min(2048, base_count); + params["buckets_count"] = + first == second ? JsonType(first) : JsonType::array({first, second}); + } +} + +void +fill_hgraph_search(JsonType& search_params, uint64_t top_k) { + if (!search_params.contains("hgraph")) { + search_params["hgraph"] = JsonType::object(); + } + auto& params = search_params["hgraph"]; + if (!params.is_object()) { + throw std::invalid_argument("hgraph search_params.hgraph must be an object"); + } + if (!params.contains("ef_search")) { + std::set values{std::max(40, top_k), + std::max(80, top_k * 2), + std::max(120, top_k * 4)}; + params["ef_search"] = values; + } +} + +void +fill_pyramid_search(JsonType& search_params, uint64_t top_k) { + if (!search_params.contains("pyramid")) { + search_params["pyramid"] = JsonType::object(); + } + auto& params = search_params["pyramid"]; + if (!params.is_object()) { + throw std::invalid_argument("pyramid search_params.pyramid must be an object"); + } + if (!params.contains("ef_search")) { + std::set values{std::max(40, top_k), + std::max(80, top_k * 2), + std::max(120, top_k * 4)}; + params["ef_search"] = values; + } +} + +void +fill_ivf_search(JsonType& search_params, const JsonType& create_params) { + if (!search_params.contains("ivf")) { + search_params["ivf"] = JsonType::object(); + } + auto& params = search_params["ivf"]; + if (!params.is_object()) { + throw std::invalid_argument("ivf search_params.ivf must be an object"); + } + if (params.contains("scan_buckets_count")) { + return; + } + + std::set values; + if (!create_params.contains("index_param") || + !create_params["index_param"].contains("buckets_count")) { + params["scan_buckets_count"] = std::set{1, 4, 16, 64}; + return; + } + + const auto buckets = create_params["index_param"]["buckets_count"].get(); + if (buckets <= 16) { + values = { + 1, std::max(1, buckets / 4), std::max(1, buckets / 2), buckets}; + } else { + values = {std::min(16, buckets), + std::min(32, buckets), + std::min(64, buckets)}; + } + params["scan_buckets_count"] = values; +} + +bool +supports_adaptive_ef_search_objective(const std::string& objective) { + return objective == "latency_avg_ms" || objective == "latency_p99_ms" || objective == "qps" || + objective == "search_seconds" || objective == "build_and_search_seconds"; +} + +int64_t +positive_int64(const JsonType& value, const std::string& path) { + if (!value.is_number_integer()) { + throw std::invalid_argument(path + " must be a positive integer"); + } + if (value.is_number_unsigned()) { + const auto number = value.get(); + if (number == 0 || number > static_cast(std::numeric_limits::max())) { + throw std::invalid_argument(path + " must be a positive integer"); + } + return static_cast(number); + } + const auto number = value.get(); + if (number <= 0) { + throw std::invalid_argument(path + " must be a positive integer"); + } + return number; +} + +std::optional +take_hgraph_ef_search_range(JsonType& search_params, const RequestContext& request) { + if (!search_params.contains("hgraph") || !search_params["hgraph"].is_object()) { + return std::nullopt; + } + auto& hgraph = search_params["hgraph"]; + if (!hgraph.contains("ef_search") || !hgraph["ef_search"].is_object() || + !hgraph["ef_search"].contains("$range")) { + return std::nullopt; + } + + auto& expression = hgraph["ef_search"]; + const auto& range = expression["$range"]; + if (range.is_object() && range.contains("step")) { + return std::nullopt; + } + if (expression.size() != 1 || !range.is_object() || range.size() != 2 || + !range.contains("start") || !range.contains("stop")) { + throw std::invalid_argument( + "hgraph ef_search $range without step requires exactly start and stop"); + } + if (request.constraints.find("recall_at_k") == request.constraints.end()) { + throw std::invalid_argument( + "hgraph ef_search $range without step requires a recall_at_k constraint"); + } + if (!supports_adaptive_ef_search_objective(request.objective)) { + throw std::invalid_argument( + "hgraph ef_search $range without step requires a query-cost objective"); + } + if (hgraph.contains("timeout_ms") || hgraph.contains("hops_limit")) { + throw std::invalid_argument( + "hgraph ef_search $range without step does not support timeout_ms or hops_limit"); + } + + HGraphEfSearchRange result{positive_int64(range["start"], "hgraph ef_search $range.start"), + positive_int64(range["stop"], "hgraph ef_search $range.stop")}; + if (result.start > result.stop) { + throw std::invalid_argument("hgraph ef_search $range.start must not exceed stop"); + } + hgraph.erase("ef_search"); + return result; +} + +uint64_t +maximum_trial_count(const std::optional& range) { + if (!range.has_value() || range->start == range->stop) { + return 1; + } + + uint64_t probes = 1; + uint64_t maximum = 1; + auto low = range->start; + while (low < range->stop) { + const auto high = low > range->stop / 2 ? range->stop : low * 2; + ++probes; + + uint64_t binary_trials = 0; + auto width = static_cast(high - low); + while (width > 1) { + ++binary_trials; + width = (width + 1) / 2; + } + maximum = std::max(maximum, probes + binary_trials); + low = high; + } + return maximum; +} + +std::vector +generate_candidates(const RequestContext& context, + const std::vector& indexes, + bool generate_create_candidates) { + std::vector candidates; + std::set seen; + uint64_t maximum_trials = 0; + for (const auto& index : indexes) { + auto create_space = index.create_params; + if (generate_create_candidates) { + if (index.name == "hgraph") { + fill_hgraph_create(create_space); + } else if (index.name == "ivf") { + fill_ivf_create(create_space, context.base_count); + } + } + + const auto expand_search = [&](const JsonType& create_params) { + auto search_space = index.search_params; + if (index.name == "hgraph") { + fill_hgraph_search(search_space, context.top_k); + } else if (index.name == "pyramid") { + fill_pyramid_search(search_space, context.top_k); + } else { + fill_ivf_search(search_space, create_params); + } + const auto ef_search_range = index.name == "hgraph" + ? take_hgraph_ef_search_range(search_space, context) + : std::nullopt; + expand(search_space, [&](const JsonType& search_params) { + Candidate candidate{index.name, create_params, search_params, ef_search_range}; + JsonType identity{{"index_name", candidate.index_name}, + {"create_params", candidate.create_params}, + {"search_params", candidate.search_params}}; + if (ef_search_range.has_value()) { + identity["ef_search_range"] = {{"start", ef_search_range->start}, + {"stop", ef_search_range->stop}}; + } + const auto key = identity.dump(); + if (!seen.emplace(key).second) { + return; + } + const auto trial_count = maximum_trial_count(ef_search_range); + if (trial_count > context.max_trials - maximum_trials) { + throw std::invalid_argument( + "planned trial count exceeds tuning_config.max_trials"); + } + maximum_trials += trial_count; + candidates.emplace_back(std::move(candidate)); + }); + }; + if (!generate_create_candidates) { + expand_search(create_space); + } else { + expand(create_space, expand_search); + } + } + if (candidates.empty()) { + throw std::invalid_argument("candidate generation produced no trials"); + } + return candidates; +} + +} // namespace + +std::vector +GenerateCandidates(const IndexTuningRequest& request) { + return generate_candidates(request.context, request.indexes, true); +} + +std::vector +GenerateCandidates(const SearchTuningRequest& request) { + return generate_candidates(request.context, {request.index_input}, false); +} + +} // namespace vsag::autotune::internal diff --git a/tools/autotune/autotune_evaluation.cpp b/tools/autotune/autotune_evaluation.cpp new file mode 100644 index 0000000000..1e9d8e1725 --- /dev/null +++ b/tools/autotune/autotune_evaluation.cpp @@ -0,0 +1,374 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "autotune_internal.h" +#include "eval_config.h" +#include "evaluator.h" +#include "vsag/factory.h" + +namespace vsag::autotune::internal { + +namespace { + +constexpr double BYTES_PER_MEBIBYTE = 1024.0 * 1024.0; + +class ScopedOpenMpThreads { +public: + ScopedOpenMpThreads() : original_(omp_get_max_threads()) { + } + + ~ScopedOpenMpThreads() { + omp_set_num_threads(original_); + } + +private: + int original_; +}; + +double +elapsed(const std::chrono::steady_clock::time_point& start) { + return std::chrono::duration(std::chrono::steady_clock::now() - start).count(); +} + +std::optional +number(const JsonType& value, const std::string& key) { + if (!value.is_object() || !value.contains(key) || !value[key].is_number()) { + return std::nullopt; + } + return value[key].get(); +} + +void +set_metric(MetricMap& metrics, const std::string& name, const std::optional& value) { + if (value.has_value()) { + metrics[name] = *value; + } +} + +MetricMap +build_metrics(const JsonType& raw, const std::string& index_path) { + MetricMap metrics; + set_metric(metrics, "build_seconds", number(raw, "duration(s)")); + const auto memory = number(raw, "index_memory(B)"); + if (memory.has_value() && *memory > 0.0) { + metrics["index_memory_mb"] = *memory / BYTES_PER_MEBIBYTE; + } + std::error_code error; + const auto bytes = std::filesystem::file_size(index_path, error); + if (!error) { + metrics["index_size_mb"] = static_cast(bytes) / BYTES_PER_MEBIBYTE; + } + return metrics; +} + +MetricMap +search_metrics(const JsonType& raw, double seconds) { + MetricMap metrics; + set_metric(metrics, "recall_at_k", number(raw, "recall_avg")); + set_metric(metrics, "latency_avg_ms", number(raw, "latency_avg(ms)")); + set_metric(metrics, "qps", number(raw, "qps")); + if (raw.contains("latency_detail(ms)") && raw["latency_detail(ms)"].is_object()) { + set_metric(metrics, "latency_p99_ms", number(raw["latency_detail(ms)"], "p99")); + } + const auto memory = number(raw, "index_memory(B)"); + if (memory.has_value() && *memory > 0.0) { + metrics["index_memory_mb"] = *memory / BYTES_PER_MEBIBYTE; + } + metrics["search_seconds"] = seconds; + return metrics; +} + +JsonType +metrics_json(const MetricMap& metrics) { + JsonType result = JsonType::object(); + for (const auto& [name, value] : metrics) { + result[name] = value; + } + return result; +} + +eval::EvalConfig +build_config(const Candidate& candidate) { + eval::EvalConfig config; + config.index_name = candidate.index_name; + config.build_param = candidate.create_params.dump(); + config.enable_memory = false; + return config; +} + +eval::EvalConfig +search_config(const RequestContext& request, const Candidate& candidate) { + auto config = build_config(candidate); + config.search_param = candidate.search_params.dump(); + config.search_mode = "knn"; + config.top_k = static_cast(request.top_k); + config.search_query_count = request.query_count; + config.num_threads_searching = static_cast(request.concurrency); + config.enable_memory = false; + config.enable_recall = request.enable_recall; + config.enable_percent_recall = false; + return config; +} + +IndexPtr +create_index(const Candidate& candidate) { + auto created = Factory::CreateIndex(candidate.index_name, candidate.create_params.dump()); + if (!created.has_value()) { + throw std::runtime_error(created.error().message); + } + return created.value(); +} + +void +serialize_index(const IndexPtr& index, const std::string& path) { + const auto parent = std::filesystem::path(path).parent_path(); + std::filesystem::create_directories(parent); + std::ofstream output(path, std::ios::binary); + if (!output.good()) { + throw std::runtime_error("failed to open index artifact: " + path); + } + auto serialized = index->Serialize(output); + if (!serialized.has_value()) { + throw std::runtime_error(serialized.error().message); + } +} + +} // namespace + +void +EvaluateEfSearchRange(const HGraphEfSearchRange& range, + double recall_target, + const std::function(int64_t)>& evaluate) { + auto low = range.start; + const auto low_recall = evaluate(low); + if (!low_recall.has_value() || *low_recall >= recall_target || low == range.stop) { + return; + } + + auto high = low; + while (low < range.stop) { + high = low > range.stop / 2 ? range.stop : low * 2; + const auto high_recall = evaluate(high); + if (!high_recall.has_value()) { + return; + } + if (*high_recall >= recall_target) { + break; + } + if (high == range.stop) { + return; + } + low = high; + } + + while (high - low > 1) { + const auto middle = low + (high - low) / 2; + const auto middle_recall = evaluate(middle); + if (!middle_recall.has_value()) { + return; + } + if (*middle_recall >= recall_target) { + high = middle; + } else { + low = middle; + } + } +} + +Evaluation +EvaluateCandidates(const IndexTuningRequest& tuning_request, + const std::vector& candidates, + const std::string& run_path) { + const auto& request = tuning_request.context; + ScopedOpenMpThreads openmp_threads; + Evaluation evaluation; + std::map> groups; + for (uint64_t i = 0; i < candidates.size(); ++i) { + const auto key = candidates[i].index_name + "\n" + candidates[i].create_params.dump(); + groups[key].emplace_back(i); + } + + uint64_t build_number = 0; + uint64_t trial_number = 0; + for (const auto& [unused, indexes] : groups) { + (void)unused; + const auto& first = candidates[indexes.front()]; + const auto build_id = "build-" + std::to_string(build_number++); + const auto index_path = + (std::filesystem::path(run_path) / "artifacts" / (build_id + ".index")).string(); + JsonType build{{"build_id", build_id}, + {"index_name", first.index_name}, + {"create_params", first.create_params}, + {"status", "failed"}, + {"metrics", JsonType::object()}, + {"failure", nullptr}, + {"artifacts", + {{"index_path", index_path}, + {"source", "generated"}, + {"use_existing_index", false}, + {"retained", request.keep_intermediate}}}}; + + MetricMap shared_metrics; + IndexPtr index; + const auto build_start = std::chrono::steady_clock::now(); + try { + index = create_index(first); + auto raw = eval::EvaluateBuild(index, request.dataset, build_config(first)); + serialize_index(index, index_path); + shared_metrics = build_metrics(raw, index_path); + if (request.include_raw_eval) { + build["raw_eval_result"] = std::move(raw); + } + build["metrics"] = metrics_json(shared_metrics); + build["status"] = "success"; + } catch (const std::exception& error) { + build["failure"] = Failure("build", "build_evaluation_failed", error.what()); + std::error_code cleanup_error; + std::filesystem::remove(index_path, cleanup_error); + build["artifacts"]["retained"] = false; + } + build["elapsed_seconds"] = elapsed(build_start); + evaluation.builds.emplace_back(build); + + const auto evaluate = [&](const Candidate& candidate) -> std::optional { + const auto trial_id = "trial-" + std::to_string(trial_number++); + JsonType trial{{"trial_id", trial_id}, + {"build_id", build_id}, + {"index_name", candidate.index_name}, + {"create_params", candidate.create_params}, + {"search_params", candidate.search_params}, + {"status", "failed"}, + {"metrics", metrics_json(shared_metrics)}, + {"failure", nullptr}, + {"artifacts", build["artifacts"]}}; + std::optional recall; + const auto search_start = std::chrono::steady_clock::now(); + if (build["status"] != "success") { + trial["failure"] = + Failure("search", "build_failed", "search skipped because build failed"); + } else { + try { + const auto measured_start = std::chrono::steady_clock::now(); + auto raw = eval::EvaluateSearch( + index, request.dataset, search_config(request, candidate)); + auto metrics = search_metrics(raw, elapsed(measured_start)); + for (const auto& [name, value] : shared_metrics) { + metrics.emplace(name, value); + } + if (metrics.find("build_seconds") != metrics.end()) { + metrics["build_and_search_seconds"] = + metrics["build_seconds"] + metrics["search_seconds"]; + } + trial["metrics"] = metrics_json(metrics); + trial["status"] = "success"; + recall = number(trial["metrics"], "recall_at_k"); + if (request.include_raw_eval) { + trial["raw_eval_result"] = std::move(raw); + } + } catch (const std::exception& error) { + trial["failure"] = Failure("search", "search_evaluation_failed", error.what()); + } + } + trial["elapsed_seconds"] = elapsed(search_start); + evaluation.trials.emplace_back(std::move(trial)); + return recall; + }; + + for (const auto candidate_index : indexes) { + const auto& candidate = candidates[candidate_index]; + if (!candidate.ef_search_range.has_value()) { + evaluate(candidate); + continue; + } + + const auto recall_target = request.constraints.at("recall_at_k"); + const auto evaluate_ef_search = [&](int64_t ef_search) { + auto concrete = candidate; + concrete.search_params["hgraph"]["ef_search"] = ef_search; + return evaluate(concrete); + }; + EvaluateEfSearchRange(*candidate.ef_search_range, recall_target, evaluate_ef_search); + } + + if (!request.keep_intermediate) { + std::error_code error; + std::filesystem::remove(index_path, error); + } + } + return evaluation; +} + +Evaluation +EvaluateCandidates(const SearchTuningRequest& tuning_request, + const std::vector& candidates) { + ScopedOpenMpThreads openmp_threads; + const auto& request = tuning_request.context; + Evaluation evaluation; + uint64_t trial_number = 0; + + const auto evaluate = [&](const Candidate& candidate) -> std::optional { + JsonType trial{{"trial_id", "trial-" + std::to_string(trial_number++)}, + {"index_name", candidate.index_name}, + {"search_params", candidate.search_params}, + {"status", "failed"}, + {"metrics", JsonType::object()}, + {"failure", nullptr}}; + std::optional recall; + const auto start = std::chrono::steady_clock::now(); + try { + const auto measured_start = std::chrono::steady_clock::now(); + auto raw = eval::EvaluateSearch( + tuning_request.index, request.dataset, search_config(request, candidate)); + const auto metrics = search_metrics(raw, elapsed(measured_start)); + trial["metrics"] = metrics_json(metrics); + trial["status"] = "success"; + recall = number(trial["metrics"], "recall_at_k"); + if (request.include_raw_eval) { + trial["raw_eval_result"] = std::move(raw); + } + } catch (const std::exception& error) { + trial["failure"] = Failure("search", "search_evaluation_failed", error.what()); + } + trial["elapsed_seconds"] = elapsed(start); + evaluation.trials.emplace_back(std::move(trial)); + return recall; + }; + + for (const auto& candidate : candidates) { + if (!candidate.ef_search_range.has_value()) { + evaluate(candidate); + continue; + } + const auto recall_target = request.constraints.at("recall_at_k"); + const auto evaluate_ef_search = [&](int64_t ef_search) { + auto concrete = candidate; + concrete.search_params["hgraph"]["ef_search"] = ef_search; + return evaluate(concrete); + }; + EvaluateEfSearchRange(*candidate.ef_search_range, recall_target, evaluate_ef_search); + } + return evaluation; +} + +} // namespace vsag::autotune::internal diff --git a/tools/autotune/autotune_internal.h b/tools/autotune/autotune_internal.h new file mode 100644 index 0000000000..9355bdc504 --- /dev/null +++ b/tools/autotune/autotune_internal.h @@ -0,0 +1,129 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "autotune.h" + +namespace vsag::eval { +class EvalDataset; +} + +namespace vsag::autotune::internal { + +namespace eval = vsag::eval; + +using MetricMap = std::map; + +struct IndexInput { + std::string name; + JsonType create_params = JsonType::object(); + JsonType search_params = JsonType::object(); +}; + +struct RequestContext { + JsonType effective_request = JsonType::object(); + std::shared_ptr dataset; + std::string workspace_path{"/tmp/vsag_autotune"}; + std::string result_path; + std::string objective; + MetricMap constraints; + uint64_t top_k{0}; + uint64_t concurrency{1}; + uint64_t max_trials{1000}; + uint64_t base_count{0}; + uint64_t query_count{0}; + uint64_t ground_truth_k{0}; + bool enable_recall{false}; + bool keep_intermediate{false}; + bool include_raw_eval{false}; +}; + +struct IndexTuningRequest { + RequestContext context; + std::vector indexes; +}; + +struct SearchTuningRequest { + RequestContext context; + IndexPtr index; + IndexInput index_input; +}; + +using ParsedRequest = std::variant; + +struct HGraphEfSearchRange { + int64_t start; + int64_t stop; +}; + +struct Candidate { + std::string index_name; + JsonType create_params; + JsonType search_params; + std::optional ef_search_range; +}; + +struct Evaluation { + std::vector builds; + std::vector trials; +}; + +ParsedRequest +ParseRequest(const JsonType& input); + +IndexTuningRequest +ParseRequest(const IndexRequest& input); + +SearchTuningRequest +ParseRequest(const SearchRequest& input); + +std::vector +GenerateCandidates(const IndexTuningRequest& request); + +std::vector +GenerateCandidates(const SearchTuningRequest& request); + +void +EvaluateEfSearchRange(const HGraphEfSearchRange& range, + double recall_target, + const std::function(int64_t)>& evaluate); + +Evaluation +EvaluateCandidates(const IndexTuningRequest& request, + const std::vector& candidates, + const std::string& run_path); + +Evaluation +EvaluateCandidates(const SearchTuningRequest& request, const std::vector& candidates); + +JsonType +SelectResult(const RequestContext& request, const Evaluation& evaluation); + +JsonType +Failure(const std::string& stage, const std::string& code, const std::string& message); + +void +WriteJson(const std::string& path, const JsonType& value); + +} // namespace vsag::autotune::internal diff --git a/tools/autotune/autotune_test.cpp b/tools/autotune/autotune_test.cpp new file mode 100644 index 0000000000..f82ed2968e --- /dev/null +++ b/tools/autotune/autotune_test.cpp @@ -0,0 +1,1001 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "autotune.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "autotune_internal.h" +#include "eval_dataset.h" +#include "vsag/options.h" +#include "vsag/vsag.h" + +namespace { + +using vsag::autotune::JsonType; + +std::string +temp_path(const std::string& stem, const std::string& extension = "") { + static std::atomic serial{0}; + const auto now = std::chrono::steady_clock::now().time_since_epoch().count(); + return (std::filesystem::temp_directory_path() / + (stem + "-" + std::to_string(now) + "-" + std::to_string(serial++) + extension)) + .string(); +} + +class ScopedPath { +public: + explicit ScopedPath(std::string path) : path_(std::move(path)) { + } + + ~ScopedPath() { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + const std::string& + Get() const { + return path_; + } + +private: + std::string path_; +}; + +class ScopedBlockSizeLimit { +public: + explicit ScopedBlockSizeLimit(uint64_t value) + : original_(vsag::Options::Instance().block_size_limit()) { + vsag::Options::Instance().set_block_size_limit(value); + } + + ~ScopedBlockSizeLimit() { + vsag::Options::Instance().set_block_size_limit(original_); + } + +private: + uint64_t original_; +}; + +class ScopedOpenMpThreads { +public: + explicit ScopedOpenMpThreads(int value) : original_(omp_get_max_threads()) { + omp_set_num_threads(value); + } + + ~ScopedOpenMpThreads() { + omp_set_num_threads(original_); + } + +private: + int original_; +}; + +uint64_t +count_index_artifacts(const std::string& path) { + if (!std::filesystem::exists(path)) { + return 0; + } + uint64_t count = 0; + for (const auto& entry : std::filesystem::recursive_directory_iterator(path)) { + if (entry.is_regular_file() && entry.path().extension() == ".index") { + ++count; + } + } + return count; +} + +std::vector +load_json_reports(const std::string& path) { + std::vector reports; + if (!std::filesystem::exists(path)) { + return reports; + } + for (const auto& entry : std::filesystem::recursive_directory_iterator(path)) { + if (!entry.is_regular_file() || entry.path().extension() != ".json") { + continue; + } + std::ifstream input(entry.path()); + JsonType report; + input >> report; + reports.emplace_back(std::move(report)); + } + return reports; +} + +float +l2(const std::vector& train, + const std::vector& test, + int64_t base, + int64_t query, + int64_t dim) { + float distance = 0.0F; + for (int64_t i = 0; i < dim; ++i) { + const auto difference = train[base * dim + i] - test[query * dim + i]; + distance += difference * difference; + } + return std::sqrt(distance); +} + +void +write_dataset(const std::string& path) { + constexpr int64_t BASE_COUNT = 64; + constexpr int64_t QUERY_COUNT = 6; + constexpr int64_t DIM = 8; + constexpr int64_t GROUND_TRUTH_K = 6; + std::vector train(BASE_COUNT * DIM); + std::vector test(QUERY_COUNT * DIM); + for (int64_t i = 0; i < BASE_COUNT; ++i) { + for (int64_t j = 0; j < DIM; ++j) { + train[i * DIM + j] = static_cast((i * 17 + j * 13) % 101) / 101.0F; + } + } + for (int64_t i = 0; i < QUERY_COUNT; ++i) { + for (int64_t j = 0; j < DIM; ++j) { + test[i * DIM + j] = train[((i * 7) % BASE_COUNT) * DIM + j]; + } + } + + std::vector neighbors(QUERY_COUNT * GROUND_TRUTH_K); + std::vector distances(QUERY_COUNT * GROUND_TRUTH_K); + for (int64_t query = 0; query < QUERY_COUNT; ++query) { + std::vector> ranked; + for (int64_t base = 0; base < BASE_COUNT; ++base) { + ranked.emplace_back(l2(train, test, base, query, DIM), base); + } + std::sort(ranked.begin(), ranked.end()); + for (int64_t k = 0; k < GROUND_TRUTH_K; ++k) { + neighbors[query * GROUND_TRUTH_K + k] = ranked[k].second; + distances[query * GROUND_TRUTH_K + k] = ranked[k].first; + } + } + + H5::H5File file(path, H5F_ACC_TRUNC); + H5::StrType string_type(H5::PredType::C_S1, H5T_VARIABLE); + auto distance = file.createAttribute("distance", string_type, H5::DataSpace(H5S_SCALAR)); + std::string metric = "euclidean"; + distance.write(string_type, metric); + const auto write_matrix = [&](const std::string& name, + const auto* data, + const H5::DataType& type, + int64_t rows, + int64_t columns) { + hsize_t dimensions[2] = {static_cast(rows), static_cast(columns)}; + H5::DataSpace space(2, dimensions); + auto dataset = file.createDataSet(name, type, space); + dataset.write(data, type); + }; + write_matrix("/train", train.data(), H5::PredType::NATIVE_FLOAT, BASE_COUNT, DIM); + write_matrix("/test", test.data(), H5::PredType::NATIVE_FLOAT, QUERY_COUNT, DIM); + write_matrix( + "/neighbors", neighbors.data(), H5::PredType::NATIVE_INT64, QUERY_COUNT, GROUND_TRUTH_K); + write_matrix( + "/distances", distances.data(), H5::PredType::NATIVE_FLOAT, QUERY_COUNT, GROUND_TRUTH_K); +} + +JsonType +request(const std::string& dataset, const std::string& workspace) { + return {{"version", 1}, + {"data_path", dataset}, + {"indexes", + JsonType::array({{{"name", "hgraph"}, + {"create_params", + {{"index_param", + {{"base_quantization_type", "fp32"}, + {"max_degree", 8}, + {"ef_construction", 40}, + {"build_thread_count", 2}}}}}, + {"search_params", {{"hgraph", {{"ef_search", {8, 16}}}}}}}, + {{"name", "ivf"}, + {"create_params", + {{"index_param", + {{"base_quantization_type", "fp32"}, + {"buckets_count", 4}, + {"thread_count", 2}}}}}, + {"search_params", {{"ivf", {{"scan_buckets_count", {1, 4}}}}}}}})}, + {"workload", {{"top_k", 3}, {"concurrency", 2}}}, + {"constraints", {{"recall_at_k", 0.0}, {"build_seconds", 1000.0}}}, + {"objective", {{"metric", "latency_avg_ms"}}}, + {"tuning_config", + {{"workspace_path", workspace}, {"keep_intermediate", true}, {"max_trials", 4}}}}; +} + +struct MemoryFixture { + static constexpr int64_t BASE_COUNT = 64; + static constexpr int64_t QUERY_COUNT = 6; + static constexpr int64_t DIM = 8; + static constexpr int64_t GROUND_TRUTH_K = 6; + + MemoryFixture() + : base_ids(BASE_COUNT), + train(BASE_COUNT * DIM), + test(QUERY_COUNT * DIM), + neighbors(QUERY_COUNT * GROUND_TRUTH_K), + distances(QUERY_COUNT * GROUND_TRUTH_K) { + for (int64_t i = 0; i < BASE_COUNT; ++i) { + base_ids[i] = 1001 + i * 17; + for (int64_t j = 0; j < DIM; ++j) { + train[i * DIM + j] = static_cast((i * 17 + j * 13) % 101) / 101.0F; + } + } + for (int64_t i = 0; i < QUERY_COUNT; ++i) { + for (int64_t j = 0; j < DIM; ++j) { + test[i * DIM + j] = train[((i * 7) % BASE_COUNT) * DIM + j]; + } + } + for (int64_t query = 0; query < QUERY_COUNT; ++query) { + std::vector> ranked; + for (int64_t row = 0; row < BASE_COUNT; ++row) { + ranked.emplace_back(l2(train, test, row, query, DIM), row); + } + std::sort(ranked.begin(), ranked.end()); + for (int64_t k = 0; k < GROUND_TRUTH_K; ++k) { + neighbors[query * GROUND_TRUTH_K + k] = base_ids[ranked[k].second]; + distances[query * GROUND_TRUTH_K + k] = ranked[k].first; + } + } + base = vsag::Dataset::Make() + ->NumElements(BASE_COUNT) + ->Dim(DIM) + ->Ids(base_ids.data()) + ->Float32Vectors(train.data()) + ->Owner(false); + queries = vsag::Dataset::Make() + ->NumElements(QUERY_COUNT) + ->Dim(DIM) + ->Float32Vectors(test.data()) + ->Owner(false); + ground_truth = vsag::Dataset::Make() + ->NumElements(QUERY_COUNT) + ->Dim(GROUND_TRUTH_K) + ->Ids(neighbors.data()) + ->Distances(distances.data()) + ->Owner(false); + } + + vsag::autotune::IndexRequest + Request(const std::string& workspace) const { + vsag::autotune::IndexRequest result; + result.base = base; + result.metric_type = vsag::METRIC_L2; + result.workload = {queries, ground_truth, 3, 2}; + result.index_spaces = { + {"hgraph", + R"({"index_param":{"base_quantization_type":"fp32",)" + R"("max_degree":[8,12],"ef_construction":40,"build_thread_count":2}})", + R"({"hgraph":{"ef_search":16}})"}}; + result.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.0}, + {vsag::autotune::Metric::BUILD_SECONDS, 1000.0}}; + result.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + result.config.workspace_path = workspace; + result.config.max_trials = 2; + return result; + } + + std::vector base_ids; + std::vector train; + std::vector test; + std::vector neighbors; + std::vector distances; + vsag::DatasetPtr base; + vsag::DatasetPtr queries; + vsag::DatasetPtr ground_truth; +}; + +} // namespace + +TEST_CASE("AutoTune candidate rules only fill missing fields") { + vsag::autotune::internal::IndexTuningRequest request; + request.context.base_count = 10000; + request.context.top_k = 10; + request.context.max_trials = 10; + request.indexes = { + {"hgraph", + {{"dim", 8}, + {"dtype", "float32"}, + {"metric_type", "l2"}, + {"index_param", + {{"base_quantization_type", "fp32"}, {"max_degree", 24}, {"ef_construction", 80}}}}, + JsonType::object()}}; + + const auto candidates = vsag::autotune::internal::GenerateCandidates(request); + REQUIRE(candidates.size() == 3); + for (const auto& candidate : candidates) { + REQUIRE(candidate.create_params["index_param"]["max_degree"] == 24); + REQUIRE(candidate.create_params["index_param"]["ef_construction"] == 80); + } + + request.indexes[0].create_params = JsonType::object(); + request.context.max_trials = 24; + REQUIRE(vsag::autotune::internal::GenerateCandidates(request).size() == 24); + + request.indexes[0].create_params = candidates[0].create_params; + request.indexes[0].search_params = { + {"hgraph", {{"ef_search", {{"$range", {{"start", 0.1}, {"stop", 0.3}, {"step", 0.1}}}}}}}}; + const auto float_range = vsag::autotune::internal::GenerateCandidates(request); + REQUIRE(float_range.size() == 3); + REQUIRE(float_range.back().search_params["hgraph"]["ef_search"] == 0.3); + REQUIRE_FALSE(float_range.back().ef_search_range.has_value()); + + const auto minimum = std::numeric_limits::min(); + request.indexes[0].search_params = { + {"hgraph", + {{"ef_search", {{"$range", {{"start", minimum}, {"stop", minimum + 5}, {"step", 10}}}}}}}}; + const auto integer_range = vsag::autotune::internal::GenerateCandidates(request); + REQUIRE(integer_range.size() == 1); + REQUIRE(integer_range[0].search_params["hgraph"]["ef_search"] == minimum); +} + +TEST_CASE("AutoTune proposes conservative defaults for existing IVF indexes") { + vsag::autotune::internal::SearchTuningRequest request; + request.context.top_k = 10; + request.context.max_trials = 4; + request.index_input.name = "ivf"; + + const auto candidates = vsag::autotune::internal::GenerateCandidates(request); + REQUIRE(candidates.size() == 4); + const std::vector expected{1, 4, 16, 64}; + for (uint64_t i = 0; i < candidates.size(); ++i) { + REQUIRE(candidates[i].search_params["ivf"]["scan_buckets_count"] == expected[i]); + } +} + +TEST_CASE("AutoTune plans an adaptive HGraph ef_search range") { + vsag::autotune::internal::IndexTuningRequest request; + request.context.base_count = 10000; + request.context.top_k = 10; + request.context.objective = "latency_avg_ms"; + request.context.constraints = {{"recall_at_k", 0.95}}; + request.context.max_trials = 30; + request.indexes = { + {"hgraph", + {{"dim", 8}, + {"dtype", "float32"}, + {"metric_type", "l2"}, + {"index_param", + {{"base_quantization_type", "fp32"}, {"max_degree", 24}, {"ef_construction", 80}}}}, + {{"hgraph", + {{"ef_search", {{"$range", {{"start", 40}, {"stop", 1000}}}}}, + {"use_reorder", {true, false}}}}}}}; + + const auto candidates = vsag::autotune::internal::GenerateCandidates(request); + REQUIRE(candidates.size() == 2); + for (const auto& candidate : candidates) { + REQUIRE(candidate.ef_search_range.has_value()); + const auto range = candidate.ef_search_range.value_or( + vsag::autotune::internal::HGraphEfSearchRange{-1, -1}); + REQUIRE(range.start == 40); + REQUIRE(range.stop == 1000); + REQUIRE_FALSE(candidate.search_params["hgraph"].contains("ef_search")); + REQUIRE(candidate.search_params["hgraph"]["use_reorder"].is_boolean()); + } + + request.context.max_trials = 29; + REQUIRE_THROWS_WITH( + vsag::autotune::internal::GenerateCandidates(request), + Catch::Matchers::ContainsSubstring("planned trial count exceeds tuning_config.max_trials")); + + request.context.max_trials = 1; + request.indexes[0].search_params = { + {"hgraph", {{"ef_search", {{"$range", {{"start", 8}, {"stop", 8}}}}}}}}; + REQUIRE(vsag::autotune::internal::GenerateCandidates(request).size() == 1); + + request.context.constraints = {{"latency_avg_ms", 1.0}}; + request.indexes[0].search_params = { + {"hgraph", {{"ef_search", {{"$range", {{"start", 8}, {"stop", 16}}}}}}}}; + request.context.max_trials = 5; + REQUIRE_THROWS_WITH(vsag::autotune::internal::GenerateCandidates(request), + Catch::Matchers::ContainsSubstring("requires a recall_at_k constraint")); + + request.context.constraints = {{"recall_at_k", 0.95}}; + request.context.objective = "recall_at_k"; + REQUIRE_THROWS_WITH(vsag::autotune::internal::GenerateCandidates(request), + Catch::Matchers::ContainsSubstring("requires a query-cost objective")); + + request.context.objective = "latency_avg_ms"; + request.indexes[0].search_params["hgraph"]["hops_limit"] = 100; + REQUIRE_THROWS_WITH( + vsag::autotune::internal::GenerateCandidates(request), + Catch::Matchers::ContainsSubstring("does not support timeout_ms or hops_limit")); + + request.indexes[0].search_params["hgraph"].erase("hops_limit"); + request.indexes[0].search_params["hgraph"]["ef_search"]["$range"]["stop"] = 1001; + request.context.max_trials = 30; + const auto unbounded = vsag::autotune::internal::GenerateCandidates(request); + REQUIRE(unbounded.size() == 1); + REQUIRE(unbounded[0].ef_search_range.has_value()); + REQUIRE(unbounded[0].ef_search_range->stop == 1001); +} + +TEST_CASE("AutoTune brackets and binary-searches the smallest passing ef_search") { + const vsag::autotune::internal::HGraphEfSearchRange range{8, 64}; + std::vector evaluated; + const auto evaluate = [&evaluated](int64_t ef_search) -> std::optional { + evaluated.emplace_back(ef_search); + return ef_search >= 25 ? 0.95 : 0.90; + }; + + vsag::autotune::internal::EvaluateEfSearchRange(range, 0.95, evaluate); + REQUIRE(evaluated == std::vector{8, 16, 32, 24, 28, 26, 25}); + + evaluated.clear(); + vsag::autotune::internal::EvaluateEfSearchRange(range, 0.90, evaluate); + REQUIRE(evaluated == std::vector{8}); + + evaluated.clear(); + vsag::autotune::internal::EvaluateEfSearchRange({8, 50}, 1.0, evaluate); + REQUIRE(evaluated == std::vector{8, 16, 32, 50}); + + evaluated.clear(); + const auto fail_at_middle = [&evaluated](int64_t ef_search) -> std::optional { + evaluated.emplace_back(ef_search); + if (ef_search == 24) { + return std::nullopt; + } + return ef_search >= 32 ? 0.95 : 0.90; + }; + vsag::autotune::internal::EvaluateEfSearchRange(range, 0.95, fail_at_middle); + REQUIRE(evaluated == std::vector{8, 16, 32, 24}); +} + +TEST_CASE("AutoTune validates typed requests and optional ground truth") { + MemoryFixture fixture; + auto input = fixture.Request(temp_path("autotune-typed-validation")); + + const auto parsed = vsag::autotune::internal::ParseRequest(input); + REQUIRE(parsed.context.dataset->GetTrainIds()[0] == fixture.base_ids[0]); + REQUIRE(parsed.context.dataset->GetTrainIds()[1] == fixture.base_ids[1]); + + input.constraints = {{vsag::autotune::Metric::LATENCY_AVG_MS, 1000.0}}; + input.workload.ground_truth = nullptr; + REQUIRE_NOTHROW(vsag::autotune::internal::ParseRequest(input)); + + input.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.9}}; + REQUIRE_THROWS_WITH(vsag::autotune::internal::ParseRequest(input), + Catch::Matchers::ContainsSubstring("ground_truth is required")); + + input.workload.ground_truth = fixture.ground_truth; + input.constraints.push_back({vsag::autotune::Metric::RECALL_AT_K, 0.8}); + REQUIRE_THROWS_WITH(vsag::autotune::internal::ParseRequest(input), + Catch::Matchers::ContainsSubstring("duplicate metric")); + + input = fixture.Request(temp_path("autotune-typed-duplicate-ids")); + auto duplicate_ids = fixture.base_ids; + duplicate_ids[1] = duplicate_ids[0]; + input.base = vsag::Dataset::Make() + ->NumElements(MemoryFixture::BASE_COUNT) + ->Dim(MemoryFixture::DIM) + ->Ids(duplicate_ids.data()) + ->Float32Vectors(fixture.train.data()) + ->Owner(false); + REQUIRE_THROWS_WITH(vsag::autotune::internal::ParseRequest(input), + Catch::Matchers::ContainsSubstring("duplicate ids")); + + input = fixture.Request(temp_path("autotune-typed-unknown-ground-truth")); + auto unknown_ids = fixture.neighbors; + unknown_ids[0] = -1; + input.workload.ground_truth = vsag::Dataset::Make() + ->NumElements(MemoryFixture::QUERY_COUNT) + ->Dim(MemoryFixture::GROUND_TRUTH_K) + ->Ids(unknown_ids.data()) + ->Owner(false); + REQUIRE_THROWS_WITH(vsag::autotune::internal::ParseRequest(input), + Catch::Matchers::ContainsSubstring("not present in base")); +} + +TEST_CASE("AutoTune returns structured offline validation failures") { + ScopedPath invalid_dataset(temp_path("autotune-invalid-dataset", ".hdf5")); + { + std::ofstream output(invalid_dataset.Get(), std::ios::binary); + output << "not an HDF5 file"; + } + H5::Exception::dontPrint(); + + const auto invalid_file = + vsag::autotune::RunAutoTune({{"version", 1}, {"data_path", invalid_dataset.Get()}}); + REQUIRE(invalid_file["status"] == "failed"); + REQUIRE(invalid_file["failure"]["stage"] == "validation"); + REQUIRE(invalid_file["failure"]["code"] == "invalid_request"); + REQUIRE_THAT(invalid_file["failure"]["message"].get(), + Catch::Matchers::ContainsSubstring("failed to load evaluation dataset")); + + const auto invalid_version = vsag::autotune::RunAutoTune({{"version", 2}}); + REQUIRE(invalid_version["status"] == "failed"); + REQUIRE(invalid_version["failure"]["stage"] == "validation"); + REQUIRE(invalid_version["failure"]["code"] == "invalid_request"); +} + +TEST_CASE("AutoTune serializes concurrent offline dataset loading") { + ScopedPath dataset(temp_path("autotune-concurrent-dataset", ".hdf5")); + write_dataset(dataset.Get()); + const JsonType input{{"version", 1}, {"data_path", dataset.Get()}}; + std::atomic ready{0}; + std::atomic start{false}; + JsonType results[2]; + + const auto run = [&](uint64_t position) { + ready.fetch_add(1); + while (!start.load()) { + std::this_thread::yield(); + } + results[position] = vsag::autotune::RunAutoTune(input); + }; + std::thread first(run, 0); + std::thread second(run, 1); + while (ready.load() != 2) { + std::this_thread::yield(); + } + start.store(true); + first.join(); + second.join(); + + for (const auto& result : results) { + REQUIRE(result["status"] == "failed"); + REQUIRE(result["failure"]["stage"] == "validation"); + REQUIRE(result["failure"]["code"] == "invalid_request"); + REQUIRE_THAT(result["failure"]["message"].get(), + Catch::Matchers::ContainsSubstring("request.workload is required")); + } +} + +TEST_CASE("AutoTune keeps normalized offline request metadata") { + ScopedPath dataset(temp_path("autotune-normalized-dataset", ".hdf5")); + ScopedPath workspace(temp_path("autotune-normalized-workspace")); + write_dataset(dataset.Get()); + auto input = request(dataset.Get(), workspace.Get()); + input.erase("indexes"); + + const auto parsed = vsag::autotune::internal::ParseRequest(input); + const auto& index_request = std::get(parsed); + const auto& effective = index_request.context.effective_request; + REQUIRE(effective["data_path"] == dataset.Get()); + REQUIRE(effective["dataset"]["dim"] == 8); + REQUIRE(effective["dataset"]["dtype"] == "float32"); + REQUIRE(effective["dataset"]["metric_type"] == "l2"); + REQUIRE(effective["index_spaces"].size() == 2); + REQUIRE(effective["index_spaces"][0]["name"] == "hgraph"); + REQUIRE(effective["index_spaces"][1]["name"] == "ivf"); + for (const auto& space : effective["index_spaces"]) { + REQUIRE(space["create_parameter_space"]["dim"] == 8); + REQUIRE(space["create_parameter_space"]["dtype"] == "float32"); + REQUIRE(space["create_parameter_space"]["metric_type"] == "l2"); + } + REQUIRE(effective["workload"]["concurrency"] == 2); + REQUIRE(effective["config"]["keep_intermediate"] == true); + REQUIRE_FALSE(effective.contains("tuning_config")); +} + +TEST_CASE("AutoTune rejects report paths that alias an existing index") { + ScopedPath dataset(temp_path("autotune-alias-dataset", ".hdf5")); + ScopedPath workspace(temp_path("autotune-alias-workspace")); + ScopedPath index_path(temp_path("autotune-alias-index", ".index")); + ScopedPath report_path(temp_path("autotune-alias-report", ".json")); + write_dataset(dataset.Get()); + { + std::ofstream output(index_path.Get(), std::ios::binary); + output << "serialized-index-placeholder"; + } + std::error_code link_error; + std::filesystem::create_hard_link(index_path.Get(), report_path.Get(), link_error); + REQUIRE_FALSE(link_error); + const auto original_size = std::filesystem::file_size(index_path.Get()); + + auto input = request(dataset.Get(), workspace.Get()); + input["indexes"] = JsonType::array({input["indexes"][0]}); + input["index_path"] = index_path.Get(); + input["output"] = {{"result_path", report_path.Get()}}; + const auto result = vsag::autotune::RunAutoTune(input); + + REQUIRE(result["status"] == "failed"); + REQUIRE(result["failure"]["stage"] == "validation"); + REQUIRE(result["failure"]["code"] == "invalid_request"); + REQUIRE_THAT(result["failure"]["message"].get(), + Catch::Matchers::ContainsSubstring("result_path must not alias index_path")); + REQUIRE(std::filesystem::file_size(index_path.Get()) == original_size); +} + +TEST_CASE("AutoTune reports the closest successful trial when constraints are infeasible") { + vsag::autotune::internal::RequestContext request; + request.top_k = 10; + request.objective = "latency_avg_ms"; + request.constraints = {{"recall_at_k", 0.95}}; + vsag::autotune::internal::Evaluation evaluation; + const auto trial = [](const std::string& id, double recall, double latency) { + return JsonType{{"trial_id", id}, + {"build_id", "build-0"}, + {"index_name", "hgraph"}, + {"create_params", JsonType::object()}, + {"search_params", JsonType::object()}, + {"status", "success"}, + {"metrics", {{"recall_at_k", recall}, {"latency_avg_ms", latency}}}, + {"artifacts", JsonType::object()}}; + }; + evaluation.trials = { + trial("trial-0", 0.90, 1.0), trial("trial-1", 0.80, 2.0), trial("trial-2", 0.85, 3.0)}; + + const auto result = vsag::autotune::internal::SelectResult(request, evaluation); + REQUIRE(result["status"] == "no_feasible_candidate"); + REQUIRE(result["recommendation"].is_null()); + REQUIRE(result["best_effort"]["evidence"]["trial_id"] == "trial-0"); +} + +TEST_CASE("AutoTune distinguishes a missing objective metric from failed trials") { + vsag::autotune::internal::RequestContext request; + request.top_k = 10; + request.objective = "index_memory_mb"; + request.constraints = {{"recall_at_k", 0.0}}; + vsag::autotune::internal::Evaluation evaluation; + evaluation.trials = {{{"trial_id", "trial-0"}, + {"index_name", "pyramid"}, + {"search_params", JsonType::object()}, + {"status", "success"}, + {"metrics", {{"recall_at_k", 1.0}}}}}; + + const auto result = vsag::autotune::internal::SelectResult(request, evaluation); + REQUIRE(result["status"] == "failed"); + REQUIRE(result["failure"]["stage"] == "selection"); + REQUIRE(result["failure"]["code"] == "objective_metric_unavailable"); + REQUIRE_THAT(result["failure"]["message"].get(), + Catch::Matchers::ContainsSubstring("index_memory_mb")); +} + +TEST_CASE("AutoTune builds once per create candidate and supports an existing index") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedOpenMpThreads openmp_threads(3); + ScopedPath dataset(temp_path("autotune-dataset", ".hdf5")); + ScopedPath workspace(temp_path("autotune-workspace")); + write_dataset(dataset.Get()); + + const auto result = vsag::autotune::RunAutoTune(request(dataset.Get(), workspace.Get())); + INFO(result.dump(2)); + REQUIRE(result["status"] == "success"); + REQUIRE(omp_get_max_threads() == 3); + REQUIRE(result["builds"].size() == 2); + REQUIRE(result["trials"].size() == 4); + REQUIRE(result["recommendation"]["create_params"]["dim"] == 8); + for (const auto& build : result["builds"]) { + REQUIRE(build["status"] == "success"); + REQUIRE(build["metrics"].contains("build_seconds")); + REQUIRE(build["metrics"].contains("index_size_mb")); + REQUIRE(build["metrics"].contains("index_memory_mb")); + } + for (const auto& trial : result["trials"]) { + REQUIRE(trial["status"] == "success"); + REQUIRE(trial["metrics"].contains("recall_at_k")); + REQUIRE(trial["metrics"].contains("latency_avg_ms")); + REQUIRE(trial["metrics"].contains("qps")); + } + + const auto hgraph = + std::find_if(result["builds"].begin(), result["builds"].end(), [](const auto& build) { + return build["index_name"] == "hgraph"; + }); + REQUIRE(hgraph != result["builds"].end()); + auto existing = request(dataset.Get(), workspace.Get()); + existing["index_path"] = (*hgraph)["artifacts"]["index_path"]; + existing["indexes"] = + JsonType::array({{{"name", "hgraph"}, + {"create_params", (*hgraph)["create_params"]}, + {"search_params", {{"hgraph", {{"ef_search", {8, 16}}}}}}}}); + existing["constraints"] = {{"recall_at_k", 0.0}}; + existing["tuning_config"]["max_trials"] = 2; + + const auto existing_result = vsag::autotune::RunAutoTune(existing); + INFO(existing_result.dump(2)); + REQUIRE(existing_result["status"] == "success"); + REQUIRE(existing_result["builds"].empty()); + REQUIRE(existing_result["trials"].size() == 2); + REQUIRE(existing_result["request"]["data_path"] == dataset.Get()); + REQUIRE(existing_result["request"]["index_path"] == existing["index_path"]); + REQUIRE(existing_result["request"]["index_name"] == "hgraph"); + REQUIRE(existing_result["request"]["create_params"]["dim"] == 8); + REQUIRE(existing_result["request"]["create_params"]["dtype"] == "float32"); + REQUIRE(existing_result["request"]["create_params"]["metric_type"] == "l2"); +} + +TEST_CASE("AutoTune searches an in-memory existing index") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath workspace(temp_path("autotune-memory-index-workspace")); + MemoryFixture fixture; + const std::string create_params = + R"({"dim":8,"dtype":"float32","metric_type":"l2","index_param":{)" + R"("base_quantization_type":"fp32","max_degree":8,"ef_construction":40,)" + R"("build_thread_count":2}})"; + auto created = vsag::Factory::CreateIndex("hgraph", create_params); + REQUIRE(created.has_value()); + REQUIRE(created.value()->Build(fixture.base).has_value()); + + vsag::autotune::SearchRequest input; + input.index = created.value(); + input.index_name = "hgraph"; + input.base = fixture.base; + input.metric_type = vsag::METRIC_L2; + input.workload = {fixture.queries, fixture.ground_truth, 3, 2}; + input.parameter_space = R"({"hgraph":{"ef_search":[8,16]}})"; + input.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.0}}; + input.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + input.config.workspace_path = workspace.Get(); + input.config.max_trials = 2; + + const auto tuned = vsag::autotune::TuneSearch(input); + REQUIRE(tuned.has_value()); + const auto result = JsonType::parse(tuned.value().report); + REQUIRE(result["status"] == "success"); + REQUIRE(result["builds"].empty()); + REQUIRE(result["trials"].size() == 2); + for (const auto& trial : result["trials"]) { + REQUIRE(trial["status"] == "success"); + REQUIRE(trial["metrics"].contains("recall_at_k")); + REQUIRE(trial["metrics"].contains("latency_avg_ms")); + } +} + +TEST_CASE("AutoTune uses default candidates for an in-memory IVF index") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath workspace(temp_path("autotune-memory-ivf-workspace")); + MemoryFixture fixture; + const std::string create_params = + R"({"dim":8,"dtype":"float32","metric_type":"l2","index_param":{)" + R"("base_quantization_type":"fp32","buckets_count":4,"thread_count":2}})"; + auto created = vsag::Factory::CreateIndex("ivf", create_params); + REQUIRE(created.has_value()); + REQUIRE(created.value()->Build(fixture.base).has_value()); + + vsag::autotune::SearchRequest input; + input.index = created.value(); + input.index_name = "ivf"; + input.base = fixture.base; + input.metric_type = vsag::METRIC_L2; + input.workload = {fixture.queries, fixture.ground_truth, 3, 2}; + input.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.0}}; + input.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + input.config.workspace_path = workspace.Get(); + input.config.max_trials = 4; + + const auto tuned = vsag::autotune::TuneSearch(input); + REQUIRE(tuned.has_value()); + const auto result = JsonType::parse(tuned.value().report); + REQUIRE(result["status"] == "success"); + REQUIRE(result["trials"].size() == 4); + REQUIRE(result["recommendation"]["search_params"]["ivf"].contains("scan_buckets_count")); +} + +TEST_CASE("AutoTune searches an existing Pyramid index for one path workload") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath workspace(temp_path("autotune-pyramid-workspace")); + MemoryFixture fixture; + std::vector base_paths(MemoryFixture::BASE_COUNT, "a/d/f"); + std::vector query_paths(MemoryFixture::QUERY_COUNT, "a/d/f"); + auto base = vsag::Dataset::Make() + ->NumElements(MemoryFixture::BASE_COUNT) + ->Dim(MemoryFixture::DIM) + ->Ids(fixture.base_ids.data()) + ->Float32Vectors(fixture.train.data()) + ->Paths(base_paths.data()) + ->Owner(false); + auto queries = vsag::Dataset::Make() + ->NumElements(MemoryFixture::QUERY_COUNT) + ->Dim(MemoryFixture::DIM) + ->Float32Vectors(fixture.test.data()) + ->Paths(query_paths.data()) + ->Owner(false); + const std::string create_params = + R"({"dim":8,"dtype":"float32","metric_type":"l2","index_param":{)" + R"("base_quantization_type":"fp32","max_degree":8,"alpha":1.2,)" + R"("graph_iter_turn":5,"neighbor_sample_rate":0.5,"no_build_levels":[0,1],)" + R"("use_reorder":true,"graph_type":"odescent","build_thread_count":2}})"; + auto created = vsag::Factory::CreateIndex("pyramid", create_params); + REQUIRE(created.has_value()); + REQUIRE(created.value()->Build(base).has_value()); + REQUIRE(created.value()->GetMemoryUsage() == 0); + + vsag::autotune::SearchRequest input; + input.index = created.value(); + input.index_name = "pyramid"; + input.base = base; + input.metric_type = vsag::METRIC_L2; + input.workload.queries = queries; + input.workload.ground_truth = fixture.ground_truth; + input.workload.top_k = 3; + input.workload.concurrency = 2; + input.parameter_space = R"({"pyramid":{"ef_search":[4,8]}})"; + input.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.0}}; + input.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + input.config.workspace_path = workspace.Get(); + input.config.max_trials = 2; + + const auto tuned = vsag::autotune::TuneSearch(input); + REQUIRE(tuned.has_value()); + const auto result = JsonType::parse(tuned.value().report); + REQUIRE(result["status"] == "success"); + REQUIRE(result["trials"].size() == 2); + REQUIRE(result["recommendation"]["index_name"] == "pyramid"); + REQUIRE(result["recommendation"]["search_params"].contains("pyramid")); + REQUIRE(result["builds"].empty()); + + input.constraints.push_back({vsag::autotune::Metric::INDEX_MEMORY_MB, 1.0}); + const auto memory_constrained = vsag::autotune::TuneSearch(input); + REQUIRE_FALSE(memory_constrained.has_value()); + REQUIRE(memory_constrained.error().type == vsag::ErrorType::INVALID_ARGUMENT); + + input.constraints.pop_back(); + input.workload.queries = fixture.queries; + REQUIRE_NOTHROW(vsag::autotune::internal::ParseRequest(input)); +} + +TEST_CASE("AutoTune writes concrete trials for an adaptive ef_search range") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath dataset(temp_path("autotune-binary-dataset", ".hdf5")); + ScopedPath workspace(temp_path("autotune-binary-workspace")); + write_dataset(dataset.Get()); + + auto input = request(dataset.Get(), workspace.Get()); + auto hgraph = input["indexes"][0]; + hgraph["search_params"] = { + {"hgraph", {{"ef_search", {{"$range", {{"start", 3}, {"stop", 35}}}}}}}}; + input["indexes"] = JsonType::array({std::move(hgraph)}); + input["constraints"] = {{"recall_at_k", 0.0}}; + input["tuning_config"]["max_trials"] = 9; + + const auto result = vsag::autotune::RunAutoTune(input); + INFO(result.dump(2)); + REQUIRE(result["status"] == "success"); + REQUIRE(result["builds"].size() == 1); + REQUIRE(result["trials"].size() == 1); + REQUIRE(result["trials"][0]["search_params"]["hgraph"]["ef_search"] == 3); + REQUIRE(result["trials"][0]["search_params"]["hgraph"]["ef_search"].is_number_integer()); + REQUIRE(result["recommendation"]["search_params"]["hgraph"]["ef_search"] == 3); +} + +TEST_CASE("TuneIndex returns a queryable selected index") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath workspace(temp_path("autotune-factory-workspace")); + MemoryFixture fixture; + auto input = fixture.Request(workspace.Get()); + + auto tuned = vsag::autotune::TuneIndex(input); + REQUIRE(tuned.has_value()); + const auto& result = tuned.value(); + REQUIRE(result.index != nullptr); + REQUIRE(result.index->GetNumElements() == 64); + REQUIRE(result.index_name == "hgraph"); + REQUIRE(JsonType::parse(result.create_parameters)["dim"] == 8); + REQUIRE(JsonType::parse(result.search_parameters)["hgraph"]["ef_search"] == 16); + REQUIRE(JsonType::parse(result.metrics).contains("recall_at_k")); + REQUIRE(std::filesystem::is_regular_file(result.artifact_path)); + REQUIRE(std::filesystem::is_regular_file(result.report_path)); + + auto query = vsag::Dataset::Make(); + query->NumElements(1) + ->Dim(MemoryFixture::DIM) + ->Float32Vectors(fixture.test.data()) + ->Owner(false); + auto neighbors = result.index->KnnSearch(query, 3, result.search_parameters); + REQUIRE(neighbors.has_value()); + REQUIRE(neighbors.value()->GetDim() == 3); + REQUIRE(std::find(fixture.base_ids.begin(), + fixture.base_ids.end(), + neighbors.value()->GetIds()[0]) != fixture.base_ids.end()); + + auto restored = vsag::Factory::CreateIndex(result.index_name, result.create_parameters); + REQUIRE(restored.has_value()); + std::ifstream artifact(result.artifact_path, std::ios::binary); + REQUIRE(artifact.good()); + REQUIRE(restored.value()->Deserialize(artifact).has_value()); + REQUIRE(restored.value()->KnnSearch(query, 3, result.search_parameters).has_value()); + + const auto report = JsonType::parse(result.report); + REQUIRE(report["status"] == "success"); + REQUIRE(report["builds"].size() == 2); + REQUIRE(std::count_if(report["builds"].begin(), report["builds"].end(), [](const auto& build) { + return build["artifacts"]["retained"].template get(); + }) == 1); + REQUIRE(count_index_artifacts(workspace.Get()) == 1); + REQUIRE(report["recommendation"]["create_params"]["dim"] == 8); + REQUIRE(report["recommendation"]["artifacts"]["retained"] == true); + REQUIRE(report["request"]["config"]["workspace_path"] == workspace.Get()); +} + +TEST_CASE("TuneIndex rejects an infeasible recommendation") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath workspace(temp_path("autotune-infeasible-workspace")); + MemoryFixture fixture; + auto input = fixture.Request(workspace.Get()); + input.index_spaces[0].create_parameter_space = + R"({"index_param":{"base_quantization_type":"fp32","max_degree":8,)" + R"("ef_construction":40,"build_thread_count":2}})"; + input.workload.ground_truth = nullptr; + input.constraints = {{vsag::autotune::Metric::QPS, 1e100}}; + input.config.max_trials = 1; + + const auto tuned = vsag::autotune::TuneIndex(input); + REQUIRE_FALSE(tuned.has_value()); + REQUIRE(tuned.error().type == vsag::ErrorType::INVALID_ARGUMENT); + REQUIRE_THAT(tuned.error().message, Catch::Matchers::ContainsSubstring("report:")); + REQUIRE(count_index_artifacts(workspace.Get()) == 0); + const auto reports = load_json_reports(workspace.Get()); + REQUIRE(reports.size() == 1); + REQUIRE(reports[0]["status"] == "no_feasible_candidate"); + REQUIRE(reports[0]["request"]["config"]["keep_intermediate"] == false); + REQUIRE(reports[0]["builds"][0]["artifacts"]["retained"] == false); + REQUIRE(reports[0]["best_effort"]["artifacts"]["retained"] == false); +} + +TEST_CASE("TuneIndex cleans artifacts when all search trials fail") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath workspace(temp_path("autotune-all-failed-workspace")); + MemoryFixture fixture; + auto input = fixture.Request(workspace.Get()); + input.index_spaces[0].create_parameter_space = + R"({"index_param":{"base_quantization_type":"fp32","max_degree":8,)" + R"("ef_construction":40,"build_thread_count":2}})"; + input.index_spaces[0].search_parameter_space = R"({"hgraph":{"ef_search":0}})"; + input.config.max_trials = 1; + + const auto tuned = vsag::autotune::TuneIndex(input); + REQUIRE_FALSE(tuned.has_value()); + REQUIRE(tuned.error().type == vsag::ErrorType::INTERNAL_ERROR); + REQUIRE_THAT(tuned.error().message, Catch::Matchers::ContainsSubstring("report:")); + REQUIRE(count_index_artifacts(workspace.Get()) == 0); + const auto reports = load_json_reports(workspace.Get()); + REQUIRE(reports.size() == 1); + REQUIRE(reports[0]["status"] == "failed"); + REQUIRE(reports[0]["failure"]["code"] == "all_trials_failed"); + REQUIRE(reports[0]["request"]["config"]["keep_intermediate"] == false); + REQUIRE(reports[0]["builds"][0]["artifacts"]["retained"] == false); +} + +TEST_CASE("TuneIndex reports report-writing failures") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath workspace(temp_path("autotune-report-failure-workspace")); + MemoryFixture fixture; + + const auto report_directory = workspace.Get() + "/report-is-a-directory"; + std::filesystem::create_directories(report_directory); + auto input = fixture.Request(workspace.Get()); + input.index_spaces[0].create_parameter_space = + R"({"index_param":{"base_quantization_type":"fp32","max_degree":8,)" + R"("ef_construction":40,"build_thread_count":2}})"; + input.config.max_trials = 1; + input.config.report_path = report_directory; + + const auto tuned = vsag::autotune::TuneIndex(input); + REQUIRE_FALSE(tuned.has_value()); + REQUIRE(tuned.error().type == vsag::ErrorType::INTERNAL_ERROR); + REQUIRE(count_index_artifacts(workspace.Get()) == 0); +} diff --git a/tools/autotune/examples/README.md b/tools/autotune/examples/README.md new file mode 100644 index 0000000000..49a8cc2b5d --- /dev/null +++ b/tools/autotune/examples/README.md @@ -0,0 +1,19 @@ +# AutoTune Examples + +The SIFT example expects the ann-benchmarks dataset at +`/tmp/sift-128-euclidean.hdf5`. Edit `data_path` if the dataset is stored elsewhere. + +From the VSAG repository root, run: + +```bash +./build-release/tools/autotune/autotune \ + tools/autotune/examples/sift_hgraph_autotune_request.json +``` + +The command prints a compact summary. The complete report is written to +`/tmp/vsag_autotune_sift/report.json`. + +This example expands six HGraph build configurations. For each build, AutoTune doubles +`ef_search` from `40` until recall passes, then binary-searches that interval for the smallest +passing value, capped at `1000`. Every evaluated point uses all queries. Add arrays or stepped +`$range` expressions to other parameter leaves to change the tuning space. diff --git a/tools/autotune/examples/sift_hgraph_autotune_request.json b/tools/autotune/examples/sift_hgraph_autotune_request.json new file mode 100644 index 0000000000..19febc2422 --- /dev/null +++ b/tools/autotune/examples/sift_hgraph_autotune_request.json @@ -0,0 +1,48 @@ +{ + "version": 1, + "data_path": "/tmp/sift-128-euclidean.hdf5", + "indexes": [ + { + "name": "hgraph", + "create_params": { + "index_param": { + "base_quantization_type": ["fp32","sq8"], + "max_degree": [16,32,48], + "ef_construction": 400, + "build_thread_count": 48 + } + }, + "search_params": { + "hgraph": { + "ef_search": { + "$range": { + "start": 40, + "stop": 1000 + } + } + } + } + } + ], + "workload": { + "top_k": 10, + "concurrency": 1 + }, + "constraints": { + "recall_at_k": 0.95, + "index_memory_mb": 1024, + "index_size_mb": 1024 + }, + "objective": { + "metric": "latency_avg_ms" + }, + "tuning_config": { + "workspace_path": "/tmp/vsag_autotune_sift", + "keep_intermediate": false, + "max_trials": 1000 + }, + "output": { + "result_path": "/tmp/vsag_autotune_sift/report.json", + "include_raw_eval": false + } +} diff --git a/tools/autotune/main.cpp b/tools/autotune/main.cpp new file mode 100644 index 0000000000..a43ce4d5b4 --- /dev/null +++ b/tools/autotune/main.cpp @@ -0,0 +1,54 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "autotune.h" +#include "vsag/options.h" + +int +main(int argc, char** argv) try { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + if (argc != 2) { + std::cerr << "Usage: " << argv[0] << " " << std::endl; + return 1; + } + + vsag::autotune::JsonType result; + try { + std::ifstream input(argv[1]); + if (!input.good()) { + throw std::runtime_error("failed to open request file: " + std::string(argv[1])); + } + vsag::autotune::JsonType request; + input >> request; + result = vsag::autotune::RunAutoTune(request); + } catch (const std::exception& error) { + result = {{"version", 1}, + {"status", "failed"}, + {"failure", + {{"stage", "cli"}, {"code", "request_file_error"}, {"message", error.what()}}}}; + } + + std::cout << vsag::autotune::FormatResultSummaryForCli(result) << std::endl; + return result.value("status", std::string()) == "failed" ? 1 : 0; +} catch (const std::exception& error) { + std::cerr << "AutoTune failed: " << error.what() << std::endl; + return 1; +} catch (...) { + std::cerr << "AutoTune failed with an unknown error" << std::endl; + return 1; +} diff --git a/tools/eval/CMakeLists.txt b/tools/eval/CMakeLists.txt index 3b87f885a5..7fb630dd69 100644 --- a/tools/eval/CMakeLists.txt +++ b/tools/eval/CMakeLists.txt @@ -28,6 +28,7 @@ set (eval_srcs monitor/http_server_monitor.cpp eval_config.cpp eval_dataset.cpp + evaluator.cpp eval_job.cpp) add_library (vsag_eval_common INTERFACE) diff --git a/tools/eval/case/build_eval_case.cpp b/tools/eval/case/build_eval_case.cpp index 3eb243ec46..3bd135fac1 100644 --- a/tools/eval/case/build_eval_case.cpp +++ b/tools/eval/case/build_eval_case.cpp @@ -15,7 +15,6 @@ #include "./build_eval_case.h" -#include #include #include @@ -28,8 +27,10 @@ namespace vsag::eval { BuildEvalCase::BuildEvalCase(const std::string& dataset_path, const std::string& index_path, vsag::IndexPtr index, - EvalConfig config) - : EvalCase(dataset_path, index_path, index), config_(std::move(config)) { + EvalConfig config, + EvalDatasetPtr dataset) + : EvalCase(dataset_path, index_path, std::move(index), std::move(dataset)), + config_(std::move(config)) { this->init_monitors(); } @@ -49,16 +50,23 @@ JsonType BuildEvalCase::Run() { this->do_build(); this->serialize(); - auto result = this->process_result(); - return result; + return this->process_result(); } + +JsonType +BuildEvalCase::RunInMemory() { + this->do_build(); + return this->process_result(); +} + void BuildEvalCase::do_build() { auto base = vsag::Dataset::Make(); int64_t total_base = this->dataset_ptr_->GetNumberOfBase(); - std::vector ids(total_base); - std::iota(ids.begin(), ids.end(), 0); - base->NumElements(total_base)->Dim(this->dataset_ptr_->GetDim())->Ids(ids.data())->Owner(false); + base->NumElements(total_base) + ->Dim(this->dataset_ptr_->GetDim()) + ->Ids(this->dataset_ptr_->GetTrainIds()) + ->Owner(false); if (this->dataset_ptr_->GetVectorType() == DENSE_VECTORS) { if (this->dataset_ptr_->GetTrainDataType() == vsag::DATATYPE_FLOAT32) { base->Float32Vectors((const float*)this->dataset_ptr_->GetTrain()); @@ -68,6 +76,9 @@ BuildEvalCase::do_build() { } else { base->SparseVectors((const SparseVector*)this->dataset_ptr_->GetTrain()); } + if (this->dataset_ptr_->GetTrainPaths() != nullptr) { + base->Paths(this->dataset_ptr_->GetTrainPaths()); + } for (auto& monitor : monitors_) { monitor->Start(); } @@ -105,6 +116,7 @@ BuildEvalCase::process_result() { result["index_info"] = JsonType::parse(config_.build_param); result["action"] = "build"; result["index"] = config_.index_name; + result["index_memory(B)"] = this->index_->GetMemoryUsage(); try { auto detail = this->index_->GetMemoryUsageDetail(); for (const auto& [name, size] : detail) { diff --git a/tools/eval/case/build_eval_case.h b/tools/eval/case/build_eval_case.h index d43267cbe7..7fb837fd37 100644 --- a/tools/eval/case/build_eval_case.h +++ b/tools/eval/case/build_eval_case.h @@ -25,13 +25,17 @@ class BuildEvalCase : public EvalCase { BuildEvalCase(const std::string& dataset_path, const std::string& index_path, vsag::IndexPtr index, - EvalConfig config); + EvalConfig config, + EvalDatasetPtr dataset = nullptr); ~BuildEvalCase() override = default; JsonType Run() override; + JsonType + RunInMemory(); + private: void init_monitors(); diff --git a/tools/eval/case/build_search_eval_case.h b/tools/eval/case/build_search_eval_case.h index c9f4f2e47f..e4da73b76e 100644 --- a/tools/eval/case/build_search_eval_case.h +++ b/tools/eval/case/build_search_eval_case.h @@ -25,10 +25,11 @@ class BuildSearchEvalCase : public EvalCase { BuildSearchEvalCase(const std::string& dataset_path, const std::string& index_path, vsag::IndexPtr index, - EvalConfig config) - : EvalCase(dataset_path, index_path, index) { - build_ = EvalCase::MakeInstance(config, "build"); - search_ = EvalCase::MakeInstance(config, "search"); + EvalConfig config, + EvalDatasetPtr dataset = nullptr) + : EvalCase(dataset_path, index_path, std::move(index), dataset) { + build_ = EvalCase::MakeInstance(config, "build", dataset); + search_ = EvalCase::MakeInstance(config, "search", dataset); config_ = std::move(config); } diff --git a/tools/eval/case/eval_case.cpp b/tools/eval/case/eval_case.cpp index ef99325181..a92a89fe94 100644 --- a/tools/eval/case/eval_case.cpp +++ b/tools/eval/case/eval_case.cpp @@ -26,15 +26,23 @@ namespace vsag::eval { -EvalCase::EvalCase(std::string dataset_path, std::string index_path, vsag::IndexPtr index) - : dataset_path_(std::move(dataset_path)), index_path_(std::move(index_path)), index_(index) { - this->dataset_ptr_ = EvalDataset::Load(dataset_path_); +EvalCase::EvalCase(std::string dataset_path, + std::string index_path, + vsag::IndexPtr index, + EvalDatasetPtr dataset) + : dataset_path_(std::move(dataset_path)), + index_path_(std::move(index_path)), + dataset_ptr_(std::move(dataset)), + index_(std::move(index)) { + if (this->dataset_ptr_ == nullptr) { + this->dataset_ptr_ = EvalDataset::Load(dataset_path_); + } this->logger_ = vsag::Options::Instance().logger(); this->basic_info_ = this->dataset_ptr_->GetInfo(); } EvalCasePtr -EvalCase::MakeInstance(const EvalConfig& config, std::string type) { +EvalCase::MakeInstance(const EvalConfig& config, std::string type, const EvalDatasetPtr& dataset) { auto dataset_path = config.dataset_path; auto index_path = config.index_path; auto index_name = config.index_name; @@ -48,14 +56,16 @@ EvalCase::MakeInstance(const EvalConfig& config, std::string type) { } if (type == "build") { - return std::make_shared(dataset_path, index_path, index.value(), config); + return std::make_shared( + dataset_path, index_path, index.value(), config, dataset); } if (type == "search") { - return std::make_shared(dataset_path, index_path, index.value(), config); + return std::make_shared( + dataset_path, index_path, index.value(), config, dataset); } if (type == "build,search") { return std::make_shared( - dataset_path, index_path, index.value(), config); + dataset_path, index_path, index.value(), config, dataset); } return nullptr; } diff --git a/tools/eval/case/eval_case.h b/tools/eval/case/eval_case.h index 8a7316242f..855a3098c5 100644 --- a/tools/eval/case/eval_case.h +++ b/tools/eval/case/eval_case.h @@ -31,7 +31,9 @@ using EvalCasePtr = std::shared_ptr; class EvalCase { public: static EvalCasePtr - MakeInstance(const EvalConfig& config, std::string type = "none"); + MakeInstance(const EvalConfig& config, + std::string type = "none", + const EvalDatasetPtr& dataset = nullptr); static void MergeJsonType(const JsonType& input, JsonType& output) { @@ -86,7 +88,10 @@ class EvalCase { } public: - explicit EvalCase(std::string dataset_path, std::string index_path, vsag::IndexPtr index); + explicit EvalCase(std::string dataset_path, + std::string index_path, + vsag::IndexPtr index, + EvalDatasetPtr dataset = nullptr); virtual ~EvalCase() = default; diff --git a/tools/eval/case/search_eval_case.cpp b/tools/eval/case/search_eval_case.cpp index 209cee9576..0219ea3bb9 100644 --- a/tools/eval/case/search_eval_case.cpp +++ b/tools/eval/case/search_eval_case.cpp @@ -17,12 +17,14 @@ #include +#include #include #include #include #include -#include +#include #include +#include #include #include "../monitor/latency_monitor.h" @@ -35,6 +37,41 @@ namespace vsag::eval { +namespace { + +class SearchFailure { +public: + bool + Failed() const { + return failed_.load(std::memory_order_acquire); + } + + void + Record(const std::string& message) { + bool expected = false; + if (failed_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) { + std::lock_guard lock(mutex_); + message_ = message; + } + } + + void + ThrowIfFailed() { + if (not Failed()) { + return; + } + std::lock_guard lock(mutex_); + throw std::runtime_error("query error: " + message_); + } + +private: + std::atomic failed_{false}; + std::mutex mutex_; + std::string message_; +}; + +} // namespace + class FilterObj : public vsag::Filter { public: FilterObj(const std::shared_ptr& train_labels, int64_t test_label, float valid_ratio) @@ -60,8 +97,10 @@ class FilterObj : public vsag::Filter { SearchEvalCase::SearchEvalCase(const std::string& dataset_path, const std::string& index_path, vsag::IndexPtr index, - EvalConfig config) - : EvalCase(dataset_path, index_path, index), config_(std::move(config)) { + EvalConfig config, + EvalDatasetPtr dataset) + : EvalCase(dataset_path, index_path, std::move(index), std::move(dataset)), + config_(std::move(config)) { auto search_mode = config_.search_mode; if (search_mode == "knn") { this->search_type_ = SearchType::KNN; @@ -84,7 +123,7 @@ SearchEvalCase::init_monitor() { void SearchEvalCase::init_latency_monitor() { - if (config_.enable_latency or config_.enable_tps or config_.enable_percent_latency) { + if (config_.enable_latency or config_.enable_qps or config_.enable_percent_latency) { this->latency_monitor_ = std::make_shared(); if (config_.enable_qps) { this->latency_monitor_->SetMetrics("qps"); @@ -126,6 +165,15 @@ JsonType SearchEvalCase::Run() { std::ifstream infile(this->index_path_, std::ios::binary); this->deserialize(infile); + auto result = this->RunInMemory(); + if (config_.delete_index_after_search) { + std::remove(this->index_path_.c_str()); + } + return result; +} + +JsonType +SearchEvalCase::RunInMemory() { switch (this->search_type_) { case KNN: this->do_knn_search(); @@ -140,11 +188,7 @@ SearchEvalCase::Run() { this->do_range_filter_search(); break; } - auto result = this->process_result(); - if (config_.delete_index_after_search) { - std::remove(this->index_path_.c_str()); - } - return result; + return this->process_result(); } void @@ -172,6 +216,9 @@ SearchEvalCase::do_knn_search() { } else { query->SparseVectors((const SparseVector*)query_vector); } + if (this->dataset_ptr_->GetTestPaths() != nullptr) { + query->Paths(this->dataset_ptr_->GetTestPaths() + query_id); + } return std::make_pair(std::move(query), query_vector); }; @@ -187,20 +234,25 @@ SearchEvalCase::do_knn_search() { if (is_latency_monitor) { using Clock = std::chrono::steady_clock; std::vector latency_records(min_query); + SearchFailure search_failure; const auto wall_start = Clock::now(); #pragma omp parallel for schedule(dynamic) for (int64_t id = 0; id < min_query; ++id) { + if (search_failure.Failed()) { + continue; + } auto i = static_cast(id) % query_count; auto query_and_vector = prepare_query(i); auto& query = query_and_vector.first; auto [result, latency_ms] = MeasureSearch( [&]() { return this->index_->KnnSearch(query, topk, config_.search_param); }); if (not result.has_value()) { - std::cerr << "query error: " << result.error().message << std::endl; - exit(-1); + search_failure.Record(result.error().message); + continue; } latency_records[static_cast(id)] = latency_ms; } + search_failure.ThrowIfFailed(); const auto wall_end = Clock::now(); auto timing_batch = LatencyTimingBatch{ std::move(latency_records), @@ -212,47 +264,56 @@ SearchEvalCase::do_knn_search() { continue; } + SearchFailure search_failure; #pragma omp parallel for schedule(dynamic) for (int64_t id = 0; id < min_query; ++id) { + if (search_failure.Failed()) { + continue; + } auto i = static_cast(id) % query_count; auto query_and_vector = prepare_query(i); auto& query = query_and_vector.first; const void* query_vector = query_and_vector.second; auto result = this->index_->KnnSearch(query, topk, config_.search_param); if (not result.has_value()) { - std::cerr << "query error: " << result.error().message << std::endl; - exit(-1); + search_failure.Record(result.error().message); + continue; } if (collect_statistics) { this->record_statistics(result.value()); } - const int64_t* neighbors = result.value()->GetIds(); - int64_t* ground_truth_neighbors = dataset_ptr_->GetNeighbors(i); - auto record = std::make_tuple(neighbors, - ground_truth_neighbors, - dataset_ptr_.get(), - query_vector, - result.value()->GetDim()); + SearchRecord record{result.value()->GetIds(), + dataset_ptr_->GetNeighbors(i), + dataset_ptr_.get(), + query_vector, + static_cast(result.value()->GetDim()), + topk}; monitor->Record(&record); } + search_failure.ThrowIfFailed(); monitor->Stop(); statistics_collected = statistics_collected or collect_statistics; } if (not statistics_collected and not this->monitors_.empty()) { omp_set_num_threads(config_.num_threads_searching); + SearchFailure search_failure; #pragma omp parallel for schedule(dynamic) for (int64_t id = 0; id < min_query; ++id) { + if (search_failure.Failed()) { + continue; + } auto i = static_cast(id) % query_count; auto query_and_vector = prepare_query(i); auto& query = query_and_vector.first; auto result = this->index_->KnnSearch(query, topk, config_.search_param); if (not result.has_value()) { - std::cerr << "query error: " << result.error().message << std::endl; - exit(-1); + search_failure.Record(result.error().message); + continue; } this->record_statistics(result.value()); } + search_failure.ThrowIfFailed(); } } @@ -299,8 +360,7 @@ SearchEvalCase::do_knn_filter_search() { return this->index_->KnnSearch(query, topk, config_.search_param, filter); }); if (not result.has_value()) { - std::cerr << "query error: " << result.error().message << std::endl; - exit(-1); + throw std::runtime_error("query error: " + result.error().message); } latency_records[static_cast(id)] = latency_ms; } @@ -330,13 +390,14 @@ SearchEvalCase::do_knn_filter_search() { train_labels, test_label, this->dataset_ptr_->GetValidRatio(test_label)); auto result = this->index_->KnnSearch(query, topk, config_.search_param, filter); if (not result.has_value()) { - std::cerr << "query error: " << result.error().message << std::endl; - exit(-1); + throw std::runtime_error("query error: " + result.error().message); } - const int64_t* neighbors = result.value()->GetIds(); - int64_t* ground_truth_neighbors = dataset_ptr_->GetNeighbors(i); - auto record = std::make_tuple( - neighbors, ground_truth_neighbors, dataset_ptr_.get(), query_vector, topk); + SearchRecord record{result.value()->GetIds(), + dataset_ptr_->GetNeighbors(i), + dataset_ptr_.get(), + query_vector, + static_cast(result.value()->GetDim()), + topk}; monitor->Record(&record); } monitor->Stop(); @@ -359,6 +420,7 @@ SearchEvalCase::process_result() { result["index_info"] = JsonType::parse(config_.build_param); result["search_param"] = config_.search_param; result["index"] = config_.index_name; + result["index_memory(B)"] = this->index_->GetMemoryUsage(); try { auto detail = this->index_->GetMemoryUsageDetail(); for (const auto& [name, size] : detail) { diff --git a/tools/eval/case/search_eval_case.h b/tools/eval/case/search_eval_case.h index ca8d3376ba..bad9a4c000 100644 --- a/tools/eval/case/search_eval_case.h +++ b/tools/eval/case/search_eval_case.h @@ -30,13 +30,17 @@ class SearchEvalCase : public EvalCase { SearchEvalCase(const std::string& dataset_path, const std::string& index_path, vsag::IndexPtr index, - EvalConfig config); + EvalConfig config, + EvalDatasetPtr dataset = nullptr); ~SearchEvalCase() override = default; JsonType Run() override; + JsonType + RunInMemory(); + private: enum SearchType { KNN, diff --git a/tools/eval/eval_dataset.cpp b/tools/eval/eval_dataset.cpp index f23021ab71..e5da1b2ead 100644 --- a/tools/eval/eval_dataset.cpp +++ b/tools/eval/eval_dataset.cpp @@ -15,6 +15,9 @@ #include "eval_dataset.h" +#include +#include + #include "impl/logger/logger.h" using namespace H5; @@ -207,6 +210,113 @@ get_distance(const SparseVector* vector1, const SparseVector* vector2, const voi return sum; } +void +EvalDataset::InitializeTrainIds(const int64_t* ids) { + identity_train_ids_.reset(); + train_ids_are_identity_ = true; + train_id_to_row_.clear(); + if (ids == nullptr) { + identity_train_ids_ = std::shared_ptr(new int64_t[number_of_base_]); + for (int64_t i = 0; i < number_of_base_; ++i) { + identity_train_ids_[i] = i; + } + return; + } + + for (int64_t i = 0; i < number_of_base_; ++i) { + if (ids[i] != i) { + train_ids_are_identity_ = false; + break; + } + } + if (train_ids_are_identity_) { + return; + } + + train_id_to_row_.reserve(static_cast(number_of_base_)); + for (int64_t i = 0; i < number_of_base_; ++i) { + if (!train_id_to_row_.emplace(ids[i], i).second) { + throw std::invalid_argument("base dataset contains duplicate ids"); + } + } +} + +EvalDatasetPtr +EvalDataset::FromDatasets(const vsag::DatasetPtr& base, + const vsag::DatasetPtr& queries, + const vsag::DatasetPtr& ground_truth, + const std::string& metric_type) { + if (base == nullptr || queries == nullptr) { + throw std::invalid_argument("base and queries datasets are required"); + } + if (base->GetNumElements() <= 0 || queries->GetNumElements() <= 0) { + throw std::invalid_argument("base and queries datasets must not be empty"); + } + if (base->GetDim() <= 0 || base->GetDim() != queries->GetDim()) { + throw std::invalid_argument("base and queries datasets must have the same positive dim"); + } + if (base->GetFloat32Vectors() == nullptr || queries->GetFloat32Vectors() == nullptr) { + throw std::invalid_argument("in-memory eval supports only float32 dense vectors"); + } + if (ground_truth != nullptr && + (ground_truth->GetNumElements() != queries->GetNumElements() || + ground_truth->GetDim() <= 0 || ground_truth->GetIds() == nullptr)) { + throw std::invalid_argument( + "ground_truth must contain one non-empty id row for every query"); + } + + auto dataset = std::make_shared(); + dataset->base_dataset_ = base; + dataset->query_dataset_ = queries; + dataset->ground_truth_dataset_ = ground_truth; + dataset->vector_type_ = DENSE_VECTORS; + dataset->train_data_type_ = vsag::DATATYPE_FLOAT32; + dataset->test_data_type_ = vsag::DATATYPE_FLOAT32; + dataset->train_data_size_ = sizeof(float); + dataset->test_data_size_ = sizeof(float); + dataset->number_of_base_ = base->GetNumElements(); + dataset->number_of_query_ = queries->GetNumElements(); + dataset->dim_ = base->GetDim(); + dataset->train_shape_ = {dataset->number_of_base_, dataset->dim_}; + dataset->test_shape_ = {dataset->number_of_query_, dataset->dim_}; + dataset->neighbors_shape_ = + ground_truth == nullptr ? shape_t{0, 0} + : shape_t{ground_truth->GetNumElements(), ground_truth->GetDim()}; + dataset->file_path_ = ""; + + if (metric_type == "l2") { + dataset->metric_ = "euclidean"; + dataset->distance_func_ = + [](const void* left, const void* right, const void* dim) -> float { + return std::sqrt(vsag::L2Sqr(left, right, dim)); + }; + } else if (metric_type == "ip") { + dataset->metric_ = "ip"; + dataset->distance_func_ = vsag::InnerProductDistance; + } else if (metric_type == "cosine") { + dataset->metric_ = "angular"; + dataset->distance_func_ = + [](const void* left, const void* right, const void* dim) -> float { + return 1 - vsag::InnerProduct(left, right, dim) / + std::sqrt(vsag::InnerProduct(left, left, dim) * + vsag::InnerProduct(right, right, dim)); + }; + } else { + throw std::invalid_argument("unsupported in-memory eval metric_type: " + metric_type); + } + + dataset->InitializeTrainIds(base->GetIds()); + if (ground_truth != nullptr) { + const auto ground_truth_count = ground_truth->GetNumElements() * ground_truth->GetDim(); + for (int64_t i = 0; i < ground_truth_count; ++i) { + if (dataset->GetOneTrainById(ground_truth->GetIds()[i]) == nullptr) { + throw std::invalid_argument("ground_truth contains an id not present in base"); + } + } + } + return dataset; +} + EvalDatasetPtr EvalDataset::Load(const std::string& filename) { H5::H5File file(filename, H5F_ACC_RDONLY); @@ -222,14 +332,27 @@ EvalDataset::Load(const std::string& filename) { has_multi_vectors = datasets.count("train_multi_vectors") && datasets.count("test_multi_vectors") && datasets.count("train_vector_counts") && datasets.count("test_vector_counts"); + auto require_dataset = [&datasets](const std::string& name) { + if (datasets.count(name) == 0) { + throw std::invalid_argument("missing required HDF5 dataset '" + name + "'"); + } + }; if (not has_multi_vectors) { - assert(datasets.count("train")); - assert(datasets.count("test")); + require_dataset("train"); + require_dataset("test"); + } + require_dataset("neighbors"); + require_dataset("distances"); + const bool has_train_labels = datasets.count("train_labels") > 0; + const bool has_test_labels = datasets.count("test_labels") > 0; + if (has_train_labels != has_test_labels) { + throw std::invalid_argument("train_labels and test_labels must be provided together"); } - assert(datasets.count("neighbors")); - assert(datasets.count("distances")); - has_labels = datasets.count("train_labels") && datasets.count("test_labels"); + has_labels = has_train_labels; has_valid_ratio = datasets.count("valid_ratios") > 0; + if (has_valid_ratio && not has_labels) { + throw std::invalid_argument("valid_ratios requires train_labels and test_labels"); + } } auto obj = std::make_shared(); @@ -250,9 +373,11 @@ EvalDataset::Load(const std::string& filename) { obj->vector_type_ = SPARSE_VECTORS; } else if (type == "multi_vector") { obj->vector_type_ = MULTI_VECTORS; + } else { + throw std::invalid_argument("unsupported HDF5 dataset type '" + type + "'"); } } catch (H5::Exception& err) { - throw std::runtime_error("fail to read metric: there is no 'type' in the dataset"); + throw std::runtime_error("failed to read HDF5 dataset type attribute"); } } @@ -270,7 +395,16 @@ EvalDataset::Load(const std::string& filename) { shape_t train_shape(0, 0); shape_t test_shape(0, 0); auto neighbors_shape = get_shape(file, "neighbors"); + auto distances_shape = get_shape(file, "distances"); logger::debug("neighbors.shape: {}", to_string(neighbors_shape)); + if (neighbors_shape.first <= 0 || neighbors_shape.second <= 0) { + throw std::invalid_argument("neighbors must have shape (query_count, K) with K > 0"); + } + if (distances_shape != neighbors_shape) { + throw std::invalid_argument( + "distances shape must match neighbors shape: neighbors=" + to_string(neighbors_shape) + + ", distances=" + to_string(distances_shape)); + } if (obj->vector_type_ == MULTI_VECTORS) { // For multi-vector datasets, N and Q come from vector_counts @@ -283,7 +417,17 @@ EvalDataset::Load(const std::string& filename) { logger::debug("train.shape: {}", to_string(train_shape)); test_shape = get_shape(file, "test"); logger::debug("test.shape: {}", to_string(test_shape)); - assert(train_shape.second == test_shape.second); + if (train_shape.second != test_shape.second) { + throw std::invalid_argument("train and test vector dimensions must match: train=" + + to_string(train_shape) + ", test=" + to_string(test_shape)); + } + } + + if (train_shape.first <= 0 || test_shape.first <= 0) { + throw std::invalid_argument("train and test datasets must not be empty"); + } + if (obj->vector_type_ == DENSE_VECTORS && (train_shape.second <= 0 || test_shape.second <= 0)) { + throw std::invalid_argument("dense train and test vector dimensions must be positive"); } obj->train_shape_ = train_shape; @@ -598,6 +742,35 @@ EvalDataset::Load(const std::string& filename) { } } + if (obj->number_of_base_ <= 0 || obj->number_of_query_ <= 0 || obj->dim_ <= 0) { + throw std::invalid_argument( + "train and test datasets must contain vectors with a positive dimension"); + } + if (neighbors_shape.first != obj->number_of_query_) { + throw std::invalid_argument("neighbors row count must match query count: neighbors=" + + std::to_string(neighbors_shape.first) + + ", queries=" + std::to_string(obj->number_of_query_)); + } + if (has_labels) { + const auto train_labels_shape = get_shape(file, "train_labels"); + const auto test_labels_shape = get_shape(file, "test_labels"); + if (train_labels_shape.second != 0 || train_labels_shape.first != obj->number_of_base_) { + throw std::invalid_argument( + "train_labels must be one-dimensional with one label per " + "base vector"); + } + if (test_labels_shape.second != 0 || test_labels_shape.first != obj->number_of_query_) { + throw std::invalid_argument( + "test_labels must be one-dimensional with one label per query"); + } + } + if (has_valid_ratio) { + const auto valid_ratios_shape = get_shape(file, "valid_ratios"); + if (valid_ratios_shape.second != 0 || valid_ratios_shape.first <= 0) { + throw std::invalid_argument("valid_ratios must be a non-empty one-dimensional dataset"); + } + } + try { H5::Attribute attr = file.openAttribute("distance"); H5::StrType str_type = attr.getStrType(); @@ -625,6 +798,9 @@ EvalDataset::Load(const std::string& filename) { std::sqrt(vsag::InnerProduct(query1, query1, qty_ptr) * vsag::InnerProduct(query2, query2, qty_ptr)); }; + } else { + throw std::invalid_argument("unsupported HDF5 distance '" + metric + + "' for dense vectors"); } } else { if (metric == "ip") { @@ -635,8 +811,8 @@ EvalDataset::Load(const std::string& filename) { qty_ptr); }; } else { - throw std::runtime_error("no support for sparse vectors with " + metric + - " distance"); + throw std::invalid_argument("unsupported HDF5 distance '" + metric + + "' for sparse vectors"); } } } catch (H5::Exception& err) { @@ -650,6 +826,12 @@ EvalDataset::Load(const std::string& filename) { H5::DataSpace dataspace = dataset.getSpace(); H5::FloatType datatype(H5::PredType::NATIVE_INT64); dataset.read(obj->neighbors_.get(), datatype, dataspace); + const auto neighbor_count = neighbors_shape.first * neighbors_shape.second; + for (int64_t i = 0; i < neighbor_count; ++i) { + if (obj->neighbors_[i] < 0 || obj->neighbors_[i] >= obj->number_of_base_) { + throw std::invalid_argument("neighbors contains an id outside the base dataset"); + } + } } { @@ -687,6 +869,7 @@ EvalDataset::Load(const std::string& filename) { } } + obj->InitializeTrainIds(nullptr); return obj; } diff --git a/tools/eval/eval_dataset.h b/tools/eval/eval_dataset.h index 8aa85341fb..1db524a728 100644 --- a/tools/eval/eval_dataset.h +++ b/tools/eval/eval_dataset.h @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include "H5Cpp.h" @@ -42,12 +43,21 @@ class EvalDataset { static EvalDatasetPtr Load(const std::string& filename); + static EvalDatasetPtr + FromDatasets(const vsag::DatasetPtr& base, + const vsag::DatasetPtr& queries, + const vsag::DatasetPtr& ground_truth, + const std::string& metric_type); + static void Save(const EvalDatasetPtr& dataset, const std::string& filename); public: [[nodiscard]] const void* GetTrain() const { + if (base_dataset_ != nullptr) { + return base_dataset_->GetFloat32Vectors(); + } if (vector_type_ == DENSE_VECTORS) { return train_.get(); } @@ -59,6 +69,9 @@ class EvalDataset { [[nodiscard]] const void* GetTest() const { + if (query_dataset_ != nullptr) { + return query_dataset_->GetFloat32Vectors(); + } if (vector_type_ == DENSE_VECTORS) { return test_.get(); } @@ -111,7 +124,7 @@ class EvalDataset { [[nodiscard]] const void* GetOneTrain(int64_t id) const { if (vector_type_ == DENSE_VECTORS) { - return train_.get() + id * dim_ * train_data_size_; + return static_cast(GetTrain()) + id * dim_ * train_data_size_; } if (vector_type_ == SPARSE_VECTORS) { return sparse_train_.data() + id; @@ -122,7 +135,7 @@ class EvalDataset { [[nodiscard]] const void* GetOneTest(int64_t id) const { if (vector_type_ == DENSE_VECTORS) { - return test_.get() + id * dim_ * test_data_size_; + return static_cast(GetTest()) + id * dim_ * test_data_size_; } if (vector_type_ == SPARSE_VECTORS) { return sparse_test_.data() + id; @@ -132,17 +145,55 @@ class EvalDataset { [[nodiscard]] int64_t GetNearestNeighbor(int64_t i) const { - return neighbors_[i * neighbors_shape_.second]; + const auto* neighbors = GetNeighbors(i); + return neighbors == nullptr ? -1 : neighbors[0]; } - [[nodiscard]] int64_t* + [[nodiscard]] const int64_t* GetNeighbors(int64_t i) const { - return neighbors_.get() + i * neighbors_shape_.second; + if (ground_truth_dataset_ != nullptr) { + return ground_truth_dataset_->GetIds() + i * neighbors_shape_.second; + } + return neighbors_ == nullptr ? nullptr : neighbors_.get() + i * neighbors_shape_.second; } - [[nodiscard]] float* + [[nodiscard]] const float* GetDistances(int64_t i) const { - return distances_.get() + i * neighbors_shape_.second; + if (ground_truth_dataset_ != nullptr) { + const auto* distances = ground_truth_dataset_->GetDistances(); + return distances == nullptr ? nullptr : distances + i * neighbors_shape_.second; + } + return distances_ == nullptr ? nullptr : distances_.get() + i * neighbors_shape_.second; + } + + [[nodiscard]] const int64_t* + GetTrainIds() const { + if (base_dataset_ != nullptr && base_dataset_->GetIds() != nullptr) { + return base_dataset_->GetIds(); + } + return identity_train_ids_.get(); + } + + [[nodiscard]] const std::string* + GetTrainPaths() const { + return base_dataset_ == nullptr ? nullptr : base_dataset_->GetPaths(); + } + + [[nodiscard]] const std::string* + GetTestPaths() const { + return query_dataset_ == nullptr ? nullptr : query_dataset_->GetPaths(); + } + + [[nodiscard]] const void* + GetOneTrainById(int64_t id) const { + if (train_ids_are_identity_) { + if (id < 0 || id >= number_of_base_) { + return nullptr; + } + return GetOneTrain(id); + } + const auto found = train_id_to_row_.find(id); + return found == train_id_to_row_.end() ? nullptr : GetOneTrain(found->second); } [[nodiscard]] int64_t @@ -155,6 +206,16 @@ class EvalDataset { return number_of_query_; } + [[nodiscard]] uint64_t + GetGroundTruthK() const { + return neighbors_shape_.second > 0 ? static_cast(neighbors_shape_.second) : 0; + } + + [[nodiscard]] const std::string& + GetMetric() const { + return metric_; + } + [[nodiscard]] int64_t GetDim() const { return dim_; @@ -246,6 +307,9 @@ class EvalDataset { } private: + void + InitializeTrainIds(const int64_t* ids); + using shape_t = std::pair; static std::unordered_set get_datasets(const H5::H5File& file) { @@ -267,14 +331,15 @@ class EvalDataset { get_shape(const H5::H5File& file, const std::string& dataset_name) { H5::DataSet dataset = file.openDataSet(dataset_name); H5::DataSpace dataspace = dataset.getSpace(); - hsize_t dims_out[2]; - int ndims = dataspace.getSimpleExtentDims(dims_out, NULL); + const int ndims = dataspace.getSimpleExtentNdims(); + if (ndims != 1 && ndims != 2) { + throw std::runtime_error("unsupported dataset rank: " + std::to_string(ndims)); + } + hsize_t dims_out[2] = {0, 0}; + dataspace.getSimpleExtentDims(dims_out, NULL); if (ndims == 1) { return std::make_pair(dims_out[0], 0); } - if (ndims != 2) { - throw std::runtime_error("unsupported dataset rank: " + std::to_string(ndims)); - } return std::make_pair(dims_out[0], dims_out[1]); } @@ -291,6 +356,8 @@ class EvalDataset { std::shared_ptr test_; std::shared_ptr neighbors_; std::shared_ptr distances_; + std::shared_ptr identity_train_ids_; + bool train_ids_are_identity_{false}; std::shared_ptr train_labels_{nullptr}; std::shared_ptr test_labels_{nullptr}; std::shared_ptr valid_ratio_; @@ -308,6 +375,11 @@ class EvalDataset { std::string file_path_; std::string metric_; + vsag::DatasetPtr base_dataset_; + vsag::DatasetPtr query_dataset_; + vsag::DatasetPtr ground_truth_dataset_; + std::unordered_map train_id_to_row_; + std::vector sparse_train_; std::vector sparse_test_; diff --git a/tools/eval/eval_dataset_test.cpp b/tools/eval/eval_dataset_test.cpp index 7a66d52048..77fef9ea9e 100644 --- a/tools/eval/eval_dataset_test.cpp +++ b/tools/eval/eval_dataset_test.cpp @@ -18,11 +18,18 @@ #include #include +#include +#include #include #include #include #include +#include "case/eval_case.h" +#include "evaluator.h" +#include "monitor/recall_monitor.h" +#include "vsag/factory.h" + namespace { using vsag::SparseVector; @@ -141,6 +148,373 @@ TempPath(const std::string& tag) { } // namespace +TEST_CASE("EvalDataset builds a dense in-memory view with original ids", "[ut][eval_dataset]") { + constexpr int64_t dim = 2; + std::vector base_vectors{1.0F, 2.0F, 3.0F, 4.0F, 5.0F, 6.0F}; + std::vector base_ids{101, 7, 42}; + auto base = vsag::Dataset::Make(); + base->NumElements(3) + ->Dim(dim) + ->Ids(base_ids.data()) + ->Float32Vectors(base_vectors.data()) + ->Owner(false); + + std::vector query_vectors{1.0F, 2.0F, 5.0F, 6.0F}; + auto queries = vsag::Dataset::Make(); + queries->NumElements(2)->Dim(dim)->Float32Vectors(query_vectors.data())->Owner(false); + + std::vector ground_truth_ids{101, 7, 42, 7}; + auto ground_truth = vsag::Dataset::Make(); + ground_truth->NumElements(2)->Dim(2)->Ids(ground_truth_ids.data())->Owner(false); + + auto dataset = EvalDataset::FromDatasets(base, queries, ground_truth, "l2"); + REQUIRE(dataset->GetTrain() == base_vectors.data()); + REQUIRE(dataset->GetTest() == query_vectors.data()); + REQUIRE(dataset->GetTrainIds() == base_ids.data()); + REQUIRE(dataset->GetMetric() == "euclidean"); + REQUIRE(dataset->GetGroundTruthK() == 2); + REQUIRE(dataset->GetNeighbors(1)[0] == 42); + REQUIRE(static_cast(dataset->GetOneTrainById(7))[0] == 3.0F); + REQUIRE(dataset->GetOneTrainById(999) == nullptr); + + int64_t one_result = 101; + vsag::eval::SearchRecord record{ + &one_result, dataset->GetNeighbors(0), dataset.get(), dataset->GetOneTest(0), 1, 2}; + vsag::eval::RecallMonitor recall_monitor(1); + recall_monitor.SetMetrics("avg_recall"); + recall_monitor.Record(&record); + REQUIRE(recall_monitor.GetResult()["recall_avg"].get() == 0.5); + + auto without_ground_truth = EvalDataset::FromDatasets(base, queries, nullptr, "cosine"); + REQUIRE(without_ground_truth->GetGroundTruthK() == 0); + REQUIRE(without_ground_truth->GetNeighbors(0) == nullptr); +} + +TEST_CASE("EvalDataset rejects incomplete and incompatible HDF5 schemas", "[ut][eval_dataset]") { + const std::vector required_datasets{"train", "test", "neighbors", "distances"}; + for (const auto& missing : required_datasets) { + const auto path = TempPath("missing_" + missing); + { + H5::H5File file(path, H5F_ACC_TRUNC); + hsize_t dims[2] = {1, 1}; + H5::DataSpace space(2, dims); + for (const auto& name : required_datasets) { + if (name != missing) { + file.createDataSet(name, H5::PredType::NATIVE_FLOAT, space); + } + } + } + REQUIRE_THROWS_WITH( + EvalDataset::Load(path), + Catch::Matchers::ContainsSubstring("missing required HDF5 dataset '" + missing + "'")); + std::remove(path.c_str()); + } + + const auto path = TempPath("dimension_mismatch"); + { + H5::H5File file(path, H5F_ACC_TRUNC); + hsize_t train_dims[2] = {1, 2}; + hsize_t test_dims[2] = {1, 3}; + hsize_t result_dims[2] = {1, 1}; + H5::DataSpace train_space(2, train_dims); + H5::DataSpace test_space(2, test_dims); + H5::DataSpace result_space(2, result_dims); + file.createDataSet("train", H5::PredType::NATIVE_FLOAT, train_space); + file.createDataSet("test", H5::PredType::NATIVE_FLOAT, test_space); + file.createDataSet("neighbors", H5::PredType::NATIVE_INT64, result_space); + file.createDataSet("distances", H5::PredType::NATIVE_FLOAT, result_space); + } + REQUIRE_THROWS_WITH( + EvalDataset::Load(path), + Catch::Matchers::ContainsSubstring("train and test vector dimensions must match")); + std::remove(path.c_str()); + + const auto result_shape_path = TempPath("result_shape_mismatch"); + { + H5::H5File file(result_shape_path, H5F_ACC_TRUNC); + hsize_t vector_dims[2] = {2, 2}; + hsize_t neighbor_dims[2] = {2, 1}; + hsize_t distance_dims[2] = {1, 1}; + H5::DataSpace vector_space(2, vector_dims); + H5::DataSpace neighbor_space(2, neighbor_dims); + H5::DataSpace distance_space(2, distance_dims); + file.createDataSet("train", H5::PredType::NATIVE_FLOAT, vector_space); + file.createDataSet("test", H5::PredType::NATIVE_FLOAT, vector_space); + file.createDataSet("neighbors", H5::PredType::NATIVE_INT64, neighbor_space); + file.createDataSet("distances", H5::PredType::NATIVE_FLOAT, distance_space); + } + REQUIRE_THROWS_WITH( + EvalDataset::Load(result_shape_path), + Catch::Matchers::ContainsSubstring("distances shape must match neighbors shape")); + std::remove(result_shape_path.c_str()); + + const auto query_count_path = TempPath("query_count_mismatch"); + { + H5::H5File file(query_count_path, H5F_ACC_TRUNC); + hsize_t train_dims[2] = {2, 2}; + hsize_t test_dims[2] = {2, 2}; + hsize_t result_dims[2] = {1, 1}; + H5::DataSpace train_space(2, train_dims); + H5::DataSpace test_space(2, test_dims); + H5::DataSpace result_space(2, result_dims); + file.createDataSet("train", H5::PredType::NATIVE_FLOAT, train_space); + file.createDataSet("test", H5::PredType::NATIVE_FLOAT, test_space); + file.createDataSet("neighbors", H5::PredType::NATIVE_INT64, result_space); + file.createDataSet("distances", H5::PredType::NATIVE_FLOAT, result_space); + } + REQUIRE_THROWS_WITH( + EvalDataset::Load(query_count_path), + Catch::Matchers::ContainsSubstring("neighbors row count must match query count")); + std::remove(query_count_path.c_str()); + + const auto unsupported_type_path = TempPath("unsupported_type"); + { + H5::H5File file(unsupported_type_path, H5F_ACC_TRUNC); + hsize_t dims[2] = {1, 1}; + H5::DataSpace space(2, dims); + file.createDataSet("train", H5::PredType::NATIVE_FLOAT, space); + file.createDataSet("test", H5::PredType::NATIVE_FLOAT, space); + file.createDataSet("neighbors", H5::PredType::NATIVE_INT64, space); + file.createDataSet("distances", H5::PredType::NATIVE_FLOAT, space); + H5::StrType string_type(H5::PredType::C_S1, H5T_VARIABLE); + auto attribute = file.createAttribute("type", string_type, H5::DataSpace(H5S_SCALAR)); + std::string type = "unsupported"; + attribute.write(string_type, type); + } + REQUIRE_THROWS_WITH( + EvalDataset::Load(unsupported_type_path), + Catch::Matchers::ContainsSubstring("unsupported HDF5 dataset type 'unsupported'")); + std::remove(unsupported_type_path.c_str()); + + const auto unsupported_rank_path = TempPath("unsupported_rank"); + { + H5::H5File file(unsupported_rank_path, H5F_ACC_TRUNC); + hsize_t train_dims[3] = {1, 1, 1}; + hsize_t matrix_dims[2] = {1, 1}; + H5::DataSpace train_space(3, train_dims); + H5::DataSpace matrix_space(2, matrix_dims); + file.createDataSet("train", H5::PredType::NATIVE_FLOAT, train_space); + file.createDataSet("test", H5::PredType::NATIVE_FLOAT, matrix_space); + file.createDataSet("neighbors", H5::PredType::NATIVE_INT64, matrix_space); + file.createDataSet("distances", H5::PredType::NATIVE_FLOAT, matrix_space); + } + REQUIRE_THROWS_WITH(EvalDataset::Load(unsupported_rank_path), + Catch::Matchers::ContainsSubstring("unsupported dataset rank: 3")); + std::remove(unsupported_rank_path.c_str()); + + const auto label_shape_path = TempPath("label_shape_mismatch"); + { + H5::H5File file(label_shape_path, H5F_ACC_TRUNC); + hsize_t train_dims[2] = {2, 2}; + hsize_t test_dims[2] = {1, 2}; + hsize_t result_dims[2] = {1, 1}; + hsize_t train_label_dims[1] = {1}; + hsize_t test_label_dims[1] = {1}; + H5::DataSpace train_space(2, train_dims); + H5::DataSpace test_space(2, test_dims); + H5::DataSpace result_space(2, result_dims); + H5::DataSpace train_label_space(1, train_label_dims); + H5::DataSpace test_label_space(1, test_label_dims); + file.createDataSet("train", H5::PredType::NATIVE_FLOAT, train_space); + file.createDataSet("test", H5::PredType::NATIVE_FLOAT, test_space); + file.createDataSet("neighbors", H5::PredType::NATIVE_INT64, result_space); + file.createDataSet("distances", H5::PredType::NATIVE_FLOAT, result_space); + file.createDataSet("train_labels", H5::PredType::NATIVE_INT64, train_label_space); + file.createDataSet("test_labels", H5::PredType::NATIVE_INT64, test_label_space); + } + REQUIRE_THROWS_WITH(EvalDataset::Load(label_shape_path), + Catch::Matchers::ContainsSubstring( + "train_labels must be one-dimensional with one label per base vector")); + std::remove(label_shape_path.c_str()); + + const auto unsupported_distance_path = TempPath("unsupported_distance"); + { + H5::H5File file(unsupported_distance_path, H5F_ACC_TRUNC); + hsize_t dims[2] = {1, 1}; + H5::DataSpace space(2, dims); + file.createDataSet("train", H5::PredType::NATIVE_FLOAT, space); + file.createDataSet("test", H5::PredType::NATIVE_FLOAT, space); + file.createDataSet("neighbors", H5::PredType::NATIVE_INT64, space); + file.createDataSet("distances", H5::PredType::NATIVE_FLOAT, space); + H5::StrType string_type(H5::PredType::C_S1, H5T_VARIABLE); + auto attribute = file.createAttribute("distance", string_type, H5::DataSpace(H5S_SCALAR)); + std::string distance = "unsupported"; + attribute.write(string_type, distance); + } + REQUIRE_THROWS_WITH(EvalDataset::Load(unsupported_distance_path), + Catch::Matchers::ContainsSubstring( + "unsupported HDF5 distance 'unsupported' for dense vectors")); + std::remove(unsupported_distance_path.c_str()); +} + +TEST_CASE("EvaluateSearch validates inputs and propagates search errors", "[ut][eval_dataset]") { + constexpr int64_t dim = 2; + std::vector base_vectors{0.0F, 0.0F, 1.0F, 0.0F, 0.0F, 1.0F, 1.0F, 1.0F}; + std::vector base_ids{0, 1, 2, 3}; + auto base = vsag::Dataset::Make(); + base->NumElements(4) + ->Dim(dim) + ->Ids(base_ids.data()) + ->Float32Vectors(base_vectors.data()) + ->Owner(false); + + std::vector query_vectors{0.0F, 0.0F, 1.0F, 1.0F}; + auto queries = vsag::Dataset::Make(); + queries->NumElements(2)->Dim(dim)->Float32Vectors(query_vectors.data())->Owner(false); + + std::vector ground_truth_ids{0, 3}; + auto ground_truth = vsag::Dataset::Make(); + ground_truth->NumElements(2)->Dim(1)->Ids(ground_truth_ids.data())->Owner(false); + auto dataset = EvalDataset::FromDatasets(base, queries, ground_truth, "l2"); + + const std::string create_params = R"( + { + "dim": 2, + "dtype": "float32", + "metric_type": "l2", + "index_param": { + "base_quantization_type": "fp32", + "max_degree": 8, + "ef_construction": 20 + } + })"; + auto created = vsag::Factory::CreateIndex("hgraph", create_params); + REQUIRE(created.has_value()); + auto index = created.value(); + REQUIRE(index->Build(base).has_value()); + + vsag::eval::EvalConfig config; + config.index_name = "hgraph"; + config.build_param = create_params; + config.search_param = R"({"hgraph":{"ef_search":8}})"; + config.top_k = 1; + config.search_query_count = 2; + config.num_threads_searching = 2; + config.enable_recall = false; + config.enable_percent_recall = false; + config.enable_qps = true; + config.enable_tps = false; + config.enable_memory = false; + config.enable_latency = false; + config.enable_percent_latency = false; + + const auto qps_only = vsag::eval::EvaluateSearch(index, dataset, config); + REQUIRE(qps_only.contains("qps")); + REQUIRE(qps_only["measurement_sample_count"].get() == 2); + + config.search_param = R"({"hgraph":{"ef_search":0}})"; + REQUIRE_THROWS_WITH( + vsag::eval::EvaluateSearch(index, dataset, config), + Catch::Matchers::ContainsSubstring("query error: ef_search(0) must be at least 1")); + + config.search_param = R"({"hgraph":{"ef_search":8}})"; + config.top_k = 0; + REQUIRE_THROWS_WITH(vsag::eval::EvaluateSearch(index, dataset, config), + "evaluation top_k must be positive"); + + config.top_k = 1; + config.search_mode = "range"; + REQUIRE_THROWS_WITH(vsag::eval::EvaluateSearch(index, dataset, config), + "in-memory evaluation supports only knn search mode"); + + config.search_mode = "knn"; + config.num_threads_searching = 0; + REQUIRE_THROWS_WITH(vsag::eval::EvaluateSearch(index, dataset, config), + "evaluation search thread count must be positive"); + + config.num_threads_searching = 2; + config.search_query_count = 0; + REQUIRE_THROWS_WITH(vsag::eval::EvaluateSearch(index, dataset, config), + "evaluation search query count must be positive"); + + config.search_query_count = 2; + config.top_k = 1; + config.enable_recall = true; + auto without_ground_truth = EvalDataset::FromDatasets(base, queries, nullptr, "l2"); + REQUIRE_THROWS_WITH(vsag::eval::EvaluateSearch(index, without_ground_truth, config), + "evaluation ground truth must contain at least top_k neighbors per query"); + + config.top_k = 2; + REQUIRE_THROWS_WITH(vsag::eval::EvaluateSearch(index, dataset, config), + "evaluation ground truth must contain at least top_k neighbors per query"); +} + +TEST_CASE("EvalCase builds and searches an in-memory dataset with original ids", + "[ut][eval_dataset]") { + constexpr int64_t dim = 2; + std::vector base_vectors{1.0F, 2.0F, 3.0F, 4.0F, 5.0F, 6.0F}; + std::vector base_ids{101, 7, 42}; + auto base = vsag::Dataset::Make(); + base->NumElements(3) + ->Dim(dim) + ->Ids(base_ids.data()) + ->Float32Vectors(base_vectors.data()) + ->Owner(false); + + std::vector query_vectors{5.0F, 6.0F}; + auto queries = vsag::Dataset::Make(); + queries->NumElements(1)->Dim(dim)->Float32Vectors(query_vectors.data())->Owner(false); + + std::vector ground_truth_ids{42}; + auto ground_truth = vsag::Dataset::Make(); + ground_truth->NumElements(1)->Dim(1)->Ids(ground_truth_ids.data())->Owner(false); + auto dataset = EvalDataset::FromDatasets(base, queries, ground_truth, "l2"); + + vsag::eval::EvalConfig config; + config.index_name = "brute_force"; + config.index_path = "/tmp/eval_dataset_memory.index"; + config.build_param = R"({ + "dtype": "float32", + "metric_type": "l2", + "dim": 2, + "index_param": { + "base_quantization_type": "fp32", + "store_raw_vector": true + } + })"; + config.top_k = 1; + config.search_query_count = 1; + config.enable_memory = false; + std::remove(config.index_path.c_str()); + + auto build = vsag::eval::EvalCase::MakeInstance(config, "build", dataset); + REQUIRE(build->Run()["action"] == "build"); + auto search = vsag::eval::EvalCase::MakeInstance(config, "search", dataset); + const auto result = search->Run(); + REQUIRE(result["action"] == "search"); + REQUIRE(result["recall_avg"].get() == 1.0); + REQUIRE(result["statistics_query_count"].get() == 1); + REQUIRE(std::isfinite(result["qps"].get())); + REQUIRE(std::isfinite(result["latency_avg(ms)"].get())); + REQUIRE(std::isfinite(result["latency_detail(ms)"]["p99"].get())); + + std::vector concurrent_query_vectors{5.0F, 6.0F, 1.0F, 2.0F}; + auto concurrent_queries = vsag::Dataset::Make(); + concurrent_queries->NumElements(2) + ->Dim(dim) + ->Float32Vectors(concurrent_query_vectors.data()) + ->Owner(false); + std::vector concurrent_ground_truth_ids{42, 101}; + auto concurrent_ground_truth = vsag::Dataset::Make(); + concurrent_ground_truth->NumElements(2) + ->Dim(1) + ->Ids(concurrent_ground_truth_ids.data()) + ->Owner(false); + auto concurrent_dataset = + EvalDataset::FromDatasets(base, concurrent_queries, concurrent_ground_truth, "l2"); + config.search_query_count = 2; + config.num_threads_searching = 2; + auto concurrent_search = + vsag::eval::EvalCase::MakeInstance(config, "search", concurrent_dataset); + const auto concurrent_result = concurrent_search->Run(); + REQUIRE(concurrent_result["recall_avg"].get() == 1.0); + REQUIRE(concurrent_result["statistics_query_count"].get() == 2); + REQUIRE(std::isfinite(concurrent_result["qps"].get())); + REQUIRE(std::isfinite(concurrent_result["latency_avg(ms)"].get())); + REQUIRE(std::isfinite(concurrent_result["latency_detail(ms)"]["p99"].get())); + std::remove(config.index_path.c_str()); +} + TEST_CASE("EvalDataset sparse round-trip without token sequences", "[ut][eval_dataset]") { auto path = TempPath("nosqry"); { @@ -172,6 +546,9 @@ TEST_CASE("EvalDataset sparse round-trip without token sequences", "[ut][eval_da REQUIRE(loaded->GetNumberOfBase() == 3); REQUIRE(loaded->GetNumberOfQuery() == 2); const auto* train = static_cast(loaded->GetTrain()); + REQUIRE(loaded->GetTrainIds()[0] == 0); + REQUIRE(loaded->GetTrainIds()[2] == 2); + REQUIRE(loaded->GetOneTrainById(1) == train + 1); for (int i = 0; i < 3; ++i) { REQUIRE(train[i].token_seq_len_ == 0); REQUIRE(train[i].token_sequence_ == nullptr); diff --git a/tools/eval/evaluator.cpp b/tools/eval/evaluator.cpp new file mode 100644 index 0000000000..2c3a252aca --- /dev/null +++ b/tools/eval/evaluator.cpp @@ -0,0 +1,79 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "evaluator.h" + +#include + +#include "case/build_eval_case.h" +#include "case/search_eval_case.h" + +namespace vsag::eval { + +namespace { + +void +validate(const IndexPtr& index, const EvalDatasetPtr& dataset) { + if (index == nullptr) { + throw std::invalid_argument("evaluation index must not be null"); + } + if (dataset == nullptr) { + throw std::invalid_argument("evaluation dataset must not be null"); + } +} + +void +validate_search(const EvalDatasetPtr& dataset, const EvalConfig& config) { + if (config.search_mode != "knn") { + throw std::invalid_argument("in-memory evaluation supports only knn search mode"); + } + if (config.top_k <= 0) { + throw std::invalid_argument("evaluation top_k must be positive"); + } + if (dataset->GetNumberOfQuery() <= 0) { + throw std::invalid_argument("evaluation dataset must contain at least one query"); + } + if (config.num_threads_searching <= 0) { + throw std::invalid_argument("evaluation search thread count must be positive"); + } + if (config.search_query_count == 0) { + throw std::invalid_argument("evaluation search query count must be positive"); + } + if (config.enable_recall or config.enable_percent_recall) { + const auto requested_top_k = static_cast(config.top_k); + if (dataset->GetGroundTruthK() < requested_top_k || dataset->GetNeighbors(0) == nullptr) { + throw std::invalid_argument( + "evaluation ground truth must contain at least top_k neighbors per query"); + } + } +} + +} // namespace + +JsonType +EvaluateBuild(const IndexPtr& index, const EvalDatasetPtr& dataset, const EvalConfig& config) { + validate(index, dataset); + BuildEvalCase eval_case("", "", index, config, dataset); + return eval_case.RunInMemory(); +} + +JsonType +EvaluateSearch(const IndexPtr& index, const EvalDatasetPtr& dataset, const EvalConfig& config) { + validate(index, dataset); + validate_search(dataset, config); + SearchEvalCase eval_case("", "", index, config, dataset); + return eval_case.RunInMemory(); +} + +} // namespace vsag::eval diff --git a/tools/eval/evaluator.h b/tools/eval/evaluator.h new file mode 100644 index 0000000000..aa5bccd0eb --- /dev/null +++ b/tools/eval/evaluator.h @@ -0,0 +1,32 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "eval_config.h" +#include "eval_dataset.h" +#include "nlohmann/json.hpp" +#include "vsag/index.h" + +namespace vsag::eval { + +using JsonType = nlohmann::json; + +JsonType +EvaluateBuild(const IndexPtr& index, const EvalDatasetPtr& dataset, const EvalConfig& config); + +JsonType +EvaluateSearch(const IndexPtr& index, const EvalDatasetPtr& dataset, const EvalConfig& config); + +} // namespace vsag::eval diff --git a/tools/eval/monitor/monitor.h b/tools/eval/monitor/monitor.h index b963e1e975..cf4cb0e318 100644 --- a/tools/eval/monitor/monitor.h +++ b/tools/eval/monitor/monitor.h @@ -15,6 +15,7 @@ #pragma once +#include #include #include #include @@ -24,6 +25,17 @@ namespace vsag::eval { +class EvalDataset; + +struct SearchRecord { + const int64_t* neighbors{nullptr}; + const int64_t* ground_truth_neighbors{nullptr}; + EvalDataset* dataset{nullptr}; + const void* query_data{nullptr}; + uint64_t result_count{0}; + uint64_t requested_top_k{0}; +}; + class Monitor { public: using JsonType = nlohmann::json; diff --git a/tools/eval/monitor/recall_monitor.cpp b/tools/eval/monitor/recall_monitor.cpp index bbf7db4e28..c2d2690df9 100644 --- a/tools/eval/monitor/recall_monitor.cpp +++ b/tools/eval/monitor/recall_monitor.cpp @@ -15,8 +15,11 @@ #include "recall_monitor.h" +#include +#include #include -#include +#include +#include #include "../eval_dataset.h" namespace vsag::eval { @@ -26,13 +29,13 @@ static const double THRESHOLD_ERROR = 2e-6; static double get_recall(const float* distances, const float* ground_truth_distances, - size_t recall_num, - size_t top_k) { + uint64_t recall_num, + uint64_t top_k) { std::vector gt_distances(ground_truth_distances, ground_truth_distances + top_k); std::sort(gt_distances.begin(), gt_distances.end()); float threshold = gt_distances[top_k - 1]; - size_t count = 0; - for (size_t i = 0; i < recall_num; ++i) { + uint64_t count = 0; + for (uint64_t i = 0; i < recall_num; ++i) { if (distances[i] <= threshold + THRESHOLD_ERROR) { ++count; } @@ -65,28 +68,42 @@ void RecallMonitor::Record(void* input) { std::lock_guard lock(record_mutex_); - auto [neighbors, gt_neighbors, dataset, query_data, topk] = - *(reinterpret_cast*>( - input)); - size_t dim = dataset->GetDim(); + const auto& record = *static_cast(input); + const auto* neighbors = record.neighbors; + const auto* gt_neighbors = record.ground_truth_neighbors; + auto* dataset = record.dataset; + const auto* query_data = record.query_data; + const auto requested_top_k = record.requested_top_k; + const auto result_count = std::min(record.result_count, requested_top_k); + if (requested_top_k == 0) { + this->recall_records_.emplace_back(0.0); + return; + } + uint64_t dim = dataset->GetDim(); auto distance_func = dataset->GetDistanceFunc(); - auto gt_distances = std::shared_ptr(new float[topk]); - auto distances = std::shared_ptr(new float[topk]); - for (int i = 0; i < topk; ++i) { - distances[i] = distance_func(query_data, dataset->GetOneTrain(neighbors[i]), &dim); - gt_distances[i] = distance_func(query_data, dataset->GetOneTrain(gt_neighbors[i]), &dim); + std::vector gt_distances(requested_top_k); + std::vector distances(result_count); + for (uint64_t i = 0; i < result_count; ++i) { + const auto* result_vector = dataset->GetOneTrainById(neighbors[i]); + distances[i] = result_vector == nullptr ? std::numeric_limits::infinity() + : distance_func(query_data, result_vector, &dim); } - - float val = 0; - if (topk != 0) { - val = get_recall(distances.get(), gt_distances.get(), topk, topk); + for (uint64_t i = 0; i < requested_top_k; ++i) { + const auto* ground_truth_vector = dataset->GetOneTrainById(gt_neighbors[i]); + gt_distances[i] = ground_truth_vector == nullptr + ? std::numeric_limits::infinity() + : distance_func(query_data, ground_truth_vector, &dim); } + + const double val = + get_recall(distances.data(), gt_distances.data(), result_count, requested_top_k); this->recall_records_.emplace_back(val); } void RecallMonitor::SetMetrics(std::string metric) { this->metrics_.emplace_back(std::move(metric)); } + void RecallMonitor::cal_and_set_result(const std::string& metric, Monitor::JsonType& result) { if (metric == "avg_recall") { From 07e0918f363385e1cd0ea4656c9d3485c79f1093 Mon Sep 17 00:00:00 2001 From: jc543239 Date: Wed, 29 Jul 2026 14:09:10 +0000 Subject: [PATCH 2/8] fix: address AutoTune review and ASan failures Signed-off-by: jc543239 Assisted-by: Codex:gpt-5 --- tools/eval/CMakeLists.txt | 1 - tools/eval/case/build_eval_case.cpp | 11 ++++++----- tools/eval/eval_dataset_test.cpp | 10 +++++++++- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tools/eval/CMakeLists.txt b/tools/eval/CMakeLists.txt index 7fb630dd69..78cbbd25f6 100644 --- a/tools/eval/CMakeLists.txt +++ b/tools/eval/CMakeLists.txt @@ -68,7 +68,6 @@ if (ENABLE_TESTS) target_link_libraries (eval_dataset_test PRIVATE Catch2::Catch2WithMain eval_obj - simd vsag vsag_eval_common ${HDF5_CPP_STATIC_LIBRARY} diff --git a/tools/eval/case/build_eval_case.cpp b/tools/eval/case/build_eval_case.cpp index 3bd135fac1..e11e1ed35d 100644 --- a/tools/eval/case/build_eval_case.cpp +++ b/tools/eval/case/build_eval_case.cpp @@ -40,10 +40,8 @@ BuildEvalCase::init_monitors() { auto memory_peak_monitor = std::make_shared("build"); this->monitors_.emplace_back(std::move(memory_peak_monitor)); } - if (config_.enable_tps) { - auto duration_monitor = std::make_shared(); - this->monitors_.emplace_back(std::move(duration_monitor)); - } + auto duration_monitor = std::make_shared(); + this->monitors_.emplace_back(std::move(duration_monitor)); } JsonType @@ -111,7 +109,10 @@ BuildEvalCase::process_result() { EvalCase::MergeJsonType(one_result, eval_result); } result = eval_result; - result["tps"] = double(this->dataset_ptr_->GetNumberOfBase()) / double(result["duration(s)"]); + if (config_.enable_tps) { + result["tps"] = + double(this->dataset_ptr_->GetNumberOfBase()) / double(result["duration(s)"]); + } EvalCase::MergeJsonType(this->basic_info_, result); result["index_info"] = JsonType::parse(config_.build_param); result["action"] = "build"; diff --git a/tools/eval/eval_dataset_test.cpp b/tools/eval/eval_dataset_test.cpp index 77fef9ea9e..0a34dcb9df 100644 --- a/tools/eval/eval_dataset_test.cpp +++ b/tools/eval/eval_dataset_test.cpp @@ -381,7 +381,15 @@ TEST_CASE("EvaluateSearch validates inputs and propagates search errors", "[ut][ auto created = vsag::Factory::CreateIndex("hgraph", create_params); REQUIRE(created.has_value()); auto index = created.value(); - REQUIRE(index->Build(base).has_value()); + + vsag::eval::EvalConfig build_config; + build_config.index_name = "hgraph"; + build_config.build_param = create_params; + build_config.enable_tps = false; + build_config.enable_memory = false; + const auto build_result = vsag::eval::EvaluateBuild(index, dataset, build_config); + REQUIRE(build_result.contains("duration(s)")); + REQUIRE_FALSE(build_result.contains("tps")); vsag::eval::EvalConfig config; config.index_name = "hgraph"; From 5a65190a3a4ca14a15854822c51cfd8182fcd2bf Mon Sep 17 00:00:00 2001 From: jc543239 Date: Wed, 29 Jul 2026 15:19:57 +0000 Subject: [PATCH 3/8] fix: harden in-memory eval contracts Signed-off-by: jc543239 Assisted-by: Codex:gpt-5 --- tools/eval/case/search_eval_case.cpp | 3 ++- tools/eval/eval_dataset.cpp | 4 ++++ tools/eval/eval_dataset_test.cpp | 9 ++++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tools/eval/case/search_eval_case.cpp b/tools/eval/case/search_eval_case.cpp index 0219ea3bb9..af4033dfc8 100644 --- a/tools/eval/case/search_eval_case.cpp +++ b/tools/eval/case/search_eval_case.cpp @@ -417,7 +417,8 @@ SearchEvalCase::process_result() { } result["action"] = "search"; result["search_mode"] = config_.search_mode; - result["index_info"] = JsonType::parse(config_.build_param); + result["index_info"] = + config_.build_param.empty() ? JsonType::object() : JsonType::parse(config_.build_param); result["search_param"] = config_.search_param; result["index"] = config_.index_name; result["index_memory(B)"] = this->index_->GetMemoryUsage(); diff --git a/tools/eval/eval_dataset.cpp b/tools/eval/eval_dataset.cpp index e5da1b2ead..4b7660391e 100644 --- a/tools/eval/eval_dataset.cpp +++ b/tools/eval/eval_dataset.cpp @@ -957,6 +957,10 @@ serialize_token_sequences(const std::vector& vectors, void EvalDataset::Save(const EvalDatasetPtr& dataset, const std::string& filename) { + if (dataset->base_dataset_ != nullptr || dataset->query_dataset_ != nullptr || + dataset->ground_truth_dataset_ != nullptr) { + throw std::invalid_argument("saving an in-memory EvalDataset view is not supported"); + } H5File file(filename, H5F_ACC_TRUNC); // write vector type attribute diff --git a/tools/eval/eval_dataset_test.cpp b/tools/eval/eval_dataset_test.cpp index 0a34dcb9df..3c75969d31 100644 --- a/tools/eval/eval_dataset_test.cpp +++ b/tools/eval/eval_dataset_test.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -188,6 +189,11 @@ TEST_CASE("EvalDataset builds a dense in-memory view with original ids", "[ut][e auto without_ground_truth = EvalDataset::FromDatasets(base, queries, nullptr, "cosine"); REQUIRE(without_ground_truth->GetGroundTruthK() == 0); REQUIRE(without_ground_truth->GetNeighbors(0) == nullptr); + + const auto save_path = TempPath("in_memory_save"); + REQUIRE_THROWS_WITH(EvalDataset::Save(dataset, save_path), + "saving an in-memory EvalDataset view is not supported"); + REQUIRE_FALSE(std::filesystem::exists(save_path)); } TEST_CASE("EvalDataset rejects incomplete and incompatible HDF5 schemas", "[ut][eval_dataset]") { @@ -393,7 +399,6 @@ TEST_CASE("EvaluateSearch validates inputs and propagates search errors", "[ut][ vsag::eval::EvalConfig config; config.index_name = "hgraph"; - config.build_param = create_params; config.search_param = R"({"hgraph":{"ef_search":8}})"; config.top_k = 1; config.search_query_count = 2; @@ -409,6 +414,8 @@ TEST_CASE("EvaluateSearch validates inputs and propagates search errors", "[ut][ const auto qps_only = vsag::eval::EvaluateSearch(index, dataset, config); REQUIRE(qps_only.contains("qps")); REQUIRE(qps_only["measurement_sample_count"].get() == 2); + REQUIRE(qps_only["index_info"].is_object()); + REQUIRE(qps_only["index_info"].empty()); config.search_param = R"({"hgraph":{"ef_search":0}})"; REQUIRE_THROWS_WITH( From 4e3e8fa9629947e7344455a36f5d4e5ee7bcd781 Mon Sep 17 00:00:00 2001 From: jc543239 Date: Thu, 30 Jul 2026 03:16:00 +0000 Subject: [PATCH 4/8] fix: handle eval edge cases Signed-off-by: jc543239 Assisted-by: Codex:gpt-5 --- tools/eval/case/build_eval_case.cpp | 3 +- tools/eval/eval_dataset.cpp | 8 ++-- tools/eval/eval_dataset_test.cpp | 57 +++++++++++++++++++++-------- 3 files changed, 48 insertions(+), 20 deletions(-) diff --git a/tools/eval/case/build_eval_case.cpp b/tools/eval/case/build_eval_case.cpp index e11e1ed35d..749776ece5 100644 --- a/tools/eval/case/build_eval_case.cpp +++ b/tools/eval/case/build_eval_case.cpp @@ -114,7 +114,8 @@ BuildEvalCase::process_result() { double(this->dataset_ptr_->GetNumberOfBase()) / double(result["duration(s)"]); } EvalCase::MergeJsonType(this->basic_info_, result); - result["index_info"] = JsonType::parse(config_.build_param); + result["index_info"] = + config_.build_param.empty() ? JsonType::object() : JsonType::parse(config_.build_param); result["action"] = "build"; result["index"] = config_.index_name; result["index_memory(B)"] = this->index_->GetMemoryUsage(); diff --git a/tools/eval/eval_dataset.cpp b/tools/eval/eval_dataset.cpp index 4b7660391e..3886030f79 100644 --- a/tools/eval/eval_dataset.cpp +++ b/tools/eval/eval_dataset.cpp @@ -742,9 +742,11 @@ EvalDataset::Load(const std::string& filename) { } } - if (obj->number_of_base_ <= 0 || obj->number_of_query_ <= 0 || obj->dim_ <= 0) { - throw std::invalid_argument( - "train and test datasets must contain vectors with a positive dimension"); + if (obj->number_of_base_ <= 0 || obj->number_of_query_ <= 0) { + throw std::invalid_argument("train and test datasets must contain vectors"); + } + if (obj->vector_type_ != SPARSE_VECTORS && obj->dim_ <= 0) { + throw std::invalid_argument("train and test vector dimensions must be positive"); } if (neighbors_shape.first != obj->number_of_query_) { throw std::invalid_argument("neighbors row count must match query count: neighbors=" + diff --git a/tools/eval/eval_dataset_test.cpp b/tools/eval/eval_dataset_test.cpp index 3c75969d31..d3eccaba43 100644 --- a/tools/eval/eval_dataset_test.cpp +++ b/tools/eval/eval_dataset_test.cpp @@ -38,28 +38,29 @@ using vsag::eval::EvalDataset; using vsag::eval::EvalDatasetPtr; EvalDatasetPtr -BuildSparseDataset(bool with_token_sequences) { +BuildSparseDataset(bool with_token_sequences, bool all_empty = false) { // Build a tiny sparse dataset (3 train, 2 test) and optionally attach // the original tokenized term-id sequences. auto ds = std::make_shared(); // Sparse train vectors. std::vector train(3); - train[0].len_ = 2; - train[0].ids_ = new uint32_t[2]{1, 5}; - train[0].vals_ = new float[2]{0.5f, 1.0f}; - train[1].len_ = 1; - train[1].ids_ = new uint32_t[1]{3}; - train[1].vals_ = new float[1]{0.25f}; - train[2].len_ = 0; // empty sparse vector - std::vector test(2); - test[0].len_ = 2; - test[0].ids_ = new uint32_t[2]{2, 7}; - test[0].vals_ = new float[2]{0.4f, 0.6f}; - test[1].len_ = 1; - test[1].ids_ = new uint32_t[1]{9}; - test[1].vals_ = new float[1]{1.0f}; + if (not all_empty) { + train[0].len_ = 2; + train[0].ids_ = new uint32_t[2]{1, 5}; + train[0].vals_ = new float[2]{0.5f, 1.0f}; + train[1].len_ = 1; + train[1].ids_ = new uint32_t[1]{3}; + train[1].vals_ = new float[1]{0.25f}; + + test[0].len_ = 2; + test[0].ids_ = new uint32_t[2]{2, 7}; + test[0].vals_ = new float[2]{0.4f, 0.6f}; + test[1].len_ = 1; + test[1].ids_ = new uint32_t[1]{9}; + test[1].vals_ = new float[1]{1.0f}; + } if (with_token_sequences) { train[0].token_seq_len_ = 4; @@ -390,12 +391,13 @@ TEST_CASE("EvaluateSearch validates inputs and propagates search errors", "[ut][ vsag::eval::EvalConfig build_config; build_config.index_name = "hgraph"; - build_config.build_param = create_params; build_config.enable_tps = false; build_config.enable_memory = false; const auto build_result = vsag::eval::EvaluateBuild(index, dataset, build_config); REQUIRE(build_result.contains("duration(s)")); REQUIRE_FALSE(build_result.contains("tps")); + REQUIRE(build_result["index_info"].is_object()); + REQUIRE(build_result["index_info"].empty()); vsag::eval::EvalConfig config; config.index_name = "hgraph"; @@ -571,6 +573,29 @@ TEST_CASE("EvalDataset sparse round-trip without token sequences", "[ut][eval_da std::remove(path.c_str()); } +TEST_CASE("EvalDataset sparse round-trip preserves all-empty records", "[ut][eval_dataset]") { + auto path = TempPath("all_empty_sparse"); + { + auto ds = BuildSparseDataset(/*with_token_sequences=*/false, /*all_empty=*/true); + EvalDataset::Save(ds, path); + } + + auto loaded = EvalDataset::Load(path); + REQUIRE(loaded->GetVectorType() == vsag::SPARSE_VECTORS); + REQUIRE(loaded->GetNumberOfBase() == 3); + REQUIRE(loaded->GetNumberOfQuery() == 2); + REQUIRE(loaded->GetDim() == 0); + const auto* train = static_cast(loaded->GetTrain()); + const auto* test = static_cast(loaded->GetTest()); + for (uint64_t i = 0; i < 3; ++i) { + REQUIRE(train[i].len_ == 0); + } + for (uint64_t i = 0; i < 2; ++i) { + REQUIRE(test[i].len_ == 0); + } + std::remove(path.c_str()); +} + TEST_CASE("EvalDataset sparse round-trip with token sequences", "[ut][eval_dataset]") { auto path = TempPath("withseq"); { From 3225578965cb8a3ad00aba29e82949356355ab47 Mon Sep 17 00:00:00 2001 From: jc543239 Date: Thu, 30 Jul 2026 05:47:48 +0000 Subject: [PATCH 5/8] fix: restore OpenMP state after search evaluation Signed-off-by: jc543239 Assisted-by: Codex:gpt-5 --- tools/eval/eval_dataset_test.cpp | 6 +++++- tools/eval/evaluator.cpp | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tools/eval/eval_dataset_test.cpp b/tools/eval/eval_dataset_test.cpp index d3eccaba43..e8c4f02cc8 100644 --- a/tools/eval/eval_dataset_test.cpp +++ b/tools/eval/eval_dataset_test.cpp @@ -16,6 +16,7 @@ #include "eval_dataset.h" #include +#include #include #include @@ -404,7 +405,8 @@ TEST_CASE("EvaluateSearch validates inputs and propagates search errors", "[ut][ config.search_param = R"({"hgraph":{"ef_search":8}})"; config.top_k = 1; config.search_query_count = 2; - config.num_threads_searching = 2; + const auto caller_thread_count = omp_get_max_threads(); + config.num_threads_searching = caller_thread_count == 1 ? 2 : 1; config.enable_recall = false; config.enable_percent_recall = false; config.enable_qps = true; @@ -418,11 +420,13 @@ TEST_CASE("EvaluateSearch validates inputs and propagates search errors", "[ut][ REQUIRE(qps_only["measurement_sample_count"].get() == 2); REQUIRE(qps_only["index_info"].is_object()); REQUIRE(qps_only["index_info"].empty()); + REQUIRE(omp_get_max_threads() == caller_thread_count); config.search_param = R"({"hgraph":{"ef_search":0}})"; REQUIRE_THROWS_WITH( vsag::eval::EvaluateSearch(index, dataset, config), Catch::Matchers::ContainsSubstring("query error: ef_search(0) must be at least 1")); + REQUIRE(omp_get_max_threads() == caller_thread_count); config.search_param = R"({"hgraph":{"ef_search":8}})"; config.top_k = 0; diff --git a/tools/eval/evaluator.cpp b/tools/eval/evaluator.cpp index 2c3a252aca..e09c64b1fb 100644 --- a/tools/eval/evaluator.cpp +++ b/tools/eval/evaluator.cpp @@ -14,6 +14,8 @@ #include "evaluator.h" +#include + #include #include "case/build_eval_case.h" @@ -23,6 +25,19 @@ namespace vsag::eval { namespace { +class ScopedOpenMpThreads { +public: + ScopedOpenMpThreads() : original_(omp_get_max_threads()) { + } + + ~ScopedOpenMpThreads() { + omp_set_num_threads(original_); + } + +private: + int original_; +}; + void validate(const IndexPtr& index, const EvalDatasetPtr& dataset) { if (index == nullptr) { @@ -72,6 +87,7 @@ JsonType EvaluateSearch(const IndexPtr& index, const EvalDatasetPtr& dataset, const EvalConfig& config) { validate(index, dataset); validate_search(dataset, config); + ScopedOpenMpThreads openmp_threads; SearchEvalCase eval_case("", "", index, config, dataset); return eval_case.RunInMemory(); } From 0160541258083abb8bf4798b090b4aabe6faefb1 Mon Sep 17 00:00:00 2001 From: jc543239 Date: Fri, 31 Jul 2026 08:38:44 +0000 Subject: [PATCH 6/8] fix: simplify AutoTune online API usage Signed-off-by: jc543239 Assisted-by: Codex:gpt-5 --- docs/docs/en/src/resources/autotune.md | 45 ++- docs/docs/en/src/resources/autotune_api_v1.md | 74 +++-- docs/docs/zh/src/resources/autotune.md | 38 ++- docs/docs/zh/src/resources/autotune_api_v1.md | 68 ++-- ..._feature_create_index_with_constraints.cpp | 9 +- .../327_feature_autotune_existing_index.cpp | 13 +- .../328_feature_autotune_existing_pyramid.cpp | 29 +- tools/autotune/autotune.cpp | 309 +++++++++++------- tools/autotune/autotune.h | 50 +-- tools/autotune/autotune_evaluation.cpp | 8 +- tools/autotune/autotune_test.cpp | 134 +++++--- tools/eval/case/search_eval_case.cpp | 4 +- tools/eval/eval_config.h | 1 + tools/eval/eval_dataset.cpp | 36 ++ tools/eval/eval_dataset.h | 3 + tools/eval/eval_dataset_test.cpp | 32 ++ tools/eval/evaluator.cpp | 3 + tools/eval/monitor/recall_monitor.cpp | 22 +- tools/eval/monitor/recall_monitor.h | 3 +- 19 files changed, 585 insertions(+), 296 deletions(-) diff --git a/docs/docs/en/src/resources/autotune.md b/docs/docs/en/src/resources/autotune.md index f1aa3ecad5..434460fe93 100644 --- a/docs/docs/en/src/resources/autotune.md +++ b/docs/docs/en/src/resources/autotune.md @@ -70,13 +70,15 @@ and IVF spaces; otherwise arrays and `$range` expressions use the same semantics contract. The result contains the loaded `index`, `index_name`, complete concrete `create_parameters`, -validated `search_parameters`, metrics, artifact path, and the complete report. The recommended -artifact is retained, so another process can recreate an empty index from -`index_name + create_parameters` and deserialize that file. Unselected intermediate artifacts are -removed by default. +validated `search_parameters`, metrics, artifact path, and the complete report. `metrics` and +`report` are `JsonType` objects. The typed API returns the report in memory and never writes a +report file. The recommended artifact is retained, so another process can recreate an empty index +from `index_name + create_parameters` and deserialize that file. Unselected intermediate +artifacts are removed by default. -If no candidate satisfies every constraint, `TuneIndex` returns an `INVALID_ARGUMENT` error -instead of loading `best_effort`. +If no candidate satisfies every constraint, the completed call has +`status=TuneStatus::NO_FEASIBLE_CANDIDATE` and a structured `best_effort`; recommendation fields +such as `index` are not valid. Invalid requests and execution failures still return an error. See [`examples/cpp/326_feature_create_index_with_constraints.cpp`][factory-example] for a complete @@ -169,9 +171,6 @@ Programmatic callers use a separate `SearchRequest` and call `TuneSearch`: ```cpp vsag::autotune::SearchRequest request; request.index = existing_index; -request.index_name = "hgraph"; -request.base = base; -request.metric_type = vsag::METRIC_L2; request.workload = {queries, ground_truth, 10, 48}; request.parameter_space = R"({"hgraph":{"ef_search":[40,80,120]}})"; request.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.95}}; @@ -182,8 +181,14 @@ auto neighbors = existing_index->KnnSearch(query, 10, result.parameters).value() ``` This avoids serialization and file I/O and reuses the caller's index for all search trials. -`base` is currently required by the shared recall evaluator. The CLI's `index_path` field remains -an offline adapter: it creates and deserializes an index, then enters the same search-only flow. +AutoTune derives the index type and element count from `IndexPtr`. Search tuning therefore needs +queries but not the base dataset or metric type. When recall is requested, it also needs ground +truth and defines per-query `recall_at_k` as the intersection of the returned and ground-truth IDs +in their first `top_k` entries, divided by `top_k`; the reported value is the average over all +queries. Ground truth is optional when recall is neither a constraint nor the objective. + +The CLI's `index_path` field remains an offline adapter: it uses the concrete create parameters to +create and deserialize an index, then enters the same search-only flow. Pyramid is supported in this search-only mode. The HDF5 CLI adapter does not provide query paths, so it can evaluate only Pyramid's native default/root search. Typed requests take paths directly @@ -215,8 +220,9 @@ The following metrics can be used as constraints or as the objective: | `build_seconds`, `search_seconds`, `build_and_search_seconds` | At most the requested value | `build_seconds`, `index_size_mb`, and `build_and_search_seconds` are unavailable in -existing-index mode. The objective direction is inferred from the metric, so no `direction` field -is needed. +existing-index mode. `index_memory_mb` may be a constraint, but not the objective, because it is +constant across search candidates for the same existing index and cannot rank them. The objective +direction is inferred from the metric, so no `direction` field is needed. AutoTune uses a dedicated latency/QPS pass and a separate recall/statistics pass, so a logical query runs more than once per trial. `search_seconds` covers both in-memory passes and metric @@ -236,10 +242,15 @@ Standard output is a short summary ordered for copying: The full report also contains every build and trial, constraint violations, the normalized request, and structured per-stage failures. In search-only mode `builds` is empty and trials contain no -artifact or build fields. For the CLI and `RunAutoTune`, `keep_intermediate=false` removes every -generated index after evaluation. Typed `TuneIndex` always retains the recommended artifact so it -can return a reproducible result, while removing unselected artifacts when the option is false. -The report itself is retained in both cases. +artifact or build fields. For both the CLI and typed `TuneIndex`, `keep_intermediate=false` retains +only the recommended generated index and removes unselected artifacts. Setting it to `true` +retains every generated index. + +After initial request validation succeeds, the CLI and `RunAutoTune` persist the full report: +`output.result_path` overrides the path, otherwise a generated path under the workspace is used. +Early validation and request-file failures do not write a report. Typed `TuneIndex` and +`TuneSearch` never write a report file; callers receive the complete `JsonType` report in the +result. ## V1 Boundaries diff --git a/docs/docs/en/src/resources/autotune_api_v1.md b/docs/docs/en/src/resources/autotune_api_v1.md index 0bcc84824b..edfc90fce1 100644 --- a/docs/docs/en/src/resources/autotune_api_v1.md +++ b/docs/docs/en/src/resources/autotune_api_v1.md @@ -156,13 +156,16 @@ With `index_path`: - missing search fields may still receive built-in search candidates; - `build_seconds`, `index_size_mb`, and `build_and_search_seconds` cannot be constraints or objectives; +- `index_memory_mb` can be a constraint but not the objective; - the caller-owned index is never deleted. The concrete create parameters are needed to instantiate the correct VSAG index before deserialization. AutoTune does not infer them from the serialized file in V1. -For a typed `SearchRequest`, the index is already instantiated and loaded. It therefore needs an -index name and search `parameter_space`, but no create parameters. +For a typed `SearchRequest`, the index is already instantiated and loaded. AutoTune derives its +index type and element count from `IndexPtr`; the request supplies only the index, workload, search +`parameter_space`, constraints, objective, and config. It needs neither create parameters nor the +base dataset or metric type. The HDF5 adapter does not provide Pyramid query paths, so the CLI can evaluate only Pyramid's native default/root search. Path-specific Pyramid tuning requires a typed `SearchRequest` whose @@ -188,6 +191,8 @@ Pyramid hierarchies are not supported by AutoTune. V1 always evaluates the complete query set in the HDF5 file and supports only KNN search. The benchmark machine, system load, and runtime environment are part of the measured workload. Latency and QPS are not portable across different machine specifications or load conditions. +For a typed `SearchRequest`, `top_k` is checked against `IndexPtr::GetNumElements()` instead of a +base dataset. Ground truth may be null unless recall is a constraint or the objective. ## Constraints and Objective @@ -207,13 +212,13 @@ comparison and objective direction are inferred from the metric and cannot be ov } ``` -| Metric | Constraint | Objective direction | Existing index | +| Metric | Constraint | Objective direction | Existing-index use | | --- | --- | --- | --- | | `recall_at_k` | actual ≥ threshold | Maximize | Yes | | `qps` | actual ≥ threshold | Maximize | Yes | | `latency_avg_ms` | actual ≤ threshold | Minimize | Yes | | `latency_p99_ms` | actual ≤ threshold | Minimize | Yes | -| `index_memory_mb` | actual ≤ threshold | Minimize | Yes | +| `index_memory_mb` | actual ≤ threshold | Minimize | Constraint only | | `index_size_mb` | actual ≤ threshold | Minimize | No | | `build_seconds` | actual ≤ threshold | Minimize | No | | `search_seconds` | actual ≤ threshold | Minimize | Yes | @@ -223,6 +228,9 @@ Thresholds must be finite and non-negative. `recall_at_k` must also be at most ` Metric meanings in V1: +- For each query, `recall_at_k` is the size of the intersection between the first `top_k` + returned IDs and ground-truth IDs, divided by `top_k`; the reported metric is the average over + all queries. It needs ground truth but does not need base vectors or a metric type. - `build_seconds` measures the index `Build` operation reported by the evaluation tool. - It excludes index serialization, dataset loading, and candidate orchestration. - `search_seconds` measures the wall time of a complete in-memory search evaluation trial. It @@ -263,20 +271,27 @@ repeat runs when latency or QPS drives selection. | Field | Default | Meaning | | --- | --- | --- | -| `tuning_config.workspace_path` | `/tmp/vsag_autotune` | Run artifacts and default reports. | -| `tuning_config.keep_intermediate` | `false` | Retain generated intermediate index artifacts. | +| `tuning_config.workspace_path` | `/tmp/vsag_autotune` | Run artifacts and CLI default reports. | +| `tuning_config.keep_intermediate` | `false` | Keep all generated indexes. | | `tuning_config.max_trials` | `1000` | Maximum planned worst-case trials; hard maximum `100000`. | -| `output.result_path` | `/run-.json` | Full report path. | +| `output.result_path` | `/run-.json` | CLI full report path. | | `output.include_raw_eval` | `false` | Include native eval JSON in build and trial records. | -`output.result_path` cannot alias `data_path` or `index_path`. For the offline JSON/CLI entry, -generated indexes are removed after evaluation when `keep_intermediate` is false. Typed -`TuneIndex` always retains the recommended artifact so it can return a reproducible result; the -option controls retention of unselected artifacts. The report is retained independently. +`output.result_path` cannot alias `data_path` or `index_path`. After initial validation succeeds, +the offline JSON/CLI entry writes the full report. An explicit `output.result_path` overrides its +location; otherwise AutoTune generates a report path under `workspace_path`. Early validation and +request-file failures do not write one. + +Typed `TuneIndex` and `TuneSearch` do not accept a report path and never write report files. Their +result objects return the complete report as `JsonType`. For both the CLI and typed `TuneIndex`, +`keep_intermediate=false` retains only the recommended generated index and removes unselected +artifacts; `true` retains every generated index. When there is no recommendation, the default is +to remove all generated indexes. Because `TuneSearch` creates no artifacts, +`workspace_path` and `keep_intermediate` have no effect on it. ## Full Report -A completed evaluation writes and returns this top-level shape: +A completed evaluation returns this top-level shape. The CLI also writes it to disk: ```json { @@ -302,12 +317,13 @@ A completed evaluation writes and returns this top-level shape: | `trials` | One record per executed concrete search candidate. | | `request` | Effective normalized request used by the tuning engine. | | `elapsed_seconds` | AutoTune wall time through selection, excluding report writing and cleanup. | -| `report_path` | Persisted full report path. | +| `report_path` | Persisted full report path; CLI/JSON adapter only. | | `failure` | Present when the overall run failed. | Early validation failures do not have `builds`, `trials`, or `report_path`, and no report file is -written. A run in which all candidate evaluations fail does contain the attempted build and trial -records and is written when the report path itself is usable. +written. A CLI run in which all candidate evaluations fail does contain the attempted build and +trial records and is written when the report path itself is usable. Typed API reports never +contain `report_path`. The normalized `request` contains derived dataset metadata, workload defaults, constraints, objective, config, and output fields. Build-and-search mode adds `index_spaces`; search-only mode @@ -318,7 +334,8 @@ present, `index_path` as top-level fields inside this normalized object. - `success`: at least one successful trial satisfies every constraint; `recommendation` is set. - `no_feasible_candidate`: successful trials exist, but none satisfies every constraint; - `best_effort` is set for explanation and is not a valid recommendation. + `best_effort` is set for explanation and is not a valid recommendation. Typed calls return a + value with `TuneStatus::NO_FEASIBLE_CANDIDATE`, not an error. - `failed`: the request is invalid, execution/reporting fails, or no trial can be evaluated successfully with the objective metric. @@ -396,7 +413,7 @@ Artifact fields appear only in build-and-search records in V1. `artifacts.source and `artifacts.index_path` is evidence of where the evaluated index was stored, not a promise that the path still exists. Check `artifacts.retained`: -- `true`: this is the selected artifact returned by typed `TuneIndex`, or retention was requested; +- `true`: this is the selected artifact, or retention of all generated indexes was requested; - `false`: AutoTune planned to remove or already removed the generated artifact. ### Structured Failures @@ -442,9 +459,9 @@ The CLI prints a compact subset of the full report in this order: Null result branches and the detailed build/trial arrays are omitted from standard output. Read `report_path` for complete evidence. -The CLI exits with code `1` for `status=failed` and command-line/request-file errors. It currently -exits with code `0` for both `success` and `no_feasible_candidate`; callers must inspect `status` -rather than using the exit code to decide whether a recommendation exists. +The CLI exits with code `1` for `status=failed` and command-line/request-file errors. It exits with +code `0` for both `success` and `no_feasible_candidate`; callers must inspect `status` rather than +using only the exit code to decide whether a recommendation exists. ## Build-Tree C++ Entry Point @@ -476,14 +493,12 @@ auto result = TuneIndex(request); ``` It returns a loaded, query-ready selected index together with concrete create and search -parameters. `TuneSearch` tunes an already built or loaded index: +parameters. `metrics` and the complete `report` are returned as `JsonType`; typed calls never +persist the report. `TuneSearch` tunes an already built or loaded index: ```cpp SearchRequest request; request.index = existing_index; -request.index_name = "hgraph"; -request.base = base; -request.metric_type = METRIC_L2; request.workload = {queries, ground_truth, 10, 48}; request.parameter_space = search_candidate_space; request.constraints = {{Metric::RECALL_AT_K, 0.95}}; @@ -491,4 +506,13 @@ request.objective = Metric::LATENCY_AVG_MS; auto result = TuneSearch(request); ``` -Build-time metrics and `index_size_mb` are unavailable to `TuneSearch`. +AutoTune derives the type and element count of `existing_index`. `TuneSearch` therefore does not +need base vectors or a metric type. Ground truth is required only when recall is a constraint or +the objective. Build-time metrics and `index_size_mb` are unavailable to `TuneSearch`. +`index_memory_mb` may be a constraint but cannot be the objective because it does not vary across +search candidates. + +Both typed calls return `tl::unexpected` only for invalid requests and execution failures. +If evaluation completes but no candidate satisfies every constraint, they return a result with +`status=TuneStatus::NO_FEASIBLE_CANDIDATE` and structured `best_effort`; recommendation fields are +invalid in that state. diff --git a/docs/docs/zh/src/resources/autotune.md b/docs/docs/zh/src/resources/autotune.md index 87edc22093..2ccd5c82ad 100644 --- a/docs/docs/zh/src/resources/autotune.md +++ b/docs/docs/zh/src/resources/autotune.md @@ -66,11 +66,14 @@ auto neighbors = IVF 候选；显式参数中的数组和 `$range` 与 CLI 契约语义相同。 返回值包括已加载的 `index`、`index_name`、完整具体 `create_parameters`、经过验证的 -`search_parameters`、metrics、artifact 路径和完整报告。最终推荐 artifact 会保留，因此 -可以用 `index_name + create_parameters` 重新创建空索引并反序列化该文件；未选中的中间 -artifact 默认删除。 +`search_parameters`、metrics、artifact 路径和完整报告。`metrics` 和 `report` 都是 +`JsonType` 对象。typed API 在内存中返回报告，绝不会写报告文件。最终推荐 artifact 会 +保留，因此可以用 `index_name + create_parameters` 重新创建空索引并反序列化该文件； +未选中的中间 artifact 默认删除。 -如果没有候选满足全部约束，`TuneIndex` 返回 `INVALID_ARGUMENT`，不会加载 `best_effort`。 +如果没有候选满足全部约束，调用仍正常完成，并返回 +`status=TuneStatus::NO_FEASIBLE_CANDIDATE` 和结构化 `best_effort`；此时 `index` 等推荐 +字段无效。请求非法或执行失败时仍然返回 error。 完整示例见 [`examples/cpp/326_feature_create_index_with_constraints.cpp`][factory-example]。同时启用工具和 @@ -160,9 +163,6 @@ cmake --build build-release --target 326_feature_create_index_with_constraints - ```cpp vsag::autotune::SearchRequest request; request.index = existing_index; -request.index_name = "hgraph"; -request.base = base; -request.metric_type = vsag::METRIC_L2; request.workload = {queries, ground_truth, 10, 48}; request.parameter_space = R"({"hgraph":{"ef_search":[40,80,120]}})"; request.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.95}}; @@ -172,8 +172,13 @@ auto result = vsag::autotune::TuneSearch(request).value(); auto neighbors = existing_index->KnnSearch(query, 10, result.parameters).value(); ``` -该接口避免序列化和文件 I/O，并让全部 search trial 复用调用方的索引。共享 recall evaluator -目前仍要求提供 `base`。CLI 的 `index_path` 只是离线适配：先创建并反序列化 Index，再进入 +该接口避免序列化和文件 I/O，并让全部 search trial 复用调用方的索引。AutoTune 从 +`IndexPtr` 推导索引类型和元素数量，因此 search tuning 只需要 query，不需要 base dataset +或 metric type。使用 recall 时还需要 ground truth：每条 query 的 `recall_at_k` 定义为 +返回结果和 ground truth 前 `top_k` 个 ID 的交集大小除以 `top_k`，最终指标是所有 query +的平均值。recall 既不是约束也不是目标时，ground truth 可省略。 + +CLI 的 `index_path` 仍是离线适配：它先使用具体 create 参数创建并反序列化 Index，再进入 同一条 search-only 流程。 Pyramid 通过这个 search-only 模式接入。HDF5 CLI 适配器不提供 query path，因此只能评测 @@ -205,7 +210,8 @@ Pyramid path 调优示例见 | `build_seconds`、`search_seconds`、`build_and_search_seconds` | 不高于请求值 | 已有索引模式不提供 `build_seconds`、`index_size_mb` 和 `build_and_search_seconds`。 -目标方向由指标本身决定，因此不需要 `direction` 字段。 +`index_memory_mb` 可以作为约束，但不能作为目标，因为同一个已有索引的所有 search 候选 +内存相同，无法据此排序。目标方向由指标本身决定，因此不需要 `direction` 字段。 AutoTune 使用独立的 latency/QPS pass 和 recall/statistics pass，因此同一条逻辑 query 在 每个 trial 中会执行多次。`search_seconds` 包含两个内存 pass 和指标采集，但不包含索引 @@ -223,10 +229,14 @@ AutoTune 使用独立的 latency/QPS pass 和 recall/statistics pass，因此同 - `report_path`：完整可复现报告的位置。 完整报告还包含每个 build 和 trial、约束 violation、补齐后的请求，以及分阶段的结构化 -失败。search-only 模式的 `builds` 为空，trial 不包含 artifact 或 build 字段。当 -CLI 或 `RunAutoTune` 设置 `keep_intermediate=false` 时，评测后会删除全部生成索引。typed -`TuneIndex` 为返回可复现结果始终保留推荐 artifact；该选项为 false 时只删除未选中的 -artifact。两种入口都会保留报告文件。 +失败。search-only 模式的 `builds` 为空，trial 不包含 artifact 或 build 字段。CLI 和 typed +`TuneIndex` 设置 `keep_intermediate=false` 时都只保留最终推荐索引，并删除未选中的 +artifact；设置为 `true` 时保留所有生成索引。 + +请求通过初始校验后，CLI 和 `RunAutoTune` 会持久化完整报告：显式 `output.result_path` +会覆盖输出路径，否则在 workspace 下生成默认路径。早期校验和请求文件错误不会写报告。 +typed `TuneIndex` 和 `TuneSearch` 绝不会写报告文件，而是通过返回值提供完整的 `JsonType` +报告。 ## V1 边界 diff --git a/docs/docs/zh/src/resources/autotune_api_v1.md b/docs/docs/zh/src/resources/autotune_api_v1.md index 41a9dbe3c0..d28d85b3d9 100644 --- a/docs/docs/zh/src/resources/autotune_api_v1.md +++ b/docs/docs/zh/src/resources/autotune_api_v1.md @@ -146,13 +146,15 @@ bucket 数量。AutoTune 无法推断已有 IVF 的 bucket 数量，因此 searc - AutoTune 不补充 create 候选，也不会执行 build； - 缺失的 search 字段仍可使用内置查询候选； - `build_seconds`、`index_size_mb` 和 `build_and_search_seconds` 不能作为约束或目标； +- `index_memory_mb` 可以作为约束，但不能作为目标； - 不会删除调用方提供的索引。 V1 反序列化前需要使用具体创建参数实例化正确的 VSAG 索引，目前不会从序列化文件推导 这些参数。 -对于 typed `SearchRequest`，索引已经完成实例化和加载，因此只需索引名和 search -`parameter_space`，不需要 create 参数。 +对于 typed `SearchRequest`，索引已经完成实例化和加载。AutoTune 从 `IndexPtr` 推导索引 +类型和元素数量；请求只提供 index、workload、search `parameter_space`、constraints、 +objective 和 config，不需要 create 参数、base dataset 或 metric type。 HDF5 适配器不提供 Pyramid query path，因此 CLI 只能评测 Pyramid 原生的默认/root 查询。 按 path 调优 Pyramid 需要使用 typed `SearchRequest`，并通过 query Dataset 的 @@ -177,6 +179,8 @@ hierarchy。 V1 始终评测 HDF5 文件中的全量 query，并且只支持 KNN。benchmark 机器、系统负载和运行 环境都属于 workload 的一部分；延迟和 QPS 不能直接移植到不同机器规格或负载条件。 +对于 typed `SearchRequest`，`top_k` 会和 `IndexPtr::GetNumElements()` 比较，不依赖 base +dataset。只有 recall 是约束或目标时才必须提供 ground truth。 ## 约束和优化目标 @@ -196,13 +200,13 @@ V1 始终评测 HDF5 文件中的全量 query，并且只支持 KNN。benchmark } ``` -| 指标 | 约束 | 目标方向 | 已有索引可用 | +| 指标 | 约束 | 目标方向 | 已有索引用法 | | --- | --- | --- | --- | | `recall_at_k` | actual ≥ threshold | 最大化 | 是 | | `qps` | actual ≥ threshold | 最大化 | 是 | | `latency_avg_ms` | actual ≤ threshold | 最小化 | 是 | | `latency_p99_ms` | actual ≤ threshold | 最小化 | 是 | -| `index_memory_mb` | actual ≤ threshold | 最小化 | 是 | +| `index_memory_mb` | actual ≤ threshold | 最小化 | 仅约束 | | `index_size_mb` | actual ≤ threshold | 最小化 | 否 | | `build_seconds` | actual ≤ threshold | 最小化 | 否 | | `search_seconds` | actual ≤ threshold | 最小化 | 是 | @@ -212,6 +216,9 @@ V1 始终评测 HDF5 文件中的全量 query，并且只支持 KNN。benchmark V1 指标口径： +- 每条 query 的 `recall_at_k` 是返回结果和 ground truth 前 `top_k` 个 ID 的交集大小除以 + `top_k`，最终指标是所有 query 的平均值。它需要 ground truth，但不需要 base vector 或 + metric type。 - `build_seconds` 是 eval 工具报告的索引 `Build` 操作耗时。 - 它不包含索引序列化、数据集加载和候选编排。 - `search_seconds` 是完整内存 search eval trial 的墙钟时间，不包含索引反序列化，但包含 @@ -247,19 +254,25 @@ V1 指标口径： | 字段 | 默认值 | 含义 | | --- | --- | --- | -| `tuning_config.workspace_path` | `/tmp/vsag_autotune` | run 产物和默认报告目录。 | -| `tuning_config.keep_intermediate` | `false` | 是否保留生成的中间索引产物。 | +| `tuning_config.workspace_path` | `/tmp/vsag_autotune` | run 产物和 CLI 默认报告目录。 | +| `tuning_config.keep_intermediate` | `false` | 是否保留所有生成索引。 | | `tuning_config.max_trials` | `1000` | 计划最坏 trial 数上限；硬上限为 `100000`。 | -| `output.result_path` | `/run-.json` | 完整报告路径。 | +| `output.result_path` | `/run-.json` | CLI 完整报告路径。 | | `output.include_raw_eval` | `false` | 在 build/trial 中包含原生 eval JSON。 | -`output.result_path` 不能与 `data_path` 或 `index_path` 指向同一文件。离线 JSON/CLI 入口 -设置 `keep_intermediate=false` 时，评测后删除所有生成索引。typed `TuneIndex` 为返回可复现 -结果始终保留推荐 artifact；该选项只控制未选中 artifact 的保留。报告单独保留。 +`output.result_path` 不能与 `data_path` 或 `index_path` 指向同一文件。通过初始校验后， +离线 JSON/CLI 入口会写完整报告：显式 `output.result_path` 会覆盖输出位置，否则 AutoTune +在 `workspace_path` 下生成报告路径。早期校验和请求文件错误不会写报告。 + +typed `TuneIndex` 和 `TuneSearch` 不接收报告路径，也绝不会写报告文件，而是通过结果对象 +返回完整的 `JsonType` 报告。CLI 和 typed `TuneIndex` 设置 +`keep_intermediate=false` 时都只保留推荐索引并删除未选中 artifact；设置为 `true` 时保留 +所有生成索引。没有 recommendation 时，默认删除所有生成索引。`TuneSearch` 不生成 +artifact，因此 `workspace_path` 和 `keep_intermediate` 对它没有影响。 ## 完整报告 -完成评测后会写入并返回以下顶层结构： +完成评测后会返回以下顶层结构；CLI 还会把它写入磁盘： ```json { @@ -285,11 +298,12 @@ V1 指标口径： | `trials` | 每组已执行的具体 search 候选一条记录。 | | `request` | 调优引擎实际使用的有效规范化请求。 | | `elapsed_seconds` | 截止结果选择的 AutoTune 墙钟时间，不含报告写入和清理。 | -| `report_path` | 持久化完整报告路径。 | +| `report_path` | 持久化完整报告路径；只在 CLI/JSON 适配器中存在。 | | `failure` | 整体执行失败时出现。 | 早期校验失败不包含 `builds`、`trials` 或 `report_path`，也不会写报告。如果所有候选评测 -失败，只要报告路径仍然可用，就会保留尝试过的 build/trial 记录并写报告。 +失败，CLI 只要报告路径仍然可用，就会保留尝试过的 build/trial 记录并写报告。typed API +的报告绝不会包含 `report_path`。 规范化 `request` 包含推导出的 dataset 元数据、workload 默认值、constraints、objective、 config 和 output。build-and-search 模式还包含 `index_spaces`；search-only 模式包含 @@ -300,7 +314,8 @@ config 和 output。build-and-search 模式还包含 `index_spaces`；search-onl - `success`：至少一个成功 trial 满足全部约束，设置 `recommendation`。 - `no_feasible_candidate`：存在成功 trial，但没有任何一个满足全部约束；设置 - `best_effort` 用于解释，它不是有效推荐。 + `best_effort` 用于解释，它不是有效推荐。typed 调用会返回带 + `TuneStatus::NO_FEASIBLE_CANDIDATE` 的值，而不是 error。 - `failed`：请求非法、执行或报告阶段失败，或者没有成功且带目标指标的 trial。 ### Recommendation 和 Best Effort @@ -376,7 +391,7 @@ V1 只在 build-and-search 记录中提供 artifact 字段。`artifacts.source` `artifacts.index_path` 用于说明被评测索引曾存放在哪里，不保证响应返回时路径仍然存在。 需要检查 `artifacts.retained`： -- `true`：这是 typed `TuneIndex` 返回的推荐产物，或者请求保留； +- `true`：这是最终推荐产物，或者请求保留所有生成索引； - `false`：AutoTune 计划删除或已经删除生成产物。 ### 结构化失败 @@ -422,9 +437,9 @@ CLI 按以下顺序输出完整报告的紧凑子集： 标准输出会省略值为 null 的结果分支和详细 build/trial 数组。完整证据请读取 `report_path`。 -CLI 在 `status=failed` 或命令行/请求文件出错时返回 `1`。目前 `success` 和 -`no_feasible_candidate` 都返回 `0`；调用方必须检查 `status`，不能只用退出码判断是否 -存在 recommendation。 +CLI 在 `status=failed` 或命令行/请求文件出错时返回 `1`。`success` 和 +`no_feasible_candidate` 都返回 `0`；调用方必须检查 `status`，不能只用退出码判断是否存在 +recommendation。 ## Build-tree C++ 入口 @@ -454,15 +469,13 @@ request.objective = Metric::LATENCY_AVG_MS; auto result = TuneIndex(request); ``` -返回值包含已加载、可查询的推荐索引，以及具体 create 和 search 参数。`TuneSearch` 对已经 -构建或加载的索引调查询参数： +返回值包含已加载、可查询的推荐索引，以及具体 create 和 search 参数。`metrics` 和完整 +`report` 都以 `JsonType` 返回；typed 调用绝不会持久化报告。`TuneSearch` 对已经构建或 +加载的索引调查询参数： ```cpp SearchRequest request; request.index = existing_index; -request.index_name = "hgraph"; -request.base = base; -request.metric_type = METRIC_L2; request.workload = {queries, ground_truth, 10, 48}; request.parameter_space = search_candidate_space; request.constraints = {{Metric::RECALL_AT_K, 0.95}}; @@ -470,4 +483,11 @@ request.objective = Metric::LATENCY_AVG_MS; auto result = TuneSearch(request); ``` -`TuneSearch` 不提供构建阶段指标和 `index_size_mb`。 +AutoTune 从 `existing_index` 推导类型和元素数量，因此 `TuneSearch` 不需要 base vector 或 +metric type。只有 recall 是约束或目标时才需要 ground truth。`TuneSearch` 不提供构建阶段 +指标和 `index_size_mb`。`index_memory_mb` 可以作为约束，但不能作为目标，因为它不会随 +search 候选变化。 + +两个 typed 调用仅在请求非法或执行失败时返回 `tl::unexpected`。如果评测正常完成但 +没有候选满足全部约束，则返回 `status=TuneStatus::NO_FEASIBLE_CANDIDATE` 和结构化 +`best_effort`；该状态下推荐字段无效。 diff --git a/examples/cpp/326_feature_create_index_with_constraints.cpp b/examples/cpp/326_feature_create_index_with_constraints.cpp index f480496f81..501d4a2aa2 100644 --- a/examples/cpp/326_feature_create_index_with_constraints.cpp +++ b/examples/cpp/326_feature_create_index_with_constraints.cpp @@ -62,14 +62,19 @@ main() { std::cerr << "Failed to tune index: " << tuned.error().message << std::endl; return 1; } + if (tuned->status == vsag::autotune::TuneStatus::NO_FEASIBLE_CANDIDATE) { + std::cerr << "No candidate satisfied the constraints. Best effort:\n" + << tuned->best_effort.dump(2) << std::endl; + return 2; + } const auto& result = tuned.value(); std::cout << "index_name: " << result.index_name << '\n' << "create_params: " << result.create_parameters << '\n' << "search_params: " << result.search_parameters << '\n' - << "validated_metrics: " << result.metrics << '\n' + << "validated_metrics: " << result.metrics.dump() << '\n' << "index_artifact: " << result.artifact_path << '\n' - << "report: " << result.report_path << std::endl; + << "trials_evaluated: " << result.report["trials"].size() << std::endl; auto query = vsag::Dataset::Make(); query->NumElements(1)->Dim(DIM)->Float32Vectors(query_vectors.data())->Owner(false); diff --git a/examples/cpp/327_feature_autotune_existing_index.cpp b/examples/cpp/327_feature_autotune_existing_index.cpp index 7bf5eb8e2d..79bc5bc1e1 100644 --- a/examples/cpp/327_feature_autotune_existing_index.cpp +++ b/examples/cpp/327_feature_autotune_existing_index.cpp @@ -83,14 +83,10 @@ main() { vsag::autotune::SearchRequest request; request.index = index; - request.base = base; - request.metric_type = vsag::METRIC_L2; request.workload = {queries, ground_truth, 1, 1}; - request.index_name = "hgraph"; request.parameter_space = R"({"hgraph":{"ef_search":[4,8,16]}})"; request.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 1.0}}; request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; - request.config.workspace_path = "/tmp/vsag_autotune_existing_index_example"; request.config.max_trials = 3; const auto tuned = vsag::autotune::TuneSearch(request); @@ -98,11 +94,16 @@ main() { std::cerr << "AutoTune failed: " << tuned.error().message << std::endl; return 1; } + if (tuned->status == vsag::autotune::TuneStatus::NO_FEASIBLE_CANDIDATE) { + std::cerr << "No candidate satisfied the constraints. Best effort:\n" + << tuned->best_effort.dump(2) << std::endl; + return 2; + } const auto& result = tuned.value(); std::cout << "recommended search_params: " << result.parameters << '\n' - << "validated metrics: " << result.metrics << '\n' - << "report: " << result.report_path << std::endl; + << "validated metrics: " << result.metrics.dump() << '\n' + << "trials evaluated: " << result.report["trials"].size() << std::endl; auto query = vsag::Dataset::Make() ->NumElements(1) diff --git a/examples/cpp/328_feature_autotune_existing_pyramid.cpp b/examples/cpp/328_feature_autotune_existing_pyramid.cpp index 3dcae015b1..152602d8b0 100644 --- a/examples/cpp/328_feature_autotune_existing_pyramid.cpp +++ b/examples/cpp/328_feature_autotune_existing_pyramid.cpp @@ -94,20 +94,13 @@ PrepareWorkload(PathWorkload& workload, } tl::expected -TunePath(const vsag::IndexPtr& index, - const vsag::DatasetPtr& base, - const PathWorkload& workload, - const std::string& workspace) { +TunePath(const vsag::IndexPtr& index, const PathWorkload& workload) { vsag::autotune::SearchRequest request; request.index = index; - request.index_name = "pyramid"; - request.base = base; - request.metric_type = vsag::METRIC_L2; request.workload = {workload.queries, workload.ground_truth, TOP_K, 1}; request.parameter_space = R"({"pyramid":{"ef_search":[10,20,40,80,160]}})"; request.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.80}}; request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; - request.config.workspace_path = workspace; request.config.max_trials = 5; return vsag::autotune::TuneSearch(request); } @@ -116,8 +109,8 @@ void PrintResult(const std::string& path, const vsag::autotune::SearchResult& result) { std::cout << "\npath: " << path << '\n' << "recommended search_params: " << result.parameters << '\n' - << "validated metrics: " << result.metrics << '\n' - << "report: " << result.report_path << std::endl; + << "validated metrics: " << result.metrics.dump() << '\n' + << "trials evaluated: " << result.report["trials"].size() << std::endl; } } // namespace @@ -185,16 +178,24 @@ main() { PrepareWorkload( hard_workload, base_ids, base_vectors, EASY_COUNT, HARD_COUNT, "catalog/hard", random); - auto easy_result = - TunePath(index, base, easy_workload, "/tmp/vsag_autotune_existing_pyramid_example/easy"); - auto hard_result = - TunePath(index, base, hard_workload, "/tmp/vsag_autotune_existing_pyramid_example/hard"); + auto easy_result = TunePath(index, easy_workload); + auto hard_result = TunePath(index, hard_workload); if (!easy_result.has_value() || !hard_result.has_value()) { const auto message = !easy_result.has_value() ? easy_result.error().message : hard_result.error().message; std::cerr << "AutoTune failed: " << message << std::endl; return 1; } + if (easy_result->status == vsag::autotune::TuneStatus::NO_FEASIBLE_CANDIDATE || + hard_result->status == vsag::autotune::TuneStatus::NO_FEASIBLE_CANDIDATE) { + const auto& result = + easy_result->status == vsag::autotune::TuneStatus::NO_FEASIBLE_CANDIDATE + ? easy_result.value() + : hard_result.value(); + std::cerr << "No candidate satisfied one path workload. Best effort:\n" + << result.best_effort.dump(2) << std::endl; + return 2; + } std::cout << "The same Pyramid index is tuned once per representative path workload." << std::endl; diff --git a/tools/autotune/autotune.cpp b/tools/autotune/autotune.cpp index 630d88ba57..ed9e78d83f 100644 --- a/tools/autotune/autotune.cpp +++ b/tools/autotune/autotune.cpp @@ -172,7 +172,13 @@ higher_is_better(const std::string& metric) { bool available_for_existing_index(const std::string& metric) { - return metric != "build_seconds" && metric != "build_and_search_seconds"; + return metric != "build_seconds" && metric != "build_and_search_seconds" && + metric != "index_size_mb"; +} + +bool +useful_existing_index_objective(const std::string& metric) { + return available_for_existing_index(metric) && metric != "index_memory_mb"; } bool @@ -378,7 +384,8 @@ parse_parameters(const std::string& value, const std::string& path) { } RequestContext -make_context(const DatasetPtr& base, +make_context(eval::EvalDatasetPtr dataset, + uint64_t base_count, const std::string& metric_type, const Workload& workload, const std::vector& constraints, @@ -388,16 +395,16 @@ make_context(const DatasetPtr& base, RequestContext request; request.workspace_path = config.workspace_path.empty() ? "/tmp/vsag_autotune" : config.workspace_path; - request.result_path = config.report_path; request.keep_intermediate = config.keep_intermediate; request.include_raw_eval = config.include_raw_evaluation; request.max_trials = config.max_trials; request.top_k = workload.top_k; request.concurrency = workload.concurrency; - require(base != nullptr, "request.base is required"); - require(base->GetIds() != nullptr, "request.base IDs are required"); - require(workload.queries != nullptr, "request.workload.queries is required"); + request.dataset = std::move(dataset); + request.base_count = base_count; + request.query_count = static_cast(request.dataset->GetNumberOfQuery()); + request.ground_truth_k = request.dataset->GetGroundTruthK(); require(request.top_k > 0, "request.workload.top_k must be positive"); require(request.top_k <= static_cast(std::numeric_limits::max()), "request.workload.top_k is too large"); @@ -417,27 +424,14 @@ make_context(const DatasetPtr& base, "request.constraints.recall_at_k must be in [0, 1]"); require(!search_only || available_for_existing_index(name), "request.constraints." + name + " is unavailable for search tuning"); - require(!search_only || name != "index_size_mb", - "request.constraints.index_size_mb is unavailable for an in-memory index"); require(request.constraints.emplace(name, constraint.value).second, "request.constraints contains duplicate metric: " + name); request.enable_recall = request.enable_recall || constraint.metric == Metric::RECALL_AT_K; } - require(!search_only || available_for_existing_index(request.objective), - "request.objective is unavailable for search tuning"); - require(!search_only || request.objective != "index_size_mb", - "request.objective index_size_mb is unavailable for an in-memory index"); + require(!search_only || useful_existing_index_objective(request.objective), + "request.objective cannot rank search candidates: " + request.objective); - const auto metric = normalize(metric_type); - require(metric == "l2" || metric == "ip" || metric == "cosine", - "request.metric_type must be l2, ip, or cosine"); - request.dataset = - eval::EvalDataset::FromDatasets(base, workload.queries, workload.ground_truth, metric); - request.base_count = static_cast(request.dataset->GetNumberOfBase()); - request.query_count = static_cast(request.dataset->GetNumberOfQuery()); - request.ground_truth_k = request.dataset->GetGroundTruthK(); - require(request.top_k <= request.base_count, - "request.workload.top_k exceeds the base dataset size"); + require(request.top_k <= request.base_count, "request.workload.top_k exceeds the index size"); if (request.enable_recall) { require(workload.ground_truth != nullptr, "request.workload.ground_truth is required for recall_at_k"); @@ -445,15 +439,17 @@ make_context(const DatasetPtr& base, "request.workload.top_k exceeds ground truth k"); } + JsonType dataset_info{{"base_count", request.base_count}, + {"query_count", request.query_count}, + {"ground_truth_k", request.ground_truth_k}, + {"dim", request.dataset->GetDim()}, + {"dtype", vsag::DATATYPE_FLOAT32}}; + if (!metric_type.empty()) { + dataset_info["metric_type"] = metric_type; + } request.effective_request = { {"version", 1}, - {"dataset", - {{"base_count", request.base_count}, - {"query_count", request.query_count}, - {"ground_truth_k", request.ground_truth_k}, - {"dim", request.dataset->GetDim()}, - {"dtype", vsag::DATATYPE_FLOAT32}, - {"metric_type", metric}}}, + {"dataset", std::move(dataset_info)}, {"workload", {{"top_k", request.top_k}, {"concurrency", request.concurrency}}}, {"constraints", JsonType::object()}, {"objective", {{"metric", request.objective}}}, @@ -461,15 +457,27 @@ make_context(const DatasetPtr& base, {{"workspace_path", request.workspace_path}, {"keep_intermediate", request.keep_intermediate}, {"max_trials", request.max_trials}}}, - {"output", - {{"report_path", request.result_path}, - {"include_raw_evaluation", request.include_raw_eval}}}}; + {"output", {{"include_raw_evaluation", request.include_raw_eval}}}}; for (const auto& [name, value] : request.constraints) { request.effective_request["constraints"][name] = value; } return request; } +std::string +index_name(const IndexPtr& index) { + switch (index->GetIndexType()) { + case IndexType::HGRAPH: + return INDEX_HGRAPH; + case IndexType::IVF: + return INDEX_IVF; + case IndexType::PYRAMID: + return INDEX_PYRAMID; + default: + throw std::invalid_argument("request.index type is unsupported by AutoTune"); + } +} + IndexInput parse_index_space(const IndexSpace& value, uint64_t position, bool search_only) { const auto path = "request.index_spaces[" + std::to_string(position) + "]"; @@ -495,9 +503,17 @@ parse_index_space(const IndexSpace& value, uint64_t position, bool search_only) IndexTuningRequest ParseRequest(const IndexRequest& input) { + require(input.base != nullptr, "request.base is required"); + require(input.base->GetIds() != nullptr, "request.base IDs are required"); + const auto metric = normalize(input.metric_type); + require(metric == "l2" || metric == "ip" || metric == "cosine", + "request.metric_type must be l2, ip, or cosine"); + auto dataset = eval::EvalDataset::FromDatasets( + input.base, input.workload.queries, input.workload.ground_truth, metric); IndexTuningRequest request; - request.context = make_context(input.base, - input.metric_type, + request.context = make_context(std::move(dataset), + static_cast(input.base->GetNumElements()), + metric, input.workload, input.constraints, input.objective, @@ -511,7 +527,6 @@ ParseRequest(const IndexRequest& input) { } } const auto dim = static_cast(request.context.dataset->GetDim()); - const auto metric = normalize(input.metric_type); for (auto& index : request.indexes) { merge_dataset_field(index.create_params, "dim", dim, index.name); merge_dataset_field(index.create_params, "dtype", vsag::DATATYPE_FLOAT32, index.name); @@ -527,16 +542,21 @@ ParseRequest(const IndexRequest& input) { SearchTuningRequest ParseRequest(const SearchRequest& input) { require(input.index != nullptr, "request.index is required"); + const auto element_count = input.index->GetNumElements(); + require(element_count > 0, "request.index must not be empty"); + auto dataset = + eval::EvalDataset::FromSearchDatasets(input.workload.queries, input.workload.ground_truth); SearchTuningRequest request; - request.context = make_context(input.base, - input.metric_type, + request.context = make_context(std::move(dataset), + static_cast(element_count), + "", input.workload, input.constraints, input.objective, input.config, true); IndexSpace space; - space.name = input.index_name; + space.name = index_name(input.index); space.search_parameter_space = input.parameter_space; request.index_input = parse_index_space(space, 0, true); request.index = input.index; @@ -563,6 +583,7 @@ ParseRequest(const JsonType& input) { check_file(data_path, "data_path"); IndexRequest typed; + std::string report_path; attach_offline_dataset(typed, eval::EvalDataset::Load(data_path)); if (input.contains("indexes")) { require(input["indexes"].is_array() && !input["indexes"].empty(), @@ -628,8 +649,8 @@ ParseRequest(const JsonType& input) { const auto& output = input["output"]; known_keys(output, {"result_path", "include_raw_eval"}, "request.output"); if (output.contains("result_path")) { - typed.config.report_path = required_string(output, "result_path", "request.output"); - require(!same_path(typed.config.report_path, data_path), + report_path = required_string(output, "result_path", "request.output"); + require(!same_path(report_path, data_path), "request.output.result_path must not alias data_path"); } if (output.contains("include_raw_eval")) { @@ -641,13 +662,17 @@ ParseRequest(const JsonType& input) { if (!input.contains("index_path")) { auto parsed = ParseRequest(typed); + parsed.context.result_path = report_path; parsed.context.effective_request["data_path"] = data_path; + if (!report_path.empty()) { + parsed.context.effective_request["output"]["report_path"] = report_path; + } return parsed; } const auto index_path = required_string(input, "index_path", "request"); check_file(index_path, "index_path"); - require(!same_path(typed.config.report_path, index_path), + require(!same_path(report_path, index_path), "request.output.result_path must not alias index_path"); require(typed.index_spaces.size() == 1, "index_path requires exactly one indexes[] specification"); @@ -672,18 +697,19 @@ ParseRequest(const JsonType& input) { SearchRequest search; search.index = created.value(); - search.index_name = space.name; - search.base = typed.base; - search.metric_type = typed.metric_type; search.workload = typed.workload; search.parameter_space = space.search_parameter_space; search.constraints = typed.constraints; search.objective = typed.objective; search.config = typed.config; auto parsed = ParseRequest(search); + parsed.context.result_path = report_path; parsed.context.effective_request["data_path"] = data_path; parsed.context.effective_request["index_path"] = index_path; parsed.context.effective_request["create_params"] = create_params; + if (!report_path.empty()) { + parsed.context.effective_request["output"]["report_path"] = report_path; + } return parsed; } @@ -792,36 +818,68 @@ namespace vsag::autotune { namespace { +void +prepare_report_path(const std::string& path) { + const auto parent = std::filesystem::path(path).parent_path(); + if (!parent.empty()) { + std::filesystem::create_directories(parent); + } + std::ofstream output(path, std::ios::app); + if (!output.good()) { + throw std::runtime_error("failed to open report path: " + path); + } +} + +void +finalize_artifacts(JsonType& report, bool keep_all, const std::optional& selected); + +std::optional +selected_artifact(const JsonType& report); + template JsonType -run_tuning_locked(Parser parser, - bool cleanup_run_on_failure, - std::chrono::steady_clock::time_point start) { +run_tuning_locked(Parser parser, bool persist_report, std::chrono::steady_clock::time_point start) { std::string stage = "validation"; std::string run_path; + std::string report_path; bool keep_intermediate = false; const auto failure = [&](const std::string& message) { - if ((cleanup_run_on_failure || !keep_intermediate) && !run_path.empty()) { + if (!keep_intermediate && !run_path.empty()) { std::error_code cleanup_error; std::filesystem::remove_all(run_path, cleanup_error); } const char* const code = stage == "validation" || stage == "candidate_generation" ? "invalid_request" : "execution_failed"; - return JsonType{{"version", 1}, + JsonType report{{"version", 1}, {"status", "failed"}, {"recommendation", nullptr}, {"best_effort", nullptr}, {"elapsed_seconds", internal::elapsed(start)}, {"failure", internal::Failure(stage, code, message)}}; + if (!report_path.empty()) { + report["report_path"] = report_path; + try { + internal::WriteJson(report_path, report); + } catch (...) { + } + } + return report; }; try { auto request = parser(); auto& context = request.context; keep_intermediate = context.keep_intermediate; + const auto run_name = internal::new_run_name(); + if (persist_report) { + report_path = context.result_path.empty() + ? context.workspace_path + "/" + run_name + ".json" + : context.result_path; + stage = "report"; + prepare_report_path(report_path); + } stage = "candidate_generation"; const auto candidates = internal::GenerateCandidates(request); - const auto run_name = internal::new_run_name(); stage = "evaluation"; internal::Evaluation evaluation; if constexpr (std::is_same_v) { @@ -834,15 +892,22 @@ run_tuning_locked(Parser parser, auto report = internal::SelectResult(context, evaluation); report["version"] = 1; report["request"] = context.effective_request; + if (!report_path.empty()) { + report["report_path"] = report_path; + report["request"]["output"]["report_path"] = report_path; + } + if constexpr (std::is_same_v) { + const auto selected = selected_artifact(report); + finalize_artifacts(report, context.keep_intermediate, selected); + if (!context.keep_intermediate && !selected.has_value() && !run_path.empty()) { + std::error_code error; + std::filesystem::remove_all(run_path, error); + } + } report["elapsed_seconds"] = internal::elapsed(start); - report["report_path"] = context.result_path.empty() - ? context.workspace_path + "/" + run_name + ".json" - : context.result_path; stage = "report"; - internal::WriteJson(report["report_path"].template get(), report); - if (!context.keep_intermediate && !run_path.empty()) { - std::error_code error; - std::filesystem::remove_all(run_path, error); + if (!report_path.empty()) { + internal::WriteJson(report_path, report); } return report; } catch (const std::exception& error) { @@ -855,20 +920,15 @@ run_tuning_locked(Parser parser, template JsonType run_tuning(Parser parser, - bool cleanup_run_on_failure = false, std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now()) { std::lock_guard lock(internal::run_mutex()); - return run_tuning_locked(std::move(parser), cleanup_run_on_failure, start); + return run_tuning_locked(std::move(parser), false, start); } Error report_error(const JsonType& report) { - const auto status = report.value("status", std::string("failed")); - auto type = - status == "no_feasible_candidate" ? ErrorType::INVALID_ARGUMENT : ErrorType::INTERNAL_ERROR; - auto message = status == "no_feasible_candidate" - ? std::string("no candidate satisfies all constraints") - : std::string("AutoTune failed"); + auto type = ErrorType::INTERNAL_ERROR; + auto message = std::string("AutoTune failed"); if (report.contains("failure") && report["failure"].is_object()) { const auto& failure = report["failure"]; if (failure.value("stage", std::string()) == "validation" || @@ -877,9 +937,6 @@ report_error(const JsonType& report) { } message = failure.value("message", message); } - if (report.contains("report_path") && report["report_path"].is_string()) { - message += " (report: " + report["report_path"].get() + ")"; - } return {type, std::move(message)}; } @@ -913,40 +970,44 @@ update_artifact(JsonType& value, } void -finalize_index_report(JsonType& report, - const Config& config, - const std::optional& selected) { - if (report.contains("request") && report["request"].is_object() && - report["request"].contains("config") && report["request"]["config"].is_object()) { - report["request"]["config"]["keep_intermediate"] = config.keep_intermediate; - } - +finalize_artifacts(JsonType& report, bool keep_all, const std::optional& selected) { std::unordered_set removed; if (report.contains("builds") && report["builds"].is_array()) { for (auto& build : report["builds"]) { - update_artifact(build, config.keep_intermediate, selected, removed); + update_artifact(build, keep_all, selected, removed); } } if (report.contains("trials") && report["trials"].is_array()) { for (auto& trial : report["trials"]) { - update_artifact(trial, config.keep_intermediate, selected, removed); + update_artifact(trial, keep_all, selected, removed); } } for (const auto* key : {"recommendation", "best_effort"}) { if (report.contains(key)) { - update_artifact(report[key], config.keep_intermediate, selected, removed); + update_artifact(report[key], keep_all, selected, removed); } } +} - if (report.contains("report_path") && report["report_path"].is_string()) { - internal::WriteJson(report["report_path"].get(), report); +std::optional +selected_artifact(const JsonType& report) { + if (report.value("status", std::string()) != "success" || !report.contains("recommendation") || + !report["recommendation"].is_object()) { + return std::nullopt; + } + const auto& recommendation = report["recommendation"]; + if (!recommendation.contains("artifacts") || !recommendation["artifacts"].is_object() || + !recommendation["artifacts"].contains("index_path") || + !recommendation["artifacts"]["index_path"].is_string()) { + return std::nullopt; } + return recommendation["artifacts"]["index_path"].get(); } void -finalize_index_report_noexcept(JsonType& report, const Config& config) noexcept { +finalize_artifacts_noexcept(JsonType& report, bool keep_all) noexcept { try { - finalize_index_report(report, config, std::nullopt); + finalize_artifacts(report, keep_all, std::nullopt); } catch (...) { } } @@ -965,20 +1026,25 @@ validation_failure(const std::chrono::steady_clock::time_point& start, const std tl::expected TuneIndex(const IndexRequest& request) { - auto retained_request = request; - retained_request.config.keep_intermediate = true; - auto report = - run_tuning([&retained_request]() { return internal::ParseRequest(retained_request); }, - !request.config.keep_intermediate); - if (report.value("status", std::string("failed")) != "success") { - try { - finalize_index_report(report, request.config, std::nullopt); - } catch (const std::exception& error) { - finalize_index_report_noexcept(report, request.config); - return tl::unexpected(Error(ErrorType::INTERNAL_ERROR, error.what())); - } + const auto start = std::chrono::steady_clock::now(); + auto report = run_tuning([&request]() { return internal::ParseRequest(request); }, start); + const auto status = report.value("status", std::string("failed")); + if (status == "failed") { return tl::unexpected(report_error(report)); } + if (status == "no_feasible_candidate") { + IndexResult result; + result.status = TuneStatus::NO_FEASIBLE_CANDIDATE; + result.report = report; + result.best_effort = report.value("best_effort", JsonType(nullptr)); + return result; + } + + const auto artifact_failure = [&](Error error) -> tl::expected { + finalize_artifacts_noexcept(report, request.config.keep_intermediate); + return tl::unexpected(std::move(error)); + }; + try { const auto& recommendation = report.at("recommendation"); const auto index_name = recommendation.at("index_name").get(); @@ -988,46 +1054,53 @@ TuneIndex(const IndexRequest& request) { recommendation.at("artifacts").at("index_path").get(); auto created = Factory::CreateIndex(index_name, create_parameters); if (!created.has_value()) { - finalize_index_report(report, request.config, std::nullopt); - return tl::unexpected(created.error()); + return artifact_failure(created.error()); } std::ifstream input(artifact_path, std::ios::binary); if (!input.good()) { - finalize_index_report(report, request.config, std::nullopt); - return tl::unexpected( + return artifact_failure( Error(ErrorType::MISSING_FILE, "failed to open index artifact: " + artifact_path)); } auto loaded = created.value()->Deserialize(input); if (!loaded.has_value()) { - finalize_index_report(report, request.config, std::nullopt); - return tl::unexpected(loaded.error()); + return artifact_failure(loaded.error()); } - finalize_index_report(report, request.config, artifact_path); - return IndexResult{created.value(), - index_name, - create_parameters, - search_parameters, - recommendation.at("metrics").dump(), - artifact_path, - report.at("report_path").get(), - report.dump()}; + + IndexResult result; + result.index = created.value(); + result.index_name = index_name; + result.create_parameters = create_parameters; + result.search_parameters = search_parameters; + result.metrics = recommendation.at("metrics"); + result.artifact_path = artifact_path; + result.report = report; + return result; } catch (const std::exception& error) { - finalize_index_report_noexcept(report, request.config); - return tl::unexpected(Error(ErrorType::INTERNAL_ERROR, error.what())); + return artifact_failure(Error(ErrorType::INTERNAL_ERROR, error.what())); } } tl::expected TuneSearch(const SearchRequest& request) { auto report = run_tuning([&request]() { return internal::ParseRequest(request); }); - if (report.value("status", std::string("failed")) != "success") { + const auto status = report.value("status", std::string("failed")); + if (status == "failed") { return tl::unexpected(report_error(report)); } + if (status == "no_feasible_candidate") { + SearchResult result; + result.status = TuneStatus::NO_FEASIBLE_CANDIDATE; + result.report = report; + result.best_effort = report.value("best_effort", JsonType(nullptr)); + return result; + } + const auto& recommendation = report["recommendation"]; - return SearchResult{recommendation["search_params"].dump(), - recommendation["metrics"].dump(), - report["report_path"].get(), - report.dump()}; + SearchResult result; + result.parameters = recommendation["search_params"].dump(); + result.metrics = recommendation["metrics"]; + result.report = report; + return result; } JsonType @@ -1038,7 +1111,7 @@ RunAutoTune(const JsonType& request) { return std::visit( [start](auto parsed) { return run_tuning_locked( - [parsed = std::move(parsed)]() { return parsed; }, false, start); + [parsed = std::move(parsed)]() { return parsed; }, true, start); }, internal::ParseRequest(request)); } catch (const H5::Exception& error) { diff --git a/tools/autotune/autotune.h b/tools/autotune/autotune.h index 8157bf0456..c89507fae5 100644 --- a/tools/autotune/autotune.h +++ b/tools/autotune/autotune.h @@ -61,12 +61,10 @@ struct Workload { uint64_t concurrency{1}; }; -/// Execution and report options shared by index and search tuning. +/// Evaluation options shared by index and search tuning. struct Config { - /// Directory for generated reports and index artifacts. + /// Directory for generated index artifacts. std::string workspace_path{"/tmp/vsag_autotune"}; - /// Optional report destination; an empty value generates a path under workspace_path. - std::string report_path; /// Maximum number of concrete search trials planned for the request. uint64_t max_trials{1000}; /// Keep every generated index artifact instead of only the selected artifact. @@ -101,12 +99,6 @@ struct IndexRequest { struct SearchRequest { /// Existing index reused in place; TuneSearch neither rebuilds nor takes exclusive ownership. IndexPtr index; - /// Concrete type of index, including pyramid for search-only tuning. - std::string index_name; - /// Base vectors and IDs used to evaluate recall and validate ground truth. - DatasetPtr base; - /// l2, ip, or cosine; it must match the existing index and datasets. - std::string metric_type; Workload workload; /// JSON search candidate space; missing supported fields receive built-in proposals. std::string parameter_space{"{}"}; @@ -115,8 +107,14 @@ struct SearchRequest { Config config; }; -/// Successful index-tuning result. +enum class TuneStatus { + SUCCESS = 0, + NO_FEASIBLE_CANDIDATE, +}; + +/// Completed index-tuning result. Recommendation fields are valid only on SUCCESS. struct IndexResult { + TuneStatus status{TuneStatus::SUCCESS}; /// Loaded selected index, ready for queries. IndexPtr index; std::string index_name; @@ -125,31 +123,34 @@ struct IndexResult { /// Concrete JSON parameters recommended for queries. std::string search_parameters; /// Validated metric values as a JSON object. - std::string metrics; + JsonType metrics = JsonType::object(); /// Selected serialized artifact; the caller removes it when it is no longer needed. std::string artifact_path; - std::string report_path; /// Complete report as JSON. - std::string report; + JsonType report = JsonType::object(); + /// Closest evaluated candidate when status is NO_FEASIBLE_CANDIDATE. + JsonType best_effort = nullptr; }; -/// Successful search-only tuning result. +/// Completed search-tuning result. Recommendation fields are valid only on SUCCESS. struct SearchResult { + TuneStatus status{TuneStatus::SUCCESS}; /// Concrete JSON parameters recommended for the existing index. std::string parameters; /// Validated metric values as a JSON object. - std::string metrics; - std::string report_path; + JsonType metrics = JsonType::object(); /// Complete report as JSON. - std::string report; + JsonType report = JsonType::object(); + /// Closest evaluated candidate when status is NO_FEASIBLE_CANDIDATE. + JsonType best_effort = nullptr; }; /** * Experimental, synchronous build-tree tool API. It is not installed as part of the VSAG SDK. * - * Returns INVALID_ARGUMENT for invalid requests or no feasible candidate, and an execution error - * for evaluation, artifact, or report failures. On success, only the selected artifact is retained - * unless Config::keep_intermediate is true. + * Returns an error for invalid requests or execution failures. A completed request with no + * feasible candidate returns TuneStatus::NO_FEASIBLE_CANDIDATE and a structured best_effort. On + * success, only the selected artifact is retained unless Config::keep_intermediate is true. */ tl::expected TuneIndex(const IndexRequest& request); @@ -157,13 +158,14 @@ TuneIndex(const IndexRequest& request); /** * Synchronously evaluates search candidates against SearchRequest::index without rebuilding it. * - * Returns INVALID_ARGUMENT for invalid requests or no feasible candidate, and an execution error - * for evaluation or report failures. + * Returns an error for invalid requests or execution failures. A completed request with no + * feasible candidate returns TuneStatus::NO_FEASIBLE_CANDIDATE and a structured best_effort. */ tl::expected TuneSearch(const SearchRequest& request); -/// Offline JSON adapter used by the CLI. Validation failures are returned as structured JSON. +/// Offline JSON adapter used by the CLI. Parsed requests persist a full report; validation +/// failures are returned as structured JSON. JsonType RunAutoTune(const JsonType& request); diff --git a/tools/autotune/autotune_evaluation.cpp b/tools/autotune/autotune_evaluation.cpp index 1e9d8e1725..35244668b3 100644 --- a/tools/autotune/autotune_evaluation.cpp +++ b/tools/autotune/autotune_evaluation.cpp @@ -128,6 +128,7 @@ search_config(const RequestContext& request, const Candidate& candidate) { config.enable_memory = false; config.enable_recall = request.enable_recall; config.enable_percent_recall = false; + config.use_id_based_recall = true; return config; } @@ -227,7 +228,7 @@ EvaluateCandidates(const IndexTuningRequest& tuning_request, {{"index_path", index_path}, {"source", "generated"}, {"use_existing_index", false}, - {"retained", request.keep_intermediate}}}}; + {"retained", true}}}}; MetricMap shared_metrics; IndexPtr index; @@ -310,11 +311,6 @@ EvaluateCandidates(const IndexTuningRequest& tuning_request, }; EvaluateEfSearchRange(*candidate.ef_search_range, recall_target, evaluate_ef_search); } - - if (!request.keep_intermediate) { - std::error_code error; - std::filesystem::remove(index_path, error); - } } return evaluation; } diff --git a/tools/autotune/autotune_test.cpp b/tools/autotune/autotune_test.cpp index f82ed2968e..27de6628f1 100644 --- a/tools/autotune/autotune_test.cpp +++ b/tools/autotune/autotune_test.cpp @@ -720,6 +720,28 @@ TEST_CASE("AutoTune builds once per create candidate and supports an existing in REQUIRE(existing_result["request"]["create_params"]["metric_type"] == "l2"); } +TEST_CASE("AutoTune CLI keeps only the recommended artifact by default") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath dataset(temp_path("autotune-retained-dataset", ".hdf5")); + ScopedPath workspace(temp_path("autotune-retained-workspace")); + write_dataset(dataset.Get()); + auto input = request(dataset.Get(), workspace.Get()); + input["indexes"] = JsonType::array({input["indexes"][0]}); + input["indexes"][0]["search_params"]["hgraph"]["ef_search"] = 8; + input["tuning_config"].erase("keep_intermediate"); + input["tuning_config"]["max_trials"] = 1; + + const auto result = vsag::autotune::RunAutoTune(input); + INFO(result.dump(2)); + REQUIRE(result["status"] == "success"); + REQUIRE(result["recommendation"]["artifacts"]["retained"] == true); + REQUIRE(std::filesystem::is_regular_file( + result["recommendation"]["artifacts"]["index_path"].get())); + REQUIRE(count_index_artifacts(workspace.Get()) == 1); + REQUIRE(std::filesystem::is_regular_file(result["report_path"].get())); +} + TEST_CASE("AutoTune searches an in-memory existing index") { vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); ScopedBlockSizeLimit block_size_limit(256UL * 1024); @@ -735,9 +757,6 @@ TEST_CASE("AutoTune searches an in-memory existing index") { vsag::autotune::SearchRequest input; input.index = created.value(); - input.index_name = "hgraph"; - input.base = fixture.base; - input.metric_type = vsag::METRIC_L2; input.workload = {fixture.queries, fixture.ground_truth, 3, 2}; input.parameter_space = R"({"hgraph":{"ef_search":[8,16]}})"; input.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.0}}; @@ -747,7 +766,8 @@ TEST_CASE("AutoTune searches an in-memory existing index") { const auto tuned = vsag::autotune::TuneSearch(input); REQUIRE(tuned.has_value()); - const auto result = JsonType::parse(tuned.value().report); + REQUIRE(tuned->status == vsag::autotune::TuneStatus::SUCCESS); + const auto& result = tuned->report; REQUIRE(result["status"] == "success"); REQUIRE(result["builds"].empty()); REQUIRE(result["trials"].size() == 2); @@ -756,6 +776,26 @@ TEST_CASE("AutoTune searches an in-memory existing index") { REQUIRE(trial["metrics"].contains("recall_at_k")); REQUIRE(trial["metrics"].contains("latency_avg_ms")); } + + input.workload.ground_truth = nullptr; + input.constraints = {{vsag::autotune::Metric::QPS, 0.0}}; + const auto latency_only = vsag::autotune::TuneSearch(input); + REQUIRE(latency_only.has_value()); + REQUIRE(latency_only->status == vsag::autotune::TuneStatus::SUCCESS); + REQUIRE_FALSE(latency_only->metrics.contains("recall_at_k")); + REQUIRE_FALSE(latency_only->report.contains("report_path")); + REQUIRE(load_json_reports(workspace.Get()).empty()); + + input.objective = vsag::autotune::Metric::INDEX_MEMORY_MB; + REQUIRE_THROWS_WITH( + vsag::autotune::internal::ParseRequest(input), + Catch::Matchers::ContainsSubstring("objective cannot rank search candidates")); + input.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + + input.parameter_space = R"({"ivf":{"scan_buckets_count":1}})"; + REQUIRE_THROWS_WITH( + vsag::autotune::internal::ParseRequest(input), + Catch::Matchers::ContainsSubstring("search_parameter_space.ivf is unsupported")); } TEST_CASE("AutoTune uses default candidates for an in-memory IVF index") { @@ -772,9 +812,6 @@ TEST_CASE("AutoTune uses default candidates for an in-memory IVF index") { vsag::autotune::SearchRequest input; input.index = created.value(); - input.index_name = "ivf"; - input.base = fixture.base; - input.metric_type = vsag::METRIC_L2; input.workload = {fixture.queries, fixture.ground_truth, 3, 2}; input.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.0}}; input.objective = vsag::autotune::Metric::LATENCY_AVG_MS; @@ -783,7 +820,8 @@ TEST_CASE("AutoTune uses default candidates for an in-memory IVF index") { const auto tuned = vsag::autotune::TuneSearch(input); REQUIRE(tuned.has_value()); - const auto result = JsonType::parse(tuned.value().report); + REQUIRE(tuned->status == vsag::autotune::TuneStatus::SUCCESS); + const auto& result = tuned->report; REQUIRE(result["status"] == "success"); REQUIRE(result["trials"].size() == 4); REQUIRE(result["recommendation"]["search_params"]["ivf"].contains("scan_buckets_count")); @@ -821,9 +859,6 @@ TEST_CASE("AutoTune searches an existing Pyramid index for one path workload") { vsag::autotune::SearchRequest input; input.index = created.value(); - input.index_name = "pyramid"; - input.base = base; - input.metric_type = vsag::METRIC_L2; input.workload.queries = queries; input.workload.ground_truth = fixture.ground_truth; input.workload.top_k = 3; @@ -836,7 +871,8 @@ TEST_CASE("AutoTune searches an existing Pyramid index for one path workload") { const auto tuned = vsag::autotune::TuneSearch(input); REQUIRE(tuned.has_value()); - const auto result = JsonType::parse(tuned.value().report); + REQUIRE(tuned->status == vsag::autotune::TuneStatus::SUCCESS); + const auto& result = tuned->report; REQUIRE(result["status"] == "success"); REQUIRE(result["trials"].size() == 2); REQUIRE(result["recommendation"]["index_name"] == "pyramid"); @@ -845,8 +881,9 @@ TEST_CASE("AutoTune searches an existing Pyramid index for one path workload") { input.constraints.push_back({vsag::autotune::Metric::INDEX_MEMORY_MB, 1.0}); const auto memory_constrained = vsag::autotune::TuneSearch(input); - REQUIRE_FALSE(memory_constrained.has_value()); - REQUIRE(memory_constrained.error().type == vsag::ErrorType::INVALID_ARGUMENT); + REQUIRE(memory_constrained.has_value()); + REQUIRE(memory_constrained->status == vsag::autotune::TuneStatus::NO_FEASIBLE_CANDIDATE); + REQUIRE(memory_constrained->best_effort["constraint_evaluation"]["satisfied"] == false); input.constraints.pop_back(); input.workload.queries = fixture.queries; @@ -888,14 +925,16 @@ TEST_CASE("TuneIndex returns a queryable selected index") { auto tuned = vsag::autotune::TuneIndex(input); REQUIRE(tuned.has_value()); const auto& result = tuned.value(); + REQUIRE(result.status == vsag::autotune::TuneStatus::SUCCESS); + REQUIRE(result.best_effort.is_null()); REQUIRE(result.index != nullptr); REQUIRE(result.index->GetNumElements() == 64); REQUIRE(result.index_name == "hgraph"); REQUIRE(JsonType::parse(result.create_parameters)["dim"] == 8); REQUIRE(JsonType::parse(result.search_parameters)["hgraph"]["ef_search"] == 16); - REQUIRE(JsonType::parse(result.metrics).contains("recall_at_k")); + REQUIRE(result.metrics.contains("recall_at_k")); REQUIRE(std::filesystem::is_regular_file(result.artifact_path)); - REQUIRE(std::filesystem::is_regular_file(result.report_path)); + REQUIRE_FALSE(result.report.contains("report_path")); auto query = vsag::Dataset::Make(); query->NumElements(1) @@ -916,7 +955,7 @@ TEST_CASE("TuneIndex returns a queryable selected index") { REQUIRE(restored.value()->Deserialize(artifact).has_value()); REQUIRE(restored.value()->KnnSearch(query, 3, result.search_parameters).has_value()); - const auto report = JsonType::parse(result.report); + const auto& report = result.report; REQUIRE(report["status"] == "success"); REQUIRE(report["builds"].size() == 2); REQUIRE(std::count_if(report["builds"].begin(), report["builds"].end(), [](const auto& build) { @@ -928,7 +967,7 @@ TEST_CASE("TuneIndex returns a queryable selected index") { REQUIRE(report["request"]["config"]["workspace_path"] == workspace.Get()); } -TEST_CASE("TuneIndex rejects an infeasible recommendation") { +TEST_CASE("TuneIndex reports an infeasible recommendation") { vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); ScopedBlockSizeLimit block_size_limit(256UL * 1024); ScopedPath workspace(temp_path("autotune-infeasible-workspace")); @@ -942,16 +981,17 @@ TEST_CASE("TuneIndex rejects an infeasible recommendation") { input.config.max_trials = 1; const auto tuned = vsag::autotune::TuneIndex(input); - REQUIRE_FALSE(tuned.has_value()); - REQUIRE(tuned.error().type == vsag::ErrorType::INVALID_ARGUMENT); - REQUIRE_THAT(tuned.error().message, Catch::Matchers::ContainsSubstring("report:")); + REQUIRE(tuned.has_value()); + REQUIRE(tuned->status == vsag::autotune::TuneStatus::NO_FEASIBLE_CANDIDATE); + REQUIRE(tuned->index == nullptr); + REQUIRE(tuned->best_effort["constraint_evaluation"]["satisfied"] == false); + REQUIRE_FALSE(tuned->report.contains("report_path")); REQUIRE(count_index_artifacts(workspace.Get()) == 0); - const auto reports = load_json_reports(workspace.Get()); - REQUIRE(reports.size() == 1); - REQUIRE(reports[0]["status"] == "no_feasible_candidate"); - REQUIRE(reports[0]["request"]["config"]["keep_intermediate"] == false); - REQUIRE(reports[0]["builds"][0]["artifacts"]["retained"] == false); - REQUIRE(reports[0]["best_effort"]["artifacts"]["retained"] == false); + REQUIRE(load_json_reports(workspace.Get()).empty()); + REQUIRE(tuned->report["status"] == "no_feasible_candidate"); + REQUIRE(tuned->report["request"]["config"]["keep_intermediate"] == false); + REQUIRE(tuned->report["builds"][0]["artifacts"]["retained"] == false); + REQUIRE(tuned->best_effort["artifacts"]["retained"] == false); } TEST_CASE("TuneIndex cleans artifacts when all search trials fail") { @@ -969,33 +1009,43 @@ TEST_CASE("TuneIndex cleans artifacts when all search trials fail") { const auto tuned = vsag::autotune::TuneIndex(input); REQUIRE_FALSE(tuned.has_value()); REQUIRE(tuned.error().type == vsag::ErrorType::INTERNAL_ERROR); - REQUIRE_THAT(tuned.error().message, Catch::Matchers::ContainsSubstring("report:")); REQUIRE(count_index_artifacts(workspace.Get()) == 0); - const auto reports = load_json_reports(workspace.Get()); - REQUIRE(reports.size() == 1); - REQUIRE(reports[0]["status"] == "failed"); - REQUIRE(reports[0]["failure"]["code"] == "all_trials_failed"); - REQUIRE(reports[0]["request"]["config"]["keep_intermediate"] == false); - REQUIRE(reports[0]["builds"][0]["artifacts"]["retained"] == false); + REQUIRE(load_json_reports(workspace.Get()).empty()); } -TEST_CASE("TuneIndex reports report-writing failures") { +TEST_CASE("TuneIndex returns its report without writing a report file") { vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); ScopedBlockSizeLimit block_size_limit(256UL * 1024); - ScopedPath workspace(temp_path("autotune-report-failure-workspace")); + ScopedPath workspace(temp_path("autotune-in-memory-report-workspace")); MemoryFixture fixture; - - const auto report_directory = workspace.Get() + "/report-is-a-directory"; - std::filesystem::create_directories(report_directory); auto input = fixture.Request(workspace.Get()); input.index_spaces[0].create_parameter_space = R"({"index_param":{"base_quantization_type":"fp32","max_degree":8,)" R"("ef_construction":40,"build_thread_count":2}})"; input.config.max_trials = 1; - input.config.report_path = report_directory; const auto tuned = vsag::autotune::TuneIndex(input); - REQUIRE_FALSE(tuned.has_value()); - REQUIRE(tuned.error().type == vsag::ErrorType::INTERNAL_ERROR); + REQUIRE(tuned.has_value()); + REQUIRE(tuned->status == vsag::autotune::TuneStatus::SUCCESS); + REQUIRE(tuned->report["status"] == "success"); + REQUIRE_FALSE(tuned->report.contains("report_path")); + REQUIRE(load_json_reports(workspace.Get()).empty()); +} + +TEST_CASE("AutoTune validates the CLI report path before evaluation") { + ScopedPath dataset(temp_path("autotune-report-path-dataset", ".hdf5")); + ScopedPath workspace(temp_path("autotune-report-path-workspace")); + write_dataset(dataset.Get()); + const auto report_directory = workspace.Get() + "/report-is-a-directory"; + std::filesystem::create_directories(report_directory); + + auto input = request(dataset.Get(), workspace.Get()); + input["output"] = {{"result_path", report_directory}}; + const auto result = vsag::autotune::RunAutoTune(input); + + REQUIRE(result["status"] == "failed"); + REQUIRE(result["failure"]["stage"] == "report"); + REQUIRE_THAT(result["failure"]["message"].get(), + Catch::Matchers::ContainsSubstring("failed to open report path")); REQUIRE(count_index_artifacts(workspace.Get()) == 0); } diff --git a/tools/eval/case/search_eval_case.cpp b/tools/eval/case/search_eval_case.cpp index af4033dfc8..dc6abe9b04 100644 --- a/tools/eval/case/search_eval_case.cpp +++ b/tools/eval/case/search_eval_case.cpp @@ -141,8 +141,8 @@ SearchEvalCase::init_latency_monitor() { void SearchEvalCase::init_recall_monitor() { if (config_.enable_recall or config_.enable_percent_recall) { - auto recall_monitor = - std::make_shared(this->dataset_ptr_->GetNumberOfQuery()); + auto recall_monitor = std::make_shared( + this->dataset_ptr_->GetNumberOfQuery(), config_.use_id_based_recall); if (config_.enable_recall) { recall_monitor->SetMetrics("avg_recall"); } diff --git a/tools/eval/eval_config.h b/tools/eval/eval_config.h index ef587f7ad4..c8eb71a926 100644 --- a/tools/eval/eval_config.h +++ b/tools/eval/eval_config.h @@ -56,6 +56,7 @@ class EvalConfig { bool enable_memory{true}; bool enable_latency{true}; bool enable_percent_latency{true}; + bool use_id_based_recall{false}; EvalConfig() = default; }; diff --git a/tools/eval/eval_dataset.cpp b/tools/eval/eval_dataset.cpp index 3886030f79..f173f2ba56 100644 --- a/tools/eval/eval_dataset.cpp +++ b/tools/eval/eval_dataset.cpp @@ -317,6 +317,42 @@ EvalDataset::FromDatasets(const vsag::DatasetPtr& base, return dataset; } +EvalDatasetPtr +EvalDataset::FromSearchDatasets(const vsag::DatasetPtr& queries, + const vsag::DatasetPtr& ground_truth) { + if (queries == nullptr || queries->GetNumElements() <= 0) { + throw std::invalid_argument("queries dataset is required and must not be empty"); + } + if (queries->GetDim() <= 0 || queries->GetFloat32Vectors() == nullptr) { + throw std::invalid_argument( + "in-memory search evaluation requires positive-dimensional float32 queries"); + } + if (ground_truth != nullptr && + (ground_truth->GetNumElements() != queries->GetNumElements() || + ground_truth->GetDim() <= 0 || ground_truth->GetIds() == nullptr)) { + throw std::invalid_argument( + "ground_truth must contain one non-empty id row for every query"); + } + + auto dataset = std::make_shared(); + dataset->query_dataset_ = queries; + dataset->ground_truth_dataset_ = ground_truth; + dataset->vector_type_ = DENSE_VECTORS; + dataset->train_data_type_ = vsag::DATATYPE_FLOAT32; + dataset->test_data_type_ = vsag::DATATYPE_FLOAT32; + dataset->train_data_size_ = sizeof(float); + dataset->test_data_size_ = sizeof(float); + dataset->number_of_query_ = queries->GetNumElements(); + dataset->dim_ = queries->GetDim(); + dataset->train_shape_ = {0, dataset->dim_}; + dataset->test_shape_ = {dataset->number_of_query_, dataset->dim_}; + dataset->neighbors_shape_ = + ground_truth == nullptr ? shape_t{0, 0} + : shape_t{ground_truth->GetNumElements(), ground_truth->GetDim()}; + dataset->file_path_ = ""; + return dataset; +} + EvalDatasetPtr EvalDataset::Load(const std::string& filename) { H5::H5File file(filename, H5F_ACC_RDONLY); diff --git a/tools/eval/eval_dataset.h b/tools/eval/eval_dataset.h index 1db524a728..2a9117158d 100644 --- a/tools/eval/eval_dataset.h +++ b/tools/eval/eval_dataset.h @@ -49,6 +49,9 @@ class EvalDataset { const vsag::DatasetPtr& ground_truth, const std::string& metric_type); + static EvalDatasetPtr + FromSearchDatasets(const vsag::DatasetPtr& queries, const vsag::DatasetPtr& ground_truth); + static void Save(const EvalDatasetPtr& dataset, const std::string& filename); diff --git a/tools/eval/eval_dataset_test.cpp b/tools/eval/eval_dataset_test.cpp index e8c4f02cc8..1d4eed2635 100644 --- a/tools/eval/eval_dataset_test.cpp +++ b/tools/eval/eval_dataset_test.cpp @@ -198,6 +198,38 @@ TEST_CASE("EvalDataset builds a dense in-memory view with original ids", "[ut][e REQUIRE_FALSE(std::filesystem::exists(save_path)); } +TEST_CASE("EvalDataset builds a query-only view for id recall", "[ut][eval_dataset]") { + constexpr int64_t dim = 2; + std::vector query_vectors{1.0F, 2.0F, 3.0F, 4.0F}; + auto queries = vsag::Dataset::Make() + ->NumElements(2) + ->Dim(dim) + ->Float32Vectors(query_vectors.data()) + ->Owner(false); + + std::vector ground_truth_ids{10, 20, 30, 40}; + auto ground_truth = + vsag::Dataset::Make()->NumElements(2)->Dim(2)->Ids(ground_truth_ids.data())->Owner(false); + + auto dataset = EvalDataset::FromSearchDatasets(queries, ground_truth); + REQUIRE(dataset->GetTrain() == nullptr); + REQUIRE(dataset->GetTest() == query_vectors.data()); + REQUIRE(dataset->GetNumberOfBase() == 0); + REQUIRE(dataset->GetNumberOfQuery() == 2); + REQUIRE(dataset->GetGroundTruthK() == 2); + + int64_t result_ids[]{10, 99}; + vsag::eval::SearchRecord record{ + result_ids, dataset->GetNeighbors(0), dataset.get(), dataset->GetOneTest(0), 2, 2}; + vsag::eval::RecallMonitor recall_monitor(1, true); + recall_monitor.SetMetrics("avg_recall"); + recall_monitor.Record(&record); + REQUIRE(recall_monitor.GetResult()["recall_avg"].get() == 0.5); + + REQUIRE_THROWS_WITH(EvalDataset::FromSearchDatasets(nullptr, ground_truth), + "queries dataset is required and must not be empty"); +} + TEST_CASE("EvalDataset rejects incomplete and incompatible HDF5 schemas", "[ut][eval_dataset]") { const std::vector required_datasets{"train", "test", "neighbors", "distances"}; for (const auto& missing : required_datasets) { diff --git a/tools/eval/evaluator.cpp b/tools/eval/evaluator.cpp index e09c64b1fb..fb3d391512 100644 --- a/tools/eval/evaluator.cpp +++ b/tools/eval/evaluator.cpp @@ -71,6 +71,9 @@ validate_search(const EvalDatasetPtr& dataset, const EvalConfig& config) { throw std::invalid_argument( "evaluation ground truth must contain at least top_k neighbors per query"); } + if (!config.use_id_based_recall && dataset->GetNumberOfBase() <= 0) { + throw std::invalid_argument("distance-based recall evaluation requires base vectors"); + } } } diff --git a/tools/eval/monitor/recall_monitor.cpp b/tools/eval/monitor/recall_monitor.cpp index c2d2690df9..79a21ceda7 100644 --- a/tools/eval/monitor/recall_monitor.cpp +++ b/tools/eval/monitor/recall_monitor.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "../eval_dataset.h" @@ -43,7 +44,21 @@ get_recall(const float* distances, return static_cast(count) / static_cast(top_k); } -RecallMonitor::RecallMonitor(uint64_t max_record_counts) : Monitor("recall_monitor") { +static double +get_id_recall(const int64_t* neighbors, + const int64_t* ground_truth_neighbors, + uint64_t recall_num, + uint64_t top_k) { + std::unordered_set expected(ground_truth_neighbors, ground_truth_neighbors + top_k); + uint64_t count = 0; + for (uint64_t i = 0; i < recall_num; ++i) { + count += expected.erase(neighbors[i]); + } + return static_cast(count) / static_cast(top_k); +} + +RecallMonitor::RecallMonitor(uint64_t max_record_counts, bool use_id_based_recall) + : Monitor("recall_monitor"), use_id_based_recall_(use_id_based_recall) { if (max_record_counts > 0) { this->recall_records_.reserve(max_record_counts); } @@ -79,6 +94,11 @@ RecallMonitor::Record(void* input) { this->recall_records_.emplace_back(0.0); return; } + if (use_id_based_recall_) { + this->recall_records_.emplace_back( + get_id_recall(neighbors, gt_neighbors, result_count, requested_top_k)); + return; + } uint64_t dim = dataset->GetDim(); auto distance_func = dataset->GetDistanceFunc(); std::vector gt_distances(requested_top_k); diff --git a/tools/eval/monitor/recall_monitor.h b/tools/eval/monitor/recall_monitor.h index 1977c55726..b3a52634e6 100644 --- a/tools/eval/monitor/recall_monitor.h +++ b/tools/eval/monitor/recall_monitor.h @@ -22,7 +22,7 @@ namespace vsag::eval { class RecallMonitor : public Monitor { public: - explicit RecallMonitor(uint64_t max_record_counts = 0); + explicit RecallMonitor(uint64_t max_record_counts = 0, bool use_id_based_recall = false); ~RecallMonitor() override = default; @@ -55,6 +55,7 @@ class RecallMonitor : public Monitor { std::vector recall_records_; std::vector metrics_; + bool use_id_based_recall_{false}; }; } // namespace vsag::eval From ff3f0f1114ec1b585514ce6d04dac657abbc2f31 Mon Sep 17 00:00:00 2001 From: jc543239 Date: Fri, 31 Jul 2026 10:32:02 +0000 Subject: [PATCH 7/8] fix: address final AutoTune review feedback Signed-off-by: jc543239 Assisted-by: Codex:gpt-5 --- docs/docs/en/src/resources/autotune.md | 17 ++++++--- docs/docs/en/src/resources/autotune_api_v1.md | 2 +- docs/docs/zh/src/resources/autotune.md | 16 +++++--- docs/docs/zh/src/resources/autotune_api_v1.md | 2 +- tools/autotune/autotune.cpp | 37 ++++++++++++------ tools/autotune/autotune_evaluation.cpp | 21 ++-------- tools/autotune/autotune_test.cpp | 38 +++++++++++++++++++ tools/eval/case/build_eval_case.cpp | 14 +++++-- tools/eval/eval_dataset.cpp | 10 ++--- tools/eval/eval_dataset.h | 4 +- tools/eval/eval_dataset_test.cpp | 28 +++++++++++++- tools/eval/monitor/recall_monitor.cpp | 8 +++- 12 files changed, 140 insertions(+), 57 deletions(-) diff --git a/docs/docs/en/src/resources/autotune.md b/docs/docs/en/src/resources/autotune.md index 434460fe93..dc720e26d5 100644 --- a/docs/docs/en/src/resources/autotune.md +++ b/docs/docs/en/src/resources/autotune.md @@ -54,9 +54,11 @@ request.constraints = { }; request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; -auto result = vsag::autotune::TuneIndex(request).value(); -auto neighbors = - result.index->KnnSearch(query, 10, result.search_parameters).value(); +auto result = vsag::autotune::TuneIndex(request); +if (result.has_value() && result->status == vsag::autotune::TuneStatus::SUCCESS) { + auto neighbors = + result->index->KnnSearch(query, 10, result->search_parameters).value(); +} ``` `base` and `queries` are dense float32 datasets. `base` IDs must be present and unique. The @@ -74,7 +76,8 @@ validated `search_parameters`, metrics, artifact path, and the complete report. `report` are `JsonType` objects. The typed API returns the report in memory and never writes a report file. The recommended artifact is retained, so another process can recreate an empty index from `index_name + create_parameters` and deserialize that file. Unselected intermediate -artifacts are removed by default. +artifacts are removed by default. The caller removes the recommended artifact when it is no longer +needed. If no candidate satisfies every constraint, the completed call has `status=TuneStatus::NO_FEASIBLE_CANDIDATE` and a structured `best_effort`; recommendation fields @@ -176,8 +179,10 @@ request.parameter_space = R"({"hgraph":{"ef_search":[40,80,120]}})"; request.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.95}}; request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; -auto result = vsag::autotune::TuneSearch(request).value(); -auto neighbors = existing_index->KnnSearch(query, 10, result.parameters).value(); +auto result = vsag::autotune::TuneSearch(request); +if (result.has_value() && result->status == vsag::autotune::TuneStatus::SUCCESS) { + auto neighbors = existing_index->KnnSearch(query, 10, result->parameters).value(); +} ``` This avoids serialization and file I/O and reuses the caller's index for all search trials. diff --git a/docs/docs/en/src/resources/autotune_api_v1.md b/docs/docs/en/src/resources/autotune_api_v1.md index edfc90fce1..6a3222715f 100644 --- a/docs/docs/en/src/resources/autotune_api_v1.md +++ b/docs/docs/en/src/resources/autotune_api_v1.md @@ -316,7 +316,7 @@ A completed evaluation returns this top-level shape. The CLI also writes it to d | `builds` | One record per concrete generated build group; empty in search-only mode. | | `trials` | One record per executed concrete search candidate. | | `request` | Effective normalized request used by the tuning engine. | -| `elapsed_seconds` | AutoTune wall time through selection, excluding report writing and cleanup. | +| `elapsed_seconds` | AutoTune wall time through artifact cleanup; excludes final report writing. | | `report_path` | Persisted full report path; CLI/JSON adapter only. | | `failure` | Present when the overall run failed. | diff --git a/docs/docs/zh/src/resources/autotune.md b/docs/docs/zh/src/resources/autotune.md index 2ccd5c82ad..378fa17ab3 100644 --- a/docs/docs/zh/src/resources/autotune.md +++ b/docs/docs/zh/src/resources/autotune.md @@ -51,9 +51,11 @@ request.constraints = { }; request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; -auto result = vsag::autotune::TuneIndex(request).value(); -auto neighbors = - result.index->KnnSearch(query, 10, result.search_parameters).value(); +auto result = vsag::autotune::TuneIndex(request); +if (result.has_value() && result->status == vsag::autotune::TuneStatus::SUCCESS) { + auto neighbors = + result->index->KnnSearch(query, 10, result->search_parameters).value(); +} ``` `base` 和 `queries` 必须是 dense float32 Dataset，base ID 必须存在且唯一。ground truth 的 @@ -69,7 +71,7 @@ IVF 候选；显式参数中的数组和 `$range` 与 CLI 契约语义相同。 `search_parameters`、metrics、artifact 路径和完整报告。`metrics` 和 `report` 都是 `JsonType` 对象。typed API 在内存中返回报告，绝不会写报告文件。最终推荐 artifact 会 保留，因此可以用 `index_name + create_parameters` 重新创建空索引并反序列化该文件； -未选中的中间 artifact 默认删除。 +未选中的中间 artifact 默认删除。调用方应在不再需要时删除推荐 artifact。 如果没有候选满足全部约束，调用仍正常完成，并返回 `status=TuneStatus::NO_FEASIBLE_CANDIDATE` 和结构化 `best_effort`；此时 `index` 等推荐 @@ -168,8 +170,10 @@ request.parameter_space = R"({"hgraph":{"ef_search":[40,80,120]}})"; request.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.95}}; request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; -auto result = vsag::autotune::TuneSearch(request).value(); -auto neighbors = existing_index->KnnSearch(query, 10, result.parameters).value(); +auto result = vsag::autotune::TuneSearch(request); +if (result.has_value() && result->status == vsag::autotune::TuneStatus::SUCCESS) { + auto neighbors = existing_index->KnnSearch(query, 10, result->parameters).value(); +} ``` 该接口避免序列化和文件 I/O，并让全部 search trial 复用调用方的索引。AutoTune 从 diff --git a/docs/docs/zh/src/resources/autotune_api_v1.md b/docs/docs/zh/src/resources/autotune_api_v1.md index d28d85b3d9..9596c9b76e 100644 --- a/docs/docs/zh/src/resources/autotune_api_v1.md +++ b/docs/docs/zh/src/resources/autotune_api_v1.md @@ -297,7 +297,7 @@ artifact，因此 `workspace_path` 和 `keep_intermediate` 对它没有影响。 | `builds` | 每组具体生成 build 一条记录；search-only 模式为空数组。 | | `trials` | 每组已执行的具体 search 候选一条记录。 | | `request` | 调优引擎实际使用的有效规范化请求。 | -| `elapsed_seconds` | 截止结果选择的 AutoTune 墙钟时间，不含报告写入和清理。 | +| `elapsed_seconds` | AutoTune 墙钟时间包含 artifact 清理，不含最终报告写入。 | | `report_path` | 持久化完整报告路径；只在 CLI/JSON 适配器中存在。 | | `failure` | 整体执行失败时出现。 | diff --git a/tools/autotune/autotune.cpp b/tools/autotune/autotune.cpp index ed9e78d83f..29eadbf4a6 100644 --- a/tools/autotune/autotune.cpp +++ b/tools/autotune/autotune.cpp @@ -216,14 +216,16 @@ metric_type(const std::string& dataset_metric) { struct offline_dataset_owner { eval::EvalDatasetPtr source; + std::vector identity_ids; DatasetPtr base; DatasetPtr queries; DatasetPtr ground_truth; }; -template void -attach_offline_dataset(Request& request, const eval::EvalDatasetPtr& dataset) { +attach_offline_dataset(IndexRequest& request, + const eval::EvalDatasetPtr& dataset, + bool include_base) { require(dataset->GetVectorType() == "dense_vectors", "AutoTune V1 supports only dense vector datasets"); require(dataset->GetTrainDataType() == vsag::DATATYPE_FLOAT32 && @@ -235,12 +237,22 @@ attach_offline_dataset(Request& request, const eval::EvalDatasetPtr& dataset) { const auto base_count = dataset->GetNumberOfBase(); const auto query_count = dataset->GetNumberOfQuery(); const auto ground_truth_k = static_cast(dataset->GetGroundTruthK()); - owner->base = Dataset::Make() - ->NumElements(base_count) - ->Dim(dataset->GetDim()) - ->Ids(dataset->GetTrainIds()) - ->Float32Vectors(static_cast(dataset->GetTrain())) - ->Owner(false); + if (include_base) { + const auto* train_ids = dataset->GetTrainIds(); + if (train_ids == nullptr) { + owner->identity_ids.resize(static_cast(base_count)); + for (int64_t i = 0; i < base_count; ++i) { + owner->identity_ids[static_cast(i)] = i; + } + train_ids = owner->identity_ids.data(); + } + owner->base = Dataset::Make() + ->NumElements(base_count) + ->Dim(dataset->GetDim()) + ->Ids(train_ids) + ->Float32Vectors(static_cast(dataset->GetTrain())) + ->Owner(false); + } owner->queries = Dataset::Make() ->NumElements(query_count) ->Dim(dataset->GetDim()) @@ -255,7 +267,9 @@ attach_offline_dataset(Request& request, const eval::EvalDatasetPtr& dataset) { ->Owner(false); } - request.base = DatasetPtr(owner, owner->base.get()); + if (owner->base != nullptr) { + request.base = DatasetPtr(owner, owner->base.get()); + } request.workload.queries = DatasetPtr(owner, owner->queries.get()); if (owner->ground_truth != nullptr) { request.workload.ground_truth = DatasetPtr(owner, owner->ground_truth.get()); @@ -584,7 +598,8 @@ ParseRequest(const JsonType& input) { IndexRequest typed; std::string report_path; - attach_offline_dataset(typed, eval::EvalDataset::Load(data_path)); + attach_offline_dataset( + typed, eval::EvalDataset::Load(data_path), !input.contains("index_path")); if (input.contains("indexes")) { require(input["indexes"].is_array() && !input["indexes"].empty(), "request.indexes must be a non-empty array"); @@ -682,7 +697,7 @@ ParseRequest(const JsonType& input) { require(create_params.contains("index_param") && create_params["index_param"].is_object(), "request.indexes[0].create_params.index_param is required"); merge_dataset_field( - create_params, "dim", static_cast(typed.base->GetDim()), space.name); + create_params, "dim", static_cast(typed.workload.queries->GetDim()), space.name); merge_dataset_field(create_params, "dtype", vsag::DATATYPE_FLOAT32, space.name); merge_dataset_field(create_params, "metric_type", normalize(typed.metric_type), space.name); auto created = Factory::CreateIndex(normalize(space.name), create_params.dump()); diff --git a/tools/autotune/autotune_evaluation.cpp b/tools/autotune/autotune_evaluation.cpp index 35244668b3..84004513f6 100644 --- a/tools/autotune/autotune_evaluation.cpp +++ b/tools/autotune/autotune_evaluation.cpp @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include #include #include @@ -33,19 +31,6 @@ namespace { constexpr double BYTES_PER_MEBIBYTE = 1024.0 * 1024.0; -class ScopedOpenMpThreads { -public: - ScopedOpenMpThreads() : original_(omp_get_max_threads()) { - } - - ~ScopedOpenMpThreads() { - omp_set_num_threads(original_); - } - -private: - int original_; -}; - double elapsed(const std::chrono::steady_clock::time_point& start) { return std::chrono::duration(std::chrono::steady_clock::now() - start).count(); @@ -153,6 +138,10 @@ serialize_index(const IndexPtr& index, const std::string& path) { if (!serialized.has_value()) { throw std::runtime_error(serialized.error().message); } + output.flush(); + if (!output.good()) { + throw std::runtime_error("failed to write index artifact: " + path); + } } } // namespace @@ -202,7 +191,6 @@ EvaluateCandidates(const IndexTuningRequest& tuning_request, const std::vector& candidates, const std::string& run_path) { const auto& request = tuning_request.context; - ScopedOpenMpThreads openmp_threads; Evaluation evaluation; std::map> groups; for (uint64_t i = 0; i < candidates.size(); ++i) { @@ -318,7 +306,6 @@ EvaluateCandidates(const IndexTuningRequest& tuning_request, Evaluation EvaluateCandidates(const SearchTuningRequest& tuning_request, const std::vector& candidates) { - ScopedOpenMpThreads openmp_threads; const auto& request = tuning_request.context; Evaluation evaluation; uint64_t trial_number = 0; diff --git a/tools/autotune/autotune_test.cpp b/tools/autotune/autotune_test.cpp index 27de6628f1..7609a4df52 100644 --- a/tools/autotune/autotune_test.cpp +++ b/tools/autotune/autotune_test.cpp @@ -574,6 +574,10 @@ TEST_CASE("AutoTune keeps normalized offline request metadata") { const auto parsed = vsag::autotune::internal::ParseRequest(input); const auto& index_request = std::get(parsed); const auto& effective = index_request.context.effective_request; + const auto* train_ids = index_request.context.dataset->GetTrainIds(); + REQUIRE(train_ids != nullptr); + REQUIRE(train_ids[0] == 0); + REQUIRE(train_ids[63] == 63); REQUIRE(effective["data_path"] == dataset.Get()); REQUIRE(effective["dataset"]["dim"] == 8); REQUIRE(effective["dataset"]["dtype"] == "float32"); @@ -591,6 +595,40 @@ TEST_CASE("AutoTune keeps normalized offline request metadata") { REQUIRE_FALSE(effective.contains("tuning_config")); } +TEST_CASE("AutoTune rejects index artifacts that fail to flush") { + if (!std::filesystem::exists("/dev/full")) { + return; + } + + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath run_path(temp_path("autotune-full-device")); + std::filesystem::create_directories(run_path.Get() + "/artifacts"); + std::error_code link_error; + std::filesystem::create_symlink( + "/dev/full", run_path.Get() + "/artifacts/build-0.index", link_error); + REQUIRE_FALSE(link_error); + + MemoryFixture fixture; + auto input = fixture.Request(run_path.Get()); + input.index_spaces[0].create_parameter_space = + R"({"index_param":{"base_quantization_type":"fp32","max_degree":8,)" + R"("ef_construction":40,"build_thread_count":2}})"; + input.config.max_trials = 1; + const auto request = vsag::autotune::internal::ParseRequest(input); + const auto candidates = vsag::autotune::internal::GenerateCandidates(request); + REQUIRE(candidates.size() == 1); + + const auto evaluation = + vsag::autotune::internal::EvaluateCandidates(request, candidates, run_path.Get()); + REQUIRE(evaluation.builds.size() == 1); + REQUIRE(evaluation.builds[0]["status"] == "failed"); + REQUIRE_THAT(evaluation.builds[0]["failure"]["message"].get(), + Catch::Matchers::ContainsSubstring("failed to write index artifact")); + REQUIRE(evaluation.trials.size() == 1); + REQUIRE(evaluation.trials[0]["failure"]["code"] == "build_failed"); +} + TEST_CASE("AutoTune rejects report paths that alias an existing index") { ScopedPath dataset(temp_path("autotune-alias-dataset", ".hdf5")); ScopedPath workspace(temp_path("autotune-alias-workspace")); diff --git a/tools/eval/case/build_eval_case.cpp b/tools/eval/case/build_eval_case.cpp index 749776ece5..146c6136c3 100644 --- a/tools/eval/case/build_eval_case.cpp +++ b/tools/eval/case/build_eval_case.cpp @@ -16,7 +16,9 @@ #include "./build_eval_case.h" #include +#include #include +#include #include "../monitor/duration_monitor.h" #include "../monitor/memory_peak_monitor.h" @@ -61,10 +63,14 @@ void BuildEvalCase::do_build() { auto base = vsag::Dataset::Make(); int64_t total_base = this->dataset_ptr_->GetNumberOfBase(); - base->NumElements(total_base) - ->Dim(this->dataset_ptr_->GetDim()) - ->Ids(this->dataset_ptr_->GetTrainIds()) - ->Owner(false); + const auto* train_ids = this->dataset_ptr_->GetTrainIds(); + std::vector identity_ids; + if (train_ids == nullptr) { + identity_ids.resize(static_cast(total_base)); + std::iota(identity_ids.begin(), identity_ids.end(), int64_t{0}); + train_ids = identity_ids.data(); + } + base->NumElements(total_base)->Dim(this->dataset_ptr_->GetDim())->Ids(train_ids)->Owner(false); if (this->dataset_ptr_->GetVectorType() == DENSE_VECTORS) { if (this->dataset_ptr_->GetTrainDataType() == vsag::DATATYPE_FLOAT32) { base->Float32Vectors((const float*)this->dataset_ptr_->GetTrain()); diff --git a/tools/eval/eval_dataset.cpp b/tools/eval/eval_dataset.cpp index f173f2ba56..780b421fbe 100644 --- a/tools/eval/eval_dataset.cpp +++ b/tools/eval/eval_dataset.cpp @@ -16,6 +16,7 @@ #include "eval_dataset.h" #include +#include #include #include "impl/logger/logger.h" @@ -212,14 +213,9 @@ get_distance(const SparseVector* vector1, const SparseVector* vector2, const voi void EvalDataset::InitializeTrainIds(const int64_t* ids) { - identity_train_ids_.reset(); train_ids_are_identity_ = true; train_id_to_row_.clear(); if (ids == nullptr) { - identity_train_ids_ = std::shared_ptr(new int64_t[number_of_base_]); - for (int64_t i = 0; i < number_of_base_; ++i) { - identity_train_ids_[i] = i; - } return; } @@ -307,6 +303,10 @@ EvalDataset::FromDatasets(const vsag::DatasetPtr& base, dataset->InitializeTrainIds(base->GetIds()); if (ground_truth != nullptr) { + if (ground_truth->GetNumElements() > + std::numeric_limits::max() / ground_truth->GetDim()) { + throw std::invalid_argument("ground_truth contains too many ids"); + } const auto ground_truth_count = ground_truth->GetNumElements() * ground_truth->GetDim(); for (int64_t i = 0; i < ground_truth_count; ++i) { if (dataset->GetOneTrainById(ground_truth->GetIds()[i]) == nullptr) { diff --git a/tools/eval/eval_dataset.h b/tools/eval/eval_dataset.h index 2a9117158d..1d117da0c4 100644 --- a/tools/eval/eval_dataset.h +++ b/tools/eval/eval_dataset.h @@ -174,7 +174,8 @@ class EvalDataset { if (base_dataset_ != nullptr && base_dataset_->GetIds() != nullptr) { return base_dataset_->GetIds(); } - return identity_train_ids_.get(); + // A null pointer represents implicit row-number IDs. + return nullptr; } [[nodiscard]] const std::string* @@ -359,7 +360,6 @@ class EvalDataset { std::shared_ptr test_; std::shared_ptr neighbors_; std::shared_ptr distances_; - std::shared_ptr identity_train_ids_; bool train_ids_are_identity_{false}; std::shared_ptr train_labels_{nullptr}; std::shared_ptr test_labels_{nullptr}; diff --git a/tools/eval/eval_dataset_test.cpp b/tools/eval/eval_dataset_test.cpp index 1d4eed2635..669091da10 100644 --- a/tools/eval/eval_dataset_test.cpp +++ b/tools/eval/eval_dataset_test.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -230,6 +231,30 @@ TEST_CASE("EvalDataset builds a query-only view for id recall", "[ut][eval_datas "queries dataset is required and must not be empty"); } +TEST_CASE("EvalDataset rejects overflowing ground-truth ID counts", "[ut][eval_dataset]") { + std::vector vectors{0.0F, 0.0F}; + int64_t base_id = 0; + auto base = vsag::Dataset::Make() + ->NumElements(1) + ->Dim(2) + ->Ids(&base_id) + ->Float32Vectors(vectors.data()) + ->Owner(false); + auto queries = vsag::Dataset::Make() + ->NumElements(std::numeric_limits::max()) + ->Dim(2) + ->Float32Vectors(vectors.data()) + ->Owner(false); + auto ground_truth = vsag::Dataset::Make() + ->NumElements(std::numeric_limits::max()) + ->Dim(2) + ->Ids(&base_id) + ->Owner(false); + + REQUIRE_THROWS_WITH(EvalDataset::FromDatasets(base, queries, ground_truth, "l2"), + "ground_truth contains too many ids"); +} + TEST_CASE("EvalDataset rejects incomplete and incompatible HDF5 schemas", "[ut][eval_dataset]") { const std::vector required_datasets{"train", "test", "neighbors", "distances"}; for (const auto& missing : required_datasets) { @@ -599,8 +624,7 @@ TEST_CASE("EvalDataset sparse round-trip without token sequences", "[ut][eval_da REQUIRE(loaded->GetNumberOfBase() == 3); REQUIRE(loaded->GetNumberOfQuery() == 2); const auto* train = static_cast(loaded->GetTrain()); - REQUIRE(loaded->GetTrainIds()[0] == 0); - REQUIRE(loaded->GetTrainIds()[2] == 2); + REQUIRE(loaded->GetTrainIds() == nullptr); REQUIRE(loaded->GetOneTrainById(1) == train + 1); for (int i = 0; i < 3; ++i) { REQUIRE(train[i].token_seq_len_ == 0); diff --git a/tools/eval/monitor/recall_monitor.cpp b/tools/eval/monitor/recall_monitor.cpp index 79a21ceda7..42e2c79274 100644 --- a/tools/eval/monitor/recall_monitor.cpp +++ b/tools/eval/monitor/recall_monitor.cpp @@ -49,10 +49,14 @@ get_id_recall(const int64_t* neighbors, const int64_t* ground_truth_neighbors, uint64_t recall_num, uint64_t top_k) { - std::unordered_set expected(ground_truth_neighbors, ground_truth_neighbors + top_k); + std::unordered_set remaining_ground_truth(ground_truth_neighbors, + ground_truth_neighbors + top_k); uint64_t count = 0; for (uint64_t i = 0; i < recall_num; ++i) { - count += expected.erase(neighbors[i]); + // Erasing a match ensures duplicate result IDs are counted only once. + if (remaining_ground_truth.erase(neighbors[i]) > 0) { + ++count; + } } return static_cast(count) / static_cast(top_k); } From dc59b20eeea51fb9774a440e705d12f783a7b33d Mon Sep 17 00:00:00 2001 From: jc543239 Date: Fri, 31 Jul 2026 13:41:35 +0000 Subject: [PATCH 8/8] fix: expose AutoTune library to CI tests Signed-off-by: jc543239 Assisted-by: Codex:gpt-5 --- .github/workflows/pr-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 4b63daeaec..b5fbc905c1 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -247,6 +247,7 @@ jobs: path: ./build - name: Run Unittests run: | + export LD_LIBRARY_PATH="$PWD/build/src:$PWD/build/tools/autotune" echo leak:libomp.so > omp.supp echo leak:libgomp.so >> omp.supp export LSAN_OPTIONS=suppressions=omp.supp @@ -309,8 +310,6 @@ jobs: needs: [changes, build-asan-x86] if: needs.changes.outputs.cpp == 'true' runs-on: ubuntu-22.04 - env: - LD_LIBRARY_PATH: ${{ github.workspace }}/build/src container: image: vsaglib/vsag:ci-x86-openblas volumes: @@ -329,6 +328,7 @@ jobs: - name: Run Examples shell: bash run: | + export LD_LIBRARY_PATH="$PWD/build/src:$PWD/build/tools/autotune" cd ./build/examples/cpp for example in ./*; do filename=$(basename "$example")