diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 98c8ae10bb..b5fbc905c1 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -247,11 +247,15 @@ jobs: path: ./build - name: Run Unittests run: | + export LD_LIBRARY_PATH="$PWD/build/src:$PWD/build/tools/autotune" echo leak:libomp.so > omp.supp echo leak:libgomp.so >> omp.supp export LSAN_OPTIONS=suppressions=omp.supp - chmod +x ./build/tests/unittests + chmod +x ./build/tests/unittests ./build/tools/eval/eval_dataset_test \ + ./build/tools/autotune/autotune_test ./scripts/testing/test_parallel_by_name.sh unittests + ./build/tools/eval/eval_dataset_test -d yes + ./build/tools/autotune/autotune_test -d yes - name: Upload Log uses: actions/upload-artifact@v5 with: @@ -306,8 +310,6 @@ jobs: needs: [changes, build-asan-x86] if: needs.changes.outputs.cpp == 'true' runs-on: ubuntu-22.04 - env: - LD_LIBRARY_PATH: ${{ github.workspace }}/build/src container: image: vsaglib/vsag:ci-x86-openblas volumes: @@ -326,6 +328,7 @@ jobs: - name: Run Examples shell: bash run: | + export LD_LIBRARY_PATH="$PWD/build/src:$PWD/build/tools/autotune" cd ./build/examples/cpp for example in ./*; do filename=$(basename "$example") diff --git a/CMakeLists.txt b/CMakeLists.txt index b4e2a200a3..86553755fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,12 +57,12 @@ add_subdirectory (src) if (ENABLE_MOCKIMPL) add_subdirectory (mockimpl) endif () -if (ENABLE_EXAMPLES) - add_subdirectory (examples/cpp) -endif () if (ENABLE_TOOLS AND ENABLE_CXX11_ABI) add_subdirectory (tools) endif () +if (ENABLE_EXAMPLES) + add_subdirectory (examples/cpp) +endif () if (ENABLE_TESTS) add_subdirectory (tests) target_compile_definitions (vsag PRIVATE ENABLE_TESTS) diff --git a/docs/docs/en/src/SUMMARY.md b/docs/docs/en/src/SUMMARY.md index f57c6a7ce6..f534fffefc 100644 --- a/docs/docs/en/src/SUMMARY.md +++ b/docs/docs/en/src/SUMMARY.md @@ -83,6 +83,8 @@ - [Optimizer (Tune)](advanced/optimizer.md) - [Benchmarks](resources/performance.md) - [Evaluation Tool](resources/eval.md) +- [AutoTune Tool](resources/autotune.md) + - [AutoTune V1 CLI JSON Contract](resources/autotune_api_v1.md) - [HDF5 Dataset Format](resources/dataset_format.md) - [Index Analysis Tool](resources/analyze_index.md) - [Compatibility Check Tool](resources/check_compatibility.md) diff --git a/docs/docs/en/src/resources/autotune.md b/docs/docs/en/src/resources/autotune.md new file mode 100644 index 0000000000..dc720e26d5 --- /dev/null +++ b/docs/docs/en/src/resources/autotune.md @@ -0,0 +1,264 @@ +# AutoTune Tool (`autotune`) + +`autotune` automates the VSAG build-and-search parameter sweep. AutoTune V1 evaluates real VSAG +indexes through the shared eval core, filters trials by constraints, and selects the best feasible +trial for one KNN workload. + +The CLI JSON request and shared report schema are defined in the +[AutoTune V1 CLI JSON contract](autotune_api_v1.md). + +V1 builds and tunes HGraph and IVF on dense float32 data. It also tunes search parameters for an +existing HGraph, IVF, or Pyramid index. The CLI reads HDF5; the C++ API accepts in-memory datasets +and indexes. The C++ API is experimental and build-tree-only; it is not installed with the VSAG +library. + +## Building + +```bash +cmake -S . -B build-release \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TOOLS=ON +cmake --build build-release --target autotune -j +``` + +The executable is `build-release/tools/autotune/autotune`. + +A runnable request template is available at [the SIFT request template][sift-example]. +It expects the dataset at `/tmp/sift-128-euclidean.hdf5`; edit `data_path` when necessary. + +[sift-example]: https://github.com/antgroup/vsag/blob/main/tools/autotune/examples/sift_hgraph_autotune_request.json + +## C++ API + +The CLI and C++ entry points share candidate generation, trial planning, evaluation, constraint +filtering, and result selection. Only their input adapters differ: + +- the CLI loads JSON and HDF5; +- `TuneIndex` accepts in-memory datasets and jointly tunes build and search parameters; +- `TuneSearch` accepts an already built or loaded `IndexPtr` and tunes search parameters only. + +Consumers that link the build-tree `vsag::autotune` target can run the complete tune, select, and +load loop directly: + +```cpp +vsag::autotune::IndexRequest request; +request.base = base; +request.metric_type = vsag::METRIC_L2; +request.workload = {queries, ground_truth, 10, 48}; +request.index_spaces = { + {"hgraph", create_candidate_space, search_candidate_space}, +}; +request.constraints = { + {vsag::autotune::Metric::RECALL_AT_K, 0.95}, + {vsag::autotune::Metric::INDEX_MEMORY_MB, 8192.0}, +}; +request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + +auto result = vsag::autotune::TuneIndex(request); +if (result.has_value() && result->status == vsag::autotune::TuneStatus::SUCCESS) { + auto neighbors = + result->index->KnnSearch(query, 10, result->search_parameters).value(); +} +``` + +`base` and `queries` are dense float32 datasets. `base` IDs must be present and unique. The +ground-truth dataset stores `query_count` rows, uses `Dim()` as ground-truth K, and contains +flattened IDs from the base ID space. It is required when `RECALL_AT_K` is a constraint or the +objective. Dataset buffers must remain valid until the synchronous call returns. + +`IndexSpace::create_parameter_space` and `search_parameter_space` remain JSON strings because each +index owns its native parameter schema. An empty `index_spaces` vector selects the built-in HGraph +and IVF spaces; otherwise arrays and `$range` expressions use the same semantics as the CLI +contract. + +The result contains the loaded `index`, `index_name`, complete concrete `create_parameters`, +validated `search_parameters`, metrics, artifact path, and the complete report. `metrics` and +`report` are `JsonType` objects. The typed API returns the report in memory and never writes a +report file. The recommended artifact is retained, so another process can recreate an empty index +from `index_name + create_parameters` and deserialize that file. Unselected intermediate +artifacts are removed by default. The caller removes the recommended artifact when it is no longer +needed. + +If no candidate satisfies every constraint, the completed call has +`status=TuneStatus::NO_FEASIBLE_CANDIDATE` and a structured `best_effort`; recommendation fields +such as `index` are not valid. Invalid requests and execution failures still return an error. + +See +[`examples/cpp/326_feature_create_index_with_constraints.cpp`][factory-example] for a complete +example. Enable both tools and examples to build it: + +```bash +cmake -S . -B build-release \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TOOLS=ON \ + -DENABLE_EXAMPLES=ON +cmake --build build-release --target 326_feature_create_index_with_constraints -j +``` + +[factory-example]: https://github.com/antgroup/vsag/blob/main/examples/cpp/326_feature_create_index_with_constraints.cpp + +## Request + +Save a request such as the following as `request.json`. `dim`, `dtype`, and `metric_type` are read +from the dataset and need not be repeated. If provided, they must match the dataset. + +```json +{ + "version": 1, + "data_path": "/tmp/sift-128-euclidean.hdf5", + "indexes": [ + { + "name": "hgraph", + "create_params": { + "index_param": { + "base_quantization_type": ["fp32", "sq8_uniform"], + "max_degree": [16, 32], + "ef_construction": 200 + } + }, + "search_params": { + "hgraph": { + "ef_search": { + "$range": {"start": 40, "stop": 1000} + } + } + } + } + ], + "workload": { + "top_k": 10, + "concurrency": 1 + }, + "constraints": { + "recall_at_k": 0.95, + "index_memory_mb": 8192 + }, + "objective": { + "metric": "latency_avg_ms" + }, + "tuning_config": { + "workspace_path": "/tmp/vsag_autotune", + "keep_intermediate": false, + "max_trials": 1000 + } +} +``` + +Run it with: + +```bash +./build-release/tools/autotune/autotune request.json +``` + +Every parameter leaf may be a scalar, an array, or a `$range` with `start`, `stop`, and `step`. +Explicit values are preserved. HGraph `ef_search` additionally accepts an integer `$range` without +`step`; AutoTune then doubles from `start` until it brackets a passing value and binary-searches +that interval for the smallest passing value. This assumes that recall does not decrease as +`ef_search` increases for the fixed configuration. Every evaluated point uses all queries. For +omitted tuning fields, V1 supplies a small built-in candidate set for HGraph or IVF. If `indexes` +is omitted, both indexes are evaluated. + +Candidates with identical concrete create parameters share one build. Each concrete search +parameter set remains a separate trial. An adaptive `ef_search` range records only the concrete +points that were actually evaluated. + +## Existing Index + +Set `index_path` and provide exactly one index specification to tune search parameters without +rebuilding. Its `create_params` must be the concrete parameters used to create the serialized +index. AutoTune does not expand `create_params` in this mode, so native index fields that happen to +be arrays remain unchanged. + +Programmatic callers use a separate `SearchRequest` and call `TuneSearch`: + +```cpp +vsag::autotune::SearchRequest request; +request.index = existing_index; +request.workload = {queries, ground_truth, 10, 48}; +request.parameter_space = R"({"hgraph":{"ef_search":[40,80,120]}})"; +request.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.95}}; +request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + +auto result = vsag::autotune::TuneSearch(request); +if (result.has_value() && result->status == vsag::autotune::TuneStatus::SUCCESS) { + auto neighbors = existing_index->KnnSearch(query, 10, result->parameters).value(); +} +``` + +This avoids serialization and file I/O and reuses the caller's index for all search trials. +AutoTune derives the index type and element count from `IndexPtr`. Search tuning therefore needs +queries but not the base dataset or metric type. When recall is requested, it also needs ground +truth and defines per-query `recall_at_k` as the intersection of the returned and ground-truth IDs +in their first `top_k` entries, divided by `top_k`; the reported value is the average over all +queries. Ground truth is optional when recall is neither a constraint nor the objective. + +The CLI's `index_path` field remains an offline adapter: it uses the concrete create parameters to +create and deserialize an index, then enters the same search-only flow. + +Pyramid is supported in this search-only mode. The HDF5 CLI adapter does not provide query paths, +so it can evaluate only Pyramid's native default/root search. Typed requests take paths directly +from `workload.queries->GetPaths()`: when paths are present, each query is evaluated against its +path; when absent, Pyramid applies its native root-search behavior. V1 supports only the default +unnamed hierarchy. To select different `ef_search` values for different paths, run one typed +AutoTune request per representative path workload, with the matching ground truth. V1 does not +aggregate path-specific recommendations into one result. + +A complete example is available at +[`examples/cpp/327_feature_autotune_existing_index.cpp`][existing-index-example]. +For Pyramid path tuning, see +[`examples/cpp/328_feature_autotune_existing_pyramid.cpp`][pyramid-example]. +It tunes the same Pyramid index separately for 512-vector and 4096-vector leaf subgraphs under the +same recall target, illustrating why different paths may need different `ef_search` values. + +[existing-index-example]: https://github.com/antgroup/vsag/blob/main/examples/cpp/327_feature_autotune_existing_index.cpp +[pyramid-example]: https://github.com/antgroup/vsag/blob/main/examples/cpp/328_feature_autotune_existing_pyramid.cpp + +## Metrics + +The following metrics can be used as constraints or as the objective: + +| Metric | Constraint comparison | +| --- | --- | +| `recall_at_k`, `qps` | At least the requested value | +| `latency_avg_ms`, `latency_p99_ms` | At most the requested value | +| `index_memory_mb`, `index_size_mb` | At most the requested value | +| `build_seconds`, `search_seconds`, `build_and_search_seconds` | At most the requested value | + +`build_seconds`, `index_size_mb`, and `build_and_search_seconds` are unavailable in +existing-index mode. `index_memory_mb` may be a constraint, but not the objective, because it is +constant across search candidates for the same existing index and cannot rank them. The objective +direction is inferred from the metric, so no `direction` field is needed. + +AutoTune uses a dedicated latency/QPS pass and a separate recall/statistics pass, so a logical +query runs more than once per trial. `search_seconds` covers both in-memory passes and metric +collection, but excludes index deserialization. Candidates run in a fixed order without warm-up +or randomization. Measure latency and QPS on a representative, controlled machine and repeat runs +when these metrics determine the recommendation. + +## Reading the Output + +Standard output is a short summary ordered for copying: + +- `recommendation`: the selected concrete parameters, metrics, and trial evidence. Build-and-search + results also include create parameters and artifact/build evidence; search-only results do not. +- `best_effort`: present only when no trial satisfies every constraint. +- `failure`: present when the request or all evaluations fail. +- `report_path`: the full reproducible report. + +The full report also contains every build and trial, constraint violations, the normalized request, +and structured per-stage failures. In search-only mode `builds` is empty and trials contain no +artifact or build fields. For both the CLI and typed `TuneIndex`, `keep_intermediate=false` retains +only the recommended generated index and removes unselected artifacts. Setting it to `true` +retains every generated index. + +After initial request validation succeeds, the CLI and `RunAutoTune` persist the full report: +`output.result_path` overrides the path, otherwise a generated path under the workspace is used. +Early validation and request-file failures do not write a report. Typed `TuneIndex` and +`TuneSearch` never write a report file; callers receive the complete `JsonType` report in the +result. + +## V1 Boundaries + +V1 evaluates one KNN workload and performs a full sweep except for the supported HGraph +`ef_search` adaptive search. It does not provide filtered or range-search workloads, adaptive query +sampling, cross-request build cache, or model-based candidate generation. diff --git a/docs/docs/en/src/resources/autotune_api_v1.md b/docs/docs/en/src/resources/autotune_api_v1.md new file mode 100644 index 0000000000..6a3222715f --- /dev/null +++ b/docs/docs/en/src/resources/autotune_api_v1.md @@ -0,0 +1,518 @@ +# AutoTune V1 CLI JSON Input and Output Contract + +This page defines the offline JSON/HDF5 adapter used by the `autotune` command-line tool. The +experimental, build-tree-only C++ API instead accepts typed `IndexRequest` and `SearchRequest` +objects. It is not installed with the VSAG library. All entry points use the same normalized +request and tuning engine after input conversion. AutoTune V1 supports one full-dataset KNN +workload on dense float32 data, with HGraph and IVF as the supported build-and-search index types. + +Unknown fields are rejected in the request, index specification, workload, objective, +`tuning_config`, and `output` objects. Native fields inside `create_params` and `search_params` are +passed to the concrete VSAG index and are validated when that index is created or searched. + +## Request Object + +| Field | Type | Required | Meaning | +| --- | --- | --- | --- | +| `version` | integer | Yes | Must be `1`. | +| `data_path` | string | Yes | Readable HDF5 evaluation dataset. | +| `index_path` | string | No | Existing serialized index for search-only tuning. | +| `indexes` | array | No | Index specifications. Defaults to HGraph and IVF in normal mode. | +| `workload` | object | Yes | The KNN workload being optimized. | +| `constraints` | object | Yes | Non-empty metric-to-threshold map. | +| `objective` | object | Yes | Metric used to rank feasible candidates. | +| `tuning_config` | object | No | Workspace, artifact retention, and trial limit. | +| `output` | object | No | Report path and raw-eval control. | + +### Dataset Rules + +`data_path` must identify a readable regular file in the HDF5 format accepted by the VSAG +evaluation tool. V1 requires: + +- dense vectors; +- float32 base and query vectors; +- at least one base vector, query vector, and ground-truth neighbor; +- `workload.top_k` no greater than the ground-truth width when recall is requested; +- a distance attribute that maps to `l2`, `ip`, or `cosine`. + +AutoTune reads `dim`, `dtype`, and `metric_type` from the dataset and injects them into every +concrete `create_params`. A request may repeat one of these fields only when it exactly matches the +dataset. + +## Index Specifications + +Each `indexes[]` item has the following shape: + +```json +{ + "name": "hgraph", + "create_params": { + "index_param": { + "max_degree": [16, 32] + } + }, + "search_params": { + "hgraph": { + "ef_search": {"$range": {"start": 40, "stop": 1000}} + } + } +} +``` + +| Field | Type | Required | Meaning | +| --- | --- | --- | --- | +| `name` | string | Yes | `hgraph`, `ivf`, or search-only `pyramid`, case-insensitive. | +| `create_params` | object | No | VSAG create parameters or candidate expressions. | +| `search_params` | object | No | Search parameters, keyed by the same index name. | + +When `indexes` is absent in normal build-and-search mode, AutoTune evaluates both HGraph and IVF. +When `index_path` is present, `indexes` is required and must contain exactly one item. +Pyramid is accepted only with an existing in-memory or serialized index. + +### Candidate Expressions + +Every leaf in `create_params` and `search_params` can be written as: + +- a scalar, which fixes that value; +- a non-empty array, which provides discrete candidate values; +- an inclusive `$range` containing exactly `start`, `stop`, and non-zero `step`. + +Examples: + +```json +{ + "fixed": 32, + "choices": [16, 32, 48], + "integer_range": {"$range": {"start": 40, "stop": 120, "step": 40}}, + "float_range": {"$range": {"start": 0.1, "stop": 0.3, "step": 0.1}} +} +``` + +HGraph `ef_search` has one additional form: + +```json +{"$range": {"start": 40, "stop": 1000}} +``` + +The bounds must be positive integers with `start <= stop`. This form requires a `recall_at_k` +constraint and an objective of `latency_avg_ms`, `latency_p99_ms`, `qps`, `search_seconds`, or +`build_and_search_seconds`. It cannot be combined with `timeout_ms` or `hops_limit`. For every +fixed combination of the other parameters, AutoTune evaluates concrete `ef_search` values and +doubles from `start`, capped at `stop`, until recall satisfies the constraint. It then +binary-searches the last failing/passing interval for the smallest passing value. Branching uses +recall only; latency and QPS remain measured metrics used by final constraint evaluation and +selection. This strategy assumes that recall does not decrease as `ef_search` increases while all +other parameters and the workload remain fixed. + +Every probe evaluates the full query workload. If a probe fails to execute or does not produce +recall, AutoTune stops that adaptive range because it cannot choose the next interval safely. + +V1 treats every JSON array and stepped range in these parameter objects as candidate choices. It +computes their full Cartesian product and removes identical complete candidates. A no-step range +is not accepted for other parameters. + +The request is rejected when the planned worst-case evaluation count exceeds +`tuning_config.max_trials`. A normal concrete candidate costs one trial. An adaptive `ef_search` +range costs one when `start == stop`. Otherwise, the planner reserves the largest probe count +through any possible passing bound plus the binary-search cost of that bound's preceding interval. +For example, `40..1000` reserves 15 trials in the worst case. + +### Built-in Missing-Field Candidates + +Explicit user fields always win. The following values are supplied only when the corresponding +field is absent: + +| Index | Missing field | V1 candidates | +| --- | --- | --- | +| HGraph | `base_quantization_type` | `fp32`, `sq8_uniform` | +| HGraph | `max_degree` | `16`, `32` | +| HGraph | `ef_construction` | `100`, `200` | +| HGraph | `ef_search` | Three values derived from `top_k`, with floors `40`, `80`, `120` | +| Pyramid, search-only | `ef_search` | Three values derived from `top_k`, with floors `40`, `80`, `120` | +| IVF | `base_quantization_type` | `fp32`, `sq8_uniform` | +| IVF | `buckets_count` | Up to `1024` and `2048`, capped by base-vector count | +| IVF, build-and-search | `scan_buckets_count` | Up to four values derived from concrete `buckets_count` | +| IVF, search-only | `scan_buckets_count` | `1`, `4`, `16`, `64` | + +The exact HGraph and Pyramid search values are `max(40, top_k)`, `max(80, 2 * top_k)`, and +`max(120, 4 * top_k)`, with duplicates removed. Build-and-search IVF candidates never exceed the +concrete bucket count. AutoTune cannot infer an existing IVF index's bucket count, so search-only +mode uses the conservative generic values above. Values rejected by the loaded index become +failed trials; callers can avoid them by providing an explicit `scan_buckets_count` space. + +Candidates with the same normalized index name and identical concrete `create_params` belong to +one build group. AutoTune builds that group once, serializes it once as evidence, and evaluates +every associated search candidate against the same in-memory index instance. + +### Existing-Index Mode + +With `index_path`: + +- the path must identify a readable regular file; +- exactly one index specification is required; +- `create_params.index_param` is required; +- `create_params` is treated as one concrete native index configuration and is not expanded; +- AutoTune does not generate missing create candidates or invoke build; +- missing search fields may still receive built-in search candidates; +- `build_seconds`, `index_size_mb`, and `build_and_search_seconds` cannot be constraints or + objectives; +- `index_memory_mb` can be a constraint but not the objective; +- the caller-owned index is never deleted. + +The concrete create parameters are needed to instantiate the correct VSAG index before +deserialization. AutoTune does not infer them from the serialized file in V1. + +For a typed `SearchRequest`, the index is already instantiated and loaded. AutoTune derives its +index type and element count from `IndexPtr`; the request supplies only the index, workload, search +`parameter_space`, constraints, objective, and config. It needs neither create parameters nor the +base dataset or metric type. + +The HDF5 adapter does not provide Pyramid query paths, so the CLI can evaluate only Pyramid's +native default/root search. Path-specific Pyramid tuning requires a typed `SearchRequest` whose +query dataset supplies `Dataset::Paths()`. V1 forwards only the default unnamed hierarchy; named +Pyramid hierarchies are not supported by AutoTune. + +## Workload + +```json +{ + "workload": { + "top_k": 10, + "concurrency": 48 + } +} +``` + +| Field | Type | Required | Default | Validation | +| --- | --- | --- | --- | --- | +| `top_k` | positive integer | Yes | — | No greater than base count, and no greater than ground-truth width when recall is requested; representable by native `int`. | +| `concurrency` | positive integer | No | `1` | Maximum `200`. | + +V1 always evaluates the complete query set in the HDF5 file and supports only KNN search. The +benchmark machine, system load, and runtime environment are part of the measured workload. +Latency and QPS are not portable across different machine specifications or load conditions. +For a typed `SearchRequest`, `top_k` is checked against `IndexPtr::GetNumElements()` instead of a +base dataset. Ground truth may be null unless recall is a constraint or the objective. + +## Constraints and Objective + +`constraints` maps metric names directly to thresholds. `objective.metric` names one metric. The +comparison and objective direction are inferred from the metric and cannot be overridden. + +```json +{ + "constraints": { + "recall_at_k": 0.95, + "latency_p99_ms": 2.0, + "index_memory_mb": 8192 + }, + "objective": { + "metric": "latency_avg_ms" + } +} +``` + +| Metric | Constraint | Objective direction | Existing-index use | +| --- | --- | --- | --- | +| `recall_at_k` | actual ≥ threshold | Maximize | Yes | +| `qps` | actual ≥ threshold | Maximize | Yes | +| `latency_avg_ms` | actual ≤ threshold | Minimize | Yes | +| `latency_p99_ms` | actual ≤ threshold | Minimize | Yes | +| `index_memory_mb` | actual ≤ threshold | Minimize | Constraint only | +| `index_size_mb` | actual ≤ threshold | Minimize | No | +| `build_seconds` | actual ≤ threshold | Minimize | No | +| `search_seconds` | actual ≤ threshold | Minimize | Yes | +| `build_and_search_seconds` | actual ≤ threshold | Minimize | No | + +Thresholds must be finite and non-negative. `recall_at_k` must also be at most `1.0`. + +Metric meanings in V1: + +- For each query, `recall_at_k` is the size of the intersection between the first `top_k` + returned IDs and ground-truth IDs, divided by `top_k`; the reported metric is the average over + all queries. It needs ground truth but does not need base vectors or a metric type. +- `build_seconds` measures the index `Build` operation reported by the evaluation tool. +- It excludes index serialization, dataset loading, and candidate orchestration. +- `search_seconds` measures the wall time of a complete in-memory search evaluation trial. It + excludes index deserialization, but includes all search passes and metric collection. Use + latency or QPS for serving-performance goals. +- `build_and_search_seconds` is the sum of those two metrics for a build-and-search trial. +- `index_memory_mb` comes from the loaded index's positive `GetMemoryUsage()` result. When an index + reports zero, AutoTune treats the metric as unavailable instead of as zero usage; a corresponding + constraint is recorded as a missing-metric violation. +- `index_size_mb` is the generated serialized artifact's file size and is unavailable in + search-only mode. +- AutoTune uses a dedicated latency/QPS pass and a separate recall/statistics pass. Latency wraps + each `KnnSearch` call with `steady_clock`; QPS is successful queries divided by the wall time of + the concurrent performance pass. The same logical query is therefore executed more than once + per trial. + +`search_seconds` remains evaluation cost because it includes both passes and monitor work; it is +not the time for one logical production query batch. Candidates run in a fixed order against a +reused index, without a warm-up or randomized ordering. Cache state and unrelated system load can +therefore affect performance metrics; benchmark in a representative, controlled environment and +repeat runs when latency or QPS drives selection. + +## Tuning and Output Configuration + +```json +{ + "tuning_config": { + "workspace_path": "/tmp/vsag_autotune", + "keep_intermediate": false, + "max_trials": 1000 + }, + "output": { + "result_path": "/tmp/vsag_autotune/report.json", + "include_raw_eval": false + } +} +``` + +| Field | Default | Meaning | +| --- | --- | --- | +| `tuning_config.workspace_path` | `/tmp/vsag_autotune` | Run artifacts and CLI default reports. | +| `tuning_config.keep_intermediate` | `false` | Keep all generated indexes. | +| `tuning_config.max_trials` | `1000` | Maximum planned worst-case trials; hard maximum `100000`. | +| `output.result_path` | `/run-.json` | CLI full report path. | +| `output.include_raw_eval` | `false` | Include native eval JSON in build and trial records. | + +`output.result_path` cannot alias `data_path` or `index_path`. After initial validation succeeds, +the offline JSON/CLI entry writes the full report. An explicit `output.result_path` overrides its +location; otherwise AutoTune generates a report path under `workspace_path`. Early validation and +request-file failures do not write one. + +Typed `TuneIndex` and `TuneSearch` do not accept a report path and never write report files. Their +result objects return the complete report as `JsonType`. For both the CLI and typed `TuneIndex`, +`keep_intermediate=false` retains only the recommended generated index and removes unselected +artifacts; `true` retains every generated index. When there is no recommendation, the default is +to remove all generated indexes. Because `TuneSearch` creates no artifacts, +`workspace_path` and `keep_intermediate` have no effect on it. + +## Full Report + +A completed evaluation returns this top-level shape. The CLI also writes it to disk: + +```json +{ + "version": 1, + "status": "success", + "recommendation": {}, + "best_effort": null, + "builds": [], + "trials": [], + "request": {}, + "elapsed_seconds": 84.2, + "report_path": "/tmp/vsag_autotune/run-....json" +} +``` + +| Field | Meaning | +| --- | --- | +| `version` | Report contract version, always `1`. | +| `status` | `success`, `no_feasible_candidate`, or `failed`. | +| `recommendation` | Best feasible trial, otherwise `null`. | +| `best_effort` | Closest successful trial when constraints are infeasible, otherwise `null`. | +| `builds` | One record per concrete generated build group; empty in search-only mode. | +| `trials` | One record per executed concrete search candidate. | +| `request` | Effective normalized request used by the tuning engine. | +| `elapsed_seconds` | AutoTune wall time through artifact cleanup; excludes final report writing. | +| `report_path` | Persisted full report path; CLI/JSON adapter only. | +| `failure` | Present when the overall run failed. | + +Early validation failures do not have `builds`, `trials`, or `report_path`, and no report file is +written. A CLI run in which all candidate evaluations fail does contain the attempted build and +trial records and is written when the report path itself is usable. Typed API reports never +contain `report_path`. + +The normalized `request` contains derived dataset metadata, workload defaults, constraints, +objective, config, and output fields. Build-and-search mode adds `index_spaces`; search-only mode +adds `index_name` and `parameter_space`. CLI reports also retain the offline `data_path` and, when +present, `index_path` as top-level fields inside this normalized object. + +### Status Semantics + +- `success`: at least one successful trial satisfies every constraint; `recommendation` is set. +- `no_feasible_candidate`: successful trials exist, but none satisfies every constraint; + `best_effort` is set for explanation and is not a valid recommendation. Typed calls return a + value with `TuneStatus::NO_FEASIBLE_CANDIDATE`, not an error. +- `failed`: the request is invalid, execution/reporting fails, or no trial can be evaluated + successfully with the objective metric. + +### Recommendation and Best Effort + +In build-and-search mode, `recommendation` contains: + +```json +{ + "index_name": "hgraph", + "create_params": {}, + "search_params": {}, + "workload": {"top_k": 10, "concurrency": 1}, + "metrics": {}, + "artifacts": {}, + "evidence": {"build_id": "build-0", "trial_id": "trial-0"} +} +``` + +In search-only mode, `create_params`, `artifacts`, and `evidence.build_id` are omitted; the +recommendation contains the concrete search parameters, workload, metrics, and +`evidence.trial_id`. `best_effort` has the same mode-specific fields plus +`constraint_evaluation`. It is selected first by the number of violated or missing constraints, +then by normalized violation magnitude. It is only explanatory. + +### Build Records + +`builds` is empty in search-only mode. In build-and-search mode, each `builds[]` item contains: + +| Field | Meaning | +| --- | --- | +| `build_id` | Stable ID referenced by trials. | +| `index_name` | Concrete index type. | +| `create_params` | Concrete create parameters. | +| `status` | `success` or `failed`. | +| `metrics` | Available build-shared metrics. | +| `artifacts` | `source`, `index_path`, `use_existing_index`, and `retained`. | +| `failure` | Structured failure or `null`. | +| `elapsed_seconds` | Build-group preparation time. | +| `raw_eval_result` | Present only when requested and a native build eval ran. | + +### Trial Records + +Each `trials[]` item contains: + +| Field | Meaning | +| --- | --- | +| `trial_id` | Stable trial ID. | +| `build_id` | Associated build group; build-and-search mode only. | +| `index_name` | Concrete index type. | +| `create_params` | Concrete create parameters; build-and-search mode only. | +| `search_params` | Concrete search parameters. | +| `status` | `success` or `failed`. | +| `metrics` | Available metrics used by constraint evaluation and selection. | +| `constraint_evaluation` | `satisfied` plus a `violations` array. | +| `artifacts` | Artifact evidence copied from the build group; build-and-search mode only. | +| `failure` | Structured failure or `null`. | +| `elapsed_seconds` | Search trial wall time. | +| `raw_eval_result` | Present only when requested and native search eval succeeded. | + +For an adaptive HGraph `ef_search` range, the report contains one trial for each point actually +evaluated. Every recorded `search_params` value is concrete; the `$range` expression remains only +in the normalized request. Every recorded trial evaluates the full query workload. + +A constraint violation contains `metric`, `comparison`, `expected`, and `actual`. `actual` is +`null` when the required metric is missing or non-finite. + +Search trials in one build group reuse the same loaded index instance. A generated index is built +once and serialized once. In search-only mode the caller's index, or the index deserialized by the +CLI adapter, is reused directly for every trial. + +### Artifact Semantics + +Artifact fields appear only in build-and-search records in V1. `artifacts.source` is `generated`, +and `artifacts.index_path` is evidence of where the evaluated index was stored, not a promise that +the path still exists. Check `artifacts.retained`: + +- `true`: this is the selected artifact, or retention of all generated indexes was requested; +- `false`: AutoTune planned to remove or already removed the generated artifact. + +### Structured Failures + +Failures use one common object: + +```json +{ + "stage": "validation", + "code": "invalid_request", + "message": "request.workload.top_k is required" +} +``` + +Typical stages are `cli`, `validation`, `candidate_generation`, `build`, `search`, `evaluation`, +`selection`, and `report`. Per-build and per-trial failures remain in their corresponding records +so other candidates can continue. + +| Stage | Code | Meaning | +| --- | --- | --- | +| `cli` | `request_file_error` | The CLI could not read or parse its request file. | +| `validation` | `invalid_request` | Request validation failed. | +| `candidate_generation` | `invalid_request` | A candidate expression or count was invalid. | +| `build` | `build_evaluation_failed` | One build group failed. | +| `search` | `build_failed` | Search was skipped because its build failed. | +| `search` | `search_evaluation_failed` | One search trial failed. | +| `evaluation` | `all_trials_failed` | Every candidate trial failed. | +| `selection` | `objective_metric_unavailable` | Trials succeeded, but none produced the objective metric. | +| `evaluation`, `report` | `execution_failed` | Top-level execution or report writing failed. | + +## CLI Summary + +The CLI prints a compact subset of the full report in this order: + +1. `recommendation`, when present; +2. `best_effort`, when present; +3. `failure`, when present; +4. `status`; +5. `elapsed_seconds`; +6. `report_path`, when present; +7. `version`. + +Null result branches and the detailed build/trial arrays are omitted from standard output. Read +`report_path` for complete evidence. + +The CLI exits with code `1` for `status=failed` and command-line/request-file errors. It exits with +code `0` for both `success` and `no_feasible_candidate`; callers must inspect `status` rather than +using only the exit code to decide whether a recommendation exists. + +## Build-Tree C++ Entry Point + +The experimental optional `vsag::autotune` CMake target exposes the declarations in +`tools/autotune/autotune.h`. The target and header are build-tree-only and are not installed: + +```cpp +tl::expected TuneIndex(const IndexRequest& request); +tl::expected TuneSearch(const SearchRequest& request); +JsonType RunAutoTune(const JsonType& request); // CLI adapter +``` + +The JSON entry point is an offline adapter: it loads `data_path`, creates or deserializes an index +when `index_path` is present, and constructs the same internal request used by the typed entry +points. The target and header are intentionally not installed in V1. Calls inside one process are +serialized because the underlying evaluation path configures process-global OpenMP state. + +`TuneIndex` jointly tunes build and search parameters: + +```cpp +IndexRequest request; +request.base = base; +request.metric_type = METRIC_L2; +request.workload = {queries, ground_truth, 10, 48}; +request.index_spaces = {{"hgraph", create_candidate_space, search_candidate_space}}; +request.constraints = {{Metric::RECALL_AT_K, 0.95}}; +request.objective = Metric::LATENCY_AVG_MS; +auto result = TuneIndex(request); +``` + +It returns a loaded, query-ready selected index together with concrete create and search +parameters. `metrics` and the complete `report` are returned as `JsonType`; typed calls never +persist the report. `TuneSearch` tunes an already built or loaded index: + +```cpp +SearchRequest request; +request.index = existing_index; +request.workload = {queries, ground_truth, 10, 48}; +request.parameter_space = search_candidate_space; +request.constraints = {{Metric::RECALL_AT_K, 0.95}}; +request.objective = Metric::LATENCY_AVG_MS; +auto result = TuneSearch(request); +``` + +AutoTune derives the type and element count of `existing_index`. `TuneSearch` therefore does not +need base vectors or a metric type. Ground truth is required only when recall is a constraint or +the objective. Build-time metrics and `index_size_mb` are unavailable to `TuneSearch`. +`index_memory_mb` may be a constraint but cannot be the objective because it does not vary across +search candidates. + +Both typed calls return `tl::unexpected` only for invalid requests and execution failures. +If evaluation completes but no candidate satisfies every constraint, they return a result with +`status=TuneStatus::NO_FEASIBLE_CANDIDATE` and structured `best_effort`; recommendation fields are +invalid in that state. diff --git a/docs/docs/zh/src/SUMMARY.md b/docs/docs/zh/src/SUMMARY.md index 8c3910f7ab..58639ff396 100644 --- a/docs/docs/zh/src/SUMMARY.md +++ b/docs/docs/zh/src/SUMMARY.md @@ -83,6 +83,8 @@ - [优化器](advanced/optimizer.md) - [标准环境性能参考](resources/performance.md) - [性能评估工具](resources/eval.md) +- [AutoTune 工具](resources/autotune.md) + - [AutoTune V1 CLI JSON 契约](resources/autotune_api_v1.md) - [HDF5 数据集格式](resources/dataset_format.md) - [索引分析工具](resources/analyze_index.md) - [兼容性检查工具](resources/check_compatibility.md) diff --git a/docs/docs/zh/src/resources/autotune.md b/docs/docs/zh/src/resources/autotune.md new file mode 100644 index 0000000000..378fa17ab3 --- /dev/null +++ b/docs/docs/zh/src/resources/autotune.md @@ -0,0 +1,249 @@ +# AutoTune 工具（`autotune`） + +`autotune` 用于自动完成 VSAG 构建参数和查询参数的枚举评测。AutoTune V1 通过共享的 eval +核心运行真实 VSAG 索引，按约束过滤 trial，并为一个 KNN workload 选择最优 +可行候选。 + +CLI JSON 请求和共用报告结构见 [AutoTune V1 CLI JSON 契约](autotune_api_v1.md)。 + +V1 支持在 dense float32 数据上构建和调优 HGraph、IVF，也支持对已有 HGraph、IVF 或 +Pyramid 索引调 search 参数。CLI 读取 HDF5；C++ API 接收内存 Dataset 和 Index。C++ API +是实验性接口，只在构建目录中提供，不会随 VSAG 库安装。 + +## 编译 + +```bash +cmake -S . -B build-release \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TOOLS=ON +cmake --build build-release --target autotune -j +``` + +可执行文件位于 `build-release/tools/autotune/autotune`。 + +仓库提供了[可运行的 SIFT 请求模板][sift-example]。 +它默认读取 `/tmp/sift-128-euclidean.hdf5`；数据集位于其他目录时请修改 `data_path`。 + +[sift-example]: https://github.com/antgroup/vsag/blob/main/tools/autotune/examples/sift_hgraph_autotune_request.json + +## C++ API + +CLI 和 C++ 入口复用同一套候选生成、trial 规划、评测、约束过滤和结果选择逻辑，区别仅在 +输入适配： + +- CLI 解析 JSON 并加载 HDF5； +- `TuneIndex` 接收内存 Dataset，同时调优构建参数和查询参数； +- `TuneSearch` 接收已经构建或加载的 `IndexPtr`，只调查询参数。 + +链接构建目录中的 `vsag::autotune` target 后，可以直接完成“调优、选择、加载”闭环： + +```cpp +vsag::autotune::IndexRequest request; +request.base = base; +request.metric_type = vsag::METRIC_L2; +request.workload = {queries, ground_truth, 10, 48}; +request.index_spaces = { + {"hgraph", create_candidate_space, search_candidate_space}, +}; +request.constraints = { + {vsag::autotune::Metric::RECALL_AT_K, 0.95}, + {vsag::autotune::Metric::INDEX_MEMORY_MB, 8192.0}, +}; +request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + +auto result = vsag::autotune::TuneIndex(request); +if (result.has_value() && result->status == vsag::autotune::TuneStatus::SUCCESS) { + auto neighbors = + result->index->KnnSearch(query, 10, result->search_parameters).value(); +} +``` + +`base` 和 `queries` 必须是 dense float32 Dataset，base ID 必须存在且唯一。ground truth 的 +`NumElements()` 等于 query 数量，`Dim()` 表示 ground-truth K，`Ids()` 是来自 base ID 空间的 +展平数组。当 `RECALL_AT_K` 是约束或目标时必须提供 ground truth。Dataset 的 buffer 需保持有效， +直到这个同步调用返回。 + +每个索引拥有不同的原生参数模式，因此 `IndexSpace::create_parameter_space` 和 +`search_parameter_space` 仍使用 JSON 字符串。`index_spaces` 为空时使用内置 HGraph 和 +IVF 候选；显式参数中的数组和 `$range` 与 CLI 契约语义相同。 + +返回值包括已加载的 `index`、`index_name`、完整具体 `create_parameters`、经过验证的 +`search_parameters`、metrics、artifact 路径和完整报告。`metrics` 和 `report` 都是 +`JsonType` 对象。typed API 在内存中返回报告，绝不会写报告文件。最终推荐 artifact 会 +保留，因此可以用 `index_name + create_parameters` 重新创建空索引并反序列化该文件； +未选中的中间 artifact 默认删除。调用方应在不再需要时删除推荐 artifact。 + +如果没有候选满足全部约束，调用仍正常完成，并返回 +`status=TuneStatus::NO_FEASIBLE_CANDIDATE` 和结构化 `best_effort`；此时 `index` 等推荐 +字段无效。请求非法或执行失败时仍然返回 error。 + +完整示例见 +[`examples/cpp/326_feature_create_index_with_constraints.cpp`][factory-example]。同时启用工具和 +示例即可编译： + +```bash +cmake -S . -B build-release \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TOOLS=ON \ + -DENABLE_EXAMPLES=ON +cmake --build build-release --target 326_feature_create_index_with_constraints -j +``` + +[factory-example]: https://github.com/antgroup/vsag/blob/main/examples/cpp/326_feature_create_index_with_constraints.cpp + +## 请求 + +将下面的请求保存为 `request.json`。`dim`、`dtype` 和 `metric_type` 会从数据集读取，用户 +不需要重复填写。 + +```json +{ + "version": 1, + "data_path": "/tmp/sift-128-euclidean.hdf5", + "indexes": [ + { + "name": "hgraph", + "create_params": { + "index_param": { + "base_quantization_type": ["fp32", "sq8_uniform"], + "max_degree": [16, 32], + "ef_construction": 200 + } + }, + "search_params": { + "hgraph": { + "ef_search": { + "$range": {"start": 40, "stop": 1000} + } + } + } + } + ], + "workload": { + "top_k": 10, + "concurrency": 1 + }, + "constraints": { + "recall_at_k": 0.95, + "index_memory_mb": 8192 + }, + "objective": { + "metric": "latency_avg_ms" + }, + "tuning_config": { + "workspace_path": "/tmp/vsag_autotune", + "keep_intermediate": false, + "max_trials": 1000 + } +} +``` + +运行方式： + +```bash +./build-release/tools/autotune/autotune request.json +``` + +参数叶子可以是标量、数组，或带有 `start`、`stop`、`step` 的 `$range`。HGraph +`ef_search` 还可以省略 `step`；AutoTune 会从 `start` 倍增，找到首个满足 `recall_at_k` +约束的区间，再在该区间内二分查找最小通过值。该策略假设固定其他配置时 recall 不会随 +`ef_search` 增大而降低。每个实际评测点都使用全部 query。用户显式提供的值保持不变； +对于缺失的调优字段，V1 会补充一小组 HGraph 或 IVF 内置候选。省略 `indexes` 时会同时 +评测这两种索引。 + +具体 create 参数相同的候选只构建一次；每组具体 search 参数仍然是独立 trial。自适应 +`ef_search` 范围只记录实际评测过的具体参数点。 + +## 已有索引 + +设置 `index_path` 并且只提供一个索引规格，可以跳过构建、只调查询参数。其中 +`create_params` 必须是序列化该索引时使用的具体参数。该模式不会展开 `create_params`， +因此索引原生的数组字段会保持不变。 + +程序调用方使用独立的 `SearchRequest` 和 `TuneSearch`： + +```cpp +vsag::autotune::SearchRequest request; +request.index = existing_index; +request.workload = {queries, ground_truth, 10, 48}; +request.parameter_space = R"({"hgraph":{"ef_search":[40,80,120]}})"; +request.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.95}}; +request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + +auto result = vsag::autotune::TuneSearch(request); +if (result.has_value() && result->status == vsag::autotune::TuneStatus::SUCCESS) { + auto neighbors = existing_index->KnnSearch(query, 10, result->parameters).value(); +} +``` + +该接口避免序列化和文件 I/O，并让全部 search trial 复用调用方的索引。AutoTune 从 +`IndexPtr` 推导索引类型和元素数量，因此 search tuning 只需要 query，不需要 base dataset +或 metric type。使用 recall 时还需要 ground truth：每条 query 的 `recall_at_k` 定义为 +返回结果和 ground truth 前 `top_k` 个 ID 的交集大小除以 `top_k`，最终指标是所有 query +的平均值。recall 既不是约束也不是目标时，ground truth 可省略。 + +CLI 的 `index_path` 仍是离线适配：它先使用具体 create 参数创建并反序列化 Index，再进入 +同一条 search-only 流程。 + +Pyramid 通过这个 search-only 模式接入。HDF5 CLI 适配器不提供 query path，因此只能评测 +Pyramid 原生的默认/root 搜索。typed 请求直接从 `workload.queries->GetPaths()` 取得 path： +提供 path 时，每条 query 按自己的 path 评测；不提供时沿用 root 搜索语义。V1 只支持默认 +未命名 hierarchy。若不同 path 需要不同 `ef_search`，应为每个代表性 path workload 分别 +发起一次 typed AutoTune 请求，并提供与该 path 对应的 ground truth。V1 不把多个 path 的 +推荐聚合成一条结果。 + +完整示例见 +[`examples/cpp/327_feature_autotune_existing_index.cpp`][existing-index-example]。 +Pyramid path 调优示例见 +[`examples/cpp/328_feature_autotune_existing_pyramid.cpp`][pyramid-example]。 +它对同一个 Pyramid 索引中的 512-vector 和 4096-vector 叶子子图使用相同 recall 目标分别 +调优，用于展示不同 path 可能需要不同的 `ef_search`。 + +[existing-index-example]: https://github.com/antgroup/vsag/blob/main/examples/cpp/327_feature_autotune_existing_index.cpp +[pyramid-example]: https://github.com/antgroup/vsag/blob/main/examples/cpp/328_feature_autotune_existing_pyramid.cpp + +## 指标 + +以下指标可以作为约束或优化目标： + +| 指标 | 约束比较方式 | +| --- | --- | +| `recall_at_k`、`qps` | 不低于请求值 | +| `latency_avg_ms`、`latency_p99_ms` | 不高于请求值 | +| `index_memory_mb`、`index_size_mb` | 不高于请求值 | +| `build_seconds`、`search_seconds`、`build_and_search_seconds` | 不高于请求值 | + +已有索引模式不提供 `build_seconds`、`index_size_mb` 和 `build_and_search_seconds`。 +`index_memory_mb` 可以作为约束，但不能作为目标，因为同一个已有索引的所有 search 候选 +内存相同，无法据此排序。目标方向由指标本身决定，因此不需要 `direction` 字段。 + +AutoTune 使用独立的 latency/QPS pass 和 recall/statistics pass，因此同一条逻辑 query 在 +每个 trial 中会执行多次。`search_seconds` 包含两个内存 pass 和指标采集，但不包含索引 +反序列化。候选按固定顺序运行，不做 warm-up 或随机排序。如果 latency 或 QPS 决定最终 +推荐，应在有代表性且受控的机器上测量，并通过重复实验确认。 + +## 阅读输出 + +标准输出是便于阅读和复制的短 summary，字段顺序如下： + +- `recommendation`：最终选中的具体参数、指标和 trial 证据。build-and-search 结果还包含 + create 参数和 artifact/build 证据；search-only 结果不包含这些字段。 +- `best_effort`：只有在没有 trial 满足全部约束时出现。 +- `failure`：请求失败或所有评测失败时出现。 +- `report_path`：完整可复现报告的位置。 + +完整报告还包含每个 build 和 trial、约束 violation、补齐后的请求，以及分阶段的结构化 +失败。search-only 模式的 `builds` 为空，trial 不包含 artifact 或 build 字段。CLI 和 typed +`TuneIndex` 设置 `keep_intermediate=false` 时都只保留最终推荐索引，并删除未选中的 +artifact；设置为 `true` 时保留所有生成索引。 + +请求通过初始校验后，CLI 和 `RunAutoTune` 会持久化完整报告：显式 `output.result_path` +会覆盖输出路径，否则在 workspace 下生成默认路径。早期校验和请求文件错误不会写报告。 +typed `TuneIndex` 和 `TuneSearch` 绝不会写报告文件，而是通过返回值提供完整的 `JsonType` +报告。 + +## V1 边界 + +V1 评测一个 KNN workload；除已支持的 HGraph `ef_search` 自适应搜索外，其他候选仍完整遍历。 +它暂不支持过滤或范围查询 workload、query sampling、跨请求 build cache，以及基于模型的 +候选生成。 diff --git a/docs/docs/zh/src/resources/autotune_api_v1.md b/docs/docs/zh/src/resources/autotune_api_v1.md new file mode 100644 index 0000000000..9596c9b76e --- /dev/null +++ b/docs/docs/zh/src/resources/autotune_api_v1.md @@ -0,0 +1,493 @@ +# AutoTune V1 CLI JSON 输入输出契约 + +本文档定义 `autotune` 命令行工具使用的离线 JSON/HDF5 适配契约。实验性的 C++ API 只在 +构建目录中提供，接收 typed `IndexRequest` 和 `SearchRequest`，不会随 VSAG 库安装。所有 +入口完成输入转换后使用同一个规范化请求和调优引擎。AutoTune V1 支持 dense float32 数据、 +HGraph/IVF 的构建和查询调优，以及一个使用全量 query 的 KNN workload。 + +请求、索引规格、workload、objective、`tuning_config` 和 `output` 中的未知字段会被拒绝。 +`create_params` 和 `search_params` 内部的原生参数会传给具体 VSAG 索引，在创建或查询时由 +索引校验。 + +## 请求对象 + +| 字段 | 类型 | 必填 | 含义 | +| --- | --- | --- | --- | +| `version` | integer | 是 | 必须为 `1`。 | +| `data_path` | string | 是 | 可读的 HDF5 评测数据集。 | +| `index_path` | string | 否 | 用于 search-only 调优的已有序列化索引。 | +| `indexes` | array | 否 | 索引规格；普通模式默认使用 HGraph 和 IVF。 | +| `workload` | object | 是 | 要优化的 KNN workload。 | +| `constraints` | object | 是 | 非空的“指标到阈值”映射。 | +| `objective` | object | 是 | 对可行候选排序的目标指标。 | +| `tuning_config` | object | 否 | workspace、产物保留和 trial 上限。 | +| `output` | object | 否 | 报告路径和原始 eval 输出开关。 | + +### 数据集规则 + +`data_path` 必须指向 VSAG eval 工具可读取的普通 HDF5 文件。V1 要求： + +- dense vector； +- base 和 query 均为 float32； +- 至少包含一个 base、一个 query 和一个 ground-truth neighbor； +- 使用 recall 指标时，`workload.top_k` 不超过 ground truth 的宽度； +- distance 属性能够映射为 `l2`、`ip` 或 `cosine`。 + +AutoTune 从数据集读取 `dim`、`dtype` 和 `metric_type`，并注入每组具体 +`create_params`。请求可以重复提供这些字段，但值必须与数据集完全一致。 + +## 索引规格 + +每个 `indexes[]` 元素的形态如下： + +```json +{ + "name": "hgraph", + "create_params": { + "index_param": { + "max_degree": [16, 32] + } + }, + "search_params": { + "hgraph": { + "ef_search": {"$range": {"start": 40, "stop": 1000}} + } + } +} +``` + +| 字段 | 类型 | 必填 | 含义 | +| --- | --- | --- | --- | +| `name` | string | 是 | `hgraph`、`ivf`，或 search-only 的 `pyramid`；不区分大小写。 | +| `create_params` | object | 否 | VSAG 创建参数或候选表达式。 | +| `search_params` | object | 否 | 查询参数，key 必须是同一个索引名。 | + +普通 build-and-search 模式省略 `indexes` 时，AutoTune 会同时评测 HGraph 和 IVF。设置 +`index_path` 时，必须提供 `indexes`，并且只能包含一个元素。 +Pyramid 只接受已有的内存索引或序列化索引。 + +### 候选表达式 + +`create_params` 和 `search_params` 中的每个叶子都可以写成： + +- 标量：固定该值； +- 非空数组：提供离散候选值； +- `$range`：必须且只能包含 `start`、`stop` 和非零 `step`，范围包含端点。 + +示例： + +```json +{ + "fixed": 32, + "choices": [16, 32, 48], + "integer_range": {"$range": {"start": 40, "stop": 120, "step": 40}}, + "float_range": {"$range": {"start": 0.1, "stop": 0.3, "step": 0.1}} +} +``` + +HGraph `ef_search` 还支持一种特殊写法： + +```json +{"$range": {"start": 40, "stop": 1000}} +``` + +上下界必须是正整数并满足 `start <= stop`。这种写法要求存在 `recall_at_k` 约束，且目标为 +`latency_avg_ms`、`latency_p99_ms`、`qps`、`search_seconds` 或 +`build_and_search_seconds`；不能同时设置 `timeout_ms` 或 `hops_limit`。对于其他参数的每个 +固定组合，AutoTune 从 `start` 开始倍增 `ef_search`（不超过 `stop`），直到实测 recall 满足 +约束；随后在最后一个失败/通过区间内二分，查找最小通过值。分支方向只由 recall 决定； +latency 和 QPS 仍作为实测指标参与最终约束过滤和结果选择。该策略假设其他参数和 workload +固定时，recall 不会随 `ef_search` 增大而降低。 + +每个探测点都会评测完整 query workload。如果某个评测点执行失败或没有 recall，AutoTune +会停止该自适应范围，因为此时无法安全选择下一个区间。 + +V1 会把参数对象中的所有 JSON 数组和带 `step` 的范围解释成候选集合，计算完整笛卡尔积 +并去除完全相同的候选。其他参数不接受省略 `step` 的范围。 + +计划的最坏评测次数超过 `tuning_config.max_trials` 时，请求失败。普通具体候选占一个 +trial；自适应 `ef_search` 在 `start == stop` 时占一个。否则，planner 会为每个可能的首次 +通过探测点计算“到达该点的探测次数 + 前一区间的二分次数”，并取其中最大值。例如 +`40..1000` 最坏预留 15 个 trial。 + +### 缺失字段的内置候选 + +用户显式字段始终优先。只有字段缺失时才补充以下候选： + +| 索引 | 缺失字段 | V1 候选 | +| --- | --- | --- | +| HGraph | `base_quantization_type` | `fp32`、`sq8_uniform` | +| HGraph | `max_degree` | `16`、`32` | +| HGraph | `ef_construction` | `100`、`200` | +| HGraph | `ef_search` | 根据 `top_k` 生成三个值，下限为 `40`、`80`、`120` | +| Pyramid，search-only | `ef_search` | 根据 `top_k` 生成三个值，下限为 `40`、`80`、`120` | +| IVF | `base_quantization_type` | `fp32`、`sq8_uniform` | +| IVF | `buckets_count` | 不超过 base 数量的 `1024`、`2048` | +| IVF，build-and-search | `scan_buckets_count` | 根据具体 `buckets_count` 生成至多四个值 | +| IVF，search-only | `scan_buckets_count` | `1`、`4`、`16`、`64` | + +HGraph 和 Pyramid 查询候选具体为 `max(40, top_k)`、`max(80, 2 * top_k)` 和 +`max(120, 4 * top_k)`，重复值会删除。build-and-search 模式的 IVF 查询候选不会超过具体 +bucket 数量。AutoTune 无法推断已有 IVF 的 bucket 数量，因此 search-only 模式使用上面的 +保守通用值；被已加载索引拒绝的值会记录为 failed trial，调用方可以显式提供 +`scan_buckets_count` 候选以避免这些 trial。 + +规范化索引名和具体 `create_params` 相同的候选属于同一个 build group。AutoTune 对该组 +只构建一次、序列化一次作为证据，再使用同一个内存索引实例执行所有关联的 search 候选。 + +### 已有索引模式 + +设置 `index_path` 后： + +- 路径必须指向可读普通文件； +- 必须且只能提供一个索引规格； +- 必须提供 `create_params.index_param`； +- `create_params` 整体视为一份具体的索引原生配置，不再做候选展开； +- AutoTune 不补充 create 候选，也不会执行 build； +- 缺失的 search 字段仍可使用内置查询候选； +- `build_seconds`、`index_size_mb` 和 `build_and_search_seconds` 不能作为约束或目标； +- `index_memory_mb` 可以作为约束，但不能作为目标； +- 不会删除调用方提供的索引。 + +V1 反序列化前需要使用具体创建参数实例化正确的 VSAG 索引，目前不会从序列化文件推导 +这些参数。 + +对于 typed `SearchRequest`，索引已经完成实例化和加载。AutoTune 从 `IndexPtr` 推导索引 +类型和元素数量；请求只提供 index、workload、search `parameter_space`、constraints、 +objective 和 config，不需要 create 参数、base dataset 或 metric type。 + +HDF5 适配器不提供 Pyramid query path，因此 CLI 只能评测 Pyramid 原生的默认/root 查询。 +按 path 调优 Pyramid 需要使用 typed `SearchRequest`，并通过 query Dataset 的 +`Dataset::Paths()` 提供 path。V1 只转发默认的未命名 hierarchy，不支持 Pyramid named +hierarchy。 + +## Workload + +```json +{ + "workload": { + "top_k": 10, + "concurrency": 48 + } +} +``` + +| 字段 | 类型 | 必填 | 默认值 | 校验 | +| --- | --- | --- | --- | --- | +| `top_k` | positive integer | 是 | — | 不超过 base 数量；使用 recall 指标时不超过 ground truth 宽度；并能用原生 `int` 表示。 | +| `concurrency` | positive integer | 否 | `1` | 最大为 `200`。 | + +V1 始终评测 HDF5 文件中的全量 query，并且只支持 KNN。benchmark 机器、系统负载和运行 +环境都属于 workload 的一部分；延迟和 QPS 不能直接移植到不同机器规格或负载条件。 +对于 typed `SearchRequest`，`top_k` 会和 `IndexPtr::GetNumElements()` 比较，不依赖 base +dataset。只有 recall 是约束或目标时才必须提供 ground truth。 + +## 约束和优化目标 + +`constraints` 直接把指标名映射到阈值；`objective.metric` 指定一个目标指标。比较方式和 +目标方向由指标决定，用户不能覆盖。 + +```json +{ + "constraints": { + "recall_at_k": 0.95, + "latency_p99_ms": 2.0, + "index_memory_mb": 8192 + }, + "objective": { + "metric": "latency_avg_ms" + } +} +``` + +| 指标 | 约束 | 目标方向 | 已有索引用法 | +| --- | --- | --- | --- | +| `recall_at_k` | actual ≥ threshold | 最大化 | 是 | +| `qps` | actual ≥ threshold | 最大化 | 是 | +| `latency_avg_ms` | actual ≤ threshold | 最小化 | 是 | +| `latency_p99_ms` | actual ≤ threshold | 最小化 | 是 | +| `index_memory_mb` | actual ≤ threshold | 最小化 | 仅约束 | +| `index_size_mb` | actual ≤ threshold | 最小化 | 否 | +| `build_seconds` | actual ≤ threshold | 最小化 | 否 | +| `search_seconds` | actual ≤ threshold | 最小化 | 是 | +| `build_and_search_seconds` | actual ≤ threshold | 最小化 | 否 | + +阈值必须是有限非负数，`recall_at_k` 还必须不大于 `1.0`。 + +V1 指标口径： + +- 每条 query 的 `recall_at_k` 是返回结果和 ground truth 前 `top_k` 个 ID 的交集大小除以 + `top_k`，最终指标是所有 query 的平均值。它需要 ground truth，但不需要 base vector 或 + metric type。 +- `build_seconds` 是 eval 工具报告的索引 `Build` 操作耗时。 +- 它不包含索引序列化、数据集加载和候选编排。 +- `search_seconds` 是完整内存 search eval trial 的墙钟时间，不包含索引反序列化，但包含 + 所有 search pass 和指标采集；线上查询性能目标应使用 latency 或 QPS。 +- `build_and_search_seconds` 是一次 build-and-search trial 中前两项之和。 +- `index_memory_mb` 来自已加载索引大于零的 `GetMemoryUsage()` 结果。索引报告零时， + AutoTune 会把该指标视为不可用，而不是零内存；相关约束会记录 missing-metric violation。 +- `index_size_mb` 是生成的序列化索引文件大小，在 search-only 模式不可用。 +- AutoTune 使用独立的 latency/QPS pass 和 recall/statistics pass。latency 用 + `steady_clock` 包围每次 `KnnSearch`；QPS 是成功查询数除以并发性能 pass 的墙钟时间。 + 因此，同一条逻辑 query 在每个 trial 中会执行多次。 + +`search_seconds` 仍表示评测成本，因为它包含两个 pass 和 monitor 处理；它不是一次线上 +逻辑 query batch 的执行时间。候选会按固定顺序在复用的索引上运行，不做 warm-up 或随机 +排序，因此 cache 状态和无关系统负载会影响性能指标。如果 latency 或 QPS 决定结果，应在 +有代表性且受控的环境中运行，并通过重复实验确认。 + +## Tuning 和输出配置 + +```json +{ + "tuning_config": { + "workspace_path": "/tmp/vsag_autotune", + "keep_intermediate": false, + "max_trials": 1000 + }, + "output": { + "result_path": "/tmp/vsag_autotune/report.json", + "include_raw_eval": false + } +} +``` + +| 字段 | 默认值 | 含义 | +| --- | --- | --- | +| `tuning_config.workspace_path` | `/tmp/vsag_autotune` | run 产物和 CLI 默认报告目录。 | +| `tuning_config.keep_intermediate` | `false` | 是否保留所有生成索引。 | +| `tuning_config.max_trials` | `1000` | 计划最坏 trial 数上限；硬上限为 `100000`。 | +| `output.result_path` | `/run-.json` | CLI 完整报告路径。 | +| `output.include_raw_eval` | `false` | 在 build/trial 中包含原生 eval JSON。 | + +`output.result_path` 不能与 `data_path` 或 `index_path` 指向同一文件。通过初始校验后， +离线 JSON/CLI 入口会写完整报告：显式 `output.result_path` 会覆盖输出位置，否则 AutoTune +在 `workspace_path` 下生成报告路径。早期校验和请求文件错误不会写报告。 + +typed `TuneIndex` 和 `TuneSearch` 不接收报告路径，也绝不会写报告文件，而是通过结果对象 +返回完整的 `JsonType` 报告。CLI 和 typed `TuneIndex` 设置 +`keep_intermediate=false` 时都只保留推荐索引并删除未选中 artifact；设置为 `true` 时保留 +所有生成索引。没有 recommendation 时，默认删除所有生成索引。`TuneSearch` 不生成 +artifact，因此 `workspace_path` 和 `keep_intermediate` 对它没有影响。 + +## 完整报告 + +完成评测后会返回以下顶层结构；CLI 还会把它写入磁盘： + +```json +{ + "version": 1, + "status": "success", + "recommendation": {}, + "best_effort": null, + "builds": [], + "trials": [], + "request": {}, + "elapsed_seconds": 84.2, + "report_path": "/tmp/vsag_autotune/run-....json" +} +``` + +| 字段 | 含义 | +| --- | --- | +| `version` | 报告契约版本，固定为 `1`。 | +| `status` | `success`、`no_feasible_candidate` 或 `failed`。 | +| `recommendation` | 最优可行 trial；不存在时为 `null`。 | +| `best_effort` | 约束不可行时最接近的成功 trial，否则为 `null`。 | +| `builds` | 每组具体生成 build 一条记录；search-only 模式为空数组。 | +| `trials` | 每组已执行的具体 search 候选一条记录。 | +| `request` | 调优引擎实际使用的有效规范化请求。 | +| `elapsed_seconds` | AutoTune 墙钟时间包含 artifact 清理，不含最终报告写入。 | +| `report_path` | 持久化完整报告路径；只在 CLI/JSON 适配器中存在。 | +| `failure` | 整体执行失败时出现。 | + +早期校验失败不包含 `builds`、`trials` 或 `report_path`，也不会写报告。如果所有候选评测 +失败，CLI 只要报告路径仍然可用，就会保留尝试过的 build/trial 记录并写报告。typed API +的报告绝不会包含 `report_path`。 + +规范化 `request` 包含推导出的 dataset 元数据、workload 默认值、constraints、objective、 +config 和 output。build-and-search 模式还包含 `index_spaces`；search-only 模式包含 +`index_name` 和 `parameter_space`。CLI 报告还会在这个规范化对象顶层保留离线 +`data_path`，存在已有索引输入时也保留 `index_path`。 + +### Status 语义 + +- `success`：至少一个成功 trial 满足全部约束，设置 `recommendation`。 +- `no_feasible_candidate`：存在成功 trial，但没有任何一个满足全部约束；设置 + `best_effort` 用于解释，它不是有效推荐。typed 调用会返回带 + `TuneStatus::NO_FEASIBLE_CANDIDATE` 的值，而不是 error。 +- `failed`：请求非法、执行或报告阶段失败，或者没有成功且带目标指标的 trial。 + +### Recommendation 和 Best Effort + +build-and-search 模式下，`recommendation` 包含： + +```json +{ + "index_name": "hgraph", + "create_params": {}, + "search_params": {}, + "workload": {"top_k": 10, "concurrency": 1}, + "metrics": {}, + "artifacts": {}, + "evidence": {"build_id": "build-0", "trial_id": "trial-0"} +} +``` + +search-only 模式会省略 `create_params`、`artifacts` 和 `evidence.build_id`，只包含具体 +查询参数、workload、metrics 和 `evidence.trial_id`。`best_effort` 使用相同的模式相关 +字段，并额外包含 `constraint_evaluation`。它首先按违反或缺失约束数量选择，再比较归一化 +violation 大小，只用于解释。 + +### Build 记录 + +search-only 模式的 `builds` 为空。build-and-search 模式下，每个 `builds[]` 元素包含： + +| 字段 | 含义 | +| --- | --- | +| `build_id` | 被 trial 引用的稳定 ID。 | +| `index_name` | 具体索引类型。 | +| `create_params` | 具体创建参数。 | +| `status` | `success` 或 `failed`。 | +| `metrics` | 可用的 build 共享指标。 | +| `artifacts` | `source`、`index_path`、`use_existing_index` 和 `retained`。 | +| `failure` | 结构化失败或 `null`。 | +| `elapsed_seconds` | build group 准备耗时。 | +| `raw_eval_result` | 请求原始输出且真实 build eval 成功时出现。 | + +### Trial 记录 + +每个 `trials[]` 元素包含： + +| 字段 | 含义 | +| --- | --- | +| `trial_id` | 稳定 trial ID。 | +| `build_id` | 关联的 build group；仅 build-and-search 模式存在。 | +| `index_name` | 具体索引类型。 | +| `create_params` | 具体创建参数；仅 build-and-search 模式存在。 | +| `search_params` | 具体查询参数。 | +| `status` | `success` 或 `failed`。 | +| `metrics` | 用于约束评估和选择的可用指标。 | +| `constraint_evaluation` | `satisfied` 和 `violations` 数组。 | +| `artifacts` | 从 build group 复制的产物证据；仅 build-and-search 模式存在。 | +| `failure` | 结构化失败或 `null`。 | +| `elapsed_seconds` | search trial 墙钟时间。 | +| `raw_eval_result` | 请求原始输出且原生 search eval 成功时出现。 | + +对于自适应 HGraph `ef_search` 范围，报告会为每个实际评测点记录一条 trial。每条记录中的 +`search_params` 都是具体值；`$range` 表达式只保留在规范化请求中。每条记录都会评测完整 +query workload。 + +每条 constraint violation 包含 `metric`、`comparison`、`expected` 和 `actual`。指标缺失或 +非有限数时，`actual` 为 `null`。 + +同一 build group 的 search trial 复用同一个已加载索引实例。新生成的索引只构建和序列化 +一次。search-only 模式直接为所有 trial 复用调用方索引，或复用 CLI 适配器反序列化得到的 +索引。 + +### Artifact 语义 + +V1 只在 build-and-search 记录中提供 artifact 字段。`artifacts.source` 为 `generated`； +`artifacts.index_path` 用于说明被评测索引曾存放在哪里，不保证响应返回时路径仍然存在。 +需要检查 `artifacts.retained`： + +- `true`：这是最终推荐产物，或者请求保留所有生成索引； +- `false`：AutoTune 计划删除或已经删除生成产物。 + +### 结构化失败 + +所有失败使用统一结构： + +```json +{ + "stage": "validation", + "code": "invalid_request", + "message": "request.workload.top_k is required" +} +``` + +常见 stage 包括 `cli`、`validation`、`candidate_generation`、`build`、`search`、 +`evaluation`、`selection` 和 `report`。build/trial 失败保留在对应记录中，其他候选可以 +继续执行。 + +| Stage | Code | 含义 | +| --- | --- | --- | +| `cli` | `request_file_error` | CLI 无法读取或解析请求文件。 | +| `validation` | `invalid_request` | 请求校验失败。 | +| `candidate_generation` | `invalid_request` | 候选表达式或数量非法。 | +| `build` | `build_evaluation_failed` | 一组 build 失败。 | +| `search` | `build_failed` | 关联 build 失败，search 被跳过。 | +| `search` | `search_evaluation_failed` | 一组 search trial 失败。 | +| `evaluation` | `all_trials_failed` | 所有候选 trial 都执行失败。 | +| `selection` | `objective_metric_unavailable` | 存在成功 trial，但都没有产生目标指标。 | +| `evaluation`、`report` | `execution_failed` | 顶层执行或报告写入失败。 | + +## CLI Summary + +CLI 按以下顺序输出完整报告的紧凑子集： + +1. `recommendation`（存在时）； +2. `best_effort`（存在时）； +3. `failure`（存在时）； +4. `status`； +5. `elapsed_seconds`； +6. `report_path`（存在时）； +7. `version`。 + +标准输出会省略值为 null 的结果分支和详细 build/trial 数组。完整证据请读取 +`report_path`。 + +CLI 在 `status=failed` 或命令行/请求文件出错时返回 `1`。`success` 和 +`no_feasible_candidate` 都返回 `0`；调用方必须检查 `status`，不能只用退出码判断是否存在 +recommendation。 + +## Build-tree C++ 入口 + +实验性的可选 CMake target `vsag::autotune` 暴露 `tools/autotune/autotune.h` 中的接口。 +该 target 和头文件只在构建目录中提供，不会安装： + +```cpp +tl::expected TuneIndex(const IndexRequest& request); +tl::expected TuneSearch(const SearchRequest& request); +JsonType RunAutoTune(const JsonType& request); // CLI adapter +``` + +JSON 入口只负责离线适配：加载 `data_path`；存在 `index_path` 时创建并反序列化索引；随后 +构造与 typed 入口相同的内部请求。V1 不安装该 target 和头文件。由于底层 eval 路径会配置 +进程级 OpenMP 状态，同一进程内的多次调用会串行执行。 + +`TuneIndex` 同时调优构建参数和查询参数： + +```cpp +IndexRequest request; +request.base = base; +request.metric_type = METRIC_L2; +request.workload = {queries, ground_truth, 10, 48}; +request.index_spaces = {{"hgraph", create_candidate_space, search_candidate_space}}; +request.constraints = {{Metric::RECALL_AT_K, 0.95}}; +request.objective = Metric::LATENCY_AVG_MS; +auto result = TuneIndex(request); +``` + +返回值包含已加载、可查询的推荐索引，以及具体 create 和 search 参数。`metrics` 和完整 +`report` 都以 `JsonType` 返回；typed 调用绝不会持久化报告。`TuneSearch` 对已经构建或 +加载的索引调查询参数： + +```cpp +SearchRequest request; +request.index = existing_index; +request.workload = {queries, ground_truth, 10, 48}; +request.parameter_space = search_candidate_space; +request.constraints = {{Metric::RECALL_AT_K, 0.95}}; +request.objective = Metric::LATENCY_AVG_MS; +auto result = TuneSearch(request); +``` + +AutoTune 从 `existing_index` 推导类型和元素数量，因此 `TuneSearch` 不需要 base vector 或 +metric type。只有 recall 是约束或目标时才需要 ground truth。`TuneSearch` 不提供构建阶段 +指标和 `index_size_mb`。`index_memory_mb` 可以作为约束，但不能作为目标，因为它不会随 +search 候选变化。 + +两个 typed 调用仅在请求非法或执行失败时返回 `tl::unexpected`。如果评测正常完成但 +没有候选满足全部约束，则返回 `status=TuneStatus::NO_FEASIBLE_CANDIDATE` 和结构化 +`best_effort`；该状态下推荐字段无效。 diff --git a/examples/cpp/326_feature_create_index_with_constraints.cpp b/examples/cpp/326_feature_create_index_with_constraints.cpp new file mode 100644 index 0000000000..501d4a2aa2 --- /dev/null +++ b/examples/cpp/326_feature_create_index_with_constraints.cpp @@ -0,0 +1,90 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include + +#include "autotune.h" + +int +main() { + constexpr int64_t DIM = 2; + vsag::Options::Instance().set_block_size_limit(2UL * 1024 * 1024); + std::vector base_ids{101, 205, 309, 413}; + std::vector base_vectors{0.0F, 0.0F, 1.0F, 0.0F, 0.0F, 1.0F, 1.0F, 1.0F}; + std::vector query_vectors{0.0F, 0.0F, 1.0F, 1.0F}; + std::vector ground_truth_ids{101, 413}; + + auto base = vsag::Dataset::Make() + ->NumElements(4) + ->Dim(DIM) + ->Ids(base_ids.data()) + ->Float32Vectors(base_vectors.data()) + ->Owner(false); + auto queries = vsag::Dataset::Make() + ->NumElements(2) + ->Dim(DIM) + ->Float32Vectors(query_vectors.data()) + ->Owner(false); + auto ground_truth = + vsag::Dataset::Make()->NumElements(2)->Dim(1)->Ids(ground_truth_ids.data())->Owner(false); + + vsag::autotune::IndexRequest request; + request.base = base; + request.metric_type = vsag::METRIC_L2; + request.workload = {queries, ground_truth, 1, 1}; + request.index_spaces = { + {"hgraph", + R"({"index_param":{"base_quantization_type":"fp32","max_degree":8,"ef_construction":40}})", + R"({"hgraph":{"ef_search":[10,20]}})"}}; + request.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 1.0}, + {vsag::autotune::Metric::INDEX_MEMORY_MB, 1024.0}}; + request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + request.config.max_trials = 2; + + auto tuned = vsag::autotune::TuneIndex(request); + if (!tuned.has_value()) { + std::cerr << "Failed to tune index: " << tuned.error().message << std::endl; + return 1; + } + if (tuned->status == vsag::autotune::TuneStatus::NO_FEASIBLE_CANDIDATE) { + std::cerr << "No candidate satisfied the constraints. Best effort:\n" + << tuned->best_effort.dump(2) << std::endl; + return 2; + } + + const auto& result = tuned.value(); + std::cout << "index_name: " << result.index_name << '\n' + << "create_params: " << result.create_parameters << '\n' + << "search_params: " << result.search_parameters << '\n' + << "validated_metrics: " << result.metrics.dump() << '\n' + << "index_artifact: " << result.artifact_path << '\n' + << "trials_evaluated: " << result.report["trials"].size() << std::endl; + + auto query = vsag::Dataset::Make(); + query->NumElements(1)->Dim(DIM)->Float32Vectors(query_vectors.data())->Owner(false); + + auto neighbors = result.index->KnnSearch(query, 1, result.search_parameters); + if (!neighbors.has_value()) { + std::cerr << "The selected index could not be queried: " << neighbors.error().message + << std::endl; + return 1; + } + std::cout << "first neighbor id: " << neighbors.value()->GetIds()[0] << std::endl; + return 0; +} diff --git a/examples/cpp/327_feature_autotune_existing_index.cpp b/examples/cpp/327_feature_autotune_existing_index.cpp new file mode 100644 index 0000000000..79bc5bc1e1 --- /dev/null +++ b/examples/cpp/327_feature_autotune_existing_index.cpp @@ -0,0 +1,120 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include + +#include "autotune.h" + +int +main() { + constexpr int64_t DIM = 4; + constexpr int64_t BASE_COUNT = 16; + constexpr int64_t QUERY_COUNT = 4; + vsag::Options::Instance().set_block_size_limit(2UL * 1024 * 1024); + + std::vector base_ids(BASE_COUNT); + std::vector base_vectors(BASE_COUNT * DIM); + for (int64_t i = 0; i < BASE_COUNT; ++i) { + base_ids[i] = 1000 + i; + for (int64_t j = 0; j < DIM; ++j) { + base_vectors[i * DIM + j] = static_cast(i * DIM + j); + } + } + + std::vector query_vectors(base_vectors.begin(), + base_vectors.begin() + QUERY_COUNT * DIM); + std::vector ground_truth_ids(base_ids.begin(), base_ids.begin() + QUERY_COUNT); + + auto base = vsag::Dataset::Make() + ->NumElements(BASE_COUNT) + ->Dim(DIM) + ->Ids(base_ids.data()) + ->Float32Vectors(base_vectors.data()) + ->Owner(false); + auto queries = vsag::Dataset::Make() + ->NumElements(QUERY_COUNT) + ->Dim(DIM) + ->Float32Vectors(query_vectors.data()) + ->Owner(false); + auto ground_truth = vsag::Dataset::Make() + ->NumElements(QUERY_COUNT) + ->Dim(1) + ->Ids(ground_truth_ids.data()) + ->Owner(false); + + const std::string create_params = R"( + { + "dim": 4, + "dtype": "float32", + "metric_type": "l2", + "index_param": { + "base_quantization_type": "fp32", + "max_degree": 8, + "ef_construction": 40 + } + })"; + auto created = vsag::Factory::CreateIndex("hgraph", create_params); + if (!created.has_value()) { + std::cerr << "Failed to create index: " << created.error().message << std::endl; + return 1; + } + auto index = created.value(); + auto built = index->Build(base); + if (!built.has_value()) { + std::cerr << "Failed to build index: " << built.error().message << std::endl; + return 1; + } + + vsag::autotune::SearchRequest request; + request.index = index; + request.workload = {queries, ground_truth, 1, 1}; + request.parameter_space = R"({"hgraph":{"ef_search":[4,8,16]}})"; + request.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 1.0}}; + request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + request.config.max_trials = 3; + + const auto tuned = vsag::autotune::TuneSearch(request); + if (!tuned.has_value()) { + std::cerr << "AutoTune failed: " << tuned.error().message << std::endl; + return 1; + } + if (tuned->status == vsag::autotune::TuneStatus::NO_FEASIBLE_CANDIDATE) { + std::cerr << "No candidate satisfied the constraints. Best effort:\n" + << tuned->best_effort.dump(2) << std::endl; + return 2; + } + + const auto& result = tuned.value(); + std::cout << "recommended search_params: " << result.parameters << '\n' + << "validated metrics: " << result.metrics.dump() << '\n' + << "trials evaluated: " << result.report["trials"].size() << std::endl; + + auto query = vsag::Dataset::Make() + ->NumElements(1) + ->Dim(DIM) + ->Float32Vectors(query_vectors.data()) + ->Owner(false); + auto neighbors = index->KnnSearch(query, 1, result.parameters); + if (!neighbors.has_value()) { + std::cerr << "Search failed: " << neighbors.error().message << std::endl; + return 1; + } + std::cout << "first neighbor id: " << neighbors.value()->GetIds()[0] << std::endl; + return 0; +} diff --git a/examples/cpp/328_feature_autotune_existing_pyramid.cpp b/examples/cpp/328_feature_autotune_existing_pyramid.cpp new file mode 100644 index 0000000000..152602d8b0 --- /dev/null +++ b/examples/cpp/328_feature_autotune_existing_pyramid.cpp @@ -0,0 +1,205 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "autotune.h" + +namespace { + +constexpr int64_t DIM = 32; +constexpr int64_t EASY_COUNT = 512; +constexpr int64_t HARD_COUNT = 4096; +constexpr int64_t QUERY_COUNT = 512; +constexpr int64_t TOP_K = 10; + +struct PathWorkload { + std::vector query_vectors; + std::vector ground_truth_ids; + std::vector query_paths; + vsag::DatasetPtr queries; + vsag::DatasetPtr ground_truth; +}; + +float +L2(const float* left, const float* right) { + float result = 0.0F; + for (int64_t i = 0; i < DIM; ++i) { + const auto difference = left[i] - right[i]; + result += difference * difference; + } + return result; +} + +void +PrepareWorkload(PathWorkload& workload, + const std::vector& base_ids, + const std::vector& base_vectors, + int64_t begin, + int64_t count, + const std::string& path, + std::mt19937& random) { + std::uniform_real_distribution unit(0.0F, 1.0F); + workload.query_vectors.resize(QUERY_COUNT * DIM); + workload.ground_truth_ids.resize(QUERY_COUNT * TOP_K); + workload.query_paths.assign(QUERY_COUNT, path); + + for (int64_t query = 0; query < QUERY_COUNT; ++query) { + auto* query_vector = workload.query_vectors.data() + query * DIM; + for (int64_t column = 0; column < DIM; ++column) { + query_vector[column] = unit(random); + } + + std::vector> ranked; + ranked.reserve(count); + for (int64_t row = begin; row < begin + count; ++row) { + ranked.emplace_back(L2(query_vector, base_vectors.data() + row * DIM), base_ids[row]); + } + std::partial_sort(ranked.begin(), ranked.begin() + TOP_K, ranked.end()); + for (int64_t k = 0; k < TOP_K; ++k) { + workload.ground_truth_ids[query * TOP_K + k] = ranked[k].second; + } + } + + workload.queries = vsag::Dataset::Make() + ->NumElements(QUERY_COUNT) + ->Dim(DIM) + ->Float32Vectors(workload.query_vectors.data()) + ->Paths(workload.query_paths.data()) + ->Owner(false); + workload.ground_truth = vsag::Dataset::Make() + ->NumElements(QUERY_COUNT) + ->Dim(TOP_K) + ->Ids(workload.ground_truth_ids.data()) + ->Owner(false); +} + +tl::expected +TunePath(const vsag::IndexPtr& index, const PathWorkload& workload) { + vsag::autotune::SearchRequest request; + request.index = index; + request.workload = {workload.queries, workload.ground_truth, TOP_K, 1}; + request.parameter_space = R"({"pyramid":{"ef_search":[10,20,40,80,160]}})"; + request.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.80}}; + request.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + request.config.max_trials = 5; + return vsag::autotune::TuneSearch(request); +} + +void +PrintResult(const std::string& path, const vsag::autotune::SearchResult& result) { + std::cout << "\npath: " << path << '\n' + << "recommended search_params: " << result.parameters << '\n' + << "validated metrics: " << result.metrics.dump() << '\n' + << "trials evaluated: " << result.report["trials"].size() << std::endl; +} + +} // namespace + +int +main() { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + vsag::Options::Instance().set_block_size_limit(2UL * 1024 * 1024); + std::mt19937 random(47); + std::uniform_real_distribution unit(0.0F, 1.0F); + + constexpr int64_t BASE_COUNT = EASY_COUNT + HARD_COUNT; + std::vector base_ids(BASE_COUNT); + std::vector base_vectors(BASE_COUNT * DIM); + std::vector base_paths(BASE_COUNT); + for (int64_t row = 0; row < BASE_COUNT; ++row) { + base_ids[row] = 1000 + row; + auto* vector = base_vectors.data() + row * DIM; + base_paths[row] = row < EASY_COUNT ? "catalog/easy" : "catalog/hard"; + for (int64_t column = 0; column < DIM; ++column) { + vector[column] = unit(random); + } + } + + auto base = vsag::Dataset::Make() + ->NumElements(BASE_COUNT) + ->Dim(DIM) + ->Ids(base_ids.data()) + ->Float32Vectors(base_vectors.data()) + ->Paths(base_paths.data()) + ->Owner(false); + + const std::string create_params = + R"({ + "dim": 32, + "dtype": "float32", + "metric_type": "l2", + "index_param": { + "base_quantization_type": "fp32", + "max_degree": 16, + "alpha": 1.2, + "graph_iter_turn": 5, + "neighbor_sample_rate": 0.2, + "no_build_levels": [0, 1], + "use_reorder": true, + "graph_type": "odescent", + "build_thread_count": 8 + } + })"; + auto created = vsag::Factory::CreateIndex("pyramid", create_params); + if (!created.has_value()) { + std::cerr << "Failed to create Pyramid: " << created.error().message << std::endl; + return 1; + } + auto index = created.value(); + auto built = index->Build(base); + if (!built.has_value()) { + std::cerr << "Failed to build Pyramid: " << built.error().message << std::endl; + return 1; + } + + PathWorkload easy_workload; + PathWorkload hard_workload; + PrepareWorkload(easy_workload, base_ids, base_vectors, 0, EASY_COUNT, "catalog/easy", random); + PrepareWorkload( + hard_workload, base_ids, base_vectors, EASY_COUNT, HARD_COUNT, "catalog/hard", random); + + auto easy_result = TunePath(index, easy_workload); + auto hard_result = TunePath(index, hard_workload); + if (!easy_result.has_value() || !hard_result.has_value()) { + const auto message = + !easy_result.has_value() ? easy_result.error().message : hard_result.error().message; + std::cerr << "AutoTune failed: " << message << std::endl; + return 1; + } + if (easy_result->status == vsag::autotune::TuneStatus::NO_FEASIBLE_CANDIDATE || + hard_result->status == vsag::autotune::TuneStatus::NO_FEASIBLE_CANDIDATE) { + const auto& result = + easy_result->status == vsag::autotune::TuneStatus::NO_FEASIBLE_CANDIDATE + ? easy_result.value() + : hard_result.value(); + std::cerr << "No candidate satisfied one path workload. Best effort:\n" + << result.best_effort.dump(2) << std::endl; + return 2; + } + + std::cout << "The same Pyramid index is tuned once per representative path workload." + << std::endl; + PrintResult("catalog/easy (512 random vectors)", easy_result.value()); + PrintResult("catalog/hard (4096 random vectors)", hard_result.value()); + return 0; +} diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 3793625a87..d71f93af3a 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -157,3 +157,17 @@ target_link_libraries(325_feature_uring_io vsag) add_executable(324_feature_hgraph_mci_companion 324_feature_hgraph_mci_companion.cpp) target_link_libraries(324_feature_hgraph_mci_companion vsag) + +if (TARGET vsag::autotune) + add_executable (326_feature_create_index_with_constraints + 326_feature_create_index_with_constraints.cpp) + target_link_libraries (326_feature_create_index_with_constraints vsag::autotune) + + add_executable (327_feature_autotune_existing_index + 327_feature_autotune_existing_index.cpp) + target_link_libraries (327_feature_autotune_existing_index vsag::autotune) + + add_executable (328_feature_autotune_existing_pyramid + 328_feature_autotune_existing_pyramid.cpp) + target_link_libraries (328_feature_autotune_existing_pyramid vsag::autotune) +endif () diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index afcad94544..30408efaee 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -15,6 +15,7 @@ add_subdirectory (eval) +add_subdirectory (autotune) add_subdirectory (check_compatibility) add_subdirectory (analyze_index) add_subdirectory (visualize_index) diff --git a/tools/README.md b/tools/README.md index 5d60142128..d98103d7e8 100644 --- a/tools/README.md +++ b/tools/README.md @@ -8,6 +8,7 @@ nearby entry points for local development and AI coding agents. | `analyze_index` | [analyze_index/README.md](analyze_index/README.md) | [docs/docs/en/src/resources/analyze_index.md](../docs/docs/en/src/resources/analyze_index.md) | | | `check_compatibility` | [check_compatibility/README.md](check_compatibility/README.md) | [docs/docs/en/src/resources/check_compatibility.md](../docs/docs/en/src/resources/check_compatibility.md) | | | `eval_performance` | [eval/README.md](eval/README.md) | [docs/docs/en/src/resources/eval.md](../docs/docs/en/src/resources/eval.md) | | +| `autotune` | [autotune/README.md](autotune/README.md) | [docs/docs/en/src/resources/autotune.md](../docs/docs/en/src/resources/autotune.md) | | Chinese documentation is available under `docs/docs/zh/src/resources/` and on the Chinese website paths under . diff --git a/tools/autotune/CMakeLists.txt b/tools/autotune/CMakeLists.txt new file mode 100644 index 0000000000..1d2f277d38 --- /dev/null +++ b/tools/autotune/CMakeLists.txt @@ -0,0 +1,54 @@ +# Copyright 2024-present the vsag project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_library (vsag_autotune SHARED + autotune.cpp + autotune_candidate.cpp + autotune_evaluation.cpp + $) +add_library (vsag::autotune ALIAS vsag_autotune) +target_include_directories (vsag_autotune PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories (vsag_autotune PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../eval) +target_link_libraries (vsag_autotune PUBLIC nlohmann_json::nlohmann_json vsag) +target_link_libraries (vsag_autotune PRIVATE + simd + vsag_eval_common + ${HDF5_CPP_STATIC_LIBRARY} + ${HDF5_C_STATIC_LIBRARY} + z + ${CMAKE_DL_LIBS}) +add_dependencies (vsag_autotune hdf5) + +add_executable (autotune main.cpp) +target_include_directories (autotune PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries (autotune PRIVATE vsag_autotune vsag) + +if (ENABLE_TESTS) + add_executable (autotune_test autotune_test.cpp) + target_include_directories (autotune_test PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../eval) + target_link_libraries (autotune_test PRIVATE + Catch2::Catch2WithMain + vsag_autotune + vsag + vsag_eval_common + ${HDF5_CPP_STATIC_LIBRARY} + ${HDF5_C_STATIC_LIBRARY} + z + ${CMAKE_DL_LIBS}) + add_dependencies (autotune_test hdf5 Catch2) +endif () diff --git a/tools/autotune/README.md b/tools/autotune/README.md new file mode 100644 index 0000000000..8eabdcf64b --- /dev/null +++ b/tools/autotune/README.md @@ -0,0 +1,25 @@ +# VSAG AutoTune Tool + +The canonical documentation for `autotune` lives in the website source tree. +The typed C++ API is experimental, build-tree-only, and is not installed with VSAG. + +- Local English docs: + [docs/docs/en/src/resources/autotune.md](../../docs/docs/en/src/resources/autotune.md) +- Local Chinese docs: + [docs/docs/zh/src/resources/autotune.md](../../docs/docs/zh/src/resources/autotune.md) +- V1 CLI JSON input/output contract: + [AutoTune V1 API](../../docs/docs/en/src/resources/autotune_api_v1.md) +- Website: +- Chinese website: +- Example request: + [examples/sift_hgraph_autotune_request.json](examples/sift_hgraph_autotune_request.json) +- Build-and-search tuning example: + [examples/cpp/326_feature_create_index_with_constraints.cpp][factory-example] +- Existing-index search tuning example: + [examples/cpp/327_feature_autotune_existing_index.cpp][existing-index-example] +- Existing-Pyramid path tuning example: + [examples/cpp/328_feature_autotune_existing_pyramid.cpp][pyramid-example] + +[factory-example]: ../../examples/cpp/326_feature_create_index_with_constraints.cpp +[existing-index-example]: ../../examples/cpp/327_feature_autotune_existing_index.cpp +[pyramid-example]: ../../examples/cpp/328_feature_autotune_existing_pyramid.cpp diff --git a/tools/autotune/README_zh.md b/tools/autotune/README_zh.md new file mode 100644 index 0000000000..b1b5f1732a --- /dev/null +++ b/tools/autotune/README_zh.md @@ -0,0 +1,25 @@ +# VSAG AutoTune 工具 + +`autotune` 的规范文档位于网站源码目录。 +typed C++ API 是实验性接口，只在构建目录中提供，不会随 VSAG 安装。 + +- 本地英文文档： + [docs/docs/en/src/resources/autotune.md](../../docs/docs/en/src/resources/autotune.md) +- 本地中文文档： + [docs/docs/zh/src/resources/autotune.md](../../docs/docs/zh/src/resources/autotune.md) +- V1 CLI JSON 输入输出契约： + [AutoTune V1 API](../../docs/docs/zh/src/resources/autotune_api_v1.md) +- 网站： +- 中文网站： +- 示例请求： + [examples/sift_hgraph_autotune_request.json](examples/sift_hgraph_autotune_request.json) +- 构建和查询联合调优示例： + [examples/cpp/326_feature_create_index_with_constraints.cpp][factory-example] +- 已有索引 search 调优示例： + [examples/cpp/327_feature_autotune_existing_index.cpp][existing-index-example] +- 已有 Pyramid path 调优示例： + [examples/cpp/328_feature_autotune_existing_pyramid.cpp][pyramid-example] + +[factory-example]: ../../examples/cpp/326_feature_create_index_with_constraints.cpp +[existing-index-example]: ../../examples/cpp/327_feature_autotune_existing_index.cpp +[pyramid-example]: ../../examples/cpp/328_feature_autotune_existing_pyramid.cpp diff --git a/tools/autotune/autotune.cpp b/tools/autotune/autotune.cpp new file mode 100644 index 0000000000..29eadbf4a6 --- /dev/null +++ b/tools/autotune/autotune.cpp @@ -0,0 +1,1159 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "autotune.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "autotune_internal.h" +#include "eval_dataset.h" +#include "vsag/constants.h" +#include "vsag/factory.h" + +namespace vsag::autotune::internal { + +namespace { + +double +elapsed(const std::chrono::steady_clock::time_point& start) { + return std::chrono::duration(std::chrono::steady_clock::now() - start).count(); +} + +void +require(bool condition, const std::string& message) { + if (!condition) { + throw std::invalid_argument(message); + } +} + +void +known_keys(const JsonType& object, + std::initializer_list keys, + const std::string& path) { + require(object.is_object(), path + " must be an object"); + std::set known(keys.begin(), keys.end()); + for (const auto& item : object.items()) { + require(known.find(item.key()) != known.end(), path + "." + item.key() + " is unsupported"); + } +} + +std::string +required_string(const JsonType& object, const std::string& key, const std::string& path) { + require(object.contains(key) && object[key].is_string(), path + "." + key + " is required"); + auto value = object[key].get(); + require(!value.empty(), path + "." + key + " must not be empty"); + return value; +} + +uint64_t +positive_integer(const JsonType& object, const std::string& key, const std::string& path) { + require(object.contains(key) && object[key].is_number_integer(), + path + "." + key + " must be a positive integer"); + if (object[key].is_number_unsigned()) { + const auto value = object[key].get(); + require(value > 0, path + "." + key + " must be positive"); + return value; + } + const auto value = object[key].get(); + require(value > 0, path + "." + key + " must be positive"); + return static_cast(value); +} + +std::string +normalize(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), [](unsigned char character) { + return static_cast(std::tolower(character)); + }); + return value; +} + +std::string +metric_name(Metric metric) { + switch (metric) { + case Metric::RECALL_AT_K: + return "recall_at_k"; + case Metric::LATENCY_AVG_MS: + return "latency_avg_ms"; + case Metric::LATENCY_P99_MS: + return "latency_p99_ms"; + case Metric::QPS: + return "qps"; + case Metric::INDEX_MEMORY_MB: + return "index_memory_mb"; + case Metric::INDEX_SIZE_MB: + return "index_size_mb"; + case Metric::BUILD_SECONDS: + return "build_seconds"; + case Metric::SEARCH_SECONDS: + return "search_seconds"; + case Metric::BUILD_AND_SEARCH_SECONDS: + return "build_and_search_seconds"; + case Metric::UNSPECIFIED: + break; + } + throw std::invalid_argument("AutoTune metric must be specified"); +} + +Metric +parse_metric(const std::string& metric) { + if (metric == "recall_at_k") { + return Metric::RECALL_AT_K; + } + if (metric == "latency_avg_ms") { + return Metric::LATENCY_AVG_MS; + } + if (metric == "latency_p99_ms") { + return Metric::LATENCY_P99_MS; + } + if (metric == "qps") { + return Metric::QPS; + } + if (metric == "index_memory_mb") { + return Metric::INDEX_MEMORY_MB; + } + if (metric == "index_size_mb") { + return Metric::INDEX_SIZE_MB; + } + if (metric == "build_seconds") { + return Metric::BUILD_SECONDS; + } + if (metric == "search_seconds") { + return Metric::SEARCH_SECONDS; + } + if (metric == "build_and_search_seconds") { + return Metric::BUILD_AND_SEARCH_SECONDS; + } + throw std::invalid_argument("unsupported metric: " + metric); +} + +bool +supported_metric(const std::string& metric) { + static const std::set metrics{"build_seconds", + "index_size_mb", + "index_memory_mb", + "recall_at_k", + "latency_avg_ms", + "latency_p99_ms", + "qps", + "search_seconds", + "build_and_search_seconds"}; + return metrics.find(metric) != metrics.end(); +} + +bool +higher_is_better(const std::string& metric) { + return metric == "recall_at_k" || metric == "qps"; +} + +bool +available_for_existing_index(const std::string& metric) { + return metric != "build_seconds" && metric != "build_and_search_seconds" && + metric != "index_size_mb"; +} + +bool +useful_existing_index_objective(const std::string& metric) { + return available_for_existing_index(metric) && metric != "index_memory_mb"; +} + +bool +same_path(const std::string& left, const std::string& right) { + if (left.empty() || right.empty()) { + return false; + } + + std::error_code equivalent_error; + if (std::filesystem::equivalent(left, right, equivalent_error)) { + return true; + } + + const auto normalized = [](const std::string& value) { + std::error_code error; + const auto canonical = std::filesystem::weakly_canonical(value, error); + return error ? std::filesystem::absolute(value).lexically_normal() : canonical; + }; + return normalized(left) == normalized(right); +} + +std::string +metric_type(const std::string& dataset_metric) { + if (dataset_metric == "euclidean") { + return "l2"; + } + if (dataset_metric == "ip") { + return "ip"; + } + if (dataset_metric == "angular") { + return "cosine"; + } + throw std::invalid_argument("unsupported dataset distance: " + dataset_metric); +} + +struct offline_dataset_owner { + eval::EvalDatasetPtr source; + std::vector identity_ids; + DatasetPtr base; + DatasetPtr queries; + DatasetPtr ground_truth; +}; + +void +attach_offline_dataset(IndexRequest& request, + const eval::EvalDatasetPtr& dataset, + bool include_base) { + require(dataset->GetVectorType() == "dense_vectors", + "AutoTune V1 supports only dense vector datasets"); + require(dataset->GetTrainDataType() == vsag::DATATYPE_FLOAT32 && + dataset->GetTestDataType() == vsag::DATATYPE_FLOAT32, + "AutoTune V1 supports only float32 datasets"); + + auto owner = std::make_shared(); + owner->source = dataset; + const auto base_count = dataset->GetNumberOfBase(); + const auto query_count = dataset->GetNumberOfQuery(); + const auto ground_truth_k = static_cast(dataset->GetGroundTruthK()); + if (include_base) { + const auto* train_ids = dataset->GetTrainIds(); + if (train_ids == nullptr) { + owner->identity_ids.resize(static_cast(base_count)); + for (int64_t i = 0; i < base_count; ++i) { + owner->identity_ids[static_cast(i)] = i; + } + train_ids = owner->identity_ids.data(); + } + owner->base = Dataset::Make() + ->NumElements(base_count) + ->Dim(dataset->GetDim()) + ->Ids(train_ids) + ->Float32Vectors(static_cast(dataset->GetTrain())) + ->Owner(false); + } + owner->queries = Dataset::Make() + ->NumElements(query_count) + ->Dim(dataset->GetDim()) + ->Float32Vectors(static_cast(dataset->GetTest())) + ->Owner(false); + if (ground_truth_k > 0) { + owner->ground_truth = Dataset::Make() + ->NumElements(query_count) + ->Dim(ground_truth_k) + ->Ids(dataset->GetNeighbors(0)) + ->Distances(dataset->GetDistances(0)) + ->Owner(false); + } + + if (owner->base != nullptr) { + request.base = DatasetPtr(owner, owner->base.get()); + } + request.workload.queries = DatasetPtr(owner, owner->queries.get()); + if (owner->ground_truth != nullptr) { + request.workload.ground_truth = DatasetPtr(owner, owner->ground_truth.get()); + } + request.metric_type = metric_type(dataset->GetMetric()); +} + +void +check_file(const std::string& path, const std::string& field) { + std::error_code error; + require(std::filesystem::is_regular_file(path, error) && !error, + field + " must name a readable regular file: " + path); + std::ifstream input(path, std::ios::binary); + require(input.good(), field + " must name a readable regular file: " + path); +} + +void +merge_dataset_field(JsonType& create_params, + const std::string& name, + const JsonType& value, + const std::string& index_name) { + if (create_params.contains(name)) { + require(create_params[name] == value, + index_name + " create_params." + name + " must match the dataset"); + } + create_params[name] = value; +} + +MetricMap +read_metrics(const JsonType& value) { + MetricMap metrics; + if (!value.is_object()) { + return metrics; + } + for (const auto& item : value.items()) { + if (item.value().is_number()) { + metrics[item.key()] = item.value().get(); + } + } + return metrics; +} + +JsonType +constraint_evaluation(const RequestContext& request, const JsonType& trial) { + JsonType violations = JsonType::array(); + const auto metrics = read_metrics(trial.value("metrics", JsonType::object())); + for (const auto& [name, expected] : request.constraints) { + const auto actual = metrics.find(name); + const bool present = actual != metrics.end() && std::isfinite(actual->second); + const bool satisfied = present && (higher_is_better(name) ? actual->second >= expected + : actual->second <= expected); + if (!satisfied) { + violations.push_back( + {{"metric", name}, + {"comparison", higher_is_better(name) ? "at_least" : "at_most"}, + {"expected", expected}, + {"actual", present ? JsonType(actual->second) : JsonType(nullptr)}}); + } + } + return {{"satisfied", violations.empty()}, {"violations", std::move(violations)}}; +} + +double +objective_value(const RequestContext& request, const JsonType& trial) { + const auto metrics = read_metrics(trial.value("metrics", JsonType::object())); + const auto value = metrics.find(request.objective); + if (value == metrics.end() || !std::isfinite(value->second)) { + return higher_is_better(request.objective) ? -std::numeric_limits::infinity() + : std::numeric_limits::infinity(); + } + return value->second; +} + +bool +has_objective(const RequestContext& request, const JsonType& trial) { + const auto metrics = read_metrics(trial.value("metrics", JsonType::object())); + const auto value = metrics.find(request.objective); + return value != metrics.end() && std::isfinite(value->second); +} + +JsonType +recommendation(const RequestContext& request, const JsonType& trial) { + JsonType result{{"index_name", trial["index_name"]}, + {"search_params", trial["search_params"]}, + {"workload", {{"top_k", request.top_k}, {"concurrency", request.concurrency}}}, + {"metrics", trial["metrics"]}, + {"evidence", {{"trial_id", trial["trial_id"]}}}}; + if (trial.contains("create_params")) { + result["create_params"] = trial["create_params"]; + } + if (trial.contains("artifacts")) { + result["artifacts"] = trial["artifacts"]; + } + if (trial.contains("build_id")) { + result["evidence"]["build_id"] = trial["build_id"]; + } + return result; +} + +std::mutex& +run_mutex() { + static std::mutex mutex; + return mutex; +} + +std::string +new_run_name() { + static std::atomic serial{0}; + const auto now = std::chrono::steady_clock::now().time_since_epoch().count(); + return "run-" + std::to_string(now) + "-" + std::to_string(serial++); +} + +} // namespace + +namespace { + +JsonType +parse_parameters(const std::string& value, const std::string& path) { + try { + auto parsed = JsonType::parse(value); + require(parsed.is_object(), path + " must encode a JSON object"); + return parsed; + } catch (const nlohmann::json::exception& error) { + throw std::invalid_argument(path + " is invalid JSON: " + error.what()); + } +} + +RequestContext +make_context(eval::EvalDatasetPtr dataset, + uint64_t base_count, + const std::string& metric_type, + const Workload& workload, + const std::vector& constraints, + Metric objective, + const Config& config, + bool search_only) { + RequestContext request; + request.workspace_path = + config.workspace_path.empty() ? "/tmp/vsag_autotune" : config.workspace_path; + request.keep_intermediate = config.keep_intermediate; + request.include_raw_eval = config.include_raw_evaluation; + request.max_trials = config.max_trials; + request.top_k = workload.top_k; + request.concurrency = workload.concurrency; + + request.dataset = std::move(dataset); + request.base_count = base_count; + request.query_count = static_cast(request.dataset->GetNumberOfQuery()); + request.ground_truth_k = request.dataset->GetGroundTruthK(); + require(request.top_k > 0, "request.workload.top_k must be positive"); + require(request.top_k <= static_cast(std::numeric_limits::max()), + "request.workload.top_k is too large"); + require(request.concurrency > 0 && request.concurrency <= 200, + "request.workload.concurrency must be in [1, 200]"); + require(request.max_trials > 0 && request.max_trials <= 100000, + "request.config.max_trials must be in [1, 100000]"); + + request.objective = metric_name(objective); + request.enable_recall = objective == Metric::RECALL_AT_K; + require(!constraints.empty(), "request.constraints must not be empty"); + for (const auto& constraint : constraints) { + const auto name = metric_name(constraint.metric); + require(std::isfinite(constraint.value) && constraint.value >= 0.0, + "request.constraints." + name + " must be finite and non-negative"); + require(name != "recall_at_k" || constraint.value <= 1.0, + "request.constraints.recall_at_k must be in [0, 1]"); + require(!search_only || available_for_existing_index(name), + "request.constraints." + name + " is unavailable for search tuning"); + require(request.constraints.emplace(name, constraint.value).second, + "request.constraints contains duplicate metric: " + name); + request.enable_recall = request.enable_recall || constraint.metric == Metric::RECALL_AT_K; + } + require(!search_only || useful_existing_index_objective(request.objective), + "request.objective cannot rank search candidates: " + request.objective); + + require(request.top_k <= request.base_count, "request.workload.top_k exceeds the index size"); + if (request.enable_recall) { + require(workload.ground_truth != nullptr, + "request.workload.ground_truth is required for recall_at_k"); + require(request.ground_truth_k >= request.top_k, + "request.workload.top_k exceeds ground truth k"); + } + + JsonType dataset_info{{"base_count", request.base_count}, + {"query_count", request.query_count}, + {"ground_truth_k", request.ground_truth_k}, + {"dim", request.dataset->GetDim()}, + {"dtype", vsag::DATATYPE_FLOAT32}}; + if (!metric_type.empty()) { + dataset_info["metric_type"] = metric_type; + } + request.effective_request = { + {"version", 1}, + {"dataset", std::move(dataset_info)}, + {"workload", {{"top_k", request.top_k}, {"concurrency", request.concurrency}}}, + {"constraints", JsonType::object()}, + {"objective", {{"metric", request.objective}}}, + {"config", + {{"workspace_path", request.workspace_path}, + {"keep_intermediate", request.keep_intermediate}, + {"max_trials", request.max_trials}}}, + {"output", {{"include_raw_evaluation", request.include_raw_eval}}}}; + for (const auto& [name, value] : request.constraints) { + request.effective_request["constraints"][name] = value; + } + return request; +} + +std::string +index_name(const IndexPtr& index) { + switch (index->GetIndexType()) { + case IndexType::HGRAPH: + return INDEX_HGRAPH; + case IndexType::IVF: + return INDEX_IVF; + case IndexType::PYRAMID: + return INDEX_PYRAMID; + default: + throw std::invalid_argument("request.index type is unsupported by AutoTune"); + } +} + +IndexInput +parse_index_space(const IndexSpace& value, uint64_t position, bool search_only) { + const auto path = "request.index_spaces[" + std::to_string(position) + "]"; + IndexInput index; + index.name = normalize(value.name); + require( + index.name == "hgraph" || index.name == "ivf" || (search_only && index.name == "pyramid"), + "unsupported index: " + index.name); + index.create_params = search_only ? JsonType::object() + : parse_parameters(value.create_parameter_space, + path + ".create_parameter_space"); + const auto search = + parse_parameters(value.search_parameter_space, path + ".search_parameter_space"); + for (const auto& item : search.items()) { + require(normalize(item.key()) == index.name, + path + ".search_parameter_space." + item.key() + " is unsupported"); + index.search_params[index.name] = item.value(); + } + return index; +} + +} // namespace + +IndexTuningRequest +ParseRequest(const IndexRequest& input) { + require(input.base != nullptr, "request.base is required"); + require(input.base->GetIds() != nullptr, "request.base IDs are required"); + const auto metric = normalize(input.metric_type); + require(metric == "l2" || metric == "ip" || metric == "cosine", + "request.metric_type must be l2, ip, or cosine"); + auto dataset = eval::EvalDataset::FromDatasets( + input.base, input.workload.queries, input.workload.ground_truth, metric); + IndexTuningRequest request; + request.context = make_context(std::move(dataset), + static_cast(input.base->GetNumElements()), + metric, + input.workload, + input.constraints, + input.objective, + input.config, + false); + if (input.index_spaces.empty()) { + request.indexes = {{"hgraph"}, {"ivf"}}; + } else { + for (uint64_t i = 0; i < input.index_spaces.size(); ++i) { + request.indexes.emplace_back(parse_index_space(input.index_spaces[i], i, false)); + } + } + const auto dim = static_cast(request.context.dataset->GetDim()); + for (auto& index : request.indexes) { + merge_dataset_field(index.create_params, "dim", dim, index.name); + merge_dataset_field(index.create_params, "dtype", vsag::DATATYPE_FLOAT32, index.name); + merge_dataset_field(index.create_params, "metric_type", metric, index.name); + request.context.effective_request["index_spaces"].push_back( + {{"name", index.name}, + {"create_parameter_space", index.create_params}, + {"search_parameter_space", index.search_params}}); + } + return request; +} + +SearchTuningRequest +ParseRequest(const SearchRequest& input) { + require(input.index != nullptr, "request.index is required"); + const auto element_count = input.index->GetNumElements(); + require(element_count > 0, "request.index must not be empty"); + auto dataset = + eval::EvalDataset::FromSearchDatasets(input.workload.queries, input.workload.ground_truth); + SearchTuningRequest request; + request.context = make_context(std::move(dataset), + static_cast(element_count), + "", + input.workload, + input.constraints, + input.objective, + input.config, + true); + IndexSpace space; + space.name = index_name(input.index); + space.search_parameter_space = input.parameter_space; + request.index_input = parse_index_space(space, 0, true); + request.index = input.index; + request.context.effective_request["index_name"] = request.index_input.name; + request.context.effective_request["parameter_space"] = request.index_input.search_params; + return request; +} + +ParsedRequest +ParseRequest(const JsonType& input) { + known_keys(input, + {"version", + "data_path", + "index_path", + "indexes", + "workload", + "constraints", + "objective", + "tuning_config", + "output"}, + "request"); + require(input.contains("version") && input["version"] == 1, "request.version must be 1"); + const auto data_path = required_string(input, "data_path", "request"); + check_file(data_path, "data_path"); + + IndexRequest typed; + std::string report_path; + attach_offline_dataset( + typed, eval::EvalDataset::Load(data_path), !input.contains("index_path")); + if (input.contains("indexes")) { + require(input["indexes"].is_array() && !input["indexes"].empty(), + "request.indexes must be a non-empty array"); + for (uint64_t i = 0; i < input["indexes"].size(); ++i) { + const auto& value = input["indexes"][i]; + const auto path = "request.indexes[" + std::to_string(i) + "]"; + known_keys(value, {"name", "create_params", "search_params"}, path); + IndexSpace index; + index.name = required_string(value, "name", path); + if (value.contains("create_params")) { + require(value["create_params"].is_object(), + path + ".create_params must be an object"); + index.create_parameter_space = value["create_params"].dump(); + } + if (value.contains("search_params")) { + require(value["search_params"].is_object(), + path + ".search_params must be an object"); + index.search_parameter_space = value["search_params"].dump(); + } + typed.index_spaces.emplace_back(std::move(index)); + } + } + require(input.contains("workload"), "request.workload is required"); + known_keys(input["workload"], {"top_k", "concurrency"}, "request.workload"); + typed.workload.top_k = positive_integer(input["workload"], "top_k", "request.workload"); + if (input["workload"].contains("concurrency")) { + typed.workload.concurrency = + positive_integer(input["workload"], "concurrency", "request.workload"); + } + require(input.contains("constraints") && input["constraints"].is_object() && + !input["constraints"].empty(), + "request.constraints must be a non-empty object"); + for (const auto& item : input["constraints"].items()) { + require(supported_metric(item.key()), "unsupported metric: " + item.key()); + require(item.value().is_number(), + "request.constraints." + item.key() + " must be a number"); + typed.constraints.push_back({parse_metric(item.key()), item.value().get()}); + } + require(input.contains("objective"), "request.objective is required"); + known_keys(input["objective"], {"metric"}, "request.objective"); + typed.objective = + parse_metric(required_string(input["objective"], "metric", "request.objective")); + if (input.contains("tuning_config")) { + const auto& config = input["tuning_config"]; + known_keys( + config, {"workspace_path", "keep_intermediate", "max_trials"}, "request.tuning_config"); + if (config.contains("workspace_path")) { + typed.config.workspace_path = + required_string(config, "workspace_path", "request.tuning_config"); + } + if (config.contains("keep_intermediate")) { + require(config["keep_intermediate"].is_boolean(), + "request.tuning_config.keep_intermediate must be a boolean"); + typed.config.keep_intermediate = config["keep_intermediate"].get(); + } + if (config.contains("max_trials")) { + typed.config.max_trials = + positive_integer(config, "max_trials", "request.tuning_config"); + } + } + if (input.contains("output")) { + const auto& output = input["output"]; + known_keys(output, {"result_path", "include_raw_eval"}, "request.output"); + if (output.contains("result_path")) { + report_path = required_string(output, "result_path", "request.output"); + require(!same_path(report_path, data_path), + "request.output.result_path must not alias data_path"); + } + if (output.contains("include_raw_eval")) { + require(output["include_raw_eval"].is_boolean(), + "request.output.include_raw_eval must be a boolean"); + typed.config.include_raw_evaluation = output["include_raw_eval"].get(); + } + } + + if (!input.contains("index_path")) { + auto parsed = ParseRequest(typed); + parsed.context.result_path = report_path; + parsed.context.effective_request["data_path"] = data_path; + if (!report_path.empty()) { + parsed.context.effective_request["output"]["report_path"] = report_path; + } + return parsed; + } + + const auto index_path = required_string(input, "index_path", "request"); + check_file(index_path, "index_path"); + require(!same_path(report_path, index_path), + "request.output.result_path must not alias index_path"); + require(typed.index_spaces.size() == 1, + "index_path requires exactly one indexes[] specification"); + auto& space = typed.index_spaces.front(); + auto create_params = + parse_parameters(space.create_parameter_space, "request.indexes[0].create_params"); + require(create_params.contains("index_param") && create_params["index_param"].is_object(), + "request.indexes[0].create_params.index_param is required"); + merge_dataset_field( + create_params, "dim", static_cast(typed.workload.queries->GetDim()), space.name); + merge_dataset_field(create_params, "dtype", vsag::DATATYPE_FLOAT32, space.name); + merge_dataset_field(create_params, "metric_type", normalize(typed.metric_type), space.name); + auto created = Factory::CreateIndex(normalize(space.name), create_params.dump()); + if (!created.has_value()) { + throw std::invalid_argument(created.error().message); + } + std::ifstream serialized(index_path, std::ios::binary); + auto loaded = created.value()->Deserialize(serialized); + if (!loaded.has_value()) { + throw std::invalid_argument(loaded.error().message); + } + + SearchRequest search; + search.index = created.value(); + search.workload = typed.workload; + search.parameter_space = space.search_parameter_space; + search.constraints = typed.constraints; + search.objective = typed.objective; + search.config = typed.config; + auto parsed = ParseRequest(search); + parsed.context.result_path = report_path; + parsed.context.effective_request["data_path"] = data_path; + parsed.context.effective_request["index_path"] = index_path; + parsed.context.effective_request["create_params"] = create_params; + if (!report_path.empty()) { + parsed.context.effective_request["output"]["report_path"] = report_path; + } + return parsed; +} + +JsonType +SelectResult(const RequestContext& request, const Evaluation& evaluation) { + JsonType trials = JsonType::array(); + int64_t best = -1; + int64_t best_effort = -1; + uint64_t best_violations = std::numeric_limits::max(); + double best_violation_score = std::numeric_limits::infinity(); + bool has_successful_trial = false; + bool has_successful_objective = false; + + for (const auto& source : evaluation.trials) { + auto trial = source; + trial["constraint_evaluation"] = constraint_evaluation(request, trial); + trials.push_back(std::move(trial)); + const auto stored_index = static_cast(trials.size() - 1); + const auto& stored = trials.back(); + if (stored["status"] != "success") { + continue; + } + has_successful_trial = true; + if (!has_objective(request, stored)) { + continue; + } + has_successful_objective = true; + + const auto violation_count = + static_cast(stored["constraint_evaluation"]["violations"].size()); + if (violation_count == 0) { + if (best < 0 || + (higher_is_better(request.objective) + ? objective_value(request, stored) > objective_value(request, trials[best]) + : objective_value(request, stored) < objective_value(request, trials[best]))) { + best = stored_index; + } + continue; + } + + double score = 0.0; + for (const auto& violation : stored["constraint_evaluation"]["violations"]) { + if (violation["actual"].is_null()) { + score += 1.0; + } else { + score += std::abs(violation["actual"].get() - + violation["expected"].get()) / + std::max(violation["expected"].get(), 1e-12); + } + } + if (best_effort < 0 || violation_count < best_violations || + (violation_count == best_violations && score < best_violation_score)) { + best_effort = stored_index; + best_violations = violation_count; + best_violation_score = score; + } + } + + JsonType result{{"status", best >= 0 ? "success" : "no_feasible_candidate"}, + {"recommendation", nullptr}, + {"best_effort", nullptr}, + {"builds", evaluation.builds}, + {"trials", std::move(trials)}}; + if (best >= 0) { + result["recommendation"] = recommendation(request, result["trials"][best]); + } else if (best_effort >= 0) { + result["best_effort"] = recommendation(request, result["trials"][best_effort]); + result["best_effort"]["constraint_evaluation"] = + result["trials"][best_effort]["constraint_evaluation"]; + } else { + result["status"] = "failed"; + result["failure"] = + has_successful_trial && !has_successful_objective + ? Failure("selection", + "objective_metric_unavailable", + "objective metric is unavailable: " + request.objective) + : Failure("evaluation", "all_trials_failed", "all candidate evaluations failed"); + } + return result; +} + +JsonType +Failure(const std::string& stage, const std::string& code, const std::string& message) { + return {{"stage", stage}, {"code", code}, {"message", message}}; +} + +void +WriteJson(const std::string& path, const JsonType& value) { + const auto parent = std::filesystem::path(path).parent_path(); + if (!parent.empty()) { + std::filesystem::create_directories(parent); + } + std::ofstream output(path); + if (!output.good()) { + throw std::runtime_error("failed to open report path: " + path); + } + output << value.dump(2) << std::endl; + if (!output.good()) { + throw std::runtime_error("failed to write report: " + path); + } +} + +} // namespace vsag::autotune::internal + +namespace vsag::autotune { + +namespace { + +void +prepare_report_path(const std::string& path) { + const auto parent = std::filesystem::path(path).parent_path(); + if (!parent.empty()) { + std::filesystem::create_directories(parent); + } + std::ofstream output(path, std::ios::app); + if (!output.good()) { + throw std::runtime_error("failed to open report path: " + path); + } +} + +void +finalize_artifacts(JsonType& report, bool keep_all, const std::optional& selected); + +std::optional +selected_artifact(const JsonType& report); + +template +JsonType +run_tuning_locked(Parser parser, bool persist_report, std::chrono::steady_clock::time_point start) { + std::string stage = "validation"; + std::string run_path; + std::string report_path; + bool keep_intermediate = false; + const auto failure = [&](const std::string& message) { + if (!keep_intermediate && !run_path.empty()) { + std::error_code cleanup_error; + std::filesystem::remove_all(run_path, cleanup_error); + } + const char* const code = stage == "validation" || stage == "candidate_generation" + ? "invalid_request" + : "execution_failed"; + JsonType report{{"version", 1}, + {"status", "failed"}, + {"recommendation", nullptr}, + {"best_effort", nullptr}, + {"elapsed_seconds", internal::elapsed(start)}, + {"failure", internal::Failure(stage, code, message)}}; + if (!report_path.empty()) { + report["report_path"] = report_path; + try { + internal::WriteJson(report_path, report); + } catch (...) { + } + } + return report; + }; + try { + auto request = parser(); + auto& context = request.context; + keep_intermediate = context.keep_intermediate; + const auto run_name = internal::new_run_name(); + if (persist_report) { + report_path = context.result_path.empty() + ? context.workspace_path + "/" + run_name + ".json" + : context.result_path; + stage = "report"; + prepare_report_path(report_path); + } + stage = "candidate_generation"; + const auto candidates = internal::GenerateCandidates(request); + stage = "evaluation"; + internal::Evaluation evaluation; + if constexpr (std::is_same_v) { + run_path = context.workspace_path + "/runs/" + run_name; + std::filesystem::create_directories(run_path + "/artifacts"); + evaluation = internal::EvaluateCandidates(request, candidates, run_path); + } else { + evaluation = internal::EvaluateCandidates(request, candidates); + } + auto report = internal::SelectResult(context, evaluation); + report["version"] = 1; + report["request"] = context.effective_request; + if (!report_path.empty()) { + report["report_path"] = report_path; + report["request"]["output"]["report_path"] = report_path; + } + if constexpr (std::is_same_v) { + const auto selected = selected_artifact(report); + finalize_artifacts(report, context.keep_intermediate, selected); + if (!context.keep_intermediate && !selected.has_value() && !run_path.empty()) { + std::error_code error; + std::filesystem::remove_all(run_path, error); + } + } + report["elapsed_seconds"] = internal::elapsed(start); + stage = "report"; + if (!report_path.empty()) { + internal::WriteJson(report_path, report); + } + return report; + } catch (const std::exception& error) { + return failure(error.what()); + } catch (...) { + return failure("unknown AutoTune error"); + } +} + +template +JsonType +run_tuning(Parser parser, + std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now()) { + std::lock_guard lock(internal::run_mutex()); + return run_tuning_locked(std::move(parser), false, start); +} + +Error +report_error(const JsonType& report) { + auto type = ErrorType::INTERNAL_ERROR; + auto message = std::string("AutoTune failed"); + if (report.contains("failure") && report["failure"].is_object()) { + const auto& failure = report["failure"]; + if (failure.value("stage", std::string()) == "validation" || + failure.value("stage", std::string()) == "candidate_generation") { + type = ErrorType::INVALID_ARGUMENT; + } + message = failure.value("message", message); + } + return {type, std::move(message)}; +} + +void +update_artifact(JsonType& value, + bool keep_all, + const std::optional& selected, + std::unordered_set& removed) { + if (!value.is_object() || !value.contains("artifacts") || !value["artifacts"].is_object() || + !value["artifacts"].contains("index_path") || + !value["artifacts"]["index_path"].is_string()) { + return; + } + + auto& artifacts = value["artifacts"]; + const auto path = artifacts["index_path"].get(); + std::error_code exists_error; + const auto exists = std::filesystem::is_regular_file(path, exists_error) && !exists_error; + const auto should_retain = keep_all || (selected.has_value() && path == *selected); + if (exists && !should_retain && removed.emplace(path).second) { + std::error_code remove_error; + if (std::filesystem::remove(path, remove_error)) { + std::filesystem::remove(std::filesystem::path(path).parent_path(), remove_error); + std::filesystem::remove(std::filesystem::path(path).parent_path().parent_path(), + remove_error); + } + } + std::error_code retained_error; + artifacts["retained"] = + std::filesystem::is_regular_file(path, retained_error) && !retained_error; +} + +void +finalize_artifacts(JsonType& report, bool keep_all, const std::optional& selected) { + std::unordered_set removed; + if (report.contains("builds") && report["builds"].is_array()) { + for (auto& build : report["builds"]) { + update_artifact(build, keep_all, selected, removed); + } + } + if (report.contains("trials") && report["trials"].is_array()) { + for (auto& trial : report["trials"]) { + update_artifact(trial, keep_all, selected, removed); + } + } + for (const auto* key : {"recommendation", "best_effort"}) { + if (report.contains(key)) { + update_artifact(report[key], keep_all, selected, removed); + } + } +} + +std::optional +selected_artifact(const JsonType& report) { + if (report.value("status", std::string()) != "success" || !report.contains("recommendation") || + !report["recommendation"].is_object()) { + return std::nullopt; + } + const auto& recommendation = report["recommendation"]; + if (!recommendation.contains("artifacts") || !recommendation["artifacts"].is_object() || + !recommendation["artifacts"].contains("index_path") || + !recommendation["artifacts"]["index_path"].is_string()) { + return std::nullopt; + } + return recommendation["artifacts"]["index_path"].get(); +} + +void +finalize_artifacts_noexcept(JsonType& report, bool keep_all) noexcept { + try { + finalize_artifacts(report, keep_all, std::nullopt); + } catch (...) { + } +} + +JsonType +validation_failure(const std::chrono::steady_clock::time_point& start, const std::string& message) { + return {{"version", 1}, + {"status", "failed"}, + {"recommendation", nullptr}, + {"best_effort", nullptr}, + {"elapsed_seconds", internal::elapsed(start)}, + {"failure", internal::Failure("validation", "invalid_request", message)}}; +} + +} // namespace + +tl::expected +TuneIndex(const IndexRequest& request) { + const auto start = std::chrono::steady_clock::now(); + auto report = run_tuning([&request]() { return internal::ParseRequest(request); }, start); + const auto status = report.value("status", std::string("failed")); + if (status == "failed") { + return tl::unexpected(report_error(report)); + } + if (status == "no_feasible_candidate") { + IndexResult result; + result.status = TuneStatus::NO_FEASIBLE_CANDIDATE; + result.report = report; + result.best_effort = report.value("best_effort", JsonType(nullptr)); + return result; + } + + const auto artifact_failure = [&](Error error) -> tl::expected { + finalize_artifacts_noexcept(report, request.config.keep_intermediate); + return tl::unexpected(std::move(error)); + }; + + try { + const auto& recommendation = report.at("recommendation"); + const auto index_name = recommendation.at("index_name").get(); + const auto create_parameters = recommendation.at("create_params").dump(); + const auto search_parameters = recommendation.at("search_params").dump(); + const auto artifact_path = + recommendation.at("artifacts").at("index_path").get(); + auto created = Factory::CreateIndex(index_name, create_parameters); + if (!created.has_value()) { + return artifact_failure(created.error()); + } + std::ifstream input(artifact_path, std::ios::binary); + if (!input.good()) { + return artifact_failure( + Error(ErrorType::MISSING_FILE, "failed to open index artifact: " + artifact_path)); + } + auto loaded = created.value()->Deserialize(input); + if (!loaded.has_value()) { + return artifact_failure(loaded.error()); + } + + IndexResult result; + result.index = created.value(); + result.index_name = index_name; + result.create_parameters = create_parameters; + result.search_parameters = search_parameters; + result.metrics = recommendation.at("metrics"); + result.artifact_path = artifact_path; + result.report = report; + return result; + } catch (const std::exception& error) { + return artifact_failure(Error(ErrorType::INTERNAL_ERROR, error.what())); + } +} + +tl::expected +TuneSearch(const SearchRequest& request) { + auto report = run_tuning([&request]() { return internal::ParseRequest(request); }); + const auto status = report.value("status", std::string("failed")); + if (status == "failed") { + return tl::unexpected(report_error(report)); + } + if (status == "no_feasible_candidate") { + SearchResult result; + result.status = TuneStatus::NO_FEASIBLE_CANDIDATE; + result.report = report; + result.best_effort = report.value("best_effort", JsonType(nullptr)); + return result; + } + + const auto& recommendation = report["recommendation"]; + SearchResult result; + result.parameters = recommendation["search_params"].dump(); + result.metrics = recommendation["metrics"]; + result.report = report; + return result; +} + +JsonType +RunAutoTune(const JsonType& request) { + const auto start = std::chrono::steady_clock::now(); + try { + std::lock_guard lock(internal::run_mutex()); + return std::visit( + [start](auto parsed) { + return run_tuning_locked( + [parsed = std::move(parsed)]() { return parsed; }, true, start); + }, + internal::ParseRequest(request)); + } catch (const H5::Exception& error) { + return validation_failure(start, + "failed to load evaluation dataset: " + error.getDetailMsg()); + } catch (const std::exception& error) { + return validation_failure(start, error.what()); + } catch (...) { + return validation_failure(start, "unknown request validation error"); + } +} + +std::string +FormatResultSummaryForCli(const JsonType& report) { + nlohmann::ordered_json result; + for (const auto* key : {"recommendation", + "best_effort", + "failure", + "status", + "elapsed_seconds", + "report_path", + "version"}) { + if (report.contains(key) && !report[key].is_null()) { + result[key] = report[key]; + } + } + return result.dump(2); +} + +} // namespace vsag::autotune diff --git a/tools/autotune/autotune.h b/tools/autotune/autotune.h new file mode 100644 index 0000000000..c89507fae5 --- /dev/null +++ b/tools/autotune/autotune.h @@ -0,0 +1,176 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#include "nlohmann/json.hpp" +#include "vsag/dataset.h" +#include "vsag/errors.h" +#include "vsag/expected.hpp" +#include "vsag/index.h" + +namespace vsag::autotune { + +using JsonType = nlohmann::json; + +/// Metrics exposed by the experimental AutoTune evaluator. +enum class Metric { + UNSPECIFIED = 0, + RECALL_AT_K, + LATENCY_AVG_MS, + LATENCY_P99_MS, + QPS, + INDEX_MEMORY_MB, + INDEX_SIZE_MB, + BUILD_SECONDS, + SEARCH_SECONDS, + BUILD_AND_SEARCH_SECONDS, +}; + +/// An inclusive upper or lower bound, according to the metric's direction. +struct Constraint { + Metric metric{Metric::UNSPECIFIED}; + /// Minimum for recall/QPS and maximum for all other metrics. + double value{0.0}; +}; + +/// Query workload evaluated for every search candidate. +struct Workload { + /// Query vectors. Buffers referenced by a non-owning Dataset must outlive the tuning call. + DatasetPtr queries; + /// Ground truth required when recall is a constraint or objective. + DatasetPtr ground_truth; + /// Number of neighbors requested from every query. + uint64_t top_k{0}; + /// Number of evaluator search threads. + uint64_t concurrency{1}; +}; + +/// Evaluation options shared by index and search tuning. +struct Config { + /// Directory for generated index artifacts. + std::string workspace_path{"/tmp/vsag_autotune"}; + /// Maximum number of concrete search trials planned for the request. + uint64_t max_trials{1000}; + /// Keep every generated index artifact instead of only the selected artifact. + bool keep_intermediate{false}; + bool include_raw_evaluation{false}; +}; + +/// Candidate space for one concrete index type. +struct IndexSpace { + /// Concrete index type, currently hgraph or ivf. + std::string name; + /// JSON object whose scalar, array, and range leaves define build candidates. + std::string create_parameter_space{"{}"}; + /// JSON object whose scalar, array, and range leaves define search candidates. + std::string search_parameter_space{"{}"}; +}; + +/// Input for jointly tuning index construction and search. +struct IndexRequest { + /// Base vectors and IDs. Buffers referenced by a non-owning Dataset must outlive TuneIndex. + DatasetPtr base; + /// l2, ip, or cosine. + std::string metric_type; + Workload workload; + std::vector index_spaces; + std::vector constraints; + Metric objective{Metric::UNSPECIFIED}; + Config config; +}; + +/// Input for tuning search parameters on an already built index. +struct SearchRequest { + /// Existing index reused in place; TuneSearch neither rebuilds nor takes exclusive ownership. + IndexPtr index; + Workload workload; + /// JSON search candidate space; missing supported fields receive built-in proposals. + std::string parameter_space{"{}"}; + std::vector constraints; + Metric objective{Metric::UNSPECIFIED}; + Config config; +}; + +enum class TuneStatus { + SUCCESS = 0, + NO_FEASIBLE_CANDIDATE, +}; + +/// Completed index-tuning result. Recommendation fields are valid only on SUCCESS. +struct IndexResult { + TuneStatus status{TuneStatus::SUCCESS}; + /// Loaded selected index, ready for queries. + IndexPtr index; + std::string index_name; + /// Concrete JSON parameters required to recreate and deserialize the selected index. + std::string create_parameters; + /// Concrete JSON parameters recommended for queries. + std::string search_parameters; + /// Validated metric values as a JSON object. + JsonType metrics = JsonType::object(); + /// Selected serialized artifact; the caller removes it when it is no longer needed. + std::string artifact_path; + /// Complete report as JSON. + JsonType report = JsonType::object(); + /// Closest evaluated candidate when status is NO_FEASIBLE_CANDIDATE. + JsonType best_effort = nullptr; +}; + +/// Completed search-tuning result. Recommendation fields are valid only on SUCCESS. +struct SearchResult { + TuneStatus status{TuneStatus::SUCCESS}; + /// Concrete JSON parameters recommended for the existing index. + std::string parameters; + /// Validated metric values as a JSON object. + JsonType metrics = JsonType::object(); + /// Complete report as JSON. + JsonType report = JsonType::object(); + /// Closest evaluated candidate when status is NO_FEASIBLE_CANDIDATE. + JsonType best_effort = nullptr; +}; + +/** + * Experimental, synchronous build-tree tool API. It is not installed as part of the VSAG SDK. + * + * Returns an error for invalid requests or execution failures. A completed request with no + * feasible candidate returns TuneStatus::NO_FEASIBLE_CANDIDATE and a structured best_effort. On + * success, only the selected artifact is retained unless Config::keep_intermediate is true. + */ +tl::expected +TuneIndex(const IndexRequest& request); + +/** + * Synchronously evaluates search candidates against SearchRequest::index without rebuilding it. + * + * Returns an error for invalid requests or execution failures. A completed request with no + * feasible candidate returns TuneStatus::NO_FEASIBLE_CANDIDATE and a structured best_effort. + */ +tl::expected +TuneSearch(const SearchRequest& request); + +/// Offline JSON adapter used by the CLI. Parsed requests persist a full report; validation +/// failures are returned as structured JSON. +JsonType +RunAutoTune(const JsonType& request); + +/// Returns the compact, recommendation-first JSON shown by the CLI. +std::string +FormatResultSummaryForCli(const JsonType& report); + +} // namespace vsag::autotune diff --git a/tools/autotune/autotune_candidate.cpp b/tools/autotune/autotune_candidate.cpp new file mode 100644 index 0000000000..cdc1e3c4f7 --- /dev/null +++ b/tools/autotune/autotune_candidate.cpp @@ -0,0 +1,424 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include + +#include "autotune_internal.h" + +namespace vsag::autotune::internal { + +namespace { + +using Emit = std::function; + +void +expand(const JsonType& value, const Emit& emit); + +void +expand_object(const JsonType& object, + const JsonType::const_iterator& field, + JsonType partial, + const Emit& emit) { + if (field == object.end()) { + emit(partial); + return; + } + + const auto& key = field.key(); + const auto next = std::next(field); + expand(field.value(), [&](const JsonType& expanded) { + auto next_partial = partial; + next_partial[key] = expanded; + expand_object(object, next, std::move(next_partial), emit); + }); +} + +void +expand_range(const JsonType& range, const Emit& emit) { + if (!range.is_object() || range.size() != 3 || !range.contains("start") || + !range.contains("stop") || !range.contains("step")) { + throw std::invalid_argument("$range requires start, stop and step"); + } + if (!range["start"].is_number() || !range["stop"].is_number() || !range["step"].is_number()) { + throw std::invalid_argument("$range start, stop and step must be numbers"); + } + + if (range["start"].is_number_integer() && range["stop"].is_number_integer() && + range["step"].is_number_integer()) { + const auto start = range["start"].get(); + const auto stop = range["stop"].get(); + const auto step = range["step"].get(); + if (step == 0 || (start < stop && step < 0) || (start > stop && step > 0)) { + throw std::invalid_argument("$range step does not reach stop"); + } + auto current = start; + for (;;) { + emit(current); + if (current == stop || + (step > 0 && current > std::numeric_limits::max() - step) || + (step < 0 && current < std::numeric_limits::min() - step)) { + break; + } + const auto next = current + step; + if ((step > 0 && next > stop) || (step < 0 && next < stop)) { + break; + } + current = next; + } + return; + } + + const auto start = range["start"].get(); + const auto stop = range["stop"].get(); + const auto step = range["step"].get(); + if (!std::isfinite(start) || !std::isfinite(stop) || !std::isfinite(step) || step == 0.0 || + (start < stop && step < 0.0) || (start > stop && step > 0.0)) { + throw std::invalid_argument("$range step does not reach stop"); + } + const auto tolerance = std::numeric_limits::epsilon() * + std::max({1.0, std::abs(start), std::abs(stop)}) * 8.0; + double previous = 0.0; + bool emitted = false; + for (uint64_t i = 0;; ++i) { + auto current = start + static_cast(i) * step; + if (!std::isfinite(current) || + (step > 0.0 && current > stop && current - stop > tolerance) || + (step < 0.0 && current < stop && stop - current > tolerance)) { + return; + } + if (std::abs(current - stop) <= tolerance) { + current = stop; + } + if (emitted && current == previous) { + throw std::invalid_argument("$range step is too small to advance"); + } + emit(current); + if (current == stop) { + return; + } + previous = current; + emitted = true; + } +} + +void +expand(const JsonType& value, const Emit& emit) { + if (value.is_array()) { + if (value.empty()) { + throw std::invalid_argument("candidate array must not be empty"); + } + std::set seen; + for (const auto& item : value) { + expand(item, [&](const JsonType& expanded) { + if (seen.emplace(expanded.dump()).second) { + emit(expanded); + } + }); + } + return; + } + if (value.is_object()) { + if (value.contains("$range")) { + if (value.size() != 1) { + throw std::invalid_argument("$range cannot be mixed with other keys"); + } + expand_range(value["$range"], emit); + return; + } + expand_object(value, value.begin(), JsonType::object(), emit); + return; + } + emit(value); +} + +void +fill_hgraph_create(JsonType& create_params) { + if (!create_params.contains("index_param")) { + create_params["index_param"] = JsonType::object(); + } + auto& params = create_params["index_param"]; + if (!params.is_object()) { + throw std::invalid_argument("hgraph create_params.index_param must be an object"); + } + if (!params.contains("base_quantization_type")) { + params["base_quantization_type"] = JsonType::array({"fp32", "sq8_uniform"}); + } + if (!params.contains("max_degree")) { + params["max_degree"] = JsonType::array({16, 32}); + } + if (!params.contains("ef_construction")) { + params["ef_construction"] = JsonType::array({100, 200}); + } +} + +void +fill_ivf_create(JsonType& create_params, uint64_t base_count) { + if (!create_params.contains("index_param")) { + create_params["index_param"] = JsonType::object(); + } + auto& params = create_params["index_param"]; + if (!params.is_object()) { + throw std::invalid_argument("ivf create_params.index_param must be an object"); + } + if (!params.contains("base_quantization_type")) { + params["base_quantization_type"] = JsonType::array({"fp32", "sq8_uniform"}); + } + if (!params.contains("buckets_count")) { + const auto first = std::min(1024, base_count); + const auto second = std::min(2048, base_count); + params["buckets_count"] = + first == second ? JsonType(first) : JsonType::array({first, second}); + } +} + +void +fill_hgraph_search(JsonType& search_params, uint64_t top_k) { + if (!search_params.contains("hgraph")) { + search_params["hgraph"] = JsonType::object(); + } + auto& params = search_params["hgraph"]; + if (!params.is_object()) { + throw std::invalid_argument("hgraph search_params.hgraph must be an object"); + } + if (!params.contains("ef_search")) { + std::set values{std::max(40, top_k), + std::max(80, top_k * 2), + std::max(120, top_k * 4)}; + params["ef_search"] = values; + } +} + +void +fill_pyramid_search(JsonType& search_params, uint64_t top_k) { + if (!search_params.contains("pyramid")) { + search_params["pyramid"] = JsonType::object(); + } + auto& params = search_params["pyramid"]; + if (!params.is_object()) { + throw std::invalid_argument("pyramid search_params.pyramid must be an object"); + } + if (!params.contains("ef_search")) { + std::set values{std::max(40, top_k), + std::max(80, top_k * 2), + std::max(120, top_k * 4)}; + params["ef_search"] = values; + } +} + +void +fill_ivf_search(JsonType& search_params, const JsonType& create_params) { + if (!search_params.contains("ivf")) { + search_params["ivf"] = JsonType::object(); + } + auto& params = search_params["ivf"]; + if (!params.is_object()) { + throw std::invalid_argument("ivf search_params.ivf must be an object"); + } + if (params.contains("scan_buckets_count")) { + return; + } + + std::set values; + if (!create_params.contains("index_param") || + !create_params["index_param"].contains("buckets_count")) { + params["scan_buckets_count"] = std::set{1, 4, 16, 64}; + return; + } + + const auto buckets = create_params["index_param"]["buckets_count"].get(); + if (buckets <= 16) { + values = { + 1, std::max(1, buckets / 4), std::max(1, buckets / 2), buckets}; + } else { + values = {std::min(16, buckets), + std::min(32, buckets), + std::min(64, buckets)}; + } + params["scan_buckets_count"] = values; +} + +bool +supports_adaptive_ef_search_objective(const std::string& objective) { + return objective == "latency_avg_ms" || objective == "latency_p99_ms" || objective == "qps" || + objective == "search_seconds" || objective == "build_and_search_seconds"; +} + +int64_t +positive_int64(const JsonType& value, const std::string& path) { + if (!value.is_number_integer()) { + throw std::invalid_argument(path + " must be a positive integer"); + } + if (value.is_number_unsigned()) { + const auto number = value.get(); + if (number == 0 || number > static_cast(std::numeric_limits::max())) { + throw std::invalid_argument(path + " must be a positive integer"); + } + return static_cast(number); + } + const auto number = value.get(); + if (number <= 0) { + throw std::invalid_argument(path + " must be a positive integer"); + } + return number; +} + +std::optional +take_hgraph_ef_search_range(JsonType& search_params, const RequestContext& request) { + if (!search_params.contains("hgraph") || !search_params["hgraph"].is_object()) { + return std::nullopt; + } + auto& hgraph = search_params["hgraph"]; + if (!hgraph.contains("ef_search") || !hgraph["ef_search"].is_object() || + !hgraph["ef_search"].contains("$range")) { + return std::nullopt; + } + + auto& expression = hgraph["ef_search"]; + const auto& range = expression["$range"]; + if (range.is_object() && range.contains("step")) { + return std::nullopt; + } + if (expression.size() != 1 || !range.is_object() || range.size() != 2 || + !range.contains("start") || !range.contains("stop")) { + throw std::invalid_argument( + "hgraph ef_search $range without step requires exactly start and stop"); + } + if (request.constraints.find("recall_at_k") == request.constraints.end()) { + throw std::invalid_argument( + "hgraph ef_search $range without step requires a recall_at_k constraint"); + } + if (!supports_adaptive_ef_search_objective(request.objective)) { + throw std::invalid_argument( + "hgraph ef_search $range without step requires a query-cost objective"); + } + if (hgraph.contains("timeout_ms") || hgraph.contains("hops_limit")) { + throw std::invalid_argument( + "hgraph ef_search $range without step does not support timeout_ms or hops_limit"); + } + + HGraphEfSearchRange result{positive_int64(range["start"], "hgraph ef_search $range.start"), + positive_int64(range["stop"], "hgraph ef_search $range.stop")}; + if (result.start > result.stop) { + throw std::invalid_argument("hgraph ef_search $range.start must not exceed stop"); + } + hgraph.erase("ef_search"); + return result; +} + +uint64_t +maximum_trial_count(const std::optional& range) { + if (!range.has_value() || range->start == range->stop) { + return 1; + } + + uint64_t probes = 1; + uint64_t maximum = 1; + auto low = range->start; + while (low < range->stop) { + const auto high = low > range->stop / 2 ? range->stop : low * 2; + ++probes; + + uint64_t binary_trials = 0; + auto width = static_cast(high - low); + while (width > 1) { + ++binary_trials; + width = (width + 1) / 2; + } + maximum = std::max(maximum, probes + binary_trials); + low = high; + } + return maximum; +} + +std::vector +generate_candidates(const RequestContext& context, + const std::vector& indexes, + bool generate_create_candidates) { + std::vector candidates; + std::set seen; + uint64_t maximum_trials = 0; + for (const auto& index : indexes) { + auto create_space = index.create_params; + if (generate_create_candidates) { + if (index.name == "hgraph") { + fill_hgraph_create(create_space); + } else if (index.name == "ivf") { + fill_ivf_create(create_space, context.base_count); + } + } + + const auto expand_search = [&](const JsonType& create_params) { + auto search_space = index.search_params; + if (index.name == "hgraph") { + fill_hgraph_search(search_space, context.top_k); + } else if (index.name == "pyramid") { + fill_pyramid_search(search_space, context.top_k); + } else { + fill_ivf_search(search_space, create_params); + } + const auto ef_search_range = index.name == "hgraph" + ? take_hgraph_ef_search_range(search_space, context) + : std::nullopt; + expand(search_space, [&](const JsonType& search_params) { + Candidate candidate{index.name, create_params, search_params, ef_search_range}; + JsonType identity{{"index_name", candidate.index_name}, + {"create_params", candidate.create_params}, + {"search_params", candidate.search_params}}; + if (ef_search_range.has_value()) { + identity["ef_search_range"] = {{"start", ef_search_range->start}, + {"stop", ef_search_range->stop}}; + } + const auto key = identity.dump(); + if (!seen.emplace(key).second) { + return; + } + const auto trial_count = maximum_trial_count(ef_search_range); + if (trial_count > context.max_trials - maximum_trials) { + throw std::invalid_argument( + "planned trial count exceeds tuning_config.max_trials"); + } + maximum_trials += trial_count; + candidates.emplace_back(std::move(candidate)); + }); + }; + if (!generate_create_candidates) { + expand_search(create_space); + } else { + expand(create_space, expand_search); + } + } + if (candidates.empty()) { + throw std::invalid_argument("candidate generation produced no trials"); + } + return candidates; +} + +} // namespace + +std::vector +GenerateCandidates(const IndexTuningRequest& request) { + return generate_candidates(request.context, request.indexes, true); +} + +std::vector +GenerateCandidates(const SearchTuningRequest& request) { + return generate_candidates(request.context, {request.index_input}, false); +} + +} // namespace vsag::autotune::internal diff --git a/tools/autotune/autotune_evaluation.cpp b/tools/autotune/autotune_evaluation.cpp new file mode 100644 index 0000000000..84004513f6 --- /dev/null +++ b/tools/autotune/autotune_evaluation.cpp @@ -0,0 +1,357 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include + +#include "autotune_internal.h" +#include "eval_config.h" +#include "evaluator.h" +#include "vsag/factory.h" + +namespace vsag::autotune::internal { + +namespace { + +constexpr double BYTES_PER_MEBIBYTE = 1024.0 * 1024.0; + +double +elapsed(const std::chrono::steady_clock::time_point& start) { + return std::chrono::duration(std::chrono::steady_clock::now() - start).count(); +} + +std::optional +number(const JsonType& value, const std::string& key) { + if (!value.is_object() || !value.contains(key) || !value[key].is_number()) { + return std::nullopt; + } + return value[key].get(); +} + +void +set_metric(MetricMap& metrics, const std::string& name, const std::optional& value) { + if (value.has_value()) { + metrics[name] = *value; + } +} + +MetricMap +build_metrics(const JsonType& raw, const std::string& index_path) { + MetricMap metrics; + set_metric(metrics, "build_seconds", number(raw, "duration(s)")); + const auto memory = number(raw, "index_memory(B)"); + if (memory.has_value() && *memory > 0.0) { + metrics["index_memory_mb"] = *memory / BYTES_PER_MEBIBYTE; + } + std::error_code error; + const auto bytes = std::filesystem::file_size(index_path, error); + if (!error) { + metrics["index_size_mb"] = static_cast(bytes) / BYTES_PER_MEBIBYTE; + } + return metrics; +} + +MetricMap +search_metrics(const JsonType& raw, double seconds) { + MetricMap metrics; + set_metric(metrics, "recall_at_k", number(raw, "recall_avg")); + set_metric(metrics, "latency_avg_ms", number(raw, "latency_avg(ms)")); + set_metric(metrics, "qps", number(raw, "qps")); + if (raw.contains("latency_detail(ms)") && raw["latency_detail(ms)"].is_object()) { + set_metric(metrics, "latency_p99_ms", number(raw["latency_detail(ms)"], "p99")); + } + const auto memory = number(raw, "index_memory(B)"); + if (memory.has_value() && *memory > 0.0) { + metrics["index_memory_mb"] = *memory / BYTES_PER_MEBIBYTE; + } + metrics["search_seconds"] = seconds; + return metrics; +} + +JsonType +metrics_json(const MetricMap& metrics) { + JsonType result = JsonType::object(); + for (const auto& [name, value] : metrics) { + result[name] = value; + } + return result; +} + +eval::EvalConfig +build_config(const Candidate& candidate) { + eval::EvalConfig config; + config.index_name = candidate.index_name; + config.build_param = candidate.create_params.dump(); + config.enable_memory = false; + return config; +} + +eval::EvalConfig +search_config(const RequestContext& request, const Candidate& candidate) { + auto config = build_config(candidate); + config.search_param = candidate.search_params.dump(); + config.search_mode = "knn"; + config.top_k = static_cast(request.top_k); + config.search_query_count = request.query_count; + config.num_threads_searching = static_cast(request.concurrency); + config.enable_memory = false; + config.enable_recall = request.enable_recall; + config.enable_percent_recall = false; + config.use_id_based_recall = true; + return config; +} + +IndexPtr +create_index(const Candidate& candidate) { + auto created = Factory::CreateIndex(candidate.index_name, candidate.create_params.dump()); + if (!created.has_value()) { + throw std::runtime_error(created.error().message); + } + return created.value(); +} + +void +serialize_index(const IndexPtr& index, const std::string& path) { + const auto parent = std::filesystem::path(path).parent_path(); + std::filesystem::create_directories(parent); + std::ofstream output(path, std::ios::binary); + if (!output.good()) { + throw std::runtime_error("failed to open index artifact: " + path); + } + auto serialized = index->Serialize(output); + if (!serialized.has_value()) { + throw std::runtime_error(serialized.error().message); + } + output.flush(); + if (!output.good()) { + throw std::runtime_error("failed to write index artifact: " + path); + } +} + +} // namespace + +void +EvaluateEfSearchRange(const HGraphEfSearchRange& range, + double recall_target, + const std::function(int64_t)>& evaluate) { + auto low = range.start; + const auto low_recall = evaluate(low); + if (!low_recall.has_value() || *low_recall >= recall_target || low == range.stop) { + return; + } + + auto high = low; + while (low < range.stop) { + high = low > range.stop / 2 ? range.stop : low * 2; + const auto high_recall = evaluate(high); + if (!high_recall.has_value()) { + return; + } + if (*high_recall >= recall_target) { + break; + } + if (high == range.stop) { + return; + } + low = high; + } + + while (high - low > 1) { + const auto middle = low + (high - low) / 2; + const auto middle_recall = evaluate(middle); + if (!middle_recall.has_value()) { + return; + } + if (*middle_recall >= recall_target) { + high = middle; + } else { + low = middle; + } + } +} + +Evaluation +EvaluateCandidates(const IndexTuningRequest& tuning_request, + const std::vector& candidates, + const std::string& run_path) { + const auto& request = tuning_request.context; + Evaluation evaluation; + std::map> groups; + for (uint64_t i = 0; i < candidates.size(); ++i) { + const auto key = candidates[i].index_name + "\n" + candidates[i].create_params.dump(); + groups[key].emplace_back(i); + } + + uint64_t build_number = 0; + uint64_t trial_number = 0; + for (const auto& [unused, indexes] : groups) { + (void)unused; + const auto& first = candidates[indexes.front()]; + const auto build_id = "build-" + std::to_string(build_number++); + const auto index_path = + (std::filesystem::path(run_path) / "artifacts" / (build_id + ".index")).string(); + JsonType build{{"build_id", build_id}, + {"index_name", first.index_name}, + {"create_params", first.create_params}, + {"status", "failed"}, + {"metrics", JsonType::object()}, + {"failure", nullptr}, + {"artifacts", + {{"index_path", index_path}, + {"source", "generated"}, + {"use_existing_index", false}, + {"retained", true}}}}; + + MetricMap shared_metrics; + IndexPtr index; + const auto build_start = std::chrono::steady_clock::now(); + try { + index = create_index(first); + auto raw = eval::EvaluateBuild(index, request.dataset, build_config(first)); + serialize_index(index, index_path); + shared_metrics = build_metrics(raw, index_path); + if (request.include_raw_eval) { + build["raw_eval_result"] = std::move(raw); + } + build["metrics"] = metrics_json(shared_metrics); + build["status"] = "success"; + } catch (const std::exception& error) { + build["failure"] = Failure("build", "build_evaluation_failed", error.what()); + std::error_code cleanup_error; + std::filesystem::remove(index_path, cleanup_error); + build["artifacts"]["retained"] = false; + } + build["elapsed_seconds"] = elapsed(build_start); + evaluation.builds.emplace_back(build); + + const auto evaluate = [&](const Candidate& candidate) -> std::optional { + const auto trial_id = "trial-" + std::to_string(trial_number++); + JsonType trial{{"trial_id", trial_id}, + {"build_id", build_id}, + {"index_name", candidate.index_name}, + {"create_params", candidate.create_params}, + {"search_params", candidate.search_params}, + {"status", "failed"}, + {"metrics", metrics_json(shared_metrics)}, + {"failure", nullptr}, + {"artifacts", build["artifacts"]}}; + std::optional recall; + const auto search_start = std::chrono::steady_clock::now(); + if (build["status"] != "success") { + trial["failure"] = + Failure("search", "build_failed", "search skipped because build failed"); + } else { + try { + const auto measured_start = std::chrono::steady_clock::now(); + auto raw = eval::EvaluateSearch( + index, request.dataset, search_config(request, candidate)); + auto metrics = search_metrics(raw, elapsed(measured_start)); + for (const auto& [name, value] : shared_metrics) { + metrics.emplace(name, value); + } + if (metrics.find("build_seconds") != metrics.end()) { + metrics["build_and_search_seconds"] = + metrics["build_seconds"] + metrics["search_seconds"]; + } + trial["metrics"] = metrics_json(metrics); + trial["status"] = "success"; + recall = number(trial["metrics"], "recall_at_k"); + if (request.include_raw_eval) { + trial["raw_eval_result"] = std::move(raw); + } + } catch (const std::exception& error) { + trial["failure"] = Failure("search", "search_evaluation_failed", error.what()); + } + } + trial["elapsed_seconds"] = elapsed(search_start); + evaluation.trials.emplace_back(std::move(trial)); + return recall; + }; + + for (const auto candidate_index : indexes) { + const auto& candidate = candidates[candidate_index]; + if (!candidate.ef_search_range.has_value()) { + evaluate(candidate); + continue; + } + + const auto recall_target = request.constraints.at("recall_at_k"); + const auto evaluate_ef_search = [&](int64_t ef_search) { + auto concrete = candidate; + concrete.search_params["hgraph"]["ef_search"] = ef_search; + return evaluate(concrete); + }; + EvaluateEfSearchRange(*candidate.ef_search_range, recall_target, evaluate_ef_search); + } + } + return evaluation; +} + +Evaluation +EvaluateCandidates(const SearchTuningRequest& tuning_request, + const std::vector& candidates) { + const auto& request = tuning_request.context; + Evaluation evaluation; + uint64_t trial_number = 0; + + const auto evaluate = [&](const Candidate& candidate) -> std::optional { + JsonType trial{{"trial_id", "trial-" + std::to_string(trial_number++)}, + {"index_name", candidate.index_name}, + {"search_params", candidate.search_params}, + {"status", "failed"}, + {"metrics", JsonType::object()}, + {"failure", nullptr}}; + std::optional recall; + const auto start = std::chrono::steady_clock::now(); + try { + const auto measured_start = std::chrono::steady_clock::now(); + auto raw = eval::EvaluateSearch( + tuning_request.index, request.dataset, search_config(request, candidate)); + const auto metrics = search_metrics(raw, elapsed(measured_start)); + trial["metrics"] = metrics_json(metrics); + trial["status"] = "success"; + recall = number(trial["metrics"], "recall_at_k"); + if (request.include_raw_eval) { + trial["raw_eval_result"] = std::move(raw); + } + } catch (const std::exception& error) { + trial["failure"] = Failure("search", "search_evaluation_failed", error.what()); + } + trial["elapsed_seconds"] = elapsed(start); + evaluation.trials.emplace_back(std::move(trial)); + return recall; + }; + + for (const auto& candidate : candidates) { + if (!candidate.ef_search_range.has_value()) { + evaluate(candidate); + continue; + } + const auto recall_target = request.constraints.at("recall_at_k"); + const auto evaluate_ef_search = [&](int64_t ef_search) { + auto concrete = candidate; + concrete.search_params["hgraph"]["ef_search"] = ef_search; + return evaluate(concrete); + }; + EvaluateEfSearchRange(*candidate.ef_search_range, recall_target, evaluate_ef_search); + } + return evaluation; +} + +} // namespace vsag::autotune::internal diff --git a/tools/autotune/autotune_internal.h b/tools/autotune/autotune_internal.h new file mode 100644 index 0000000000..9355bdc504 --- /dev/null +++ b/tools/autotune/autotune_internal.h @@ -0,0 +1,129 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "autotune.h" + +namespace vsag::eval { +class EvalDataset; +} + +namespace vsag::autotune::internal { + +namespace eval = vsag::eval; + +using MetricMap = std::map; + +struct IndexInput { + std::string name; + JsonType create_params = JsonType::object(); + JsonType search_params = JsonType::object(); +}; + +struct RequestContext { + JsonType effective_request = JsonType::object(); + std::shared_ptr dataset; + std::string workspace_path{"/tmp/vsag_autotune"}; + std::string result_path; + std::string objective; + MetricMap constraints; + uint64_t top_k{0}; + uint64_t concurrency{1}; + uint64_t max_trials{1000}; + uint64_t base_count{0}; + uint64_t query_count{0}; + uint64_t ground_truth_k{0}; + bool enable_recall{false}; + bool keep_intermediate{false}; + bool include_raw_eval{false}; +}; + +struct IndexTuningRequest { + RequestContext context; + std::vector indexes; +}; + +struct SearchTuningRequest { + RequestContext context; + IndexPtr index; + IndexInput index_input; +}; + +using ParsedRequest = std::variant; + +struct HGraphEfSearchRange { + int64_t start; + int64_t stop; +}; + +struct Candidate { + std::string index_name; + JsonType create_params; + JsonType search_params; + std::optional ef_search_range; +}; + +struct Evaluation { + std::vector builds; + std::vector trials; +}; + +ParsedRequest +ParseRequest(const JsonType& input); + +IndexTuningRequest +ParseRequest(const IndexRequest& input); + +SearchTuningRequest +ParseRequest(const SearchRequest& input); + +std::vector +GenerateCandidates(const IndexTuningRequest& request); + +std::vector +GenerateCandidates(const SearchTuningRequest& request); + +void +EvaluateEfSearchRange(const HGraphEfSearchRange& range, + double recall_target, + const std::function(int64_t)>& evaluate); + +Evaluation +EvaluateCandidates(const IndexTuningRequest& request, + const std::vector& candidates, + const std::string& run_path); + +Evaluation +EvaluateCandidates(const SearchTuningRequest& request, const std::vector& candidates); + +JsonType +SelectResult(const RequestContext& request, const Evaluation& evaluation); + +JsonType +Failure(const std::string& stage, const std::string& code, const std::string& message); + +void +WriteJson(const std::string& path, const JsonType& value); + +} // namespace vsag::autotune::internal diff --git a/tools/autotune/autotune_test.cpp b/tools/autotune/autotune_test.cpp new file mode 100644 index 0000000000..7609a4df52 --- /dev/null +++ b/tools/autotune/autotune_test.cpp @@ -0,0 +1,1089 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "autotune.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "autotune_internal.h" +#include "eval_dataset.h" +#include "vsag/options.h" +#include "vsag/vsag.h" + +namespace { + +using vsag::autotune::JsonType; + +std::string +temp_path(const std::string& stem, const std::string& extension = "") { + static std::atomic serial{0}; + const auto now = std::chrono::steady_clock::now().time_since_epoch().count(); + return (std::filesystem::temp_directory_path() / + (stem + "-" + std::to_string(now) + "-" + std::to_string(serial++) + extension)) + .string(); +} + +class ScopedPath { +public: + explicit ScopedPath(std::string path) : path_(std::move(path)) { + } + + ~ScopedPath() { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + const std::string& + Get() const { + return path_; + } + +private: + std::string path_; +}; + +class ScopedBlockSizeLimit { +public: + explicit ScopedBlockSizeLimit(uint64_t value) + : original_(vsag::Options::Instance().block_size_limit()) { + vsag::Options::Instance().set_block_size_limit(value); + } + + ~ScopedBlockSizeLimit() { + vsag::Options::Instance().set_block_size_limit(original_); + } + +private: + uint64_t original_; +}; + +class ScopedOpenMpThreads { +public: + explicit ScopedOpenMpThreads(int value) : original_(omp_get_max_threads()) { + omp_set_num_threads(value); + } + + ~ScopedOpenMpThreads() { + omp_set_num_threads(original_); + } + +private: + int original_; +}; + +uint64_t +count_index_artifacts(const std::string& path) { + if (!std::filesystem::exists(path)) { + return 0; + } + uint64_t count = 0; + for (const auto& entry : std::filesystem::recursive_directory_iterator(path)) { + if (entry.is_regular_file() && entry.path().extension() == ".index") { + ++count; + } + } + return count; +} + +std::vector +load_json_reports(const std::string& path) { + std::vector reports; + if (!std::filesystem::exists(path)) { + return reports; + } + for (const auto& entry : std::filesystem::recursive_directory_iterator(path)) { + if (!entry.is_regular_file() || entry.path().extension() != ".json") { + continue; + } + std::ifstream input(entry.path()); + JsonType report; + input >> report; + reports.emplace_back(std::move(report)); + } + return reports; +} + +float +l2(const std::vector& train, + const std::vector& test, + int64_t base, + int64_t query, + int64_t dim) { + float distance = 0.0F; + for (int64_t i = 0; i < dim; ++i) { + const auto difference = train[base * dim + i] - test[query * dim + i]; + distance += difference * difference; + } + return std::sqrt(distance); +} + +void +write_dataset(const std::string& path) { + constexpr int64_t BASE_COUNT = 64; + constexpr int64_t QUERY_COUNT = 6; + constexpr int64_t DIM = 8; + constexpr int64_t GROUND_TRUTH_K = 6; + std::vector train(BASE_COUNT * DIM); + std::vector test(QUERY_COUNT * DIM); + for (int64_t i = 0; i < BASE_COUNT; ++i) { + for (int64_t j = 0; j < DIM; ++j) { + train[i * DIM + j] = static_cast((i * 17 + j * 13) % 101) / 101.0F; + } + } + for (int64_t i = 0; i < QUERY_COUNT; ++i) { + for (int64_t j = 0; j < DIM; ++j) { + test[i * DIM + j] = train[((i * 7) % BASE_COUNT) * DIM + j]; + } + } + + std::vector neighbors(QUERY_COUNT * GROUND_TRUTH_K); + std::vector distances(QUERY_COUNT * GROUND_TRUTH_K); + for (int64_t query = 0; query < QUERY_COUNT; ++query) { + std::vector> ranked; + for (int64_t base = 0; base < BASE_COUNT; ++base) { + ranked.emplace_back(l2(train, test, base, query, DIM), base); + } + std::sort(ranked.begin(), ranked.end()); + for (int64_t k = 0; k < GROUND_TRUTH_K; ++k) { + neighbors[query * GROUND_TRUTH_K + k] = ranked[k].second; + distances[query * GROUND_TRUTH_K + k] = ranked[k].first; + } + } + + H5::H5File file(path, H5F_ACC_TRUNC); + H5::StrType string_type(H5::PredType::C_S1, H5T_VARIABLE); + auto distance = file.createAttribute("distance", string_type, H5::DataSpace(H5S_SCALAR)); + std::string metric = "euclidean"; + distance.write(string_type, metric); + const auto write_matrix = [&](const std::string& name, + const auto* data, + const H5::DataType& type, + int64_t rows, + int64_t columns) { + hsize_t dimensions[2] = {static_cast(rows), static_cast(columns)}; + H5::DataSpace space(2, dimensions); + auto dataset = file.createDataSet(name, type, space); + dataset.write(data, type); + }; + write_matrix("/train", train.data(), H5::PredType::NATIVE_FLOAT, BASE_COUNT, DIM); + write_matrix("/test", test.data(), H5::PredType::NATIVE_FLOAT, QUERY_COUNT, DIM); + write_matrix( + "/neighbors", neighbors.data(), H5::PredType::NATIVE_INT64, QUERY_COUNT, GROUND_TRUTH_K); + write_matrix( + "/distances", distances.data(), H5::PredType::NATIVE_FLOAT, QUERY_COUNT, GROUND_TRUTH_K); +} + +JsonType +request(const std::string& dataset, const std::string& workspace) { + return {{"version", 1}, + {"data_path", dataset}, + {"indexes", + JsonType::array({{{"name", "hgraph"}, + {"create_params", + {{"index_param", + {{"base_quantization_type", "fp32"}, + {"max_degree", 8}, + {"ef_construction", 40}, + {"build_thread_count", 2}}}}}, + {"search_params", {{"hgraph", {{"ef_search", {8, 16}}}}}}}, + {{"name", "ivf"}, + {"create_params", + {{"index_param", + {{"base_quantization_type", "fp32"}, + {"buckets_count", 4}, + {"thread_count", 2}}}}}, + {"search_params", {{"ivf", {{"scan_buckets_count", {1, 4}}}}}}}})}, + {"workload", {{"top_k", 3}, {"concurrency", 2}}}, + {"constraints", {{"recall_at_k", 0.0}, {"build_seconds", 1000.0}}}, + {"objective", {{"metric", "latency_avg_ms"}}}, + {"tuning_config", + {{"workspace_path", workspace}, {"keep_intermediate", true}, {"max_trials", 4}}}}; +} + +struct MemoryFixture { + static constexpr int64_t BASE_COUNT = 64; + static constexpr int64_t QUERY_COUNT = 6; + static constexpr int64_t DIM = 8; + static constexpr int64_t GROUND_TRUTH_K = 6; + + MemoryFixture() + : base_ids(BASE_COUNT), + train(BASE_COUNT * DIM), + test(QUERY_COUNT * DIM), + neighbors(QUERY_COUNT * GROUND_TRUTH_K), + distances(QUERY_COUNT * GROUND_TRUTH_K) { + for (int64_t i = 0; i < BASE_COUNT; ++i) { + base_ids[i] = 1001 + i * 17; + for (int64_t j = 0; j < DIM; ++j) { + train[i * DIM + j] = static_cast((i * 17 + j * 13) % 101) / 101.0F; + } + } + for (int64_t i = 0; i < QUERY_COUNT; ++i) { + for (int64_t j = 0; j < DIM; ++j) { + test[i * DIM + j] = train[((i * 7) % BASE_COUNT) * DIM + j]; + } + } + for (int64_t query = 0; query < QUERY_COUNT; ++query) { + std::vector> ranked; + for (int64_t row = 0; row < BASE_COUNT; ++row) { + ranked.emplace_back(l2(train, test, row, query, DIM), row); + } + std::sort(ranked.begin(), ranked.end()); + for (int64_t k = 0; k < GROUND_TRUTH_K; ++k) { + neighbors[query * GROUND_TRUTH_K + k] = base_ids[ranked[k].second]; + distances[query * GROUND_TRUTH_K + k] = ranked[k].first; + } + } + base = vsag::Dataset::Make() + ->NumElements(BASE_COUNT) + ->Dim(DIM) + ->Ids(base_ids.data()) + ->Float32Vectors(train.data()) + ->Owner(false); + queries = vsag::Dataset::Make() + ->NumElements(QUERY_COUNT) + ->Dim(DIM) + ->Float32Vectors(test.data()) + ->Owner(false); + ground_truth = vsag::Dataset::Make() + ->NumElements(QUERY_COUNT) + ->Dim(GROUND_TRUTH_K) + ->Ids(neighbors.data()) + ->Distances(distances.data()) + ->Owner(false); + } + + vsag::autotune::IndexRequest + Request(const std::string& workspace) const { + vsag::autotune::IndexRequest result; + result.base = base; + result.metric_type = vsag::METRIC_L2; + result.workload = {queries, ground_truth, 3, 2}; + result.index_spaces = { + {"hgraph", + R"({"index_param":{"base_quantization_type":"fp32",)" + R"("max_degree":[8,12],"ef_construction":40,"build_thread_count":2}})", + R"({"hgraph":{"ef_search":16}})"}}; + result.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.0}, + {vsag::autotune::Metric::BUILD_SECONDS, 1000.0}}; + result.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + result.config.workspace_path = workspace; + result.config.max_trials = 2; + return result; + } + + std::vector base_ids; + std::vector train; + std::vector test; + std::vector neighbors; + std::vector distances; + vsag::DatasetPtr base; + vsag::DatasetPtr queries; + vsag::DatasetPtr ground_truth; +}; + +} // namespace + +TEST_CASE("AutoTune candidate rules only fill missing fields") { + vsag::autotune::internal::IndexTuningRequest request; + request.context.base_count = 10000; + request.context.top_k = 10; + request.context.max_trials = 10; + request.indexes = { + {"hgraph", + {{"dim", 8}, + {"dtype", "float32"}, + {"metric_type", "l2"}, + {"index_param", + {{"base_quantization_type", "fp32"}, {"max_degree", 24}, {"ef_construction", 80}}}}, + JsonType::object()}}; + + const auto candidates = vsag::autotune::internal::GenerateCandidates(request); + REQUIRE(candidates.size() == 3); + for (const auto& candidate : candidates) { + REQUIRE(candidate.create_params["index_param"]["max_degree"] == 24); + REQUIRE(candidate.create_params["index_param"]["ef_construction"] == 80); + } + + request.indexes[0].create_params = JsonType::object(); + request.context.max_trials = 24; + REQUIRE(vsag::autotune::internal::GenerateCandidates(request).size() == 24); + + request.indexes[0].create_params = candidates[0].create_params; + request.indexes[0].search_params = { + {"hgraph", {{"ef_search", {{"$range", {{"start", 0.1}, {"stop", 0.3}, {"step", 0.1}}}}}}}}; + const auto float_range = vsag::autotune::internal::GenerateCandidates(request); + REQUIRE(float_range.size() == 3); + REQUIRE(float_range.back().search_params["hgraph"]["ef_search"] == 0.3); + REQUIRE_FALSE(float_range.back().ef_search_range.has_value()); + + const auto minimum = std::numeric_limits::min(); + request.indexes[0].search_params = { + {"hgraph", + {{"ef_search", {{"$range", {{"start", minimum}, {"stop", minimum + 5}, {"step", 10}}}}}}}}; + const auto integer_range = vsag::autotune::internal::GenerateCandidates(request); + REQUIRE(integer_range.size() == 1); + REQUIRE(integer_range[0].search_params["hgraph"]["ef_search"] == minimum); +} + +TEST_CASE("AutoTune proposes conservative defaults for existing IVF indexes") { + vsag::autotune::internal::SearchTuningRequest request; + request.context.top_k = 10; + request.context.max_trials = 4; + request.index_input.name = "ivf"; + + const auto candidates = vsag::autotune::internal::GenerateCandidates(request); + REQUIRE(candidates.size() == 4); + const std::vector expected{1, 4, 16, 64}; + for (uint64_t i = 0; i < candidates.size(); ++i) { + REQUIRE(candidates[i].search_params["ivf"]["scan_buckets_count"] == expected[i]); + } +} + +TEST_CASE("AutoTune plans an adaptive HGraph ef_search range") { + vsag::autotune::internal::IndexTuningRequest request; + request.context.base_count = 10000; + request.context.top_k = 10; + request.context.objective = "latency_avg_ms"; + request.context.constraints = {{"recall_at_k", 0.95}}; + request.context.max_trials = 30; + request.indexes = { + {"hgraph", + {{"dim", 8}, + {"dtype", "float32"}, + {"metric_type", "l2"}, + {"index_param", + {{"base_quantization_type", "fp32"}, {"max_degree", 24}, {"ef_construction", 80}}}}, + {{"hgraph", + {{"ef_search", {{"$range", {{"start", 40}, {"stop", 1000}}}}}, + {"use_reorder", {true, false}}}}}}}; + + const auto candidates = vsag::autotune::internal::GenerateCandidates(request); + REQUIRE(candidates.size() == 2); + for (const auto& candidate : candidates) { + REQUIRE(candidate.ef_search_range.has_value()); + const auto range = candidate.ef_search_range.value_or( + vsag::autotune::internal::HGraphEfSearchRange{-1, -1}); + REQUIRE(range.start == 40); + REQUIRE(range.stop == 1000); + REQUIRE_FALSE(candidate.search_params["hgraph"].contains("ef_search")); + REQUIRE(candidate.search_params["hgraph"]["use_reorder"].is_boolean()); + } + + request.context.max_trials = 29; + REQUIRE_THROWS_WITH( + vsag::autotune::internal::GenerateCandidates(request), + Catch::Matchers::ContainsSubstring("planned trial count exceeds tuning_config.max_trials")); + + request.context.max_trials = 1; + request.indexes[0].search_params = { + {"hgraph", {{"ef_search", {{"$range", {{"start", 8}, {"stop", 8}}}}}}}}; + REQUIRE(vsag::autotune::internal::GenerateCandidates(request).size() == 1); + + request.context.constraints = {{"latency_avg_ms", 1.0}}; + request.indexes[0].search_params = { + {"hgraph", {{"ef_search", {{"$range", {{"start", 8}, {"stop", 16}}}}}}}}; + request.context.max_trials = 5; + REQUIRE_THROWS_WITH(vsag::autotune::internal::GenerateCandidates(request), + Catch::Matchers::ContainsSubstring("requires a recall_at_k constraint")); + + request.context.constraints = {{"recall_at_k", 0.95}}; + request.context.objective = "recall_at_k"; + REQUIRE_THROWS_WITH(vsag::autotune::internal::GenerateCandidates(request), + Catch::Matchers::ContainsSubstring("requires a query-cost objective")); + + request.context.objective = "latency_avg_ms"; + request.indexes[0].search_params["hgraph"]["hops_limit"] = 100; + REQUIRE_THROWS_WITH( + vsag::autotune::internal::GenerateCandidates(request), + Catch::Matchers::ContainsSubstring("does not support timeout_ms or hops_limit")); + + request.indexes[0].search_params["hgraph"].erase("hops_limit"); + request.indexes[0].search_params["hgraph"]["ef_search"]["$range"]["stop"] = 1001; + request.context.max_trials = 30; + const auto unbounded = vsag::autotune::internal::GenerateCandidates(request); + REQUIRE(unbounded.size() == 1); + REQUIRE(unbounded[0].ef_search_range.has_value()); + REQUIRE(unbounded[0].ef_search_range->stop == 1001); +} + +TEST_CASE("AutoTune brackets and binary-searches the smallest passing ef_search") { + const vsag::autotune::internal::HGraphEfSearchRange range{8, 64}; + std::vector evaluated; + const auto evaluate = [&evaluated](int64_t ef_search) -> std::optional { + evaluated.emplace_back(ef_search); + return ef_search >= 25 ? 0.95 : 0.90; + }; + + vsag::autotune::internal::EvaluateEfSearchRange(range, 0.95, evaluate); + REQUIRE(evaluated == std::vector{8, 16, 32, 24, 28, 26, 25}); + + evaluated.clear(); + vsag::autotune::internal::EvaluateEfSearchRange(range, 0.90, evaluate); + REQUIRE(evaluated == std::vector{8}); + + evaluated.clear(); + vsag::autotune::internal::EvaluateEfSearchRange({8, 50}, 1.0, evaluate); + REQUIRE(evaluated == std::vector{8, 16, 32, 50}); + + evaluated.clear(); + const auto fail_at_middle = [&evaluated](int64_t ef_search) -> std::optional { + evaluated.emplace_back(ef_search); + if (ef_search == 24) { + return std::nullopt; + } + return ef_search >= 32 ? 0.95 : 0.90; + }; + vsag::autotune::internal::EvaluateEfSearchRange(range, 0.95, fail_at_middle); + REQUIRE(evaluated == std::vector{8, 16, 32, 24}); +} + +TEST_CASE("AutoTune validates typed requests and optional ground truth") { + MemoryFixture fixture; + auto input = fixture.Request(temp_path("autotune-typed-validation")); + + const auto parsed = vsag::autotune::internal::ParseRequest(input); + REQUIRE(parsed.context.dataset->GetTrainIds()[0] == fixture.base_ids[0]); + REQUIRE(parsed.context.dataset->GetTrainIds()[1] == fixture.base_ids[1]); + + input.constraints = {{vsag::autotune::Metric::LATENCY_AVG_MS, 1000.0}}; + input.workload.ground_truth = nullptr; + REQUIRE_NOTHROW(vsag::autotune::internal::ParseRequest(input)); + + input.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.9}}; + REQUIRE_THROWS_WITH(vsag::autotune::internal::ParseRequest(input), + Catch::Matchers::ContainsSubstring("ground_truth is required")); + + input.workload.ground_truth = fixture.ground_truth; + input.constraints.push_back({vsag::autotune::Metric::RECALL_AT_K, 0.8}); + REQUIRE_THROWS_WITH(vsag::autotune::internal::ParseRequest(input), + Catch::Matchers::ContainsSubstring("duplicate metric")); + + input = fixture.Request(temp_path("autotune-typed-duplicate-ids")); + auto duplicate_ids = fixture.base_ids; + duplicate_ids[1] = duplicate_ids[0]; + input.base = vsag::Dataset::Make() + ->NumElements(MemoryFixture::BASE_COUNT) + ->Dim(MemoryFixture::DIM) + ->Ids(duplicate_ids.data()) + ->Float32Vectors(fixture.train.data()) + ->Owner(false); + REQUIRE_THROWS_WITH(vsag::autotune::internal::ParseRequest(input), + Catch::Matchers::ContainsSubstring("duplicate ids")); + + input = fixture.Request(temp_path("autotune-typed-unknown-ground-truth")); + auto unknown_ids = fixture.neighbors; + unknown_ids[0] = -1; + input.workload.ground_truth = vsag::Dataset::Make() + ->NumElements(MemoryFixture::QUERY_COUNT) + ->Dim(MemoryFixture::GROUND_TRUTH_K) + ->Ids(unknown_ids.data()) + ->Owner(false); + REQUIRE_THROWS_WITH(vsag::autotune::internal::ParseRequest(input), + Catch::Matchers::ContainsSubstring("not present in base")); +} + +TEST_CASE("AutoTune returns structured offline validation failures") { + ScopedPath invalid_dataset(temp_path("autotune-invalid-dataset", ".hdf5")); + { + std::ofstream output(invalid_dataset.Get(), std::ios::binary); + output << "not an HDF5 file"; + } + H5::Exception::dontPrint(); + + const auto invalid_file = + vsag::autotune::RunAutoTune({{"version", 1}, {"data_path", invalid_dataset.Get()}}); + REQUIRE(invalid_file["status"] == "failed"); + REQUIRE(invalid_file["failure"]["stage"] == "validation"); + REQUIRE(invalid_file["failure"]["code"] == "invalid_request"); + REQUIRE_THAT(invalid_file["failure"]["message"].get(), + Catch::Matchers::ContainsSubstring("failed to load evaluation dataset")); + + const auto invalid_version = vsag::autotune::RunAutoTune({{"version", 2}}); + REQUIRE(invalid_version["status"] == "failed"); + REQUIRE(invalid_version["failure"]["stage"] == "validation"); + REQUIRE(invalid_version["failure"]["code"] == "invalid_request"); +} + +TEST_CASE("AutoTune serializes concurrent offline dataset loading") { + ScopedPath dataset(temp_path("autotune-concurrent-dataset", ".hdf5")); + write_dataset(dataset.Get()); + const JsonType input{{"version", 1}, {"data_path", dataset.Get()}}; + std::atomic ready{0}; + std::atomic start{false}; + JsonType results[2]; + + const auto run = [&](uint64_t position) { + ready.fetch_add(1); + while (!start.load()) { + std::this_thread::yield(); + } + results[position] = vsag::autotune::RunAutoTune(input); + }; + std::thread first(run, 0); + std::thread second(run, 1); + while (ready.load() != 2) { + std::this_thread::yield(); + } + start.store(true); + first.join(); + second.join(); + + for (const auto& result : results) { + REQUIRE(result["status"] == "failed"); + REQUIRE(result["failure"]["stage"] == "validation"); + REQUIRE(result["failure"]["code"] == "invalid_request"); + REQUIRE_THAT(result["failure"]["message"].get(), + Catch::Matchers::ContainsSubstring("request.workload is required")); + } +} + +TEST_CASE("AutoTune keeps normalized offline request metadata") { + ScopedPath dataset(temp_path("autotune-normalized-dataset", ".hdf5")); + ScopedPath workspace(temp_path("autotune-normalized-workspace")); + write_dataset(dataset.Get()); + auto input = request(dataset.Get(), workspace.Get()); + input.erase("indexes"); + + const auto parsed = vsag::autotune::internal::ParseRequest(input); + const auto& index_request = std::get(parsed); + const auto& effective = index_request.context.effective_request; + const auto* train_ids = index_request.context.dataset->GetTrainIds(); + REQUIRE(train_ids != nullptr); + REQUIRE(train_ids[0] == 0); + REQUIRE(train_ids[63] == 63); + REQUIRE(effective["data_path"] == dataset.Get()); + REQUIRE(effective["dataset"]["dim"] == 8); + REQUIRE(effective["dataset"]["dtype"] == "float32"); + REQUIRE(effective["dataset"]["metric_type"] == "l2"); + REQUIRE(effective["index_spaces"].size() == 2); + REQUIRE(effective["index_spaces"][0]["name"] == "hgraph"); + REQUIRE(effective["index_spaces"][1]["name"] == "ivf"); + for (const auto& space : effective["index_spaces"]) { + REQUIRE(space["create_parameter_space"]["dim"] == 8); + REQUIRE(space["create_parameter_space"]["dtype"] == "float32"); + REQUIRE(space["create_parameter_space"]["metric_type"] == "l2"); + } + REQUIRE(effective["workload"]["concurrency"] == 2); + REQUIRE(effective["config"]["keep_intermediate"] == true); + REQUIRE_FALSE(effective.contains("tuning_config")); +} + +TEST_CASE("AutoTune rejects index artifacts that fail to flush") { + if (!std::filesystem::exists("/dev/full")) { + return; + } + + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath run_path(temp_path("autotune-full-device")); + std::filesystem::create_directories(run_path.Get() + "/artifacts"); + std::error_code link_error; + std::filesystem::create_symlink( + "/dev/full", run_path.Get() + "/artifacts/build-0.index", link_error); + REQUIRE_FALSE(link_error); + + MemoryFixture fixture; + auto input = fixture.Request(run_path.Get()); + input.index_spaces[0].create_parameter_space = + R"({"index_param":{"base_quantization_type":"fp32","max_degree":8,)" + R"("ef_construction":40,"build_thread_count":2}})"; + input.config.max_trials = 1; + const auto request = vsag::autotune::internal::ParseRequest(input); + const auto candidates = vsag::autotune::internal::GenerateCandidates(request); + REQUIRE(candidates.size() == 1); + + const auto evaluation = + vsag::autotune::internal::EvaluateCandidates(request, candidates, run_path.Get()); + REQUIRE(evaluation.builds.size() == 1); + REQUIRE(evaluation.builds[0]["status"] == "failed"); + REQUIRE_THAT(evaluation.builds[0]["failure"]["message"].get(), + Catch::Matchers::ContainsSubstring("failed to write index artifact")); + REQUIRE(evaluation.trials.size() == 1); + REQUIRE(evaluation.trials[0]["failure"]["code"] == "build_failed"); +} + +TEST_CASE("AutoTune rejects report paths that alias an existing index") { + ScopedPath dataset(temp_path("autotune-alias-dataset", ".hdf5")); + ScopedPath workspace(temp_path("autotune-alias-workspace")); + ScopedPath index_path(temp_path("autotune-alias-index", ".index")); + ScopedPath report_path(temp_path("autotune-alias-report", ".json")); + write_dataset(dataset.Get()); + { + std::ofstream output(index_path.Get(), std::ios::binary); + output << "serialized-index-placeholder"; + } + std::error_code link_error; + std::filesystem::create_hard_link(index_path.Get(), report_path.Get(), link_error); + REQUIRE_FALSE(link_error); + const auto original_size = std::filesystem::file_size(index_path.Get()); + + auto input = request(dataset.Get(), workspace.Get()); + input["indexes"] = JsonType::array({input["indexes"][0]}); + input["index_path"] = index_path.Get(); + input["output"] = {{"result_path", report_path.Get()}}; + const auto result = vsag::autotune::RunAutoTune(input); + + REQUIRE(result["status"] == "failed"); + REQUIRE(result["failure"]["stage"] == "validation"); + REQUIRE(result["failure"]["code"] == "invalid_request"); + REQUIRE_THAT(result["failure"]["message"].get(), + Catch::Matchers::ContainsSubstring("result_path must not alias index_path")); + REQUIRE(std::filesystem::file_size(index_path.Get()) == original_size); +} + +TEST_CASE("AutoTune reports the closest successful trial when constraints are infeasible") { + vsag::autotune::internal::RequestContext request; + request.top_k = 10; + request.objective = "latency_avg_ms"; + request.constraints = {{"recall_at_k", 0.95}}; + vsag::autotune::internal::Evaluation evaluation; + const auto trial = [](const std::string& id, double recall, double latency) { + return JsonType{{"trial_id", id}, + {"build_id", "build-0"}, + {"index_name", "hgraph"}, + {"create_params", JsonType::object()}, + {"search_params", JsonType::object()}, + {"status", "success"}, + {"metrics", {{"recall_at_k", recall}, {"latency_avg_ms", latency}}}, + {"artifacts", JsonType::object()}}; + }; + evaluation.trials = { + trial("trial-0", 0.90, 1.0), trial("trial-1", 0.80, 2.0), trial("trial-2", 0.85, 3.0)}; + + const auto result = vsag::autotune::internal::SelectResult(request, evaluation); + REQUIRE(result["status"] == "no_feasible_candidate"); + REQUIRE(result["recommendation"].is_null()); + REQUIRE(result["best_effort"]["evidence"]["trial_id"] == "trial-0"); +} + +TEST_CASE("AutoTune distinguishes a missing objective metric from failed trials") { + vsag::autotune::internal::RequestContext request; + request.top_k = 10; + request.objective = "index_memory_mb"; + request.constraints = {{"recall_at_k", 0.0}}; + vsag::autotune::internal::Evaluation evaluation; + evaluation.trials = {{{"trial_id", "trial-0"}, + {"index_name", "pyramid"}, + {"search_params", JsonType::object()}, + {"status", "success"}, + {"metrics", {{"recall_at_k", 1.0}}}}}; + + const auto result = vsag::autotune::internal::SelectResult(request, evaluation); + REQUIRE(result["status"] == "failed"); + REQUIRE(result["failure"]["stage"] == "selection"); + REQUIRE(result["failure"]["code"] == "objective_metric_unavailable"); + REQUIRE_THAT(result["failure"]["message"].get(), + Catch::Matchers::ContainsSubstring("index_memory_mb")); +} + +TEST_CASE("AutoTune builds once per create candidate and supports an existing index") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedOpenMpThreads openmp_threads(3); + ScopedPath dataset(temp_path("autotune-dataset", ".hdf5")); + ScopedPath workspace(temp_path("autotune-workspace")); + write_dataset(dataset.Get()); + + const auto result = vsag::autotune::RunAutoTune(request(dataset.Get(), workspace.Get())); + INFO(result.dump(2)); + REQUIRE(result["status"] == "success"); + REQUIRE(omp_get_max_threads() == 3); + REQUIRE(result["builds"].size() == 2); + REQUIRE(result["trials"].size() == 4); + REQUIRE(result["recommendation"]["create_params"]["dim"] == 8); + for (const auto& build : result["builds"]) { + REQUIRE(build["status"] == "success"); + REQUIRE(build["metrics"].contains("build_seconds")); + REQUIRE(build["metrics"].contains("index_size_mb")); + REQUIRE(build["metrics"].contains("index_memory_mb")); + } + for (const auto& trial : result["trials"]) { + REQUIRE(trial["status"] == "success"); + REQUIRE(trial["metrics"].contains("recall_at_k")); + REQUIRE(trial["metrics"].contains("latency_avg_ms")); + REQUIRE(trial["metrics"].contains("qps")); + } + + const auto hgraph = + std::find_if(result["builds"].begin(), result["builds"].end(), [](const auto& build) { + return build["index_name"] == "hgraph"; + }); + REQUIRE(hgraph != result["builds"].end()); + auto existing = request(dataset.Get(), workspace.Get()); + existing["index_path"] = (*hgraph)["artifacts"]["index_path"]; + existing["indexes"] = + JsonType::array({{{"name", "hgraph"}, + {"create_params", (*hgraph)["create_params"]}, + {"search_params", {{"hgraph", {{"ef_search", {8, 16}}}}}}}}); + existing["constraints"] = {{"recall_at_k", 0.0}}; + existing["tuning_config"]["max_trials"] = 2; + + const auto existing_result = vsag::autotune::RunAutoTune(existing); + INFO(existing_result.dump(2)); + REQUIRE(existing_result["status"] == "success"); + REQUIRE(existing_result["builds"].empty()); + REQUIRE(existing_result["trials"].size() == 2); + REQUIRE(existing_result["request"]["data_path"] == dataset.Get()); + REQUIRE(existing_result["request"]["index_path"] == existing["index_path"]); + REQUIRE(existing_result["request"]["index_name"] == "hgraph"); + REQUIRE(existing_result["request"]["create_params"]["dim"] == 8); + REQUIRE(existing_result["request"]["create_params"]["dtype"] == "float32"); + REQUIRE(existing_result["request"]["create_params"]["metric_type"] == "l2"); +} + +TEST_CASE("AutoTune CLI keeps only the recommended artifact by default") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath dataset(temp_path("autotune-retained-dataset", ".hdf5")); + ScopedPath workspace(temp_path("autotune-retained-workspace")); + write_dataset(dataset.Get()); + auto input = request(dataset.Get(), workspace.Get()); + input["indexes"] = JsonType::array({input["indexes"][0]}); + input["indexes"][0]["search_params"]["hgraph"]["ef_search"] = 8; + input["tuning_config"].erase("keep_intermediate"); + input["tuning_config"]["max_trials"] = 1; + + const auto result = vsag::autotune::RunAutoTune(input); + INFO(result.dump(2)); + REQUIRE(result["status"] == "success"); + REQUIRE(result["recommendation"]["artifacts"]["retained"] == true); + REQUIRE(std::filesystem::is_regular_file( + result["recommendation"]["artifacts"]["index_path"].get())); + REQUIRE(count_index_artifacts(workspace.Get()) == 1); + REQUIRE(std::filesystem::is_regular_file(result["report_path"].get())); +} + +TEST_CASE("AutoTune searches an in-memory existing index") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath workspace(temp_path("autotune-memory-index-workspace")); + MemoryFixture fixture; + const std::string create_params = + R"({"dim":8,"dtype":"float32","metric_type":"l2","index_param":{)" + R"("base_quantization_type":"fp32","max_degree":8,"ef_construction":40,)" + R"("build_thread_count":2}})"; + auto created = vsag::Factory::CreateIndex("hgraph", create_params); + REQUIRE(created.has_value()); + REQUIRE(created.value()->Build(fixture.base).has_value()); + + vsag::autotune::SearchRequest input; + input.index = created.value(); + input.workload = {fixture.queries, fixture.ground_truth, 3, 2}; + input.parameter_space = R"({"hgraph":{"ef_search":[8,16]}})"; + input.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.0}}; + input.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + input.config.workspace_path = workspace.Get(); + input.config.max_trials = 2; + + const auto tuned = vsag::autotune::TuneSearch(input); + REQUIRE(tuned.has_value()); + REQUIRE(tuned->status == vsag::autotune::TuneStatus::SUCCESS); + const auto& result = tuned->report; + REQUIRE(result["status"] == "success"); + REQUIRE(result["builds"].empty()); + REQUIRE(result["trials"].size() == 2); + for (const auto& trial : result["trials"]) { + REQUIRE(trial["status"] == "success"); + REQUIRE(trial["metrics"].contains("recall_at_k")); + REQUIRE(trial["metrics"].contains("latency_avg_ms")); + } + + input.workload.ground_truth = nullptr; + input.constraints = {{vsag::autotune::Metric::QPS, 0.0}}; + const auto latency_only = vsag::autotune::TuneSearch(input); + REQUIRE(latency_only.has_value()); + REQUIRE(latency_only->status == vsag::autotune::TuneStatus::SUCCESS); + REQUIRE_FALSE(latency_only->metrics.contains("recall_at_k")); + REQUIRE_FALSE(latency_only->report.contains("report_path")); + REQUIRE(load_json_reports(workspace.Get()).empty()); + + input.objective = vsag::autotune::Metric::INDEX_MEMORY_MB; + REQUIRE_THROWS_WITH( + vsag::autotune::internal::ParseRequest(input), + Catch::Matchers::ContainsSubstring("objective cannot rank search candidates")); + input.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + + input.parameter_space = R"({"ivf":{"scan_buckets_count":1}})"; + REQUIRE_THROWS_WITH( + vsag::autotune::internal::ParseRequest(input), + Catch::Matchers::ContainsSubstring("search_parameter_space.ivf is unsupported")); +} + +TEST_CASE("AutoTune uses default candidates for an in-memory IVF index") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath workspace(temp_path("autotune-memory-ivf-workspace")); + MemoryFixture fixture; + const std::string create_params = + R"({"dim":8,"dtype":"float32","metric_type":"l2","index_param":{)" + R"("base_quantization_type":"fp32","buckets_count":4,"thread_count":2}})"; + auto created = vsag::Factory::CreateIndex("ivf", create_params); + REQUIRE(created.has_value()); + REQUIRE(created.value()->Build(fixture.base).has_value()); + + vsag::autotune::SearchRequest input; + input.index = created.value(); + input.workload = {fixture.queries, fixture.ground_truth, 3, 2}; + input.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.0}}; + input.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + input.config.workspace_path = workspace.Get(); + input.config.max_trials = 4; + + const auto tuned = vsag::autotune::TuneSearch(input); + REQUIRE(tuned.has_value()); + REQUIRE(tuned->status == vsag::autotune::TuneStatus::SUCCESS); + const auto& result = tuned->report; + REQUIRE(result["status"] == "success"); + REQUIRE(result["trials"].size() == 4); + REQUIRE(result["recommendation"]["search_params"]["ivf"].contains("scan_buckets_count")); +} + +TEST_CASE("AutoTune searches an existing Pyramid index for one path workload") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath workspace(temp_path("autotune-pyramid-workspace")); + MemoryFixture fixture; + std::vector base_paths(MemoryFixture::BASE_COUNT, "a/d/f"); + std::vector query_paths(MemoryFixture::QUERY_COUNT, "a/d/f"); + auto base = vsag::Dataset::Make() + ->NumElements(MemoryFixture::BASE_COUNT) + ->Dim(MemoryFixture::DIM) + ->Ids(fixture.base_ids.data()) + ->Float32Vectors(fixture.train.data()) + ->Paths(base_paths.data()) + ->Owner(false); + auto queries = vsag::Dataset::Make() + ->NumElements(MemoryFixture::QUERY_COUNT) + ->Dim(MemoryFixture::DIM) + ->Float32Vectors(fixture.test.data()) + ->Paths(query_paths.data()) + ->Owner(false); + const std::string create_params = + R"({"dim":8,"dtype":"float32","metric_type":"l2","index_param":{)" + R"("base_quantization_type":"fp32","max_degree":8,"alpha":1.2,)" + R"("graph_iter_turn":5,"neighbor_sample_rate":0.5,"no_build_levels":[0,1],)" + R"("use_reorder":true,"graph_type":"odescent","build_thread_count":2}})"; + auto created = vsag::Factory::CreateIndex("pyramid", create_params); + REQUIRE(created.has_value()); + REQUIRE(created.value()->Build(base).has_value()); + REQUIRE(created.value()->GetMemoryUsage() == 0); + + vsag::autotune::SearchRequest input; + input.index = created.value(); + input.workload.queries = queries; + input.workload.ground_truth = fixture.ground_truth; + input.workload.top_k = 3; + input.workload.concurrency = 2; + input.parameter_space = R"({"pyramid":{"ef_search":[4,8]}})"; + input.constraints = {{vsag::autotune::Metric::RECALL_AT_K, 0.0}}; + input.objective = vsag::autotune::Metric::LATENCY_AVG_MS; + input.config.workspace_path = workspace.Get(); + input.config.max_trials = 2; + + const auto tuned = vsag::autotune::TuneSearch(input); + REQUIRE(tuned.has_value()); + REQUIRE(tuned->status == vsag::autotune::TuneStatus::SUCCESS); + const auto& result = tuned->report; + REQUIRE(result["status"] == "success"); + REQUIRE(result["trials"].size() == 2); + REQUIRE(result["recommendation"]["index_name"] == "pyramid"); + REQUIRE(result["recommendation"]["search_params"].contains("pyramid")); + REQUIRE(result["builds"].empty()); + + input.constraints.push_back({vsag::autotune::Metric::INDEX_MEMORY_MB, 1.0}); + const auto memory_constrained = vsag::autotune::TuneSearch(input); + REQUIRE(memory_constrained.has_value()); + REQUIRE(memory_constrained->status == vsag::autotune::TuneStatus::NO_FEASIBLE_CANDIDATE); + REQUIRE(memory_constrained->best_effort["constraint_evaluation"]["satisfied"] == false); + + input.constraints.pop_back(); + input.workload.queries = fixture.queries; + REQUIRE_NOTHROW(vsag::autotune::internal::ParseRequest(input)); +} + +TEST_CASE("AutoTune writes concrete trials for an adaptive ef_search range") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath dataset(temp_path("autotune-binary-dataset", ".hdf5")); + ScopedPath workspace(temp_path("autotune-binary-workspace")); + write_dataset(dataset.Get()); + + auto input = request(dataset.Get(), workspace.Get()); + auto hgraph = input["indexes"][0]; + hgraph["search_params"] = { + {"hgraph", {{"ef_search", {{"$range", {{"start", 3}, {"stop", 35}}}}}}}}; + input["indexes"] = JsonType::array({std::move(hgraph)}); + input["constraints"] = {{"recall_at_k", 0.0}}; + input["tuning_config"]["max_trials"] = 9; + + const auto result = vsag::autotune::RunAutoTune(input); + INFO(result.dump(2)); + REQUIRE(result["status"] == "success"); + REQUIRE(result["builds"].size() == 1); + REQUIRE(result["trials"].size() == 1); + REQUIRE(result["trials"][0]["search_params"]["hgraph"]["ef_search"] == 3); + REQUIRE(result["trials"][0]["search_params"]["hgraph"]["ef_search"].is_number_integer()); + REQUIRE(result["recommendation"]["search_params"]["hgraph"]["ef_search"] == 3); +} + +TEST_CASE("TuneIndex returns a queryable selected index") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath workspace(temp_path("autotune-factory-workspace")); + MemoryFixture fixture; + auto input = fixture.Request(workspace.Get()); + + auto tuned = vsag::autotune::TuneIndex(input); + REQUIRE(tuned.has_value()); + const auto& result = tuned.value(); + REQUIRE(result.status == vsag::autotune::TuneStatus::SUCCESS); + REQUIRE(result.best_effort.is_null()); + REQUIRE(result.index != nullptr); + REQUIRE(result.index->GetNumElements() == 64); + REQUIRE(result.index_name == "hgraph"); + REQUIRE(JsonType::parse(result.create_parameters)["dim"] == 8); + REQUIRE(JsonType::parse(result.search_parameters)["hgraph"]["ef_search"] == 16); + REQUIRE(result.metrics.contains("recall_at_k")); + REQUIRE(std::filesystem::is_regular_file(result.artifact_path)); + REQUIRE_FALSE(result.report.contains("report_path")); + + auto query = vsag::Dataset::Make(); + query->NumElements(1) + ->Dim(MemoryFixture::DIM) + ->Float32Vectors(fixture.test.data()) + ->Owner(false); + auto neighbors = result.index->KnnSearch(query, 3, result.search_parameters); + REQUIRE(neighbors.has_value()); + REQUIRE(neighbors.value()->GetDim() == 3); + REQUIRE(std::find(fixture.base_ids.begin(), + fixture.base_ids.end(), + neighbors.value()->GetIds()[0]) != fixture.base_ids.end()); + + auto restored = vsag::Factory::CreateIndex(result.index_name, result.create_parameters); + REQUIRE(restored.has_value()); + std::ifstream artifact(result.artifact_path, std::ios::binary); + REQUIRE(artifact.good()); + REQUIRE(restored.value()->Deserialize(artifact).has_value()); + REQUIRE(restored.value()->KnnSearch(query, 3, result.search_parameters).has_value()); + + const auto& report = result.report; + REQUIRE(report["status"] == "success"); + REQUIRE(report["builds"].size() == 2); + REQUIRE(std::count_if(report["builds"].begin(), report["builds"].end(), [](const auto& build) { + return build["artifacts"]["retained"].template get(); + }) == 1); + REQUIRE(count_index_artifacts(workspace.Get()) == 1); + REQUIRE(report["recommendation"]["create_params"]["dim"] == 8); + REQUIRE(report["recommendation"]["artifacts"]["retained"] == true); + REQUIRE(report["request"]["config"]["workspace_path"] == workspace.Get()); +} + +TEST_CASE("TuneIndex reports an infeasible recommendation") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath workspace(temp_path("autotune-infeasible-workspace")); + MemoryFixture fixture; + auto input = fixture.Request(workspace.Get()); + input.index_spaces[0].create_parameter_space = + R"({"index_param":{"base_quantization_type":"fp32","max_degree":8,)" + R"("ef_construction":40,"build_thread_count":2}})"; + input.workload.ground_truth = nullptr; + input.constraints = {{vsag::autotune::Metric::QPS, 1e100}}; + input.config.max_trials = 1; + + const auto tuned = vsag::autotune::TuneIndex(input); + REQUIRE(tuned.has_value()); + REQUIRE(tuned->status == vsag::autotune::TuneStatus::NO_FEASIBLE_CANDIDATE); + REQUIRE(tuned->index == nullptr); + REQUIRE(tuned->best_effort["constraint_evaluation"]["satisfied"] == false); + REQUIRE_FALSE(tuned->report.contains("report_path")); + REQUIRE(count_index_artifacts(workspace.Get()) == 0); + REQUIRE(load_json_reports(workspace.Get()).empty()); + REQUIRE(tuned->report["status"] == "no_feasible_candidate"); + REQUIRE(tuned->report["request"]["config"]["keep_intermediate"] == false); + REQUIRE(tuned->report["builds"][0]["artifacts"]["retained"] == false); + REQUIRE(tuned->best_effort["artifacts"]["retained"] == false); +} + +TEST_CASE("TuneIndex cleans artifacts when all search trials fail") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath workspace(temp_path("autotune-all-failed-workspace")); + MemoryFixture fixture; + auto input = fixture.Request(workspace.Get()); + input.index_spaces[0].create_parameter_space = + R"({"index_param":{"base_quantization_type":"fp32","max_degree":8,)" + R"("ef_construction":40,"build_thread_count":2}})"; + input.index_spaces[0].search_parameter_space = R"({"hgraph":{"ef_search":0}})"; + input.config.max_trials = 1; + + const auto tuned = vsag::autotune::TuneIndex(input); + REQUIRE_FALSE(tuned.has_value()); + REQUIRE(tuned.error().type == vsag::ErrorType::INTERNAL_ERROR); + REQUIRE(count_index_artifacts(workspace.Get()) == 0); + REQUIRE(load_json_reports(workspace.Get()).empty()); +} + +TEST_CASE("TuneIndex returns its report without writing a report file") { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + ScopedBlockSizeLimit block_size_limit(256UL * 1024); + ScopedPath workspace(temp_path("autotune-in-memory-report-workspace")); + MemoryFixture fixture; + auto input = fixture.Request(workspace.Get()); + input.index_spaces[0].create_parameter_space = + R"({"index_param":{"base_quantization_type":"fp32","max_degree":8,)" + R"("ef_construction":40,"build_thread_count":2}})"; + input.config.max_trials = 1; + + const auto tuned = vsag::autotune::TuneIndex(input); + REQUIRE(tuned.has_value()); + REQUIRE(tuned->status == vsag::autotune::TuneStatus::SUCCESS); + REQUIRE(tuned->report["status"] == "success"); + REQUIRE_FALSE(tuned->report.contains("report_path")); + REQUIRE(load_json_reports(workspace.Get()).empty()); +} + +TEST_CASE("AutoTune validates the CLI report path before evaluation") { + ScopedPath dataset(temp_path("autotune-report-path-dataset", ".hdf5")); + ScopedPath workspace(temp_path("autotune-report-path-workspace")); + write_dataset(dataset.Get()); + const auto report_directory = workspace.Get() + "/report-is-a-directory"; + std::filesystem::create_directories(report_directory); + + auto input = request(dataset.Get(), workspace.Get()); + input["output"] = {{"result_path", report_directory}}; + const auto result = vsag::autotune::RunAutoTune(input); + + REQUIRE(result["status"] == "failed"); + REQUIRE(result["failure"]["stage"] == "report"); + REQUIRE_THAT(result["failure"]["message"].get(), + Catch::Matchers::ContainsSubstring("failed to open report path")); + REQUIRE(count_index_artifacts(workspace.Get()) == 0); +} diff --git a/tools/autotune/examples/README.md b/tools/autotune/examples/README.md new file mode 100644 index 0000000000..49a8cc2b5d --- /dev/null +++ b/tools/autotune/examples/README.md @@ -0,0 +1,19 @@ +# AutoTune Examples + +The SIFT example expects the ann-benchmarks dataset at +`/tmp/sift-128-euclidean.hdf5`. Edit `data_path` if the dataset is stored elsewhere. + +From the VSAG repository root, run: + +```bash +./build-release/tools/autotune/autotune \ + tools/autotune/examples/sift_hgraph_autotune_request.json +``` + +The command prints a compact summary. The complete report is written to +`/tmp/vsag_autotune_sift/report.json`. + +This example expands six HGraph build configurations. For each build, AutoTune doubles +`ef_search` from `40` until recall passes, then binary-searches that interval for the smallest +passing value, capped at `1000`. Every evaluated point uses all queries. Add arrays or stepped +`$range` expressions to other parameter leaves to change the tuning space. diff --git a/tools/autotune/examples/sift_hgraph_autotune_request.json b/tools/autotune/examples/sift_hgraph_autotune_request.json new file mode 100644 index 0000000000..19febc2422 --- /dev/null +++ b/tools/autotune/examples/sift_hgraph_autotune_request.json @@ -0,0 +1,48 @@ +{ + "version": 1, + "data_path": "/tmp/sift-128-euclidean.hdf5", + "indexes": [ + { + "name": "hgraph", + "create_params": { + "index_param": { + "base_quantization_type": ["fp32","sq8"], + "max_degree": [16,32,48], + "ef_construction": 400, + "build_thread_count": 48 + } + }, + "search_params": { + "hgraph": { + "ef_search": { + "$range": { + "start": 40, + "stop": 1000 + } + } + } + } + } + ], + "workload": { + "top_k": 10, + "concurrency": 1 + }, + "constraints": { + "recall_at_k": 0.95, + "index_memory_mb": 1024, + "index_size_mb": 1024 + }, + "objective": { + "metric": "latency_avg_ms" + }, + "tuning_config": { + "workspace_path": "/tmp/vsag_autotune_sift", + "keep_intermediate": false, + "max_trials": 1000 + }, + "output": { + "result_path": "/tmp/vsag_autotune_sift/report.json", + "include_raw_eval": false + } +} diff --git a/tools/autotune/main.cpp b/tools/autotune/main.cpp new file mode 100644 index 0000000000..a43ce4d5b4 --- /dev/null +++ b/tools/autotune/main.cpp @@ -0,0 +1,54 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "autotune.h" +#include "vsag/options.h" + +int +main(int argc, char** argv) try { + vsag::Options::Instance().logger()->SetLevel(vsag::Logger::kOFF); + if (argc != 2) { + std::cerr << "Usage: " << argv[0] << " " << std::endl; + return 1; + } + + vsag::autotune::JsonType result; + try { + std::ifstream input(argv[1]); + if (!input.good()) { + throw std::runtime_error("failed to open request file: " + std::string(argv[1])); + } + vsag::autotune::JsonType request; + input >> request; + result = vsag::autotune::RunAutoTune(request); + } catch (const std::exception& error) { + result = {{"version", 1}, + {"status", "failed"}, + {"failure", + {{"stage", "cli"}, {"code", "request_file_error"}, {"message", error.what()}}}}; + } + + std::cout << vsag::autotune::FormatResultSummaryForCli(result) << std::endl; + return result.value("status", std::string()) == "failed" ? 1 : 0; +} catch (const std::exception& error) { + std::cerr << "AutoTune failed: " << error.what() << std::endl; + return 1; +} catch (...) { + std::cerr << "AutoTune failed with an unknown error" << std::endl; + return 1; +} diff --git a/tools/eval/CMakeLists.txt b/tools/eval/CMakeLists.txt index 3b87f885a5..78cbbd25f6 100644 --- a/tools/eval/CMakeLists.txt +++ b/tools/eval/CMakeLists.txt @@ -28,6 +28,7 @@ set (eval_srcs monitor/http_server_monitor.cpp eval_config.cpp eval_dataset.cpp + evaluator.cpp eval_job.cpp) add_library (vsag_eval_common INTERFACE) @@ -67,7 +68,6 @@ if (ENABLE_TESTS) target_link_libraries (eval_dataset_test PRIVATE Catch2::Catch2WithMain eval_obj - simd vsag vsag_eval_common ${HDF5_CPP_STATIC_LIBRARY} diff --git a/tools/eval/case/build_eval_case.cpp b/tools/eval/case/build_eval_case.cpp index 3eb243ec46..146c6136c3 100644 --- a/tools/eval/case/build_eval_case.cpp +++ b/tools/eval/case/build_eval_case.cpp @@ -15,9 +15,10 @@ #include "./build_eval_case.h" -#include #include +#include #include +#include #include "../monitor/duration_monitor.h" #include "../monitor/memory_peak_monitor.h" @@ -28,8 +29,10 @@ namespace vsag::eval { BuildEvalCase::BuildEvalCase(const std::string& dataset_path, const std::string& index_path, vsag::IndexPtr index, - EvalConfig config) - : EvalCase(dataset_path, index_path, index), config_(std::move(config)) { + EvalConfig config, + EvalDatasetPtr dataset) + : EvalCase(dataset_path, index_path, std::move(index), std::move(dataset)), + config_(std::move(config)) { this->init_monitors(); } @@ -39,26 +42,35 @@ BuildEvalCase::init_monitors() { auto memory_peak_monitor = std::make_shared("build"); this->monitors_.emplace_back(std::move(memory_peak_monitor)); } - if (config_.enable_tps) { - auto duration_monitor = std::make_shared(); - this->monitors_.emplace_back(std::move(duration_monitor)); - } + auto duration_monitor = std::make_shared(); + this->monitors_.emplace_back(std::move(duration_monitor)); } JsonType BuildEvalCase::Run() { this->do_build(); this->serialize(); - auto result = this->process_result(); - return result; + return this->process_result(); +} + +JsonType +BuildEvalCase::RunInMemory() { + this->do_build(); + return this->process_result(); } + void BuildEvalCase::do_build() { auto base = vsag::Dataset::Make(); int64_t total_base = this->dataset_ptr_->GetNumberOfBase(); - std::vector ids(total_base); - std::iota(ids.begin(), ids.end(), 0); - base->NumElements(total_base)->Dim(this->dataset_ptr_->GetDim())->Ids(ids.data())->Owner(false); + const auto* train_ids = this->dataset_ptr_->GetTrainIds(); + std::vector identity_ids; + if (train_ids == nullptr) { + identity_ids.resize(static_cast(total_base)); + std::iota(identity_ids.begin(), identity_ids.end(), int64_t{0}); + train_ids = identity_ids.data(); + } + base->NumElements(total_base)->Dim(this->dataset_ptr_->GetDim())->Ids(train_ids)->Owner(false); if (this->dataset_ptr_->GetVectorType() == DENSE_VECTORS) { if (this->dataset_ptr_->GetTrainDataType() == vsag::DATATYPE_FLOAT32) { base->Float32Vectors((const float*)this->dataset_ptr_->GetTrain()); @@ -68,6 +80,9 @@ BuildEvalCase::do_build() { } else { base->SparseVectors((const SparseVector*)this->dataset_ptr_->GetTrain()); } + if (this->dataset_ptr_->GetTrainPaths() != nullptr) { + base->Paths(this->dataset_ptr_->GetTrainPaths()); + } for (auto& monitor : monitors_) { monitor->Start(); } @@ -100,11 +115,16 @@ BuildEvalCase::process_result() { EvalCase::MergeJsonType(one_result, eval_result); } result = eval_result; - result["tps"] = double(this->dataset_ptr_->GetNumberOfBase()) / double(result["duration(s)"]); + if (config_.enable_tps) { + result["tps"] = + double(this->dataset_ptr_->GetNumberOfBase()) / double(result["duration(s)"]); + } EvalCase::MergeJsonType(this->basic_info_, result); - result["index_info"] = JsonType::parse(config_.build_param); + result["index_info"] = + config_.build_param.empty() ? JsonType::object() : JsonType::parse(config_.build_param); result["action"] = "build"; result["index"] = config_.index_name; + result["index_memory(B)"] = this->index_->GetMemoryUsage(); try { auto detail = this->index_->GetMemoryUsageDetail(); for (const auto& [name, size] : detail) { diff --git a/tools/eval/case/build_eval_case.h b/tools/eval/case/build_eval_case.h index d43267cbe7..7fb837fd37 100644 --- a/tools/eval/case/build_eval_case.h +++ b/tools/eval/case/build_eval_case.h @@ -25,13 +25,17 @@ class BuildEvalCase : public EvalCase { BuildEvalCase(const std::string& dataset_path, const std::string& index_path, vsag::IndexPtr index, - EvalConfig config); + EvalConfig config, + EvalDatasetPtr dataset = nullptr); ~BuildEvalCase() override = default; JsonType Run() override; + JsonType + RunInMemory(); + private: void init_monitors(); diff --git a/tools/eval/case/build_search_eval_case.h b/tools/eval/case/build_search_eval_case.h index c9f4f2e47f..e4da73b76e 100644 --- a/tools/eval/case/build_search_eval_case.h +++ b/tools/eval/case/build_search_eval_case.h @@ -25,10 +25,11 @@ class BuildSearchEvalCase : public EvalCase { BuildSearchEvalCase(const std::string& dataset_path, const std::string& index_path, vsag::IndexPtr index, - EvalConfig config) - : EvalCase(dataset_path, index_path, index) { - build_ = EvalCase::MakeInstance(config, "build"); - search_ = EvalCase::MakeInstance(config, "search"); + EvalConfig config, + EvalDatasetPtr dataset = nullptr) + : EvalCase(dataset_path, index_path, std::move(index), dataset) { + build_ = EvalCase::MakeInstance(config, "build", dataset); + search_ = EvalCase::MakeInstance(config, "search", dataset); config_ = std::move(config); } diff --git a/tools/eval/case/eval_case.cpp b/tools/eval/case/eval_case.cpp index ef99325181..a92a89fe94 100644 --- a/tools/eval/case/eval_case.cpp +++ b/tools/eval/case/eval_case.cpp @@ -26,15 +26,23 @@ namespace vsag::eval { -EvalCase::EvalCase(std::string dataset_path, std::string index_path, vsag::IndexPtr index) - : dataset_path_(std::move(dataset_path)), index_path_(std::move(index_path)), index_(index) { - this->dataset_ptr_ = EvalDataset::Load(dataset_path_); +EvalCase::EvalCase(std::string dataset_path, + std::string index_path, + vsag::IndexPtr index, + EvalDatasetPtr dataset) + : dataset_path_(std::move(dataset_path)), + index_path_(std::move(index_path)), + dataset_ptr_(std::move(dataset)), + index_(std::move(index)) { + if (this->dataset_ptr_ == nullptr) { + this->dataset_ptr_ = EvalDataset::Load(dataset_path_); + } this->logger_ = vsag::Options::Instance().logger(); this->basic_info_ = this->dataset_ptr_->GetInfo(); } EvalCasePtr -EvalCase::MakeInstance(const EvalConfig& config, std::string type) { +EvalCase::MakeInstance(const EvalConfig& config, std::string type, const EvalDatasetPtr& dataset) { auto dataset_path = config.dataset_path; auto index_path = config.index_path; auto index_name = config.index_name; @@ -48,14 +56,16 @@ EvalCase::MakeInstance(const EvalConfig& config, std::string type) { } if (type == "build") { - return std::make_shared(dataset_path, index_path, index.value(), config); + return std::make_shared( + dataset_path, index_path, index.value(), config, dataset); } if (type == "search") { - return std::make_shared(dataset_path, index_path, index.value(), config); + return std::make_shared( + dataset_path, index_path, index.value(), config, dataset); } if (type == "build,search") { return std::make_shared( - dataset_path, index_path, index.value(), config); + dataset_path, index_path, index.value(), config, dataset); } return nullptr; } diff --git a/tools/eval/case/eval_case.h b/tools/eval/case/eval_case.h index 8a7316242f..855a3098c5 100644 --- a/tools/eval/case/eval_case.h +++ b/tools/eval/case/eval_case.h @@ -31,7 +31,9 @@ using EvalCasePtr = std::shared_ptr; class EvalCase { public: static EvalCasePtr - MakeInstance(const EvalConfig& config, std::string type = "none"); + MakeInstance(const EvalConfig& config, + std::string type = "none", + const EvalDatasetPtr& dataset = nullptr); static void MergeJsonType(const JsonType& input, JsonType& output) { @@ -86,7 +88,10 @@ class EvalCase { } public: - explicit EvalCase(std::string dataset_path, std::string index_path, vsag::IndexPtr index); + explicit EvalCase(std::string dataset_path, + std::string index_path, + vsag::IndexPtr index, + EvalDatasetPtr dataset = nullptr); virtual ~EvalCase() = default; diff --git a/tools/eval/case/search_eval_case.cpp b/tools/eval/case/search_eval_case.cpp index 209cee9576..dc6abe9b04 100644 --- a/tools/eval/case/search_eval_case.cpp +++ b/tools/eval/case/search_eval_case.cpp @@ -17,12 +17,14 @@ #include +#include #include #include #include #include -#include +#include #include +#include #include #include "../monitor/latency_monitor.h" @@ -35,6 +37,41 @@ namespace vsag::eval { +namespace { + +class SearchFailure { +public: + bool + Failed() const { + return failed_.load(std::memory_order_acquire); + } + + void + Record(const std::string& message) { + bool expected = false; + if (failed_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) { + std::lock_guard lock(mutex_); + message_ = message; + } + } + + void + ThrowIfFailed() { + if (not Failed()) { + return; + } + std::lock_guard lock(mutex_); + throw std::runtime_error("query error: " + message_); + } + +private: + std::atomic failed_{false}; + std::mutex mutex_; + std::string message_; +}; + +} // namespace + class FilterObj : public vsag::Filter { public: FilterObj(const std::shared_ptr& train_labels, int64_t test_label, float valid_ratio) @@ -60,8 +97,10 @@ class FilterObj : public vsag::Filter { SearchEvalCase::SearchEvalCase(const std::string& dataset_path, const std::string& index_path, vsag::IndexPtr index, - EvalConfig config) - : EvalCase(dataset_path, index_path, index), config_(std::move(config)) { + EvalConfig config, + EvalDatasetPtr dataset) + : EvalCase(dataset_path, index_path, std::move(index), std::move(dataset)), + config_(std::move(config)) { auto search_mode = config_.search_mode; if (search_mode == "knn") { this->search_type_ = SearchType::KNN; @@ -84,7 +123,7 @@ SearchEvalCase::init_monitor() { void SearchEvalCase::init_latency_monitor() { - if (config_.enable_latency or config_.enable_tps or config_.enable_percent_latency) { + if (config_.enable_latency or config_.enable_qps or config_.enable_percent_latency) { this->latency_monitor_ = std::make_shared(); if (config_.enable_qps) { this->latency_monitor_->SetMetrics("qps"); @@ -102,8 +141,8 @@ SearchEvalCase::init_latency_monitor() { void SearchEvalCase::init_recall_monitor() { if (config_.enable_recall or config_.enable_percent_recall) { - auto recall_monitor = - std::make_shared(this->dataset_ptr_->GetNumberOfQuery()); + auto recall_monitor = std::make_shared( + this->dataset_ptr_->GetNumberOfQuery(), config_.use_id_based_recall); if (config_.enable_recall) { recall_monitor->SetMetrics("avg_recall"); } @@ -126,6 +165,15 @@ JsonType SearchEvalCase::Run() { std::ifstream infile(this->index_path_, std::ios::binary); this->deserialize(infile); + auto result = this->RunInMemory(); + if (config_.delete_index_after_search) { + std::remove(this->index_path_.c_str()); + } + return result; +} + +JsonType +SearchEvalCase::RunInMemory() { switch (this->search_type_) { case KNN: this->do_knn_search(); @@ -140,11 +188,7 @@ SearchEvalCase::Run() { this->do_range_filter_search(); break; } - auto result = this->process_result(); - if (config_.delete_index_after_search) { - std::remove(this->index_path_.c_str()); - } - return result; + return this->process_result(); } void @@ -172,6 +216,9 @@ SearchEvalCase::do_knn_search() { } else { query->SparseVectors((const SparseVector*)query_vector); } + if (this->dataset_ptr_->GetTestPaths() != nullptr) { + query->Paths(this->dataset_ptr_->GetTestPaths() + query_id); + } return std::make_pair(std::move(query), query_vector); }; @@ -187,20 +234,25 @@ SearchEvalCase::do_knn_search() { if (is_latency_monitor) { using Clock = std::chrono::steady_clock; std::vector latency_records(min_query); + SearchFailure search_failure; const auto wall_start = Clock::now(); #pragma omp parallel for schedule(dynamic) for (int64_t id = 0; id < min_query; ++id) { + if (search_failure.Failed()) { + continue; + } auto i = static_cast(id) % query_count; auto query_and_vector = prepare_query(i); auto& query = query_and_vector.first; auto [result, latency_ms] = MeasureSearch( [&]() { return this->index_->KnnSearch(query, topk, config_.search_param); }); if (not result.has_value()) { - std::cerr << "query error: " << result.error().message << std::endl; - exit(-1); + search_failure.Record(result.error().message); + continue; } latency_records[static_cast(id)] = latency_ms; } + search_failure.ThrowIfFailed(); const auto wall_end = Clock::now(); auto timing_batch = LatencyTimingBatch{ std::move(latency_records), @@ -212,47 +264,56 @@ SearchEvalCase::do_knn_search() { continue; } + SearchFailure search_failure; #pragma omp parallel for schedule(dynamic) for (int64_t id = 0; id < min_query; ++id) { + if (search_failure.Failed()) { + continue; + } auto i = static_cast(id) % query_count; auto query_and_vector = prepare_query(i); auto& query = query_and_vector.first; const void* query_vector = query_and_vector.second; auto result = this->index_->KnnSearch(query, topk, config_.search_param); if (not result.has_value()) { - std::cerr << "query error: " << result.error().message << std::endl; - exit(-1); + search_failure.Record(result.error().message); + continue; } if (collect_statistics) { this->record_statistics(result.value()); } - const int64_t* neighbors = result.value()->GetIds(); - int64_t* ground_truth_neighbors = dataset_ptr_->GetNeighbors(i); - auto record = std::make_tuple(neighbors, - ground_truth_neighbors, - dataset_ptr_.get(), - query_vector, - result.value()->GetDim()); + SearchRecord record{result.value()->GetIds(), + dataset_ptr_->GetNeighbors(i), + dataset_ptr_.get(), + query_vector, + static_cast(result.value()->GetDim()), + topk}; monitor->Record(&record); } + search_failure.ThrowIfFailed(); monitor->Stop(); statistics_collected = statistics_collected or collect_statistics; } if (not statistics_collected and not this->monitors_.empty()) { omp_set_num_threads(config_.num_threads_searching); + SearchFailure search_failure; #pragma omp parallel for schedule(dynamic) for (int64_t id = 0; id < min_query; ++id) { + if (search_failure.Failed()) { + continue; + } auto i = static_cast(id) % query_count; auto query_and_vector = prepare_query(i); auto& query = query_and_vector.first; auto result = this->index_->KnnSearch(query, topk, config_.search_param); if (not result.has_value()) { - std::cerr << "query error: " << result.error().message << std::endl; - exit(-1); + search_failure.Record(result.error().message); + continue; } this->record_statistics(result.value()); } + search_failure.ThrowIfFailed(); } } @@ -299,8 +360,7 @@ SearchEvalCase::do_knn_filter_search() { return this->index_->KnnSearch(query, topk, config_.search_param, filter); }); if (not result.has_value()) { - std::cerr << "query error: " << result.error().message << std::endl; - exit(-1); + throw std::runtime_error("query error: " + result.error().message); } latency_records[static_cast(id)] = latency_ms; } @@ -330,13 +390,14 @@ SearchEvalCase::do_knn_filter_search() { train_labels, test_label, this->dataset_ptr_->GetValidRatio(test_label)); auto result = this->index_->KnnSearch(query, topk, config_.search_param, filter); if (not result.has_value()) { - std::cerr << "query error: " << result.error().message << std::endl; - exit(-1); + throw std::runtime_error("query error: " + result.error().message); } - const int64_t* neighbors = result.value()->GetIds(); - int64_t* ground_truth_neighbors = dataset_ptr_->GetNeighbors(i); - auto record = std::make_tuple( - neighbors, ground_truth_neighbors, dataset_ptr_.get(), query_vector, topk); + SearchRecord record{result.value()->GetIds(), + dataset_ptr_->GetNeighbors(i), + dataset_ptr_.get(), + query_vector, + static_cast(result.value()->GetDim()), + topk}; monitor->Record(&record); } monitor->Stop(); @@ -356,9 +417,11 @@ SearchEvalCase::process_result() { } result["action"] = "search"; result["search_mode"] = config_.search_mode; - result["index_info"] = JsonType::parse(config_.build_param); + result["index_info"] = + config_.build_param.empty() ? JsonType::object() : JsonType::parse(config_.build_param); result["search_param"] = config_.search_param; result["index"] = config_.index_name; + result["index_memory(B)"] = this->index_->GetMemoryUsage(); try { auto detail = this->index_->GetMemoryUsageDetail(); for (const auto& [name, size] : detail) { diff --git a/tools/eval/case/search_eval_case.h b/tools/eval/case/search_eval_case.h index ca8d3376ba..bad9a4c000 100644 --- a/tools/eval/case/search_eval_case.h +++ b/tools/eval/case/search_eval_case.h @@ -30,13 +30,17 @@ class SearchEvalCase : public EvalCase { SearchEvalCase(const std::string& dataset_path, const std::string& index_path, vsag::IndexPtr index, - EvalConfig config); + EvalConfig config, + EvalDatasetPtr dataset = nullptr); ~SearchEvalCase() override = default; JsonType Run() override; + JsonType + RunInMemory(); + private: enum SearchType { KNN, diff --git a/tools/eval/eval_config.h b/tools/eval/eval_config.h index ef587f7ad4..c8eb71a926 100644 --- a/tools/eval/eval_config.h +++ b/tools/eval/eval_config.h @@ -56,6 +56,7 @@ class EvalConfig { bool enable_memory{true}; bool enable_latency{true}; bool enable_percent_latency{true}; + bool use_id_based_recall{false}; EvalConfig() = default; }; diff --git a/tools/eval/eval_dataset.cpp b/tools/eval/eval_dataset.cpp index f23021ab71..780b421fbe 100644 --- a/tools/eval/eval_dataset.cpp +++ b/tools/eval/eval_dataset.cpp @@ -15,6 +15,10 @@ #include "eval_dataset.h" +#include +#include +#include + #include "impl/logger/logger.h" using namespace H5; @@ -207,6 +211,148 @@ get_distance(const SparseVector* vector1, const SparseVector* vector2, const voi return sum; } +void +EvalDataset::InitializeTrainIds(const int64_t* ids) { + train_ids_are_identity_ = true; + train_id_to_row_.clear(); + if (ids == nullptr) { + return; + } + + for (int64_t i = 0; i < number_of_base_; ++i) { + if (ids[i] != i) { + train_ids_are_identity_ = false; + break; + } + } + if (train_ids_are_identity_) { + return; + } + + train_id_to_row_.reserve(static_cast(number_of_base_)); + for (int64_t i = 0; i < number_of_base_; ++i) { + if (!train_id_to_row_.emplace(ids[i], i).second) { + throw std::invalid_argument("base dataset contains duplicate ids"); + } + } +} + +EvalDatasetPtr +EvalDataset::FromDatasets(const vsag::DatasetPtr& base, + const vsag::DatasetPtr& queries, + const vsag::DatasetPtr& ground_truth, + const std::string& metric_type) { + if (base == nullptr || queries == nullptr) { + throw std::invalid_argument("base and queries datasets are required"); + } + if (base->GetNumElements() <= 0 || queries->GetNumElements() <= 0) { + throw std::invalid_argument("base and queries datasets must not be empty"); + } + if (base->GetDim() <= 0 || base->GetDim() != queries->GetDim()) { + throw std::invalid_argument("base and queries datasets must have the same positive dim"); + } + if (base->GetFloat32Vectors() == nullptr || queries->GetFloat32Vectors() == nullptr) { + throw std::invalid_argument("in-memory eval supports only float32 dense vectors"); + } + if (ground_truth != nullptr && + (ground_truth->GetNumElements() != queries->GetNumElements() || + ground_truth->GetDim() <= 0 || ground_truth->GetIds() == nullptr)) { + throw std::invalid_argument( + "ground_truth must contain one non-empty id row for every query"); + } + + auto dataset = std::make_shared(); + dataset->base_dataset_ = base; + dataset->query_dataset_ = queries; + dataset->ground_truth_dataset_ = ground_truth; + dataset->vector_type_ = DENSE_VECTORS; + dataset->train_data_type_ = vsag::DATATYPE_FLOAT32; + dataset->test_data_type_ = vsag::DATATYPE_FLOAT32; + dataset->train_data_size_ = sizeof(float); + dataset->test_data_size_ = sizeof(float); + dataset->number_of_base_ = base->GetNumElements(); + dataset->number_of_query_ = queries->GetNumElements(); + dataset->dim_ = base->GetDim(); + dataset->train_shape_ = {dataset->number_of_base_, dataset->dim_}; + dataset->test_shape_ = {dataset->number_of_query_, dataset->dim_}; + dataset->neighbors_shape_ = + ground_truth == nullptr ? shape_t{0, 0} + : shape_t{ground_truth->GetNumElements(), ground_truth->GetDim()}; + dataset->file_path_ = ""; + + if (metric_type == "l2") { + dataset->metric_ = "euclidean"; + dataset->distance_func_ = + [](const void* left, const void* right, const void* dim) -> float { + return std::sqrt(vsag::L2Sqr(left, right, dim)); + }; + } else if (metric_type == "ip") { + dataset->metric_ = "ip"; + dataset->distance_func_ = vsag::InnerProductDistance; + } else if (metric_type == "cosine") { + dataset->metric_ = "angular"; + dataset->distance_func_ = + [](const void* left, const void* right, const void* dim) -> float { + return 1 - vsag::InnerProduct(left, right, dim) / + std::sqrt(vsag::InnerProduct(left, left, dim) * + vsag::InnerProduct(right, right, dim)); + }; + } else { + throw std::invalid_argument("unsupported in-memory eval metric_type: " + metric_type); + } + + dataset->InitializeTrainIds(base->GetIds()); + if (ground_truth != nullptr) { + if (ground_truth->GetNumElements() > + std::numeric_limits::max() / ground_truth->GetDim()) { + throw std::invalid_argument("ground_truth contains too many ids"); + } + const auto ground_truth_count = ground_truth->GetNumElements() * ground_truth->GetDim(); + for (int64_t i = 0; i < ground_truth_count; ++i) { + if (dataset->GetOneTrainById(ground_truth->GetIds()[i]) == nullptr) { + throw std::invalid_argument("ground_truth contains an id not present in base"); + } + } + } + return dataset; +} + +EvalDatasetPtr +EvalDataset::FromSearchDatasets(const vsag::DatasetPtr& queries, + const vsag::DatasetPtr& ground_truth) { + if (queries == nullptr || queries->GetNumElements() <= 0) { + throw std::invalid_argument("queries dataset is required and must not be empty"); + } + if (queries->GetDim() <= 0 || queries->GetFloat32Vectors() == nullptr) { + throw std::invalid_argument( + "in-memory search evaluation requires positive-dimensional float32 queries"); + } + if (ground_truth != nullptr && + (ground_truth->GetNumElements() != queries->GetNumElements() || + ground_truth->GetDim() <= 0 || ground_truth->GetIds() == nullptr)) { + throw std::invalid_argument( + "ground_truth must contain one non-empty id row for every query"); + } + + auto dataset = std::make_shared(); + dataset->query_dataset_ = queries; + dataset->ground_truth_dataset_ = ground_truth; + dataset->vector_type_ = DENSE_VECTORS; + dataset->train_data_type_ = vsag::DATATYPE_FLOAT32; + dataset->test_data_type_ = vsag::DATATYPE_FLOAT32; + dataset->train_data_size_ = sizeof(float); + dataset->test_data_size_ = sizeof(float); + dataset->number_of_query_ = queries->GetNumElements(); + dataset->dim_ = queries->GetDim(); + dataset->train_shape_ = {0, dataset->dim_}; + dataset->test_shape_ = {dataset->number_of_query_, dataset->dim_}; + dataset->neighbors_shape_ = + ground_truth == nullptr ? shape_t{0, 0} + : shape_t{ground_truth->GetNumElements(), ground_truth->GetDim()}; + dataset->file_path_ = ""; + return dataset; +} + EvalDatasetPtr EvalDataset::Load(const std::string& filename) { H5::H5File file(filename, H5F_ACC_RDONLY); @@ -222,14 +368,27 @@ EvalDataset::Load(const std::string& filename) { has_multi_vectors = datasets.count("train_multi_vectors") && datasets.count("test_multi_vectors") && datasets.count("train_vector_counts") && datasets.count("test_vector_counts"); + auto require_dataset = [&datasets](const std::string& name) { + if (datasets.count(name) == 0) { + throw std::invalid_argument("missing required HDF5 dataset '" + name + "'"); + } + }; if (not has_multi_vectors) { - assert(datasets.count("train")); - assert(datasets.count("test")); + require_dataset("train"); + require_dataset("test"); } - assert(datasets.count("neighbors")); - assert(datasets.count("distances")); - has_labels = datasets.count("train_labels") && datasets.count("test_labels"); + require_dataset("neighbors"); + require_dataset("distances"); + const bool has_train_labels = datasets.count("train_labels") > 0; + const bool has_test_labels = datasets.count("test_labels") > 0; + if (has_train_labels != has_test_labels) { + throw std::invalid_argument("train_labels and test_labels must be provided together"); + } + has_labels = has_train_labels; has_valid_ratio = datasets.count("valid_ratios") > 0; + if (has_valid_ratio && not has_labels) { + throw std::invalid_argument("valid_ratios requires train_labels and test_labels"); + } } auto obj = std::make_shared(); @@ -250,9 +409,11 @@ EvalDataset::Load(const std::string& filename) { obj->vector_type_ = SPARSE_VECTORS; } else if (type == "multi_vector") { obj->vector_type_ = MULTI_VECTORS; + } else { + throw std::invalid_argument("unsupported HDF5 dataset type '" + type + "'"); } } catch (H5::Exception& err) { - throw std::runtime_error("fail to read metric: there is no 'type' in the dataset"); + throw std::runtime_error("failed to read HDF5 dataset type attribute"); } } @@ -270,7 +431,16 @@ EvalDataset::Load(const std::string& filename) { shape_t train_shape(0, 0); shape_t test_shape(0, 0); auto neighbors_shape = get_shape(file, "neighbors"); + auto distances_shape = get_shape(file, "distances"); logger::debug("neighbors.shape: {}", to_string(neighbors_shape)); + if (neighbors_shape.first <= 0 || neighbors_shape.second <= 0) { + throw std::invalid_argument("neighbors must have shape (query_count, K) with K > 0"); + } + if (distances_shape != neighbors_shape) { + throw std::invalid_argument( + "distances shape must match neighbors shape: neighbors=" + to_string(neighbors_shape) + + ", distances=" + to_string(distances_shape)); + } if (obj->vector_type_ == MULTI_VECTORS) { // For multi-vector datasets, N and Q come from vector_counts @@ -283,7 +453,17 @@ EvalDataset::Load(const std::string& filename) { logger::debug("train.shape: {}", to_string(train_shape)); test_shape = get_shape(file, "test"); logger::debug("test.shape: {}", to_string(test_shape)); - assert(train_shape.second == test_shape.second); + if (train_shape.second != test_shape.second) { + throw std::invalid_argument("train and test vector dimensions must match: train=" + + to_string(train_shape) + ", test=" + to_string(test_shape)); + } + } + + if (train_shape.first <= 0 || test_shape.first <= 0) { + throw std::invalid_argument("train and test datasets must not be empty"); + } + if (obj->vector_type_ == DENSE_VECTORS && (train_shape.second <= 0 || test_shape.second <= 0)) { + throw std::invalid_argument("dense train and test vector dimensions must be positive"); } obj->train_shape_ = train_shape; @@ -598,6 +778,37 @@ EvalDataset::Load(const std::string& filename) { } } + if (obj->number_of_base_ <= 0 || obj->number_of_query_ <= 0) { + throw std::invalid_argument("train and test datasets must contain vectors"); + } + if (obj->vector_type_ != SPARSE_VECTORS && obj->dim_ <= 0) { + throw std::invalid_argument("train and test vector dimensions must be positive"); + } + if (neighbors_shape.first != obj->number_of_query_) { + throw std::invalid_argument("neighbors row count must match query count: neighbors=" + + std::to_string(neighbors_shape.first) + + ", queries=" + std::to_string(obj->number_of_query_)); + } + if (has_labels) { + const auto train_labels_shape = get_shape(file, "train_labels"); + const auto test_labels_shape = get_shape(file, "test_labels"); + if (train_labels_shape.second != 0 || train_labels_shape.first != obj->number_of_base_) { + throw std::invalid_argument( + "train_labels must be one-dimensional with one label per " + "base vector"); + } + if (test_labels_shape.second != 0 || test_labels_shape.first != obj->number_of_query_) { + throw std::invalid_argument( + "test_labels must be one-dimensional with one label per query"); + } + } + if (has_valid_ratio) { + const auto valid_ratios_shape = get_shape(file, "valid_ratios"); + if (valid_ratios_shape.second != 0 || valid_ratios_shape.first <= 0) { + throw std::invalid_argument("valid_ratios must be a non-empty one-dimensional dataset"); + } + } + try { H5::Attribute attr = file.openAttribute("distance"); H5::StrType str_type = attr.getStrType(); @@ -625,6 +836,9 @@ EvalDataset::Load(const std::string& filename) { std::sqrt(vsag::InnerProduct(query1, query1, qty_ptr) * vsag::InnerProduct(query2, query2, qty_ptr)); }; + } else { + throw std::invalid_argument("unsupported HDF5 distance '" + metric + + "' for dense vectors"); } } else { if (metric == "ip") { @@ -635,8 +849,8 @@ EvalDataset::Load(const std::string& filename) { qty_ptr); }; } else { - throw std::runtime_error("no support for sparse vectors with " + metric + - " distance"); + throw std::invalid_argument("unsupported HDF5 distance '" + metric + + "' for sparse vectors"); } } } catch (H5::Exception& err) { @@ -650,6 +864,12 @@ EvalDataset::Load(const std::string& filename) { H5::DataSpace dataspace = dataset.getSpace(); H5::FloatType datatype(H5::PredType::NATIVE_INT64); dataset.read(obj->neighbors_.get(), datatype, dataspace); + const auto neighbor_count = neighbors_shape.first * neighbors_shape.second; + for (int64_t i = 0; i < neighbor_count; ++i) { + if (obj->neighbors_[i] < 0 || obj->neighbors_[i] >= obj->number_of_base_) { + throw std::invalid_argument("neighbors contains an id outside the base dataset"); + } + } } { @@ -687,6 +907,7 @@ EvalDataset::Load(const std::string& filename) { } } + obj->InitializeTrainIds(nullptr); return obj; } @@ -774,6 +995,10 @@ serialize_token_sequences(const std::vector& vectors, void EvalDataset::Save(const EvalDatasetPtr& dataset, const std::string& filename) { + if (dataset->base_dataset_ != nullptr || dataset->query_dataset_ != nullptr || + dataset->ground_truth_dataset_ != nullptr) { + throw std::invalid_argument("saving an in-memory EvalDataset view is not supported"); + } H5File file(filename, H5F_ACC_TRUNC); // write vector type attribute diff --git a/tools/eval/eval_dataset.h b/tools/eval/eval_dataset.h index 8aa85341fb..1d117da0c4 100644 --- a/tools/eval/eval_dataset.h +++ b/tools/eval/eval_dataset.h @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include "H5Cpp.h" @@ -42,12 +43,24 @@ class EvalDataset { static EvalDatasetPtr Load(const std::string& filename); + static EvalDatasetPtr + FromDatasets(const vsag::DatasetPtr& base, + const vsag::DatasetPtr& queries, + const vsag::DatasetPtr& ground_truth, + const std::string& metric_type); + + static EvalDatasetPtr + FromSearchDatasets(const vsag::DatasetPtr& queries, const vsag::DatasetPtr& ground_truth); + static void Save(const EvalDatasetPtr& dataset, const std::string& filename); public: [[nodiscard]] const void* GetTrain() const { + if (base_dataset_ != nullptr) { + return base_dataset_->GetFloat32Vectors(); + } if (vector_type_ == DENSE_VECTORS) { return train_.get(); } @@ -59,6 +72,9 @@ class EvalDataset { [[nodiscard]] const void* GetTest() const { + if (query_dataset_ != nullptr) { + return query_dataset_->GetFloat32Vectors(); + } if (vector_type_ == DENSE_VECTORS) { return test_.get(); } @@ -111,7 +127,7 @@ class EvalDataset { [[nodiscard]] const void* GetOneTrain(int64_t id) const { if (vector_type_ == DENSE_VECTORS) { - return train_.get() + id * dim_ * train_data_size_; + return static_cast(GetTrain()) + id * dim_ * train_data_size_; } if (vector_type_ == SPARSE_VECTORS) { return sparse_train_.data() + id; @@ -122,7 +138,7 @@ class EvalDataset { [[nodiscard]] const void* GetOneTest(int64_t id) const { if (vector_type_ == DENSE_VECTORS) { - return test_.get() + id * dim_ * test_data_size_; + return static_cast(GetTest()) + id * dim_ * test_data_size_; } if (vector_type_ == SPARSE_VECTORS) { return sparse_test_.data() + id; @@ -132,17 +148,56 @@ class EvalDataset { [[nodiscard]] int64_t GetNearestNeighbor(int64_t i) const { - return neighbors_[i * neighbors_shape_.second]; + const auto* neighbors = GetNeighbors(i); + return neighbors == nullptr ? -1 : neighbors[0]; } - [[nodiscard]] int64_t* + [[nodiscard]] const int64_t* GetNeighbors(int64_t i) const { - return neighbors_.get() + i * neighbors_shape_.second; + if (ground_truth_dataset_ != nullptr) { + return ground_truth_dataset_->GetIds() + i * neighbors_shape_.second; + } + return neighbors_ == nullptr ? nullptr : neighbors_.get() + i * neighbors_shape_.second; } - [[nodiscard]] float* + [[nodiscard]] const float* GetDistances(int64_t i) const { - return distances_.get() + i * neighbors_shape_.second; + if (ground_truth_dataset_ != nullptr) { + const auto* distances = ground_truth_dataset_->GetDistances(); + return distances == nullptr ? nullptr : distances + i * neighbors_shape_.second; + } + return distances_ == nullptr ? nullptr : distances_.get() + i * neighbors_shape_.second; + } + + [[nodiscard]] const int64_t* + GetTrainIds() const { + if (base_dataset_ != nullptr && base_dataset_->GetIds() != nullptr) { + return base_dataset_->GetIds(); + } + // A null pointer represents implicit row-number IDs. + return nullptr; + } + + [[nodiscard]] const std::string* + GetTrainPaths() const { + return base_dataset_ == nullptr ? nullptr : base_dataset_->GetPaths(); + } + + [[nodiscard]] const std::string* + GetTestPaths() const { + return query_dataset_ == nullptr ? nullptr : query_dataset_->GetPaths(); + } + + [[nodiscard]] const void* + GetOneTrainById(int64_t id) const { + if (train_ids_are_identity_) { + if (id < 0 || id >= number_of_base_) { + return nullptr; + } + return GetOneTrain(id); + } + const auto found = train_id_to_row_.find(id); + return found == train_id_to_row_.end() ? nullptr : GetOneTrain(found->second); } [[nodiscard]] int64_t @@ -155,6 +210,16 @@ class EvalDataset { return number_of_query_; } + [[nodiscard]] uint64_t + GetGroundTruthK() const { + return neighbors_shape_.second > 0 ? static_cast(neighbors_shape_.second) : 0; + } + + [[nodiscard]] const std::string& + GetMetric() const { + return metric_; + } + [[nodiscard]] int64_t GetDim() const { return dim_; @@ -246,6 +311,9 @@ class EvalDataset { } private: + void + InitializeTrainIds(const int64_t* ids); + using shape_t = std::pair; static std::unordered_set get_datasets(const H5::H5File& file) { @@ -267,14 +335,15 @@ class EvalDataset { get_shape(const H5::H5File& file, const std::string& dataset_name) { H5::DataSet dataset = file.openDataSet(dataset_name); H5::DataSpace dataspace = dataset.getSpace(); - hsize_t dims_out[2]; - int ndims = dataspace.getSimpleExtentDims(dims_out, NULL); + const int ndims = dataspace.getSimpleExtentNdims(); + if (ndims != 1 && ndims != 2) { + throw std::runtime_error("unsupported dataset rank: " + std::to_string(ndims)); + } + hsize_t dims_out[2] = {0, 0}; + dataspace.getSimpleExtentDims(dims_out, NULL); if (ndims == 1) { return std::make_pair(dims_out[0], 0); } - if (ndims != 2) { - throw std::runtime_error("unsupported dataset rank: " + std::to_string(ndims)); - } return std::make_pair(dims_out[0], dims_out[1]); } @@ -291,6 +360,7 @@ class EvalDataset { std::shared_ptr test_; std::shared_ptr neighbors_; std::shared_ptr distances_; + bool train_ids_are_identity_{false}; std::shared_ptr train_labels_{nullptr}; std::shared_ptr test_labels_{nullptr}; std::shared_ptr valid_ratio_; @@ -308,6 +378,11 @@ class EvalDataset { std::string file_path_; std::string metric_; + vsag::DatasetPtr base_dataset_; + vsag::DatasetPtr query_dataset_; + vsag::DatasetPtr ground_truth_dataset_; + std::unordered_map train_id_to_row_; + std::vector sparse_train_; std::vector sparse_test_; diff --git a/tools/eval/eval_dataset_test.cpp b/tools/eval/eval_dataset_test.cpp index 7a66d52048..669091da10 100644 --- a/tools/eval/eval_dataset_test.cpp +++ b/tools/eval/eval_dataset_test.cpp @@ -16,13 +16,23 @@ #include "eval_dataset.h" #include +#include #include +#include +#include #include #include +#include +#include #include #include +#include "case/eval_case.h" +#include "evaluator.h" +#include "monitor/recall_monitor.h" +#include "vsag/factory.h" + namespace { using vsag::SparseVector; @@ -30,28 +40,29 @@ using vsag::eval::EvalDataset; using vsag::eval::EvalDatasetPtr; EvalDatasetPtr -BuildSparseDataset(bool with_token_sequences) { +BuildSparseDataset(bool with_token_sequences, bool all_empty = false) { // Build a tiny sparse dataset (3 train, 2 test) and optionally attach // the original tokenized term-id sequences. auto ds = std::make_shared(); // Sparse train vectors. std::vector train(3); - train[0].len_ = 2; - train[0].ids_ = new uint32_t[2]{1, 5}; - train[0].vals_ = new float[2]{0.5f, 1.0f}; - train[1].len_ = 1; - train[1].ids_ = new uint32_t[1]{3}; - train[1].vals_ = new float[1]{0.25f}; - train[2].len_ = 0; // empty sparse vector - std::vector test(2); - test[0].len_ = 2; - test[0].ids_ = new uint32_t[2]{2, 7}; - test[0].vals_ = new float[2]{0.4f, 0.6f}; - test[1].len_ = 1; - test[1].ids_ = new uint32_t[1]{9}; - test[1].vals_ = new float[1]{1.0f}; + if (not all_empty) { + train[0].len_ = 2; + train[0].ids_ = new uint32_t[2]{1, 5}; + train[0].vals_ = new float[2]{0.5f, 1.0f}; + train[1].len_ = 1; + train[1].ids_ = new uint32_t[1]{3}; + train[1].vals_ = new float[1]{0.25f}; + + test[0].len_ = 2; + test[0].ids_ = new uint32_t[2]{2, 7}; + test[0].vals_ = new float[2]{0.4f, 0.6f}; + test[1].len_ = 1; + test[1].ids_ = new uint32_t[1]{9}; + test[1].vals_ = new float[1]{1.0f}; + } if (with_token_sequences) { train[0].token_seq_len_ = 4; @@ -141,6 +152,447 @@ TempPath(const std::string& tag) { } // namespace +TEST_CASE("EvalDataset builds a dense in-memory view with original ids", "[ut][eval_dataset]") { + constexpr int64_t dim = 2; + std::vector base_vectors{1.0F, 2.0F, 3.0F, 4.0F, 5.0F, 6.0F}; + std::vector base_ids{101, 7, 42}; + auto base = vsag::Dataset::Make(); + base->NumElements(3) + ->Dim(dim) + ->Ids(base_ids.data()) + ->Float32Vectors(base_vectors.data()) + ->Owner(false); + + std::vector query_vectors{1.0F, 2.0F, 5.0F, 6.0F}; + auto queries = vsag::Dataset::Make(); + queries->NumElements(2)->Dim(dim)->Float32Vectors(query_vectors.data())->Owner(false); + + std::vector ground_truth_ids{101, 7, 42, 7}; + auto ground_truth = vsag::Dataset::Make(); + ground_truth->NumElements(2)->Dim(2)->Ids(ground_truth_ids.data())->Owner(false); + + auto dataset = EvalDataset::FromDatasets(base, queries, ground_truth, "l2"); + REQUIRE(dataset->GetTrain() == base_vectors.data()); + REQUIRE(dataset->GetTest() == query_vectors.data()); + REQUIRE(dataset->GetTrainIds() == base_ids.data()); + REQUIRE(dataset->GetMetric() == "euclidean"); + REQUIRE(dataset->GetGroundTruthK() == 2); + REQUIRE(dataset->GetNeighbors(1)[0] == 42); + REQUIRE(static_cast(dataset->GetOneTrainById(7))[0] == 3.0F); + REQUIRE(dataset->GetOneTrainById(999) == nullptr); + + int64_t one_result = 101; + vsag::eval::SearchRecord record{ + &one_result, dataset->GetNeighbors(0), dataset.get(), dataset->GetOneTest(0), 1, 2}; + vsag::eval::RecallMonitor recall_monitor(1); + recall_monitor.SetMetrics("avg_recall"); + recall_monitor.Record(&record); + REQUIRE(recall_monitor.GetResult()["recall_avg"].get() == 0.5); + + auto without_ground_truth = EvalDataset::FromDatasets(base, queries, nullptr, "cosine"); + REQUIRE(without_ground_truth->GetGroundTruthK() == 0); + REQUIRE(without_ground_truth->GetNeighbors(0) == nullptr); + + const auto save_path = TempPath("in_memory_save"); + REQUIRE_THROWS_WITH(EvalDataset::Save(dataset, save_path), + "saving an in-memory EvalDataset view is not supported"); + REQUIRE_FALSE(std::filesystem::exists(save_path)); +} + +TEST_CASE("EvalDataset builds a query-only view for id recall", "[ut][eval_dataset]") { + constexpr int64_t dim = 2; + std::vector query_vectors{1.0F, 2.0F, 3.0F, 4.0F}; + auto queries = vsag::Dataset::Make() + ->NumElements(2) + ->Dim(dim) + ->Float32Vectors(query_vectors.data()) + ->Owner(false); + + std::vector ground_truth_ids{10, 20, 30, 40}; + auto ground_truth = + vsag::Dataset::Make()->NumElements(2)->Dim(2)->Ids(ground_truth_ids.data())->Owner(false); + + auto dataset = EvalDataset::FromSearchDatasets(queries, ground_truth); + REQUIRE(dataset->GetTrain() == nullptr); + REQUIRE(dataset->GetTest() == query_vectors.data()); + REQUIRE(dataset->GetNumberOfBase() == 0); + REQUIRE(dataset->GetNumberOfQuery() == 2); + REQUIRE(dataset->GetGroundTruthK() == 2); + + int64_t result_ids[]{10, 99}; + vsag::eval::SearchRecord record{ + result_ids, dataset->GetNeighbors(0), dataset.get(), dataset->GetOneTest(0), 2, 2}; + vsag::eval::RecallMonitor recall_monitor(1, true); + recall_monitor.SetMetrics("avg_recall"); + recall_monitor.Record(&record); + REQUIRE(recall_monitor.GetResult()["recall_avg"].get() == 0.5); + + REQUIRE_THROWS_WITH(EvalDataset::FromSearchDatasets(nullptr, ground_truth), + "queries dataset is required and must not be empty"); +} + +TEST_CASE("EvalDataset rejects overflowing ground-truth ID counts", "[ut][eval_dataset]") { + std::vector vectors{0.0F, 0.0F}; + int64_t base_id = 0; + auto base = vsag::Dataset::Make() + ->NumElements(1) + ->Dim(2) + ->Ids(&base_id) + ->Float32Vectors(vectors.data()) + ->Owner(false); + auto queries = vsag::Dataset::Make() + ->NumElements(std::numeric_limits::max()) + ->Dim(2) + ->Float32Vectors(vectors.data()) + ->Owner(false); + auto ground_truth = vsag::Dataset::Make() + ->NumElements(std::numeric_limits::max()) + ->Dim(2) + ->Ids(&base_id) + ->Owner(false); + + REQUIRE_THROWS_WITH(EvalDataset::FromDatasets(base, queries, ground_truth, "l2"), + "ground_truth contains too many ids"); +} + +TEST_CASE("EvalDataset rejects incomplete and incompatible HDF5 schemas", "[ut][eval_dataset]") { + const std::vector required_datasets{"train", "test", "neighbors", "distances"}; + for (const auto& missing : required_datasets) { + const auto path = TempPath("missing_" + missing); + { + H5::H5File file(path, H5F_ACC_TRUNC); + hsize_t dims[2] = {1, 1}; + H5::DataSpace space(2, dims); + for (const auto& name : required_datasets) { + if (name != missing) { + file.createDataSet(name, H5::PredType::NATIVE_FLOAT, space); + } + } + } + REQUIRE_THROWS_WITH( + EvalDataset::Load(path), + Catch::Matchers::ContainsSubstring("missing required HDF5 dataset '" + missing + "'")); + std::remove(path.c_str()); + } + + const auto path = TempPath("dimension_mismatch"); + { + H5::H5File file(path, H5F_ACC_TRUNC); + hsize_t train_dims[2] = {1, 2}; + hsize_t test_dims[2] = {1, 3}; + hsize_t result_dims[2] = {1, 1}; + H5::DataSpace train_space(2, train_dims); + H5::DataSpace test_space(2, test_dims); + H5::DataSpace result_space(2, result_dims); + file.createDataSet("train", H5::PredType::NATIVE_FLOAT, train_space); + file.createDataSet("test", H5::PredType::NATIVE_FLOAT, test_space); + file.createDataSet("neighbors", H5::PredType::NATIVE_INT64, result_space); + file.createDataSet("distances", H5::PredType::NATIVE_FLOAT, result_space); + } + REQUIRE_THROWS_WITH( + EvalDataset::Load(path), + Catch::Matchers::ContainsSubstring("train and test vector dimensions must match")); + std::remove(path.c_str()); + + const auto result_shape_path = TempPath("result_shape_mismatch"); + { + H5::H5File file(result_shape_path, H5F_ACC_TRUNC); + hsize_t vector_dims[2] = {2, 2}; + hsize_t neighbor_dims[2] = {2, 1}; + hsize_t distance_dims[2] = {1, 1}; + H5::DataSpace vector_space(2, vector_dims); + H5::DataSpace neighbor_space(2, neighbor_dims); + H5::DataSpace distance_space(2, distance_dims); + file.createDataSet("train", H5::PredType::NATIVE_FLOAT, vector_space); + file.createDataSet("test", H5::PredType::NATIVE_FLOAT, vector_space); + file.createDataSet("neighbors", H5::PredType::NATIVE_INT64, neighbor_space); + file.createDataSet("distances", H5::PredType::NATIVE_FLOAT, distance_space); + } + REQUIRE_THROWS_WITH( + EvalDataset::Load(result_shape_path), + Catch::Matchers::ContainsSubstring("distances shape must match neighbors shape")); + std::remove(result_shape_path.c_str()); + + const auto query_count_path = TempPath("query_count_mismatch"); + { + H5::H5File file(query_count_path, H5F_ACC_TRUNC); + hsize_t train_dims[2] = {2, 2}; + hsize_t test_dims[2] = {2, 2}; + hsize_t result_dims[2] = {1, 1}; + H5::DataSpace train_space(2, train_dims); + H5::DataSpace test_space(2, test_dims); + H5::DataSpace result_space(2, result_dims); + file.createDataSet("train", H5::PredType::NATIVE_FLOAT, train_space); + file.createDataSet("test", H5::PredType::NATIVE_FLOAT, test_space); + file.createDataSet("neighbors", H5::PredType::NATIVE_INT64, result_space); + file.createDataSet("distances", H5::PredType::NATIVE_FLOAT, result_space); + } + REQUIRE_THROWS_WITH( + EvalDataset::Load(query_count_path), + Catch::Matchers::ContainsSubstring("neighbors row count must match query count")); + std::remove(query_count_path.c_str()); + + const auto unsupported_type_path = TempPath("unsupported_type"); + { + H5::H5File file(unsupported_type_path, H5F_ACC_TRUNC); + hsize_t dims[2] = {1, 1}; + H5::DataSpace space(2, dims); + file.createDataSet("train", H5::PredType::NATIVE_FLOAT, space); + file.createDataSet("test", H5::PredType::NATIVE_FLOAT, space); + file.createDataSet("neighbors", H5::PredType::NATIVE_INT64, space); + file.createDataSet("distances", H5::PredType::NATIVE_FLOAT, space); + H5::StrType string_type(H5::PredType::C_S1, H5T_VARIABLE); + auto attribute = file.createAttribute("type", string_type, H5::DataSpace(H5S_SCALAR)); + std::string type = "unsupported"; + attribute.write(string_type, type); + } + REQUIRE_THROWS_WITH( + EvalDataset::Load(unsupported_type_path), + Catch::Matchers::ContainsSubstring("unsupported HDF5 dataset type 'unsupported'")); + std::remove(unsupported_type_path.c_str()); + + const auto unsupported_rank_path = TempPath("unsupported_rank"); + { + H5::H5File file(unsupported_rank_path, H5F_ACC_TRUNC); + hsize_t train_dims[3] = {1, 1, 1}; + hsize_t matrix_dims[2] = {1, 1}; + H5::DataSpace train_space(3, train_dims); + H5::DataSpace matrix_space(2, matrix_dims); + file.createDataSet("train", H5::PredType::NATIVE_FLOAT, train_space); + file.createDataSet("test", H5::PredType::NATIVE_FLOAT, matrix_space); + file.createDataSet("neighbors", H5::PredType::NATIVE_INT64, matrix_space); + file.createDataSet("distances", H5::PredType::NATIVE_FLOAT, matrix_space); + } + REQUIRE_THROWS_WITH(EvalDataset::Load(unsupported_rank_path), + Catch::Matchers::ContainsSubstring("unsupported dataset rank: 3")); + std::remove(unsupported_rank_path.c_str()); + + const auto label_shape_path = TempPath("label_shape_mismatch"); + { + H5::H5File file(label_shape_path, H5F_ACC_TRUNC); + hsize_t train_dims[2] = {2, 2}; + hsize_t test_dims[2] = {1, 2}; + hsize_t result_dims[2] = {1, 1}; + hsize_t train_label_dims[1] = {1}; + hsize_t test_label_dims[1] = {1}; + H5::DataSpace train_space(2, train_dims); + H5::DataSpace test_space(2, test_dims); + H5::DataSpace result_space(2, result_dims); + H5::DataSpace train_label_space(1, train_label_dims); + H5::DataSpace test_label_space(1, test_label_dims); + file.createDataSet("train", H5::PredType::NATIVE_FLOAT, train_space); + file.createDataSet("test", H5::PredType::NATIVE_FLOAT, test_space); + file.createDataSet("neighbors", H5::PredType::NATIVE_INT64, result_space); + file.createDataSet("distances", H5::PredType::NATIVE_FLOAT, result_space); + file.createDataSet("train_labels", H5::PredType::NATIVE_INT64, train_label_space); + file.createDataSet("test_labels", H5::PredType::NATIVE_INT64, test_label_space); + } + REQUIRE_THROWS_WITH(EvalDataset::Load(label_shape_path), + Catch::Matchers::ContainsSubstring( + "train_labels must be one-dimensional with one label per base vector")); + std::remove(label_shape_path.c_str()); + + const auto unsupported_distance_path = TempPath("unsupported_distance"); + { + H5::H5File file(unsupported_distance_path, H5F_ACC_TRUNC); + hsize_t dims[2] = {1, 1}; + H5::DataSpace space(2, dims); + file.createDataSet("train", H5::PredType::NATIVE_FLOAT, space); + file.createDataSet("test", H5::PredType::NATIVE_FLOAT, space); + file.createDataSet("neighbors", H5::PredType::NATIVE_INT64, space); + file.createDataSet("distances", H5::PredType::NATIVE_FLOAT, space); + H5::StrType string_type(H5::PredType::C_S1, H5T_VARIABLE); + auto attribute = file.createAttribute("distance", string_type, H5::DataSpace(H5S_SCALAR)); + std::string distance = "unsupported"; + attribute.write(string_type, distance); + } + REQUIRE_THROWS_WITH(EvalDataset::Load(unsupported_distance_path), + Catch::Matchers::ContainsSubstring( + "unsupported HDF5 distance 'unsupported' for dense vectors")); + std::remove(unsupported_distance_path.c_str()); +} + +TEST_CASE("EvaluateSearch validates inputs and propagates search errors", "[ut][eval_dataset]") { + constexpr int64_t dim = 2; + std::vector base_vectors{0.0F, 0.0F, 1.0F, 0.0F, 0.0F, 1.0F, 1.0F, 1.0F}; + std::vector base_ids{0, 1, 2, 3}; + auto base = vsag::Dataset::Make(); + base->NumElements(4) + ->Dim(dim) + ->Ids(base_ids.data()) + ->Float32Vectors(base_vectors.data()) + ->Owner(false); + + std::vector query_vectors{0.0F, 0.0F, 1.0F, 1.0F}; + auto queries = vsag::Dataset::Make(); + queries->NumElements(2)->Dim(dim)->Float32Vectors(query_vectors.data())->Owner(false); + + std::vector ground_truth_ids{0, 3}; + auto ground_truth = vsag::Dataset::Make(); + ground_truth->NumElements(2)->Dim(1)->Ids(ground_truth_ids.data())->Owner(false); + auto dataset = EvalDataset::FromDatasets(base, queries, ground_truth, "l2"); + + const std::string create_params = R"( + { + "dim": 2, + "dtype": "float32", + "metric_type": "l2", + "index_param": { + "base_quantization_type": "fp32", + "max_degree": 8, + "ef_construction": 20 + } + })"; + auto created = vsag::Factory::CreateIndex("hgraph", create_params); + REQUIRE(created.has_value()); + auto index = created.value(); + + vsag::eval::EvalConfig build_config; + build_config.index_name = "hgraph"; + build_config.enable_tps = false; + build_config.enable_memory = false; + const auto build_result = vsag::eval::EvaluateBuild(index, dataset, build_config); + REQUIRE(build_result.contains("duration(s)")); + REQUIRE_FALSE(build_result.contains("tps")); + REQUIRE(build_result["index_info"].is_object()); + REQUIRE(build_result["index_info"].empty()); + + vsag::eval::EvalConfig config; + config.index_name = "hgraph"; + config.search_param = R"({"hgraph":{"ef_search":8}})"; + config.top_k = 1; + config.search_query_count = 2; + const auto caller_thread_count = omp_get_max_threads(); + config.num_threads_searching = caller_thread_count == 1 ? 2 : 1; + config.enable_recall = false; + config.enable_percent_recall = false; + config.enable_qps = true; + config.enable_tps = false; + config.enable_memory = false; + config.enable_latency = false; + config.enable_percent_latency = false; + + const auto qps_only = vsag::eval::EvaluateSearch(index, dataset, config); + REQUIRE(qps_only.contains("qps")); + REQUIRE(qps_only["measurement_sample_count"].get() == 2); + REQUIRE(qps_only["index_info"].is_object()); + REQUIRE(qps_only["index_info"].empty()); + REQUIRE(omp_get_max_threads() == caller_thread_count); + + config.search_param = R"({"hgraph":{"ef_search":0}})"; + REQUIRE_THROWS_WITH( + vsag::eval::EvaluateSearch(index, dataset, config), + Catch::Matchers::ContainsSubstring("query error: ef_search(0) must be at least 1")); + REQUIRE(omp_get_max_threads() == caller_thread_count); + + config.search_param = R"({"hgraph":{"ef_search":8}})"; + config.top_k = 0; + REQUIRE_THROWS_WITH(vsag::eval::EvaluateSearch(index, dataset, config), + "evaluation top_k must be positive"); + + config.top_k = 1; + config.search_mode = "range"; + REQUIRE_THROWS_WITH(vsag::eval::EvaluateSearch(index, dataset, config), + "in-memory evaluation supports only knn search mode"); + + config.search_mode = "knn"; + config.num_threads_searching = 0; + REQUIRE_THROWS_WITH(vsag::eval::EvaluateSearch(index, dataset, config), + "evaluation search thread count must be positive"); + + config.num_threads_searching = 2; + config.search_query_count = 0; + REQUIRE_THROWS_WITH(vsag::eval::EvaluateSearch(index, dataset, config), + "evaluation search query count must be positive"); + + config.search_query_count = 2; + config.top_k = 1; + config.enable_recall = true; + auto without_ground_truth = EvalDataset::FromDatasets(base, queries, nullptr, "l2"); + REQUIRE_THROWS_WITH(vsag::eval::EvaluateSearch(index, without_ground_truth, config), + "evaluation ground truth must contain at least top_k neighbors per query"); + + config.top_k = 2; + REQUIRE_THROWS_WITH(vsag::eval::EvaluateSearch(index, dataset, config), + "evaluation ground truth must contain at least top_k neighbors per query"); +} + +TEST_CASE("EvalCase builds and searches an in-memory dataset with original ids", + "[ut][eval_dataset]") { + constexpr int64_t dim = 2; + std::vector base_vectors{1.0F, 2.0F, 3.0F, 4.0F, 5.0F, 6.0F}; + std::vector base_ids{101, 7, 42}; + auto base = vsag::Dataset::Make(); + base->NumElements(3) + ->Dim(dim) + ->Ids(base_ids.data()) + ->Float32Vectors(base_vectors.data()) + ->Owner(false); + + std::vector query_vectors{5.0F, 6.0F}; + auto queries = vsag::Dataset::Make(); + queries->NumElements(1)->Dim(dim)->Float32Vectors(query_vectors.data())->Owner(false); + + std::vector ground_truth_ids{42}; + auto ground_truth = vsag::Dataset::Make(); + ground_truth->NumElements(1)->Dim(1)->Ids(ground_truth_ids.data())->Owner(false); + auto dataset = EvalDataset::FromDatasets(base, queries, ground_truth, "l2"); + + vsag::eval::EvalConfig config; + config.index_name = "brute_force"; + config.index_path = "/tmp/eval_dataset_memory.index"; + config.build_param = R"({ + "dtype": "float32", + "metric_type": "l2", + "dim": 2, + "index_param": { + "base_quantization_type": "fp32", + "store_raw_vector": true + } + })"; + config.top_k = 1; + config.search_query_count = 1; + config.enable_memory = false; + std::remove(config.index_path.c_str()); + + auto build = vsag::eval::EvalCase::MakeInstance(config, "build", dataset); + REQUIRE(build->Run()["action"] == "build"); + auto search = vsag::eval::EvalCase::MakeInstance(config, "search", dataset); + const auto result = search->Run(); + REQUIRE(result["action"] == "search"); + REQUIRE(result["recall_avg"].get() == 1.0); + REQUIRE(result["statistics_query_count"].get() == 1); + REQUIRE(std::isfinite(result["qps"].get())); + REQUIRE(std::isfinite(result["latency_avg(ms)"].get())); + REQUIRE(std::isfinite(result["latency_detail(ms)"]["p99"].get())); + + std::vector concurrent_query_vectors{5.0F, 6.0F, 1.0F, 2.0F}; + auto concurrent_queries = vsag::Dataset::Make(); + concurrent_queries->NumElements(2) + ->Dim(dim) + ->Float32Vectors(concurrent_query_vectors.data()) + ->Owner(false); + std::vector concurrent_ground_truth_ids{42, 101}; + auto concurrent_ground_truth = vsag::Dataset::Make(); + concurrent_ground_truth->NumElements(2) + ->Dim(1) + ->Ids(concurrent_ground_truth_ids.data()) + ->Owner(false); + auto concurrent_dataset = + EvalDataset::FromDatasets(base, concurrent_queries, concurrent_ground_truth, "l2"); + config.search_query_count = 2; + config.num_threads_searching = 2; + auto concurrent_search = + vsag::eval::EvalCase::MakeInstance(config, "search", concurrent_dataset); + const auto concurrent_result = concurrent_search->Run(); + REQUIRE(concurrent_result["recall_avg"].get() == 1.0); + REQUIRE(concurrent_result["statistics_query_count"].get() == 2); + REQUIRE(std::isfinite(concurrent_result["qps"].get())); + REQUIRE(std::isfinite(concurrent_result["latency_avg(ms)"].get())); + REQUIRE(std::isfinite(concurrent_result["latency_detail(ms)"]["p99"].get())); + std::remove(config.index_path.c_str()); +} + TEST_CASE("EvalDataset sparse round-trip without token sequences", "[ut][eval_dataset]") { auto path = TempPath("nosqry"); { @@ -172,6 +624,8 @@ TEST_CASE("EvalDataset sparse round-trip without token sequences", "[ut][eval_da REQUIRE(loaded->GetNumberOfBase() == 3); REQUIRE(loaded->GetNumberOfQuery() == 2); const auto* train = static_cast(loaded->GetTrain()); + REQUIRE(loaded->GetTrainIds() == nullptr); + REQUIRE(loaded->GetOneTrainById(1) == train + 1); for (int i = 0; i < 3; ++i) { REQUIRE(train[i].token_seq_len_ == 0); REQUIRE(train[i].token_sequence_ == nullptr); @@ -179,6 +633,29 @@ TEST_CASE("EvalDataset sparse round-trip without token sequences", "[ut][eval_da std::remove(path.c_str()); } +TEST_CASE("EvalDataset sparse round-trip preserves all-empty records", "[ut][eval_dataset]") { + auto path = TempPath("all_empty_sparse"); + { + auto ds = BuildSparseDataset(/*with_token_sequences=*/false, /*all_empty=*/true); + EvalDataset::Save(ds, path); + } + + auto loaded = EvalDataset::Load(path); + REQUIRE(loaded->GetVectorType() == vsag::SPARSE_VECTORS); + REQUIRE(loaded->GetNumberOfBase() == 3); + REQUIRE(loaded->GetNumberOfQuery() == 2); + REQUIRE(loaded->GetDim() == 0); + const auto* train = static_cast(loaded->GetTrain()); + const auto* test = static_cast(loaded->GetTest()); + for (uint64_t i = 0; i < 3; ++i) { + REQUIRE(train[i].len_ == 0); + } + for (uint64_t i = 0; i < 2; ++i) { + REQUIRE(test[i].len_ == 0); + } + std::remove(path.c_str()); +} + TEST_CASE("EvalDataset sparse round-trip with token sequences", "[ut][eval_dataset]") { auto path = TempPath("withseq"); { diff --git a/tools/eval/evaluator.cpp b/tools/eval/evaluator.cpp new file mode 100644 index 0000000000..fb3d391512 --- /dev/null +++ b/tools/eval/evaluator.cpp @@ -0,0 +1,98 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "evaluator.h" + +#include + +#include + +#include "case/build_eval_case.h" +#include "case/search_eval_case.h" + +namespace vsag::eval { + +namespace { + +class ScopedOpenMpThreads { +public: + ScopedOpenMpThreads() : original_(omp_get_max_threads()) { + } + + ~ScopedOpenMpThreads() { + omp_set_num_threads(original_); + } + +private: + int original_; +}; + +void +validate(const IndexPtr& index, const EvalDatasetPtr& dataset) { + if (index == nullptr) { + throw std::invalid_argument("evaluation index must not be null"); + } + if (dataset == nullptr) { + throw std::invalid_argument("evaluation dataset must not be null"); + } +} + +void +validate_search(const EvalDatasetPtr& dataset, const EvalConfig& config) { + if (config.search_mode != "knn") { + throw std::invalid_argument("in-memory evaluation supports only knn search mode"); + } + if (config.top_k <= 0) { + throw std::invalid_argument("evaluation top_k must be positive"); + } + if (dataset->GetNumberOfQuery() <= 0) { + throw std::invalid_argument("evaluation dataset must contain at least one query"); + } + if (config.num_threads_searching <= 0) { + throw std::invalid_argument("evaluation search thread count must be positive"); + } + if (config.search_query_count == 0) { + throw std::invalid_argument("evaluation search query count must be positive"); + } + if (config.enable_recall or config.enable_percent_recall) { + const auto requested_top_k = static_cast(config.top_k); + if (dataset->GetGroundTruthK() < requested_top_k || dataset->GetNeighbors(0) == nullptr) { + throw std::invalid_argument( + "evaluation ground truth must contain at least top_k neighbors per query"); + } + if (!config.use_id_based_recall && dataset->GetNumberOfBase() <= 0) { + throw std::invalid_argument("distance-based recall evaluation requires base vectors"); + } + } +} + +} // namespace + +JsonType +EvaluateBuild(const IndexPtr& index, const EvalDatasetPtr& dataset, const EvalConfig& config) { + validate(index, dataset); + BuildEvalCase eval_case("", "", index, config, dataset); + return eval_case.RunInMemory(); +} + +JsonType +EvaluateSearch(const IndexPtr& index, const EvalDatasetPtr& dataset, const EvalConfig& config) { + validate(index, dataset); + validate_search(dataset, config); + ScopedOpenMpThreads openmp_threads; + SearchEvalCase eval_case("", "", index, config, dataset); + return eval_case.RunInMemory(); +} + +} // namespace vsag::eval diff --git a/tools/eval/evaluator.h b/tools/eval/evaluator.h new file mode 100644 index 0000000000..aa5bccd0eb --- /dev/null +++ b/tools/eval/evaluator.h @@ -0,0 +1,32 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "eval_config.h" +#include "eval_dataset.h" +#include "nlohmann/json.hpp" +#include "vsag/index.h" + +namespace vsag::eval { + +using JsonType = nlohmann::json; + +JsonType +EvaluateBuild(const IndexPtr& index, const EvalDatasetPtr& dataset, const EvalConfig& config); + +JsonType +EvaluateSearch(const IndexPtr& index, const EvalDatasetPtr& dataset, const EvalConfig& config); + +} // namespace vsag::eval diff --git a/tools/eval/monitor/monitor.h b/tools/eval/monitor/monitor.h index b963e1e975..cf4cb0e318 100644 --- a/tools/eval/monitor/monitor.h +++ b/tools/eval/monitor/monitor.h @@ -15,6 +15,7 @@ #pragma once +#include #include #include #include @@ -24,6 +25,17 @@ namespace vsag::eval { +class EvalDataset; + +struct SearchRecord { + const int64_t* neighbors{nullptr}; + const int64_t* ground_truth_neighbors{nullptr}; + EvalDataset* dataset{nullptr}; + const void* query_data{nullptr}; + uint64_t result_count{0}; + uint64_t requested_top_k{0}; +}; + class Monitor { public: using JsonType = nlohmann::json; diff --git a/tools/eval/monitor/recall_monitor.cpp b/tools/eval/monitor/recall_monitor.cpp index bbf7db4e28..42e2c79274 100644 --- a/tools/eval/monitor/recall_monitor.cpp +++ b/tools/eval/monitor/recall_monitor.cpp @@ -15,8 +15,12 @@ #include "recall_monitor.h" +#include +#include #include +#include #include +#include #include "../eval_dataset.h" namespace vsag::eval { @@ -26,13 +30,13 @@ static const double THRESHOLD_ERROR = 2e-6; static double get_recall(const float* distances, const float* ground_truth_distances, - size_t recall_num, - size_t top_k) { + uint64_t recall_num, + uint64_t top_k) { std::vector gt_distances(ground_truth_distances, ground_truth_distances + top_k); std::sort(gt_distances.begin(), gt_distances.end()); float threshold = gt_distances[top_k - 1]; - size_t count = 0; - for (size_t i = 0; i < recall_num; ++i) { + uint64_t count = 0; + for (uint64_t i = 0; i < recall_num; ++i) { if (distances[i] <= threshold + THRESHOLD_ERROR) { ++count; } @@ -40,7 +44,25 @@ get_recall(const float* distances, return static_cast(count) / static_cast(top_k); } -RecallMonitor::RecallMonitor(uint64_t max_record_counts) : Monitor("recall_monitor") { +static double +get_id_recall(const int64_t* neighbors, + const int64_t* ground_truth_neighbors, + uint64_t recall_num, + uint64_t top_k) { + std::unordered_set remaining_ground_truth(ground_truth_neighbors, + ground_truth_neighbors + top_k); + uint64_t count = 0; + for (uint64_t i = 0; i < recall_num; ++i) { + // Erasing a match ensures duplicate result IDs are counted only once. + if (remaining_ground_truth.erase(neighbors[i]) > 0) { + ++count; + } + } + return static_cast(count) / static_cast(top_k); +} + +RecallMonitor::RecallMonitor(uint64_t max_record_counts, bool use_id_based_recall) + : Monitor("recall_monitor"), use_id_based_recall_(use_id_based_recall) { if (max_record_counts > 0) { this->recall_records_.reserve(max_record_counts); } @@ -65,28 +87,47 @@ void RecallMonitor::Record(void* input) { std::lock_guard lock(record_mutex_); - auto [neighbors, gt_neighbors, dataset, query_data, topk] = - *(reinterpret_cast*>( - input)); - size_t dim = dataset->GetDim(); + const auto& record = *static_cast(input); + const auto* neighbors = record.neighbors; + const auto* gt_neighbors = record.ground_truth_neighbors; + auto* dataset = record.dataset; + const auto* query_data = record.query_data; + const auto requested_top_k = record.requested_top_k; + const auto result_count = std::min(record.result_count, requested_top_k); + if (requested_top_k == 0) { + this->recall_records_.emplace_back(0.0); + return; + } + if (use_id_based_recall_) { + this->recall_records_.emplace_back( + get_id_recall(neighbors, gt_neighbors, result_count, requested_top_k)); + return; + } + uint64_t dim = dataset->GetDim(); auto distance_func = dataset->GetDistanceFunc(); - auto gt_distances = std::shared_ptr(new float[topk]); - auto distances = std::shared_ptr(new float[topk]); - for (int i = 0; i < topk; ++i) { - distances[i] = distance_func(query_data, dataset->GetOneTrain(neighbors[i]), &dim); - gt_distances[i] = distance_func(query_data, dataset->GetOneTrain(gt_neighbors[i]), &dim); + std::vector gt_distances(requested_top_k); + std::vector distances(result_count); + for (uint64_t i = 0; i < result_count; ++i) { + const auto* result_vector = dataset->GetOneTrainById(neighbors[i]); + distances[i] = result_vector == nullptr ? std::numeric_limits::infinity() + : distance_func(query_data, result_vector, &dim); } - - float val = 0; - if (topk != 0) { - val = get_recall(distances.get(), gt_distances.get(), topk, topk); + for (uint64_t i = 0; i < requested_top_k; ++i) { + const auto* ground_truth_vector = dataset->GetOneTrainById(gt_neighbors[i]); + gt_distances[i] = ground_truth_vector == nullptr + ? std::numeric_limits::infinity() + : distance_func(query_data, ground_truth_vector, &dim); } + + const double val = + get_recall(distances.data(), gt_distances.data(), result_count, requested_top_k); this->recall_records_.emplace_back(val); } void RecallMonitor::SetMetrics(std::string metric) { this->metrics_.emplace_back(std::move(metric)); } + void RecallMonitor::cal_and_set_result(const std::string& metric, Monitor::JsonType& result) { if (metric == "avg_recall") { diff --git a/tools/eval/monitor/recall_monitor.h b/tools/eval/monitor/recall_monitor.h index 1977c55726..b3a52634e6 100644 --- a/tools/eval/monitor/recall_monitor.h +++ b/tools/eval/monitor/recall_monitor.h @@ -22,7 +22,7 @@ namespace vsag::eval { class RecallMonitor : public Monitor { public: - explicit RecallMonitor(uint64_t max_record_counts = 0); + explicit RecallMonitor(uint64_t max_record_counts = 0, bool use_id_based_recall = false); ~RecallMonitor() override = default; @@ -55,6 +55,7 @@ class RecallMonitor : public Monitor { std::vector recall_records_; std::vector metrics_; + bool use_id_based_recall_{false}; }; } // namespace vsag::eval