From 4cc2841aac570f64416a83883b6e2da86cc8a628 Mon Sep 17 00:00:00 2001 From: Roxanne0321 Date: Tue, 21 Jul 2026 19:26:02 +0800 Subject: [PATCH] feat(sindi): add date-window filtering Signed-off-by: Roxanne0321 Assisted-by: Codex:gpt-5 --- README.md | 3 +- .../docs/en/src/advanced/new_serialization.md | 2 + docs/docs/en/src/api/dataset.md | 1 + docs/docs/en/src/indexes/sindi.md | 32 +- .../docs/zh/src/advanced/new_serialization.md | 2 + docs/docs/zh/src/api/dataset.md | 1 + docs/docs/zh/src/indexes/sindi.md | 25 +- examples/cpp/109_index_sindi.cpp | 6 +- include/vsag/constants.h | 1 + include/vsag/dataset.h | 17 + src/algorithm/sindi/sindi.cpp | 526 +++++++++++++----- src/algorithm/sindi/sindi.h | 27 + src/algorithm/sindi/sindi_parameter.cpp | 87 +++ src/algorithm/sindi/sindi_parameter.h | 7 + src/algorithm/sindi/sindi_parameter_test.cpp | 46 ++ src/algorithm/sindi/sindi_test.cpp | 295 ++++++++++ src/analyzer/sindi_analyzer.cpp | 94 +++- src/analyzer/sindi_analyzer.h | 3 +- src/constants.cpp | 1 + src/dataset_impl.cpp | 14 + src/dataset_impl.h | 14 + src/dataset_impl_test.cpp | 43 ++ src/inner_string_params.h | 1 + src/storage/serialization_tags.h | 5 + 24 files changed, 1064 insertions(+), 189 deletions(-) diff --git a/README.md b/README.md index c9bfade54c..d77694f0c1 100644 --- a/README.md +++ b/README.md @@ -268,7 +268,8 @@ Thrive together in VSAG community with users and developers from all around the phase physically remove vectors and shrink storage instead of leaving tombstones. Vector updates use the same public API in both flat and graph phases. - **IVF (Inverted File Index)**: Optimized for large-scale search (high `k`) and batch queries. - - **SINDI (Sparse Inverted Non-redundant Distance Index)**: Optimized sparse vector index. + - **SINDI (Sparse Inverted Non-redundant Distance Index)**: Optimized sparse vector index with + date-window filtering for online queries. - **Advanced Quantization Methods** - **RaBitQ (BQ)**: Extreme compression for minimal memory usage. diff --git a/docs/docs/en/src/advanced/new_serialization.md b/docs/docs/en/src/advanced/new_serialization.md index f4389d24cd..cc76e95627 100644 --- a/docs/docs/en/src/advanced/new_serialization.md +++ b/docs/docs/en/src/advanced/new_serialization.md @@ -193,10 +193,12 @@ SINDI writes these streaming blocks in order: | `label_table` | external labels and label remap | yes | | `sindi_rerank_index` | optional rerank flat index when rerank is enabled | conditional | | `sindi_term_id_mapper` | optional term-id remapping table | conditional | +| `sindi_window_metadata` | window boundaries and date labels | no (absent in legacy streams) | `DeserializeStreaming` restores the full in-memory SINDI index. `Index::Load` can create the SINDI index directly from streaming metadata and currently loads all emitted SINDI blocks into memory. Immutable SINDI runtime serialization is not supported by this streaming path. +When `sindi_window_metadata` is absent, the reader reconstructs legacy fixed-size, undated windows. ## Pyramid Blocks diff --git a/docs/docs/en/src/api/dataset.md b/docs/docs/en/src/api/dataset.md index 06e4fd25ba..46fa17e20a 100644 --- a/docs/docs/en/src/api/dataset.md +++ b/docs/docs/en/src/api/dataset.md @@ -85,6 +85,7 @@ For documents that hold several dense sub-vectors each: | `ExtraInfoSize(int64_t)` | `GetExtraInfoSize()` | `int64_t` | Bytes per extra-info blob. | | `Paths(const std::string*)` | `GetPaths()` | `const std::string*` | Hierarchy paths (Pyramid). Default hierarchy. | | `Paths(const std::string& hierarchy, const std::string*)` | `GetPaths(const std::string& hierarchy)` | `const std::string*` | Paths for a named hierarchy. | +| `Dates(const std::string*)` | `GetDates()` | `const std::string*` | Optional per-element SINDI date labels (`YYYY`, `YYYY/MM`, or `YYYY/MM/DD`). | | `SourceID(const std::string*)` | `GetSourceID()` | `const std::string*` | Optional source identifier. | See [Attribute Filter (Hybrid Search)](../advanced/attribute_filter.md) and diff --git a/docs/docs/en/src/indexes/sindi.md b/docs/docs/en/src/indexes/sindi.md index f8444be520..1ce5bdccb7 100644 --- a/docs/docs/en/src/indexes/sindi.md +++ b/docs/docs/en/src/indexes/sindi.md @@ -12,9 +12,11 @@ pairs and is the only VSAG index that accepts `dtype: "sparse"`. ## How it works -1. **Window-based inverted lists.** Documents are grouped into fixed-size windows - (`window_size`). Within each window, an inverted list per term maps a term id - to the `(doc_id, value)` pairs that mention it. +1. **Window-based inverted lists.** Documents are grouped into windows capped by + `window_size`. When date labels are supplied, each window contains one exact + date label; a label that exceeds `window_size` is split across multiple windows. + Within each window, an inverted list per term maps a term id to the + `(doc_id, value)` pairs that mention it. 2. **Optional pruning and quantization.** During construction, `doc_prune_ratio` drops low-weight terms per document, and `use_quantization` compresses the term values to shrink memory further. @@ -51,6 +53,7 @@ auto base = vsag::Dataset::Make(); base->NumElements(n) ->SparseVectors(sparse_vectors) // vsag::SparseVector* ->Ids(ids) + ->Dates(dates) // optional std::string[n] ->Owner(false); index->Build(base); @@ -59,7 +62,7 @@ auto query = vsag::Dataset::Make(); query->NumElements(1)->SparseVectors(&query_vec)->Owner(false); auto result = index->KnnSearch( query, /*topk=*/10, - R"({"sindi": {"n_candidate": 100}})").value(); + R"({"sindi": {"n_candidate": 100, "date": "2026/07"}})").value(); ``` ## Build parameters @@ -92,6 +95,7 @@ Search-time parameters live under the `sindi` sub-object: | `n_candidate` | int | `0` | Candidate heap size. When `0`, defaults to `SPARSE_AMPLIFICATION_FACTOR · topk` (500×). If set, must satisfy `1 ≤ n_candidate ≤ SPARSE_AMPLIFICATION_FACTOR · topk`. | | `query_prune_ratio` | float | `0.0` | Fraction of lowest-weight query terms skipped (0.0 – 0.9). | | `term_prune_ratio` | float | `0.0` | Fraction of term-list entries skipped (0.0 – 0.9). | +| `date` | string | `""` | Optional date-window selector in `YYYY`, `YYYY/MM`, or `YYYY/MM/DD` form. | SINDI chooses the heap-insertion strategy automatically from the build-time `doc_prune_ratio` and search-time `query_prune_ratio`. With the current `0.1` @@ -107,6 +111,26 @@ auto result = index->KnnSearch( R"({"sindi": {"n_candidate": 200, "query_prune_ratio": 0.1}})").value(); ``` +## Date-window filtering + +Attach one optional date label to each build or add document with +`Dataset::Dates(const std::string*)`. Labels must use a canonical calendar form: +`YYYY`, `YYYY/MM`, or `YYYY/MM/DD`, including two-digit months and days. Documents +with the same exact label share windows; documents without a date (no dates array, +or an empty string entry) share undated windows. Different granularities are +different labels, so a `2026/05/17` document is stored only in a `2026/05/17` +window, not also in a `2026/05` window. + +Pass the online query selector in the SINDI search parameters. A year selects its +year, month, and day windows; a month selects its month and day windows; a day +selects only that exact day. For example, `"date": "2026"` matches `2026`, +`2026/05`, and `2026/05/17`. A missing or empty query date disables date +filtering. A non-empty query date excludes undated windows. + +Date filtering first prunes the windows to search. Any ID filter, bitset, or +`Filter` callback is then applied inside those windows, so the two filters have +AND semantics. Both KNN and range search support the date selector. + ## When to use SINDI - Sparse retrieval with BM25, SPLADE, uniCOIL, or similar learned-sparse encoders. diff --git a/docs/docs/zh/src/advanced/new_serialization.md b/docs/docs/zh/src/advanced/new_serialization.md index 1eba4cc7eb..1683b47d2c 100644 --- a/docs/docs/zh/src/advanced/new_serialization.md +++ b/docs/docs/zh/src/advanced/new_serialization.md @@ -173,10 +173,12 @@ SINDI 按顺序写入以下 streaming blocks： | `label_table` | 外部 label 和 label remap | 是 | | `sindi_rerank_index` | rerank 开启时的可选 rerank flat index | 条件必需 | | `sindi_term_id_mapper` | 可选 term-id remap 表 | 条件必需 | +| `sindi_window_metadata` | 窗口边界与日期标签 | 否（旧版 stream 中不存在） | `DeserializeStreaming` 会恢复完整的内存 SINDI 索引。`Index::Load` 可以直接从 streaming metadata 创建 SINDI 索引对象，当前会把写出的 SINDI blocks 都加载到内存中。immutable SINDI runtime 暂不支持 该 streaming 序列化路径。 +缺少 `sindi_window_metadata` 时，读取端会按旧版固定大小规则恢复为无日期窗口。 ## Pyramid Blocks diff --git a/docs/docs/zh/src/api/dataset.md b/docs/docs/zh/src/api/dataset.md index 42b0ebf57d..a709e0d837 100644 --- a/docs/docs/zh/src/api/dataset.md +++ b/docs/docs/zh/src/api/dataset.md @@ -84,6 +84,7 @@ DatasetPtr DeepCopy(Allocator* allocator = nullptr) const; // 独立副本 | `ExtraInfoSize(int64_t)` | `GetExtraInfoSize()` | `int64_t` | 每个 extra-info 数据块的字节数。 | | `Paths(const std::string*)` | `GetPaths()` | `const std::string*` | 层级路径（Pyramid）。默认层级。 | | `Paths(const std::string& hierarchy, const std::string*)` | `GetPaths(const std::string& hierarchy)` | `const std::string*` | 命名层级的路径。 | +| `Dates(const std::string*)` | `GetDates()` | `const std::string*` | 可选的逐元素 SINDI 日期标签（`YYYY`、`YYYY/MM` 或 `YYYY/MM/DD`）。 | | `SourceID(const std::string*)` | `GetSourceID()` | `const std::string*` | 可选的来源标识。 | 见 [属性过滤（混合搜索）](../advanced/attribute_filter.md) 与 diff --git a/docs/docs/zh/src/indexes/sindi.md b/docs/docs/zh/src/indexes/sindi.md index 823198b5e4..408e717924 100644 --- a/docs/docs/zh/src/indexes/sindi.md +++ b/docs/docs/zh/src/indexes/sindi.md @@ -12,8 +12,9 @@ SINDI（**S**parse **IN**verted **D**ense **I**ndex）是 VSAG 面向 **稀疏 ## 工作原理 -1. **基于窗口的倒排表。** 文档按固定窗口大小（`window_size`）分组，每个窗口独立维护一套 - 倒排表——即“词项 → `(doc_id, value)` 列表”的映射。 +1. **基于窗口的倒排表。** 文档按不超过 `window_size` 的窗口分组。提供日期标签时，每个窗口 + 只包含一个完全相同的日期标签；同一标签超过 `window_size` 时拆分为多个窗口。每个窗口独立 + 维护一套倒排表——即“词项 → `(doc_id, value)` 列表”的映射。 2. **可选的剪枝与量化。** 构建时可通过 `doc_prune_ratio` 按文档粒度丢弃权重最低的词项； 通过 `use_quantization` 压缩词项权重以进一步节省内存。 3. **打分。** 检索时，SINDI 遍历查询向量的非零项，按窗口访问对应的倒排表，使用大小为 @@ -47,6 +48,7 @@ auto base = vsag::Dataset::Make(); base->NumElements(n) ->SparseVectors(sparse_vectors) // vsag::SparseVector* ->Ids(ids) + ->Dates(dates) // 可选的 std::string[n] ->Owner(false); index->Build(base); @@ -55,7 +57,7 @@ auto query = vsag::Dataset::Make(); query->NumElements(1)->SparseVectors(&query_vec)->Owner(false); auto result = index->KnnSearch( query, /*topk=*/10, - R"({"sindi": {"n_candidate": 100}})").value(); + R"({"sindi": {"n_candidate": 100, "date": "2026/07"}})").value(); ``` ## 构建参数 @@ -87,6 +89,7 @@ auto result = index->KnnSearch( | `n_candidate` | int | `0` | 候选堆大小。为 `0` 时自动取 `SPARSE_AMPLIFICATION_FACTOR · topk`（500 倍）；若显式设置，须满足 `1 ≤ n_candidate ≤ SPARSE_AMPLIFICATION_FACTOR · topk` | | `query_prune_ratio` | float | `0.0` | 查询时丢弃权重最低查询项的比例（0.0 – 0.9） | | `term_prune_ratio` | float | `0.0` | 查询时丢弃倒排表中低权项的比例（0.0 – 0.9） | +| `date` | string | `""` | 可选的日期窗口选择条件，格式为 `YYYY`、`YYYY/MM` 或 `YYYY/MM/DD` | SINDI 会根据构建阶段的 `doc_prune_ratio` 与检索阶段的 `query_prune_ratio` 自动选择堆插入策略。按当前 `0.1` 阈值，当两个比例都 `<= 0.1` 时，SINDI 使用 @@ -99,6 +102,22 @@ auto result = index->KnnSearch( R"({"sindi": {"n_candidate": 200, "query_prune_ratio": 0.1}})").value(); ``` +## 日期窗口过滤 + +构建或 Add 时，通过 `Dataset::Dates(const std::string*)` 为每篇文档提供可选日期标签。 +标签必须使用规范日历格式：`YYYY`、`YYYY/MM` 或 `YYYY/MM/DD`，月和日固定为两位。 +完全相同的标签共用窗口；未提供 dates 数组或数组元素为空字符串时，文档进入无日期窗口。 +不同精度是不同标签，因此 `2026/05/17` 文档只存入 `2026/05/17` 窗口，不会再重复存入 +`2026/05` 窗口。 + +线上查询时在 SINDI 检索参数中传入日期。年份会选中该年的年、月、日窗口；月份会选中该月的 +月、日窗口；日期只选中完全相同的日窗口。例如 `"date": "2026"` 会匹配 `2026`、 +`2026/05` 和 `2026/05/17`。缺少 `date` 或传入空字符串表示不启用日期过滤；非空日期不会 +匹配无日期窗口。 + +日期条件先裁剪待检索窗口，已有的 ID filter、bitset 或 `Filter` callback 再在命中窗口内部 +执行，因此两者为 AND 关系。KNN 与范围检索均支持日期条件。 + ## 何时选择 SINDI - 使用 BM25、SPLADE、uniCOIL 等学习稀疏编码器的稀疏检索场景。 diff --git a/examples/cpp/109_index_sindi.cpp b/examples/cpp/109_index_sindi.cpp index 8e2748dba4..e2d0b1e669 100644 --- a/examples/cpp/109_index_sindi.cpp +++ b/examples/cpp/109_index_sindi.cpp @@ -33,6 +33,7 @@ main(int argc, char** argv) { std::vector ids(num_vectors); std::vector sparse_vectors(num_vectors); + std::vector dates(num_vectors, "2026/07/21"); for (int64_t i = 0; i < num_vectors; ++i) { ids[i] = i; @@ -56,6 +57,7 @@ main(int argc, char** argv) { base->NumElements(num_vectors) ->SparseVectors(sparse_vectors.data()) ->Ids(ids.data()) + ->Dates(dates.data()) ->Owner(false); /******************* Create SINDI Index *****************/ @@ -123,11 +125,13 @@ main(int argc, char** argv) { * - query_prune_ratio: Ratio of term pruning for the query (0 = no pruning). * - n_candidate: Number of candidates for re-ranking. Must be greater than topK. * This parameter is ignored if use_reorder is false in the build parameters. + * - date: Optional date-window filter (YYYY, YYYY/MM, or YYYY/MM/DD). */ auto sindi_search_parameters = R"({ "sindi": { "query_prune_ratio": 0, - "n_candidate": 0 + "n_candidate": 0, + "date": "2026/07" } })"; diff --git a/include/vsag/constants.h b/include/vsag/constants.h index 70abc6670d..e13458088c 100644 --- a/include/vsag/constants.h +++ b/include/vsag/constants.h @@ -39,6 +39,7 @@ extern const char* const SPARSE_VECTORS; extern const char* const INT8_VECTORS; extern const char* const ATTRIBUTE_SETS; extern const char* const DATASET_PATHS; +extern const char* const DATASET_DATES; extern const char* const EXTRA_INFOS; extern const char* const EXTRA_INFO_SIZE; extern const char* const VECTOR_COUNTS; diff --git a/include/vsag/dataset.h b/include/vsag/dataset.h index 7825acf1dd..c172ed103f 100644 --- a/include/vsag/dataset.h +++ b/include/vsag/dataset.h @@ -354,6 +354,23 @@ class Dataset : public std::enable_shared_from_this { virtual const std::string* GetPaths(const std::string& hierarchy_name) const = 0; + /** + * @brief Sets the date-label array for the dataset. + * + * @param dates Pointer to an array with one date label per element. + * @return DatasetPtr A shared pointer to the dataset with updated date labels. + */ + virtual DatasetPtr + Dates(const std::string* dates) = 0; + + /** + * @brief Retrieves the date-label array for the dataset. + * + * @return const std::string* Pointer to the date-label array, or nullptr when absent. + */ + virtual const std::string* + GetDates() const = 0; + /** * @brief Sets the extra info for the dataset. * diff --git a/src/algorithm/sindi/sindi.cpp b/src/algorithm/sindi/sindi.cpp index 0c17acd404..5ee226d6c9 100644 --- a/src/algorithm/sindi/sindi.cpp +++ b/src/algorithm/sindi/sindi.cpp @@ -16,6 +16,7 @@ #include "sindi.h" #include +#include #include #include "algorithm/sparse_distance.h" @@ -42,6 +43,36 @@ namespace { constexpr uint64_t TERM_ID_MAPPER_ENTRY_MEMORY_BYTES = 54; constexpr const char* SINDI_RERANK_FLAT_FORMAT_KEY = "sindi_rerank_flat_format"; constexpr int64_t SINDI_RERANK_FLAT_FORMAT_DATACELL = 2; +constexpr const char* SINDI_WINDOW_METADATA_FORMAT_KEY = "sindi_window_metadata_format"; +constexpr int64_t SINDI_WINDOW_METADATA_FORMAT_V1 = 1; +constexpr uint64_t MAX_SERIALIZED_DATE_LENGTH = 10; + +struct SINDIDateGroup { + std::string date; + std::vector indices; +}; + +std::vector +group_dataset_by_date(const DatasetPtr& base) { + const auto data_num = base->GetNumElements(); + const auto* dates = base->GetDates(); + std::vector groups; + groups.reserve(dates == nullptr ? 1 : static_cast(data_num)); + std::unordered_map group_by_date; + group_by_date.reserve(dates == nullptr ? 1 : static_cast(data_num)); + + for (int64_t i = 0; i < data_num; ++i) { + const std::string date = dates == nullptr ? "" : dates[i]; + CHECK_ARGUMENT(IsValidSINDIDate(date), + fmt::format("invalid SINDI date for document {}: {}", i, date)); + auto [iter, inserted] = group_by_date.emplace(date, groups.size()); + if (inserted) { + groups.push_back({date, {}}); + } + groups[iter->second].indices.push_back(i); + } + return groups; +} uint32_t sparse_value_code_size(SparseValueQuantizationType type) { @@ -195,6 +226,7 @@ SINDI::SINDI(const SINDIParameterPtr& param, const IndexCommonParam& common_para doc_prune_ratio_(param->doc_prune_ratio), doc_retain_ratio_(1.0F - param->doc_prune_ratio), window_term_list_(common_param.allocator_.get()), + window_metadata_(common_param.allocator_.get()), deserialize_without_footer_(param->deserialize_without_footer), deserialize_without_buffer_(param->deserialize_without_buffer), quantization_params_(std::make_shared()), @@ -253,6 +285,7 @@ SINDI::Add(const DatasetPtr& base) { const auto* ids = base->GetIds(); const auto* extra_info = base->GetExtraInfos(); const auto extra_info_size = base->GetExtraInfoSize(); + const auto date_groups = group_dataset_by_date(base); if (sparse_value_quant_type_ == SparseValueQuantizationType::SQ8 && cur_element_count_ == 0) { float min_val = std::numeric_limits::max(); @@ -277,74 +310,71 @@ SINDI::Add(const DatasetPtr& base) { } } - // adjust window - int64_t final_add_window = align_up(cur_element_count_ + data_num, window_size_) / window_size_; - bool window_changed = false; - while (window_term_list_.size() < final_add_window) { - window_term_list_.emplace_back( - std::make_shared(doc_retain_ratio_, - term_id_limit_, - allocator_, - sparse_value_quant_type_, - quantization_params_)); - window_changed = true; - } - - // add process + bool data_inserted = false; Vector tmp_ids(allocator_); - for (uint32_t i = 0; i < data_num; ++i) { - auto cur_window = cur_element_count_ / window_size_; - auto window_start_id = cur_window * window_size_; - const auto& sparse_vector = sparse_vectors[i]; - if (label_table_->CheckLabel(ids[i])) { - failed_ids.push_back(ids[i]); - logger::warn("id ({}) already exists", ids[i]); - continue; - } - if (sparse_vector.len_ <= 0) { - failed_ids.push_back(ids[i]); - logger::warn( - "sparse_vector.len_ ({}) is invalid for id ({})", sparse_vector.len_, ids[i]); - continue; - } - - auto inner_id = static_cast(cur_element_count_ - window_start_id); + for (const auto& group : date_groups) { + const auto previous_window_count = window_metadata_.size(); + for (const auto i : group.indices) { + const auto& sparse_vector = sparse_vectors[i]; + if (label_table_->CheckLabel(ids[i])) { + failed_ids.push_back(ids[i]); + logger::warn("id ({}) already exists", ids[i]); + continue; + } + if (sparse_vector.len_ <= 0) { + failed_ids.push_back(ids[i]); + logger::warn( + "sparse_vector.len_ ({}) is invalid for id ({})", sparse_vector.len_, ids[i]); + continue; + } - try { - if (remap_term_ids_) { - auto remapped = remap_sparse_vector_for_build(sparse_vector, tmp_ids); - window_term_list_[cur_window]->InsertVector(remapped, inner_id); - } else { - window_term_list_[cur_window]->InsertVector(sparse_vector, inner_id); + if (window_metadata_.empty() or window_metadata_.back().date != group.date or + window_metadata_.back().doc_count == window_size_) { + create_mutable_window(group.date); + } + auto& metadata = window_metadata_.back(); + const auto inner_id = static_cast(metadata.doc_count); + + try { + if (remap_term_ids_) { + auto remapped = remap_sparse_vector_for_build(sparse_vector, tmp_ids); + window_term_list_.back()->InsertVector(remapped, inner_id); + } else { + window_term_list_.back()->InsertVector(sparse_vector, inner_id); + } + } catch (const std::runtime_error& e) { + failed_ids.push_back(ids[i]); + logger::warn("runtime error: {}", e.what()); + continue; + } catch (const VsagException& e) { + failed_ids.push_back(ids[i]); + logger::warn("vsag exception: {}", e.what()); + continue; + } catch (const std::bad_alloc& e) { + failed_ids.push_back(ids[i]); + logger::warn("memory allocation failed: {}", e.what()); + continue; } - } catch (const std::runtime_error& e) { - failed_ids.push_back(ids[i]); - logger::warn("runtime error: {}", e.what()); - continue; - } catch (const VsagException& e) { - failed_ids.push_back(ids[i]); - logger::warn("vsag exception: {}", e.what()); - continue; - } catch (const std::bad_alloc& e) { - failed_ids.push_back(ids[i]); - logger::warn("memory allocation failed: {}", e.what()); - continue; - } - label_table_->Insert(cur_element_count_, ids[i]); // todo(zxy): check id exists + label_table_->Insert(cur_element_count_, ids[i]); + if (extra_info_size > 0) { + extra_infos_->InsertExtraInfo(extra_info + i * extra_info_size, cur_element_count_); + } + metadata.doc_count++; + cur_element_count_++; + data_inserted = true; - if (extra_info_size > 0) { - extra_infos_->InsertExtraInfo(extra_info + i * extra_info_size, cur_element_count_); + if (use_reorder_) { + rerank_flat_->InsertVector(sparse_vectors + i, cur_element_count_ - 1); + } } - - cur_element_count_++; - - // high precision part - if (use_reorder_) { - rerank_flat_->InsertVector(sparse_vectors + i, cur_element_count_ - 1); + if (window_metadata_.size() > previous_window_count and + window_metadata_.back().doc_count == 0) { + window_metadata_.pop_back(); + window_term_list_.pop_back(); } } - if (window_changed) { + if (data_inserted or not failed_ids.empty()) { this->cal_memory_usage(); } return failed_ids; @@ -381,6 +411,7 @@ SINDI::build_immutable(const DatasetPtr& base) { const auto* ids = base->GetIds(); const auto* extra_info = base->GetExtraInfos(); const auto extra_info_size = base->GetExtraInfoSize(); + const auto date_groups = group_dataset_by_date(base); if (sparse_value_quant_type_ == SparseValueQuantizationType::SQ8 && cur_element_count_ == 0) { float min_val = std::numeric_limits::max(); @@ -405,84 +436,92 @@ SINDI::build_immutable(const DatasetPtr& base) { immutable_data_->sparse_value_quant_type = sparse_value_quant_type_; immutable_data_->value_code_size = sparse_value_code_size(sparse_value_quant_type_); immutable_data_->windows.reserve(align_up(data_num, window_size_) / window_size_); + window_metadata_.clear(); SparseTermDataCellPtr current_window = nullptr; Vector tmp_ids(allocator_); - for (int64_t i = 0; i < data_num; ++i) { - if (current_window == nullptr) { - current_window = std::make_shared(doc_retain_ratio_, - term_id_limit_, - allocator_, - sparse_value_quant_type_, - quantization_params_); - } - - const auto& sparse_vector = sparse_vectors[i]; - if (label_table_->CheckLabel(ids[i])) { - failed_ids.push_back(ids[i]); - logger::warn("id ({}) already exists", ids[i]); - continue; - } - if (sparse_vector.len_ <= 0) { - failed_ids.push_back(ids[i]); - logger::warn( - "sparse_vector.len_ ({}) is invalid for id ({})", sparse_vector.len_, ids[i]); - continue; - } - - const auto window_start_id = - static_cast(immutable_data_->windows.size()) * window_size_; - const auto window_inner_id = cur_element_count_ - window_start_id; - if (window_inner_id < 0 or window_inner_id > std::numeric_limits::max()) { - throw VsagException(ErrorType::INVALID_ARGUMENT, - "immutable SINDI window-local doc id overflows uint16_t"); + const auto flush_window = [this, ¤t_window]() { + if (current_window == nullptr or window_metadata_.empty() or + window_metadata_.back().doc_count == 0) { + return; } - auto inner_id = static_cast(window_inner_id); + immutable_data_->windows.emplace_back(allocator_); + compact_window_to_immutable(*current_window, immutable_data_->windows.back()); + current_window.reset(); + }; - try { - if (remap_term_ids_) { - auto remapped = remap_sparse_vector_for_build(sparse_vector, tmp_ids); - current_window->InsertVector(remapped, inner_id); - } else { - current_window->InsertVector(sparse_vector, inner_id); + for (const auto& group : date_groups) { + const auto previous_window_count = window_metadata_.size(); + for (const auto i : group.indices) { + const auto& sparse_vector = sparse_vectors[i]; + if (label_table_->CheckLabel(ids[i])) { + failed_ids.push_back(ids[i]); + logger::warn("id ({}) already exists", ids[i]); + continue; + } + if (sparse_vector.len_ <= 0) { + failed_ids.push_back(ids[i]); + logger::warn( + "sparse_vector.len_ ({}) is invalid for id ({})", sparse_vector.len_, ids[i]); + continue; } - } catch (const std::runtime_error& e) { - failed_ids.push_back(ids[i]); - logger::warn("runtime error: {}", e.what()); - continue; - } catch (const VsagException& e) { - failed_ids.push_back(ids[i]); - logger::warn("vsag exception: {}", e.what()); - continue; - } catch (const std::bad_alloc& e) { - failed_ids.push_back(ids[i]); - logger::warn("memory allocation failed: {}", e.what()); - continue; - } - - label_table_->Insert(cur_element_count_, ids[i]); - if (extra_info_size > 0) { - extra_infos_->InsertExtraInfo(extra_info + i * extra_info_size, cur_element_count_); - } + if (current_window == nullptr) { + CHECK_ARGUMENT(cur_element_count_.load() <= std::numeric_limits::max(), + "SINDI element count overflows inner id"); + current_window = std::make_shared(doc_retain_ratio_, + term_id_limit_, + allocator_, + sparse_value_quant_type_, + quantization_params_); + window_metadata_.push_back( + {group.date, static_cast(cur_element_count_.load()), 0}); + } + auto& metadata = window_metadata_.back(); + const auto inner_id = static_cast(metadata.doc_count); + + try { + if (remap_term_ids_) { + auto remapped = remap_sparse_vector_for_build(sparse_vector, tmp_ids); + current_window->InsertVector(remapped, inner_id); + } else { + current_window->InsertVector(sparse_vector, inner_id); + } + } catch (const std::runtime_error& e) { + failed_ids.push_back(ids[i]); + logger::warn("runtime error: {}", e.what()); + continue; + } catch (const VsagException& e) { + failed_ids.push_back(ids[i]); + logger::warn("vsag exception: {}", e.what()); + continue; + } catch (const std::bad_alloc& e) { + failed_ids.push_back(ids[i]); + logger::warn("memory allocation failed: {}", e.what()); + continue; + } - cur_element_count_++; + label_table_->Insert(cur_element_count_, ids[i]); + if (extra_info_size > 0) { + extra_infos_->InsertExtraInfo(extra_info + i * extra_info_size, cur_element_count_); + } + metadata.doc_count++; + cur_element_count_++; - if (use_reorder_) { - rerank_flat_->InsertVector(sparse_vectors + i, cur_element_count_ - 1); + if (use_reorder_) { + rerank_flat_->InsertVector(sparse_vectors + i, cur_element_count_ - 1); + } + if (metadata.doc_count == window_size_) { + flush_window(); + } } - - if (cur_element_count_ % window_size_ == 0) { - immutable_data_->windows.emplace_back(allocator_); - compact_window_to_immutable(*current_window, immutable_data_->windows.back()); + flush_window(); + if (window_metadata_.size() > previous_window_count and + window_metadata_.back().doc_count == 0) { + window_metadata_.pop_back(); current_window.reset(); } } - - if (current_window != nullptr && current_window->total_count_ > 0) { - immutable_data_->windows.emplace_back(allocator_); - compact_window_to_immutable(*current_window, immutable_data_->windows.back()); - } current_window.reset(); window_term_list_.clear(); window_term_list_.shrink_to_fit(); @@ -578,11 +617,19 @@ SINDI::KnnSearch(const DatasetPtr& query, auto computer = std::make_shared(effective_query, search_param, allocator_); const SparseVector* rerank_query = (remap_term_ids_ && use_reorder_) ? &sparse_query : nullptr; if (immutable_data_ != nullptr) { - return immutable_search_impl( - computer, inner_param, allocator, UseTermListsHeapInsert(search_param), rerank_query); - } - return search_impl( - computer, inner_param, allocator, UseTermListsHeapInsert(search_param), rerank_query); + return immutable_search_impl(computer, + inner_param, + allocator, + UseTermListsHeapInsert(search_param), + search_param.date, + rerank_query); + } + return search_impl(computer, + inner_param, + allocator, + UseTermListsHeapInsert(search_param), + search_param.date, + rerank_query); } std::optional @@ -839,6 +886,7 @@ SINDI::immutable_search_impl(const SparseTermComputerPtr& computer, const InnerSearchParam& inner_param, Allocator* allocator, bool use_term_lists_heap_insert, + const std::string& query_date, const SparseVector* original_query) const { Allocator* search_allocator = allocator != nullptr ? allocator : allocator_; MaxHeap heap(search_allocator); @@ -852,8 +900,12 @@ SINDI::immutable_search_impl(const SparseTermComputerPtr& computer, auto filter = inner_param.is_inner_id_allowed; const auto [min_window_id, max_window_id] = this->get_min_max_window_id(filter); for (auto cur = min_window_id; cur <= max_window_id; cur++) { + const auto& metadata = window_metadata_[static_cast(cur)]; + if (not SINDIDateMatches(metadata.date, query_date)) { + continue; + } const auto& window = immutable_data_->windows[static_cast(cur)]; - const auto window_start_id = static_cast(cur) * window_size_; + const auto window_start_id = metadata.start_id; map_immutable_query_terms(window, computer, mapped_terms); std::fill(dists.begin(), dists.end(), 0.0F); scan_immutable_window_by_mapped_terms(dists.data(), window, computer, mapped_terms); @@ -877,14 +929,12 @@ SINDI::immutable_search_impl(const SparseTermComputerPtr& computer, window_start_id); } } else { - const auto window_doc_count = static_cast(std::min( - window_size_, cur_element_count_ - static_cast(window_start_id))); if (inner_param.is_inner_id_allowed) { immutable_insert_heap_by_dists( - dists.data(), window_doc_count, heap, inner_param, window_start_id); + dists.data(), metadata.doc_count, heap, inner_param, window_start_id); } else { immutable_insert_heap_by_dists( - dists.data(), window_doc_count, heap, inner_param, window_start_id); + dists.data(), metadata.doc_count, heap, inner_param, window_start_id); } } } @@ -951,6 +1001,7 @@ SINDI::search_impl(const SparseTermComputerPtr& computer, const InnerSearchParam& inner_param, Allocator* allocator, bool use_term_lists_heap_insert, + const std::string& query_date, const SparseVector* original_query) const { // computer and heap MaxHeap heap(allocator); @@ -965,7 +1016,11 @@ SINDI::search_impl(const SparseTermComputerPtr& computer, auto filter = inner_param.is_inner_id_allowed; const auto [min_window_id, max_window_id] = this->get_min_max_window_id(filter); for (auto cur = min_window_id; cur <= max_window_id; cur++) { - auto window_start_id = cur * window_size_; + const auto& metadata = window_metadata_[static_cast(cur)]; + if (not SINDIDateMatches(metadata.date, query_date)) { + continue; + } + auto window_start_id = metadata.start_id; auto term_list = this->window_term_list_[cur]; // compute @@ -983,10 +1038,10 @@ SINDI::search_impl(const SparseTermComputerPtr& computer, } else { if (inner_param.is_inner_id_allowed) { term_list->InsertHeapByDists( - dists.data(), dists.size(), heap, inner_param, window_start_id); + dists.data(), metadata.doc_count, heap, inner_param, window_start_id); } else { term_list->InsertHeapByDists( - dists.data(), dists.size(), heap, inner_param, window_start_id); + dists.data(), metadata.doc_count, heap, inner_param, window_start_id); } } } @@ -1096,11 +1151,19 @@ SINDI::RangeSearch(const DatasetPtr& query, auto computer = std::make_shared(effective_query, search_param, allocator_); const SparseVector* rerank_query = (remap_term_ids_ && use_reorder_) ? &sparse_query : nullptr; if (immutable_data_ != nullptr) { - return immutable_search_impl( - computer, inner_param, allocator_, UseTermListsHeapInsert(search_param), rerank_query); - } - return search_impl( - computer, inner_param, allocator_, UseTermListsHeapInsert(search_param), rerank_query); + return immutable_search_impl(computer, + inner_param, + allocator_, + UseTermListsHeapInsert(search_param), + search_param.date, + rerank_query); + } + return search_impl(computer, + inner_param, + allocator_, + UseTermListsHeapInsert(search_param), + search_param.date, + rerank_query); } bool @@ -1130,6 +1193,10 @@ SINDI::cal_memory_usage() { memory += window->GetMemoryUsage(); } } + memory += window_metadata_.size() * sizeof(SINDIWindowMetadata); + for (const auto& metadata : window_metadata_) { + memory += metadata.date.capacity(); + } memory += label_table_->GetMemoryUsage(); if (this->rerank_flat_ != nullptr) { memory += this->rerank_flat_->GetMemoryUsage(); @@ -1199,8 +1266,11 @@ SINDI::Serialize(StreamWriter& writer) const { term_id_mapper_->Serialize(writer); } + serialize_window_metadata(writer); + JsonType jsonify_basic_info; jsonify_basic_info[INDEX_PARAM].SetString(this->create_param_ptr_->ToString()); + jsonify_basic_info[SINDI_WINDOW_METADATA_FORMAT_KEY].SetInt(SINDI_WINDOW_METADATA_FORMAT_V1); if (use_reorder_) { jsonify_basic_info[SINDI_RERANK_FLAT_FORMAT_KEY].SetInt(SINDI_RERANK_FLAT_FORMAT_DATACELL); } @@ -1252,6 +1322,11 @@ SINDI::collect_streaming_header() const { StreamSerializationBlockCurrentVersion(tag), StreamSerializationTagCritical(tag)); } + auto window_metadata_tag = static_cast(StreamSerializationTag::SINDI_WINDOW_METADATA); + AppendStreamingManifestBlock(manifest, + window_metadata_tag, + StreamSerializationBlockCurrentVersion(window_metadata_tag), + StreamSerializationTagCritical(window_metadata_tag)); metadata->Set("block_manifest", manifest); metadata->SetEmptyIndex(this->GetNumElements() == 0); return metadata; @@ -1305,6 +1380,11 @@ SINDI::serialize_streaming_body(StreamWriter& writer) const { this->term_id_mapper_->Serialize(w); }); } + auto window_metadata_tag = static_cast(StreamSerializationTag::SINDI_WINDOW_METADATA); + WriteStreamingBlock(writer, + window_metadata_tag, + StreamSerializationTagCritical(window_metadata_tag), + [this](StreamWriter& w) { this->serialize_window_metadata(w); }); } void @@ -1381,6 +1461,7 @@ SINDI::read_streaming_body(StreamReader& reader, const MetadataPtr& metadata) { bool loaded_label_table = false; bool loaded_rerank = false; bool loaded_term_mapper = false; + bool loaded_window_metadata = false; while (true) { auto block_header = StreamBlockHeader::Read(reader); @@ -1440,6 +1521,12 @@ SINDI::read_streaming_body(StreamReader& reader, const MetadataPtr& metadata) { loaded_term_mapper = true; } break; + case StreamSerializationTag::SINDI_WINDOW_METADATA: + ReadSeekableBlockPayload(block_reader, block_header, [this](StreamReader& block) { + this->deserialize_window_metadata(block); + }); + loaded_window_metadata = true; + break; default: if (block_header.IsCritical()) { throw VsagException(ErrorType::UNSUPPORTED_INDEX_OPERATION, @@ -1469,6 +1556,13 @@ SINDI::read_streaming_body(StreamReader& reader, const MetadataPtr& metadata) { throw VsagException(ErrorType::READ_ERROR, "SINDI streaming serialization term mapper block is missing"); } + const auto window_count = static_cast( + immutable_data_ != nullptr ? immutable_data_->windows.size() : window_term_list_.size()); + if (loaded_window_metadata) { + validate_window_metadata(window_count); + } else { + infer_legacy_window_metadata(window_count); + } this->cal_memory_usage(); } @@ -1477,6 +1571,7 @@ SINDI::Deserialize(StreamReader& reader) { std::scoped_lock wlock(this->global_mutex_); bool has_datacell_rerank_format = false; + bool has_window_metadata_format = false; bool has_footer = false; if (not deserialize_without_footer_) { JsonType jsonify_basic_info; @@ -1509,6 +1604,11 @@ SINDI::Deserialize(StreamReader& reader) { jsonify_basic_info[SINDI_RERANK_FLAT_FORMAT_KEY].GetInt() == SINDI_RERANK_FLAT_FORMAT_DATACELL; } + if (jsonify_basic_info.Contains(SINDI_WINDOW_METADATA_FORMAT_KEY)) { + has_window_metadata_format = + jsonify_basic_info[SINDI_WINDOW_METADATA_FORMAT_KEY].GetInt() == + SINDI_WINDOW_METADATA_FORMAT_V1; + } } else { logger::debug("SINDI footer not found, fallback to legacy deserialize path"); } @@ -1588,6 +1688,13 @@ SINDI::Deserialize(StreamReader& reader) { term_id_mapper_->Deserialize(reader_ref); } + if (has_window_metadata_format) { + deserialize_window_metadata(reader_ref); + validate_window_metadata(window_term_list_size); + } else { + infer_legacy_window_metadata(window_term_list_size); + } + this->cal_memory_usage(); } @@ -1787,8 +1894,8 @@ SINDI::GetSparseVectorByInnerId(InnerIdType inner_id, return this->rerank_flat_->GetSparseVectorByInnerId(inner_id, data, specified_allocator); } - auto cur_window = inner_id / window_size_; - auto window_start_id = cur_window * window_size_; + auto cur_window = find_window_id(inner_id); + auto window_start_id = window_metadata_[cur_window].start_id; auto term_list = this->window_term_list_[cur_window]; term_list->GetSparseVector(inner_id - window_start_id, data, specified_allocator); @@ -1820,8 +1927,8 @@ SINDI::CalcDistanceById(const DatasetPtr& vector, } auto inner_id = this->label_table_->GetIdByLabel(id); - auto cur_window = inner_id / window_size_; - auto window_start_id = cur_window * window_size_; + auto cur_window = find_window_id(inner_id); + auto window_start_id = window_metadata_[cur_window].start_id; auto term_list = this->window_term_list_[cur_window]; auto sparse_query = vector->GetSparseVectors()[0]; @@ -1950,9 +2057,10 @@ SINDI::InitFeatures() { std::pair SINDI::get_min_max_window_id(const FilterPtr& filter) const { int64_t min_window_id = 0; - auto num_windows = - immutable_data_ != nullptr ? immutable_data_->windows.size() : window_term_list_.size(); - auto max_window_id = static_cast(num_windows) - 1; + auto max_window_id = static_cast(window_metadata_.size()) - 1; + if (max_window_id < 0) { + return {0, -1}; + } // get min and max window id if (filter) { @@ -1971,16 +2079,130 @@ SINDI::get_min_max_window_id(const FilterPtr& filter) const { } } if (min_inner_id != INT64_MAX) { - min_window_id = min_inner_id / window_size_; + min_window_id = find_window_id(static_cast(min_inner_id)); } if (max_inner_id != INT64_MIN) { - max_window_id = max_inner_id / window_size_; + max_window_id = find_window_id(static_cast(max_inner_id)); } } return {min_window_id, max_window_id}; } +uint32_t +SINDI::find_window_id(InnerIdType inner_id) const { + auto iter = std::upper_bound( + window_metadata_.begin(), + window_metadata_.end(), + inner_id, + [](InnerIdType id, const SINDIWindowMetadata& metadata) { return id < metadata.start_id; }); + CHECK_ARGUMENT(iter != window_metadata_.begin(), "SINDI inner id precedes all windows"); + --iter; + const auto end_id = static_cast(iter->start_id) + iter->doc_count; + CHECK_ARGUMENT(inner_id < end_id, "SINDI inner id does not belong to a window"); + return static_cast(iter - window_metadata_.begin()); +} + +void +SINDI::create_mutable_window(const std::string& date) { + CHECK_ARGUMENT(cur_element_count_.load() <= std::numeric_limits::max(), + "SINDI element count overflows inner id"); + auto window = std::make_shared(doc_retain_ratio_, + term_id_limit_, + allocator_, + sparse_value_quant_type_, + quantization_params_); + window_term_list_.push_back(std::move(window)); + try { + window_metadata_.push_back({date, static_cast(cur_element_count_.load()), 0}); + } catch (...) { + window_term_list_.pop_back(); + throw; + } +} + +void +SINDI::infer_legacy_window_metadata(uint32_t window_count) { + window_metadata_.clear(); + window_metadata_.reserve(window_count); + const auto element_count = static_cast(cur_element_count_.load()); + for (uint32_t i = 0; i < window_count; ++i) { + const auto start_id = static_cast(i) * window_size_; + CHECK_ARGUMENT(start_id <= std::numeric_limits::max(), + "legacy SINDI window start id overflows inner id"); + const auto doc_count = + start_id < element_count + ? static_cast(std::min(window_size_, element_count - start_id)) + : 0; + window_metadata_.push_back({"", static_cast(start_id), doc_count}); + } +} + +void +SINDI::validate_window_metadata(uint32_t window_count) const { + CHECK_ARGUMENT(window_metadata_.size() == window_count, + "SINDI window metadata count does not match window count"); + const auto element_count = static_cast(cur_element_count_.load()); + uint64_t expected_start_id = 0; + bool reached_empty_window = false; + for (const auto& metadata : window_metadata_) { + CHECK_ARGUMENT(IsValidSINDIDate(metadata.date), + "SINDI window metadata contains an invalid date"); + CHECK_ARGUMENT(metadata.doc_count <= window_size_, + "SINDI window metadata document count exceeds window size"); + if (metadata.doc_count == 0) { + reached_empty_window = true; + CHECK_ARGUMENT( + expected_start_id == element_count and metadata.start_id >= expected_start_id, + "SINDI empty window metadata has an invalid boundary"); + continue; + } + CHECK_ARGUMENT(not reached_empty_window, "SINDI non-empty window follows an empty window"); + CHECK_ARGUMENT(metadata.start_id == expected_start_id, + "SINDI window metadata boundaries are not contiguous"); + expected_start_id += metadata.doc_count; + } + CHECK_ARGUMENT(expected_start_id == element_count, + "SINDI window metadata document count does not match element count"); +} + +void +SINDI::serialize_window_metadata(StreamWriter& writer) const { + const auto window_count = static_cast(window_metadata_.size()); + StreamWriter::WriteObj(writer, window_count); + for (const auto& metadata : window_metadata_) { + StreamWriter::WriteString(writer, metadata.date); + StreamWriter::WriteObj(writer, metadata.start_id); + StreamWriter::WriteObj(writer, metadata.doc_count); + } +} + +void +SINDI::deserialize_window_metadata(StreamReader& reader) { + uint32_t window_count = 0; + StreamReader::ReadObj(reader, window_count); + const auto element_count = static_cast(cur_element_count_.load()); + CHECK_ARGUMENT(window_count <= element_count, + "serialized SINDI window metadata count exceeds element count"); + window_metadata_.clear(); + window_metadata_.reserve(window_count); + for (uint32_t i = 0; i < window_count; ++i) { + uint64_t date_length = 0; + StreamReader::ReadObj(reader, date_length); + CHECK_ARGUMENT(date_length <= MAX_SERIALIZED_DATE_LENGTH, + "serialized SINDI date exceeds maximum length"); + std::string date(date_length, '\0'); + if (date_length > 0) { + reader.Read(date.data(), date_length); + } + SINDIWindowMetadata metadata; + metadata.date = std::move(date); + StreamReader::ReadObj(reader, metadata.start_id); + StreamReader::ReadObj(reader, metadata.doc_count); + window_metadata_.push_back(std::move(metadata)); + } +} + SparseVector SINDI::remap_sparse_vector_for_query(const SparseVector& input, Vector& tmp_ids, diff --git a/src/algorithm/sindi/sindi.h b/src/algorithm/sindi/sindi.h index 3185f04543..6c237922ef 100644 --- a/src/algorithm/sindi/sindi.h +++ b/src/algorithm/sindi/sindi.h @@ -25,6 +25,12 @@ namespace vsag { +struct SINDIWindowMetadata { + std::string date; + InnerIdType start_id{0}; + uint32_t doc_count{0}; +}; + struct ImmutableSINDIWindow { explicit ImmutableSINDIWindow(Allocator* allocator) : sorted_global_terms(allocator), @@ -208,6 +214,7 @@ class SINDI : public InnerIndexInterface { const InnerSearchParam& inner_param, Allocator* allocator, bool use_term_lists_heap_insert, + const std::string& query_date, const SparseVector* original_query = nullptr) const; template @@ -216,6 +223,7 @@ class SINDI : public InnerIndexInterface { const InnerSearchParam& inner_param, Allocator* allocator, bool use_term_lists_heap_insert, + const std::string& query_date, const SparseVector* original_query = nullptr) const; bool @@ -232,6 +240,24 @@ class SINDI : public InnerIndexInterface { std::pair get_min_max_window_id(const FilterPtr& filter) const; + uint32_t + find_window_id(InnerIdType inner_id) const; + + void + create_mutable_window(const std::string& date); + + void + infer_legacy_window_metadata(uint32_t window_count); + + void + validate_window_metadata(uint32_t window_count) const; + + void + serialize_window_metadata(StreamWriter& writer) const; + + void + deserialize_window_metadata(StreamReader& reader); + MetadataPtr collect_streaming_header() const override; @@ -350,6 +376,7 @@ class SINDI : public InnerIndexInterface { uint32_t window_size_{0}; // number of vectors per window Vector window_term_list_; // one inverted list per window + Vector window_metadata_; std::atomic cur_element_count_{0}; // total inserted vectors std::atomic delete_count_{0}; // soft-deleted vectors diff --git a/src/algorithm/sindi/sindi_parameter.cpp b/src/algorithm/sindi/sindi_parameter.cpp index f44f78a575..38d0b67651 100644 --- a/src/algorithm/sindi/sindi_parameter.cpp +++ b/src/algorithm/sindi/sindi_parameter.cpp @@ -20,6 +20,84 @@ #include "utils/param_compat_macros.h" namespace vsag { + +namespace { + +bool +is_ascii_digits(const std::string& value, uint64_t begin, uint64_t count) { + for (uint64_t i = begin; i < begin + count; ++i) { + if (value[i] < '0' or value[i] > '9') { + return false; + } + } + return true; +} + +uint32_t +parse_date_component(const std::string& value, uint64_t begin, uint64_t count) { + uint32_t result = 0; + for (uint64_t i = begin; i < begin + count; ++i) { + result = result * 10 + static_cast(value[i] - '0'); + } + return result; +} + +} // namespace + +bool +IsValidSINDIDate(const std::string& date) { + if (date.empty()) { + return true; + } + if (date.size() != 4 and date.size() != 7 and date.size() != 10) { + return false; + } + if (not is_ascii_digits(date, 0, 4)) { + return false; + } + const auto year = parse_date_component(date, 0, 4); + if (year == 0) { + return false; + } + if (date.size() == 4) { + return true; + } + if (date[4] != '/' or not is_ascii_digits(date, 5, 2)) { + return false; + } + const auto month = parse_date_component(date, 5, 2); + if (month == 0 or month > 12) { + return false; + } + if (date.size() == 7) { + return true; + } + if (date[7] != '/' or not is_ascii_digits(date, 8, 2)) { + return false; + } + const auto day = parse_date_component(date, 8, 2); + constexpr uint32_t DAYS_PER_MONTH[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + auto max_day = DAYS_PER_MONTH[month - 1]; + const bool leap_year = year % 4 == 0 and (year % 100 != 0 or year % 400 == 0); + if (month == 2 and leap_year) { + max_day = 29; + } + return day > 0 and day <= max_day; +} + +bool +SINDIDateMatches(const std::string& window_date, const std::string& query_date) { + if (query_date.empty()) { + return true; + } + if (window_date == query_date) { + return true; + } + return window_date.size() > query_date.size() and + window_date.compare(0, query_date.size(), query_date) == 0 and + window_date[query_date.size()] == '/'; +} + std::string SparseValueQuantizationTypeToString(SparseValueQuantizationType type) { switch (type) { @@ -191,6 +269,12 @@ SINDISearchParameter::FromJson(const JsonType& json) { } else { n_candidate = DEFAULT_N_CANDIDATE; } + if (json[INDEX_SINDI].Contains(SPARSE_DATE)) { + date = json[INDEX_SINDI][SPARSE_DATE].GetString(); + CHECK_ARGUMENT(IsValidSINDIDate(date), fmt::format("invalid SINDI date: {}", date)); + } else { + date.clear(); + } if (json[INDEX_SINDI].Contains(LEGACY_USE_TERM_LISTS_HEAP_INSERT_KEY)) { logger::warn( @@ -206,6 +290,9 @@ SINDISearchParameter::ToJson() const { json[INDEX_SINDI][SPARSE_QUERY_PRUNE_RATIO].SetFloat(query_prune_ratio); json[INDEX_SINDI][SPARSE_N_CANDIDATE].SetInt(n_candidate); json[INDEX_SINDI][SPARSE_TERM_PRUNE_RATIO].SetFloat(term_prune_ratio); + if (not date.empty()) { + json[INDEX_SINDI][SPARSE_DATE].SetString(date); + } return json; } diff --git a/src/algorithm/sindi/sindi_parameter.h b/src/algorithm/sindi/sindi_parameter.h index e7e053a362..3ad44b0de0 100644 --- a/src/algorithm/sindi/sindi_parameter.h +++ b/src/algorithm/sindi/sindi_parameter.h @@ -34,6 +34,12 @@ enum class SparseValueQuantizationType { std::string SparseValueQuantizationTypeToString(SparseValueQuantizationType type); +bool +IsValidSINDIDate(const std::string& date); + +bool +SINDIDateMatches(const std::string& window_date, const std::string& query_date); + class SINDIParameter : public InnerIndexParameter { public: void @@ -86,6 +92,7 @@ class SINDISearchParameter : public Parameter { // data cell float query_prune_ratio{0}; float term_prune_ratio{0}; + std::string date; }; } // namespace vsag diff --git a/src/algorithm/sindi/sindi_parameter_test.cpp b/src/algorithm/sindi/sindi_parameter_test.cpp index 614beb495d..cc5c449b24 100644 --- a/src/algorithm/sindi/sindi_parameter_test.cpp +++ b/src/algorithm/sindi/sindi_parameter_test.cpp @@ -115,6 +115,52 @@ TEST_CASE("SINDI Index Parameters Test", "[ut][SINDIParameter]") { legacy_search_param->ToJson()[INDEX_SINDI].Contains("use_term_lists_heap_insert")); } +TEST_CASE("SINDI Date Parameter Test", "[ut][SINDIParameter][date]") { + SECTION("valid dates round trip") { + const std::string date = GENERATE("2026", "2026/07", "2024/02/29"); + const auto parameters = fmt::format(R"({{"sindi": {{"date": "{}"}}}})", date); + SINDISearchParameter search_param; + search_param.FromJson(JsonType::Parse(parameters)); + REQUIRE(search_param.date == date); + REQUIRE(search_param.ToJson()[INDEX_SINDI][SPARSE_DATE].GetString() == date); + } + + SECTION("missing and empty dates disable filtering") { + SINDISearchParameter missing; + missing.FromJson(JsonType::Parse(R"({"sindi": {}})")); + REQUIRE(missing.date.empty()); + REQUIRE_FALSE(missing.ToJson()[INDEX_SINDI].Contains(SPARSE_DATE)); + + SINDISearchParameter empty; + empty.FromJson(JsonType::Parse(R"({"sindi": {"date": ""}})")); + REQUIRE(empty.date.empty()); + REQUIRE_FALSE(empty.ToJson()[INDEX_SINDI].Contains(SPARSE_DATE)); + } + + SECTION("invalid dates are rejected") { + const std::string date = GENERATE("26", + "0000", + "2026/7", + "2026/13", + "2026/04/31", + "2025/02/29", + "2026/07/21/01", + "abcd/ef/gh"); + const auto parameters = fmt::format(R"({{"sindi": {{"date": "{}"}}}})", date); + SINDISearchParameter search_param; + REQUIRE_THROWS(search_param.FromJson(JsonType::Parse(parameters))); + } + + SECTION("hierarchical matching respects component boundaries") { + REQUIRE(SINDIDateMatches("2026", "2026")); + REQUIRE(SINDIDateMatches("2026/07", "2026")); + REQUIRE(SINDIDateMatches("2026/07/21", "2026/07")); + REQUIRE_FALSE(SINDIDateMatches("2026", "2026/07")); + REQUIRE_FALSE(SINDIDateMatches("2026/08/01", "2026/07")); + REQUIRE_FALSE(SINDIDateMatches("20260/07", "2026")); + } +} + TEST_CASE("SINDI Index Parameters Compatibility Test", "[ut][SINDIParameter]") { TEST_COMPATIBILITY_CASE("use_reorder compatibility", use_reorder, true, false, false); TEST_COMPATIBILITY_CASE("value quantization compatibility", diff --git a/src/algorithm/sindi/sindi_test.cpp b/src/algorithm/sindi/sindi_test.cpp index 152472179f..7dfc295a03 100644 --- a/src/algorithm/sindi/sindi_test.cpp +++ b/src/algorithm/sindi/sindi_test.cpp @@ -57,6 +57,26 @@ class SINDITestAccess { ImmutableSINDIWindow& window) { index.deserialize_immutable_window(reader, window); } + + static const Vector& + WindowMetadata(const SINDI& index) { + return index.window_metadata_; + } + + static void + SerializeLegacyMutable(const SINDI& index, StreamWriter& writer) { + REQUIRE_FALSE(index.use_reorder_); + REQUIRE_FALSE(index.remap_term_ids_); + REQUIRE(index.sparse_value_quant_type_ == SparseValueQuantizationType::FP32); + const auto element_count = index.cur_element_count_.load(); + StreamWriter::WriteObj(writer, element_count); + const auto window_count = static_cast(index.window_term_list_.size()); + StreamWriter::WriteObj(writer, window_count); + for (const auto& window : index.window_term_list_) { + window->Serialize(writer); + } + index.label_table_->Serialize(writer); + } }; } // namespace vsag @@ -105,6 +125,35 @@ class MockValidIdFilter : public Filter { std::unordered_set valid_ids_set_; }; +namespace { + +std::set +result_ids(const DatasetPtr& result) { + return std::set(result->GetIds(), result->GetIds() + result->GetDim()); +} + +SINDIParameterPtr +create_date_test_param(bool immutable, uint32_t window_size) { + auto param = std::make_shared(); + param->term_id_limit = 10; + param->window_size = window_size; + param->doc_prune_ratio = 0.0F; + param->avg_doc_term_length = 1; + param->use_reorder = true; + param->immutable = immutable; + return param; +} + +std::string +date_search_params(const std::string& date = "") { + const auto date_field = date.empty() ? "" : fmt::format(R"(,"date":"{}")", date); + return fmt::format( + R"({{"sindi":{{"query_prune_ratio":0.0,"term_prune_ratio":0.0,"n_candidate":10{}}}}})", + date_field); +} + +} // namespace + TEST_CASE("SINDI Heap Insert Strategy Test", "[ut][SINDI]") { auto allocator = SafeAllocator::FactoryDefaultAllocator(); IndexCommonParam common_param; @@ -244,12 +293,258 @@ TEST_CASE("SINDI streaming compatibility", "[ut][SINDI][streaming][compatibility REQUIRE_FALSE(Index::Load(load_stream, "{}").has_value()); } + SECTION("loads legacy stream without optional window metadata") { + auto mutated = EraseStreamingBlock(bytes, StreamSerializationTag::SINDI_WINDOW_METADATA); + auto restored = std::make_unique(index_param, common_param); + std::stringstream deserialize_stream(mutated); + REQUIRE_NOTHROW(restored->DeserializeStreaming(deserialize_stream)); + REQUIRE(restored->GetNumElements() == num_base); + + auto query = Dataset::Make(); + query->NumElements(1)->SparseVectors(sv_base.data())->Owner(false); + REQUIRE(restored->KnnSearch(query, 10, date_search_params(), nullptr)->GetDim() == 10); + REQUIRE(restored->KnnSearch(query, 10, date_search_params("2026"), nullptr)->GetDim() == 0); + } + for (auto& item : sv_base) { delete[] item.vals_; delete[] item.ids_; } } +TEST_CASE("SINDI Date Window Search Test", "[ut][SINDI][date]") { + const bool immutable = GENERATE(false, true); + CAPTURE(immutable); + + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + common_param.metric_ = MetricType::METRIC_TYPE_IP; + common_param.dim_ = 2; + + constexpr uint32_t num_base = 7; + uint32_t term_id = 1; + float term_value = 1.0F; + std::vector vectors(num_base); + for (auto& vector : vectors) { + vector.len_ = 1; + vector.ids_ = &term_id; + vector.vals_ = &term_value; + } + std::array ids = {10, 11, 12, 13, 14, 15, 16}; + std::array dates = { + "2026/05/17", "2025", "2026/05/17", "2026/05", "", "2026/06/01", "2026/05/17"}; + auto base = Dataset::Make(); + base->NumElements(num_base) + ->SparseVectors(vectors.data()) + ->Ids(ids.data()) + ->Dates(dates.data()) + ->Owner(false); + + auto index = std::make_unique(create_date_test_param(immutable, 2), common_param); + REQUIRE(index->Build(base).empty()); + + const auto& metadata = SINDITestAccess::WindowMetadata(*index); + REQUIRE(metadata.size() == 6); + REQUIRE(metadata[0].date == "2026/05/17"); + REQUIRE(metadata[0].start_id == 0); + REQUIRE(metadata[0].doc_count == 2); + REQUIRE(metadata[1].date == "2026/05/17"); + REQUIRE(metadata[1].start_id == 2); + REQUIRE(metadata[1].doc_count == 1); + REQUIRE(metadata[2].date == "2025"); + REQUIRE(metadata[3].date == "2026/05"); + REQUIRE(metadata[4].date.empty()); + REQUIRE(metadata[5].date == "2026/06/01"); + + auto query = Dataset::Make(); + query->NumElements(1)->SparseVectors(vectors.data())->Owner(false); + + REQUIRE(result_ids(index->KnnSearch(query, 10, date_search_params(), nullptr)) == + std::set{10, 11, 12, 13, 14, 15, 16}); + REQUIRE(result_ids(index->KnnSearch(query, 10, date_search_params("2026"), nullptr)) == + std::set{10, 12, 13, 15, 16}); + REQUIRE(result_ids(index->KnnSearch(query, 10, date_search_params("2026/05"), nullptr)) == + std::set{10, 12, 13, 16}); + REQUIRE(result_ids(index->KnnSearch(query, 10, date_search_params("2026/05/17"), nullptr)) == + std::set{10, 12, 16}); + REQUIRE(index->KnnSearch(query, 10, date_search_params("2024"), nullptr)->GetDim() == 0); + + auto even_filter = std::make_shared(); + REQUIRE(result_ids(index->KnnSearch(query, 10, date_search_params("2026"), even_filter)) == + std::set{10, 12, 16}); + REQUIRE(result_ids(index->RangeSearch(query, 0.0F, date_search_params("2026/05"), nullptr)) == + std::set{10, 12, 13, 16}); +} + +TEST_CASE("SINDI Date Add And Serialization Test", "[ut][SINDI][date]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + common_param.metric_ = MetricType::METRIC_TYPE_IP; + common_param.dim_ = 2; + + uint32_t term_id = 1; + float term_value = 1.0F; + SparseVector vector; + vector.len_ = 1; + vector.ids_ = &term_id; + vector.vals_ = &term_value; + + SECTION("incremental add only reuses the matching tail window") { + auto index = std::make_unique(create_date_test_param(false, 2), common_param); + const std::array ids = {20, 21, 22, 23}; + const std::array dates = {"2026", "2026", "2025", "2026"}; + for (uint32_t i = 0; i < ids.size(); ++i) { + auto data = Dataset::Make(); + data->NumElements(1) + ->SparseVectors(&vector) + ->Ids(ids.data() + i) + ->Dates(dates.data() + i) + ->Owner(false); + REQUIRE(index->Add(data).empty()); + } + + const auto& metadata = SINDITestAccess::WindowMetadata(*index); + REQUIRE(metadata.size() == 3); + REQUIRE(metadata[0].date == "2026"); + REQUIRE(metadata[0].doc_count == 2); + REQUIRE(metadata[1].date == "2025"); + REQUIRE(metadata[1].doc_count == 1); + REQUIRE(metadata[2].date == "2026"); + REQUIRE(metadata[2].doc_count == 1); + } + + SECTION("date metadata survives normal and streaming serialization") { + const bool immutable = GENERATE(false, true); + CAPTURE(immutable); + auto param = create_date_test_param(immutable, 10000); + auto source = std::make_unique(param, common_param); + std::array ids = {30, 31, 32}; + std::array dates = {"2026/05", "", "2025"}; + std::array vectors = {vector, vector, vector}; + auto base = Dataset::Make(); + base->NumElements(3) + ->SparseVectors(vectors.data()) + ->Ids(ids.data()) + ->Dates(dates.data()) + ->Owner(false); + REQUIRE(source->Build(base).empty()); + + auto restored = std::make_unique(param, common_param); + test_serializion(*source, *restored); + auto query = Dataset::Make(); + query->NumElements(1)->SparseVectors(&vector)->Owner(false); + REQUIRE(result_ids(restored->KnnSearch(query, 10, date_search_params("2026"), nullptr)) == + std::set{30}); + + if (not immutable) { + std::stringstream stream; + REQUIRE_NOTHROW(source->SerializeStreaming(stream)); + auto streaming_restored = std::make_unique(param, common_param); + REQUIRE_NOTHROW(streaming_restored->DeserializeStreaming(stream)); + REQUIRE(result_ids(streaming_restored->KnnSearch( + query, 10, date_search_params("2025"), nullptr)) == std::set{32}); + } + } + + SECTION("invalid build date fails before inserting documents") { + auto index = std::make_unique(create_date_test_param(false, 2), common_param); + int64_t id = 40; + std::string date = "2026/5"; + auto base = Dataset::Make(); + base->NumElements(1)->SparseVectors(&vector)->Ids(&id)->Dates(&date)->Owner(false); + REQUIRE_THROWS(index->Build(base)); + REQUIRE(index->GetNumElements() == 0); + REQUIRE(SINDITestAccess::WindowMetadata(*index).empty()); + } + + SECTION("legacy normal serialization restores undated windows") { + auto source_param = create_date_test_param(false, 10000); + source_param->use_reorder = false; + auto source = std::make_unique(source_param, common_param); + std::array ids = {50, 51}; + std::array vectors = {vector, vector}; + auto base = Dataset::Make(); + base->NumElements(2)->SparseVectors(vectors.data())->Ids(ids.data())->Owner(false); + REQUIRE(source->Build(base).empty()); + + std::stringstream stream; + IOStreamWriter writer(stream); + SINDITestAccess::SerializeLegacyMutable(*source, writer); + + auto restored_param = create_date_test_param(false, 10000); + restored_param->use_reorder = false; + restored_param->deserialize_without_footer = true; + auto restored = std::make_unique(restored_param, common_param); + IOStreamReader reader(stream); + REQUIRE_NOTHROW(restored->Deserialize(reader)); + + auto query = Dataset::Make(); + query->NumElements(1)->SparseVectors(&vector)->Owner(false); + REQUIRE(restored->KnnSearch(query, 10, date_search_params(), nullptr)->GetDim() == 2); + REQUIRE(restored->KnnSearch(query, 10, date_search_params("2026"), nullptr)->GetDim() == 0); + } +} + +TEST_CASE("SINDI Date Quantization And Remap Test", "[ut][SINDI][date]") { + const auto configuration = GENERATE(std::make_tuple(SparseValueQuantizationType::SQ8, false), + std::make_tuple(SparseValueQuantizationType::FP32, true)); + const auto quantization = std::get<0>(configuration); + const auto remap_term_ids = std::get<1>(configuration); + CAPTURE(static_cast(quantization), remap_term_ids); + + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + common_param.metric_ = MetricType::METRIC_TYPE_IP; + common_param.dim_ = 2; + + auto param = create_date_test_param(false, 2); + param->use_reorder = false; + param->sparse_value_quant_type = quantization; + param->remap_term_ids = remap_term_ids; + auto index = std::make_unique(param, common_param); + + uint32_t term_id = remap_term_ids ? 1000 : 1; + float term_value = 1.0F; + SparseVector vector; + vector.len_ = 1; + vector.ids_ = &term_id; + vector.vals_ = &term_value; + std::array base_values = {2.0F, 1.0F, 3.0F}; + std::array vectors = {vector, vector, vector}; + for (uint32_t i = 0; i < vectors.size(); ++i) { + vectors[i].vals_ = base_values.data() + i; + } + std::array ids = {60, 61, 62}; + std::array dates = {"2026", "2025", "2026"}; + auto base = Dataset::Make(); + base->NumElements(3) + ->SparseVectors(vectors.data()) + ->Ids(ids.data()) + ->Dates(dates.data()) + ->Owner(false); + REQUIRE(index->Build(base).empty()); + + auto query = Dataset::Make(); + query->NumElements(1)->SparseVectors(&vector)->Owner(false); + REQUIRE(result_ids(index->KnnSearch(query, 10, date_search_params("2026"), nullptr)) == + std::set{60, 62}); + REQUIRE(std::isfinite(index->CalcDistanceById(query, 61, false))); + + SparseVector restored; + index->GetSparseVectorByInnerId(2, &restored, allocator.get()); + REQUIRE(restored.len_ == 1); + REQUIRE(restored.ids_[0] == term_id); + allocator->Deallocate(restored.ids_); + allocator->Deallocate(restored.vals_); + + REQUIRE(index->Remove({60}, RemoveMode::MARK_REMOVE) == 1); + REQUIRE(result_ids(index->KnnSearch(query, 10, date_search_params("2026"), nullptr)) == + std::set{62}); +} + TEST_CASE("SINDI Basic Test", "[ut][SINDI]") { auto allocator = SafeAllocator::FactoryDefaultAllocator(); IndexCommonParam common_param; diff --git a/src/analyzer/sindi_analyzer.cpp b/src/analyzer/sindi_analyzer.cpp index 3350e8989a..e393303896 100644 --- a/src/analyzer/sindi_analyzer.cpp +++ b/src/analyzer/sindi_analyzer.cpp @@ -185,8 +185,8 @@ SINDIAnalyzer::get_pruned_sparse_vector_by_inner_id(InnerIdType inner_id, std::shared_lock rlock(sindi_->global_mutex_); Allocator* allocator = specified_allocator != nullptr ? specified_allocator : sindi_->allocator_; - auto cur_window = inner_id / sindi_->window_size_; - auto window_start_id = cur_window * sindi_->window_size_; + auto cur_window = sindi_->find_window_id(inner_id); + auto window_start_id = sindi_->window_metadata_[cur_window].start_id; auto term_list = sindi_->window_term_list_[cur_window]; term_list->GetSparseVector(inner_id - window_start_id, data, allocator); if (sindi_->remap_term_ids_ && sindi_->term_id_mapper_) { @@ -226,7 +226,11 @@ SINDIAnalyzer::collect_coarse_candidates(const SparseVector& query, Vector dists(sindi_->window_size_, 0.0F, sindi_->allocator_); for (int64_t cur = 0; cur < static_cast(sindi_->window_term_list_.size()); ++cur) { - auto window_start_id = static_cast(cur * sindi_->window_size_); + const auto& metadata = sindi_->window_metadata_[static_cast(cur)]; + if (not SINDIDateMatches(metadata.date, search_param.date)) { + continue; + } + auto window_start_id = metadata.start_id; auto term_list = sindi_->window_term_list_[cur]; term_list->Query(dists.data(), computer); if (use_term_lists_heap_insert) { @@ -234,7 +238,7 @@ SINDIAnalyzer::collect_coarse_candidates(const SparseVector& query, dists.data(), computer, heap, inner_param, window_start_id); } else { term_list->InsertHeapByDists( - dists.data(), dists.size(), heap, inner_param, window_start_id); + dists.data(), metadata.doc_count, heap, inner_param, window_start_id); } } @@ -278,7 +282,11 @@ SINDIAnalyzer::collect_doc_prune_candidates(const SparseVector& query, Vector dists(sindi_->window_size_, 0.0F, sindi_->allocator_); for (int64_t cur = 0; cur < static_cast(sindi_->window_term_list_.size()); ++cur) { - auto window_start_id = static_cast(cur * sindi_->window_size_); + const auto& metadata = sindi_->window_metadata_[static_cast(cur)]; + if (not SINDIDateMatches(metadata.date, search_param.date)) { + continue; + } + auto window_start_id = metadata.start_id; auto term_list = sindi_->window_term_list_[cur]; if (term_list == nullptr) { continue; @@ -324,7 +332,7 @@ SINDIAnalyzer::collect_doc_prune_candidates(const SparseVector& query, dists.data(), computer, heap, inner_param, window_start_id); } else { term_list->InsertHeapByDists( - dists.data(), dists.size(), heap, inner_param, window_start_id); + dists.data(), metadata.doc_count, heap, inner_param, window_start_id); } } @@ -380,7 +388,18 @@ SINDIAnalyzer::get_original_sparse_vector_by_inner_id(InnerIdType inner_id, inner_id >= base_dataset->GetNumElements()) { return false; } - copy_sparse_vector(base_dataset->GetSparseVectors()[inner_id], data, allocator); + auto base_idx = static_cast(inner_id); + const auto label = sindi_->label_table_->GetLabelById(inner_id); + if (base_dataset->GetIds() != nullptr) { + const auto* begin = base_dataset->GetIds(); + const auto* end = begin + base_dataset->GetNumElements(); + const auto* iter = std::find(begin, end, label); + if (iter == end) { + return false; + } + base_idx = iter - begin; + } + copy_sparse_vector(base_dataset->GetSparseVectors()[base_idx], data, allocator); return true; } @@ -412,7 +431,8 @@ SINDIAnalyzer::calculate_pruned_distance_by_label(const SparseVector& query, int DatasetPtr SINDIAnalyzer::calculate_ground_truth(const DatasetPtr& query_dataset, int64_t topk, - const DatasetPtr& base_dataset) const { + const DatasetPtr& base_dataset, + const SINDISearchParameter& search_param) const { if (query_dataset == nullptr || query_dataset->GetSparseVectors() == nullptr || topk <= 0) { return nullptr; } @@ -421,10 +441,20 @@ SINDIAnalyzer::calculate_ground_truth(const DatasetPtr& query_dataset, return nullptr; } + std::vector matching_inner_ids; + matching_inner_ids.reserve(static_cast(sindi_->cur_element_count_.load())); + for (uint64_t raw_inner_id = 0; + raw_inner_id < static_cast(sindi_->cur_element_count_.load()); + ++raw_inner_id) { + const auto inner_id = static_cast(raw_inner_id); + const auto window_id = sindi_->find_window_id(inner_id); + if (SINDIDateMatches(sindi_->window_metadata_[window_id].date, search_param.date)) { + matching_inner_ids.push_back(inner_id); + } + } + auto query_count = query_dataset->GetNumElements(); - auto base_count = base_dataset == nullptr ? sindi_->cur_element_count_.load() - : base_dataset->GetNumElements(); - auto effective_topk = std::min(topk, base_count); + auto effective_topk = std::min(topk, static_cast(matching_inner_ids.size())); if (effective_topk <= 0) { return nullptr; } @@ -437,20 +467,16 @@ SINDIAnalyzer::calculate_ground_truth(const DatasetPtr& query_dataset, for (int64_t query_idx = 0; query_idx < query_count; ++query_idx) { std::priority_queue> heap; const auto& query = query_dataset->GetSparseVectors()[query_idx]; - for (int64_t base_idx = 0; base_idx < base_count; ++base_idx) { + for (const auto inner_id : matching_inner_ids) { SparseVector original{}; - const SparseVector* base_vector = nullptr; - if (base_dataset != nullptr && base_dataset->GetSparseVectors() != nullptr) { - base_vector = base_dataset->GetSparseVectors() + base_idx; - } else { - sindi_->GetSparseVectorByInnerId(base_idx, &original, sindi_->allocator_); - base_vector = &original; + if (not get_original_sparse_vector_by_inner_id( + inner_id, &original, sindi_->allocator_, base_dataset)) { + sindi_->allocator_->Deallocate(ids); + sindi_->allocator_->Deallocate(distances); + return nullptr; } - - auto label = base_dataset != nullptr && base_dataset->GetIds() != nullptr - ? base_dataset->GetIds()[base_idx] - : sindi_->label_table_->GetLabelById(base_idx); - heap.emplace(get_sparse_distance(query, *base_vector), label); + auto label = sindi_->label_table_->GetLabelById(inner_id); + heap.emplace(get_sparse_distance(query, original), label); if (static_cast(heap.size()) > effective_topk) { heap.pop(); } @@ -860,7 +886,13 @@ SINDIAnalyzer::calculate_postings_scanned_stats(const DatasetPtr& query_dataset, term = compact.value(); } bool has_posting = false; - for (const auto& window : sindi_->window_term_list_) { + for (uint64_t window_id = 0; window_id < sindi_->window_term_list_.size(); + ++window_id) { + if (not SINDIDateMatches(sindi_->window_metadata_[window_id].date, + search_param.date)) { + continue; + } + const auto& window = sindi_->window_term_list_[window_id]; if (window != nullptr && term < window->term_sizes_.size() && window->term_sizes_[term] > 0) { has_posting = true; @@ -925,7 +957,8 @@ SINDIAnalyzer::get_base_search_stats(const std::string& search_param, auto search_json = parse_sindi_search_json(search_param); SINDISearchParameter parsed_search_param; parsed_search_param.FromJson(search_json); - auto ground_truth = calculate_ground_truth(query_dataset, K_ANALYZE_DEFAULT_TOPK, base_dataset); + auto ground_truth = calculate_ground_truth( + query_dataset, K_ANALYZE_DEFAULT_TOPK, base_dataset, parsed_search_param); auto recall_stats = calculate_recall_stats( query_dataset, ground_truth, parsed_search_param, K_ANALYZE_DEFAULT_TOPK); if (recall_stats.Contains("recall_query")) { @@ -1228,7 +1261,13 @@ SINDIAnalyzer::AnalyzeIndexBySearch(const SearchRequest& request) { CHECK_ARGUMENT(query_dataset->GetSparseVectors() != nullptr, "SINDI analyze requires sparse query dataset"); - auto effective_topk = std::min(request.topk_, sindi_->cur_element_count_); + int64_t matching_doc_count = 0; + for (const auto& metadata : sindi_->window_metadata_) { + if (SINDIDateMatches(metadata.date, search_param.date)) { + matching_doc_count += metadata.doc_count; + } + } + auto effective_topk = std::min(request.topk_, matching_doc_count); if (effective_topk <= 0) { return stats; } @@ -1247,7 +1286,8 @@ SINDIAnalyzer::AnalyzeIndexBySearch(const SearchRequest& request) { ground_truth = load_ground_truth(analyze_options.groundtruth_path); } if (ground_truth == nullptr && not has_explicit_groundtruth) { - ground_truth = calculate_ground_truth(query_dataset, effective_topk, base_dataset); + ground_truth = + calculate_ground_truth(query_dataset, effective_topk, base_dataset, search_param); if (ground_truth == nullptr && not sindi_->use_reorder_ && (base_dataset == nullptr || base_dataset->GetSparseVectors() == nullptr)) { ground_truth_skip_reason = diff --git a/src/analyzer/sindi_analyzer.h b/src/analyzer/sindi_analyzer.h index 410de574f3..db8999f9f3 100644 --- a/src/analyzer/sindi_analyzer.h +++ b/src/analyzer/sindi_analyzer.h @@ -75,7 +75,8 @@ class SINDIAnalyzer : public AnalyzerBase { DatasetPtr calculate_ground_truth(const DatasetPtr& query_dataset, int64_t topk, - const DatasetPtr& base_dataset = nullptr) const; + const DatasetPtr& base_dataset, + const SINDISearchParameter& search_param) const; JsonType get_active_term_count_stats() const; diff --git a/src/constants.cpp b/src/constants.cpp index f1c7b0364e..ed340bda38 100644 --- a/src/constants.cpp +++ b/src/constants.cpp @@ -41,6 +41,7 @@ const char* const SPARSE_VECTORS = "sparse_vectors"; const char* const INT8_VECTORS = "i8_vectors"; const char* const ATTRIBUTE_SETS = "attribute_sets"; const char* const DATASET_PATHS = "paths"; +const char* const DATASET_DATES = "dates"; const char* const EXTRA_INFOS = "extra_infos"; const char* const EXTRA_INFO_SIZE = "extra_info_size"; const char* const VECTOR_COUNTS = "vector_counts"; diff --git a/src/dataset_impl.cpp b/src/dataset_impl.cpp index 444d9819f7..21df04e564 100644 --- a/src/dataset_impl.cpp +++ b/src/dataset_impl.cpp @@ -288,6 +288,7 @@ DatasetImpl::~DatasetImpl() { // NOLINT release_paths(std::get(value)); } } + release_paths(DatasetImpl::GetDates()); delete[] DatasetImpl::GetSourceID(); if (DatasetImpl::GetAttributeSets() != nullptr) { const auto* attrsets = DatasetImpl::GetAttributeSets(); @@ -362,6 +363,10 @@ DatasetImpl::DeepCopy(Allocator* allocator) const { static_cast(num_elements))); } } + if (this->GetDates() != nullptr) { + copy_dataset->Dates( + allocate_and_copy_paths(this->GetDates(), static_cast(num_elements))); + } if (this->GetSourceID() != nullptr) { auto* source_ids = new std::string[num_elements]; @@ -413,6 +418,10 @@ DatasetImpl::Append(const DatasetPtr& other) { throw VsagException(ErrorType::INVALID_ARGUMENT, "Cannot append datasets with different extra info sizes"); } + if ((this->GetDates() == nullptr) != (other->GetDates() == nullptr)) { + throw VsagException(ErrorType::INVALID_ARGUMENT, + "Cannot append datasets with inconsistent date labels"); + } auto old_num_elements = this->GetNumElements(); auto new_num_elements = other->GetNumElements(); @@ -535,6 +544,11 @@ DatasetImpl::Append(const DatasetPtr& other) { append_paths(std::get(iter->second), other->GetPaths(hierarchy_name))); } + // append date labels + if (auto iter = this->data_.find(DATASET_DATES); iter != this->data_.end()) { + auto* current_dates = std::get(iter->second); + this->Dates(append_paths(current_dates, other->GetDates())); + } for (const auto* paths : replaced_paths) { delete[] paths; } diff --git a/src/dataset_impl.h b/src/dataset_impl.h index 61e7762736..383f715aca 100644 --- a/src/dataset_impl.h +++ b/src/dataset_impl.h @@ -239,6 +239,20 @@ class DatasetImpl : public Dataset { return nullptr; } + DatasetPtr + Dates(const std::string* dates) override { + this->data_[DATASET_DATES] = dates; + return shared_from_this(); + } + + const std::string* + GetDates() const override { + if (auto iter = this->data_.find(DATASET_DATES); iter != this->data_.end()) { + return std::get(iter->second); + } + return nullptr; + } + DatasetPtr ExtraInfos(const char* extra_info) override { this->data_[EXTRA_INFOS] = extra_info; diff --git a/src/dataset_impl_test.cpp b/src/dataset_impl_test.cpp index a71e081bea..84c7a4007f 100644 --- a/src/dataset_impl_test.cpp +++ b/src/dataset_impl_test.cpp @@ -531,6 +531,49 @@ TEST_CASE("Dataset Named Paths Test", "[ut][dataset]") { } } +TEST_CASE("Dataset Date Labels Test", "[ut][dataset][date]") { + SECTION("setter getter and deep copy") { + std::string dates[3] = {"2026", "2026/07", ""}; + auto dataset = vsag::Dataset::Make(); + dataset->NumElements(3)->Dim(1)->Dates(dates)->Owner(false); + + REQUIRE(dataset->GetDates() == dates); + REQUIRE(dataset->GetDates()[1] == "2026/07"); + + auto copy = dataset->DeepCopy(); + REQUIRE(copy->GetDates() != dates); + REQUIRE(copy->GetDates()[0] == "2026"); + REQUIRE(copy->GetDates()[1] == "2026/07"); + REQUIRE(copy->GetDates()[2].empty()); + } + + SECTION("append preserves date labels") { + auto* dates = CopyPathArray({"2026", "2026/07"}); + auto dataset = vsag::Dataset::Make(); + dataset->NumElements(2)->Dim(1)->Dates(dates)->Owner(true); + + auto* appended_dates = CopyPathArray({"2026/07/21"}); + auto appended = vsag::Dataset::Make(); + appended->NumElements(1)->Dim(1)->Dates(appended_dates)->Owner(true); + + dataset->Append(appended); + REQUIRE(dataset->GetNumElements() == 3); + REQUIRE(dataset->GetDates()[0] == "2026"); + REQUIRE(dataset->GetDates()[1] == "2026/07"); + REQUIRE(dataset->GetDates()[2] == "2026/07/21"); + } + + SECTION("append rejects inconsistent date fields") { + auto with_dates = vsag::Dataset::Make(); + with_dates->NumElements(1)->Dim(1)->Dates(CopyPathArray({"2026"}))->Owner(true); + auto without_dates = vsag::Dataset::Make(); + without_dates->NumElements(1)->Dim(1)->Owner(true); + + REQUIRE_THROWS(with_dates->Append(without_dates)); + REQUIRE_THROWS(without_dates->Append(with_dates)); + } +} + TEST_CASE("Dataset MultiVector Basic Test", "[ut][dataset]") { SECTION("MultiVectorDim default is 0") { auto dataset = vsag::Dataset::Make(); diff --git a/src/inner_string_params.h b/src/inner_string_params.h index 2b8ac532ba..d2373775c6 100644 --- a/src/inner_string_params.h +++ b/src/inner_string_params.h @@ -125,6 +125,7 @@ const char* const SPARSE_DESERIALIZE_WITHOUT_BUFFER = "deserialize_without_buffe const char* const SPARSE_AVG_DOC_TERM_LENGTH = "avg_doc_term_length"; const char* const SPARSE_REMAP_TERM_IDS = "remap_term_ids"; const char* const SPARSE_IMMUTABLE = "immutable"; +const char* const SPARSE_DATE = "date"; // graph param value const char* const GRAPH_PARAM_MAX_DEGREE_KEY = "max_degree"; diff --git a/src/storage/serialization_tags.h b/src/storage/serialization_tags.h index 889f823e8e..4b6497f50b 100644 --- a/src/storage/serialization_tags.h +++ b/src/storage/serialization_tags.h @@ -38,6 +38,7 @@ enum class StreamSerializationTag : uint32_t { SINDI_RERANK_INDEX = 12, SINDI_TERM_ID_MAPPER = 13, PYRAMID_HIERARCHIES = 14, + SINDI_WINDOW_METADATA = 15, }; inline const char* @@ -73,6 +74,8 @@ StreamSerializationTagName(uint32_t tag) { return "sindi_term_id_mapper"; case StreamSerializationTag::PYRAMID_HIERARCHIES: return "pyramid_hierarchies"; + case StreamSerializationTag::SINDI_WINDOW_METADATA: + return "sindi_window_metadata"; } return "unknown"; } @@ -97,6 +100,7 @@ StreamSerializationTagCritical(uint32_t tag) { case StreamSerializationTag::ATTRIBUTE_FILTER: case StreamSerializationTag::EXTRA_INFO: case StreamSerializationTag::RAW_VECTOR: + case StreamSerializationTag::SINDI_WINDOW_METADATA: return false; } return false; @@ -122,6 +126,7 @@ StreamSerializationBlockCurrentVersion(uint32_t tag) { case StreamSerializationTag::SINDI_RERANK_INDEX: case StreamSerializationTag::SINDI_TERM_ID_MAPPER: case StreamSerializationTag::PYRAMID_HIERARCHIES: + case StreamSerializationTag::SINDI_WINDOW_METADATA: return kStreamSerializationBlockVersionV1; } return kStreamSerializationBlockVersionV1;