feat(sindi): implement value-aware term pruning

benchs/indexes/hgraph-90.yml

@@ -90,14 +90,3 @@ HGRAPH/COHERE/90:
     topk: 10
     search_mode: "knn"
     delete_index_after_search: false
-
-HGRAPH/INTERNET/90:
-    datapath: "/tmp/data/wholenet-sparse-1m-ip.hdf5"
-    type: "build,search" # build, search
-    index_name: "sindi"
-    create_params: '{"dim":512,"dtype":"sparse","metric_type":"ip","index_param":{"use_reorder": true, "doc_prune_ratio": 0.0, "window_size": 60000}}'
-    search_params: '{"sindi":{"query_prune_ratio": 0.25, "term_prune_ratio": 0, "n_candidate": 35}}'
-    index_path: "/tmp/wholenet-sparse-1m-ip/index/wholenet_index"
-    topk: 10
-    search_mode: "knn"
-    delete_index_after_search: false

benchs/indexes/hgraph-95.yml

@@ -90,14 +90,3 @@ HGRAPH/COHERE/95:
     topk: 10
     search_mode: "knn"
     delete_index_after_search: false
-
-HGRAPH/INTERNET/95:
-    datapath: "/tmp/data/wholenet-sparse-1m-ip.hdf5"
-    type: "build,search" # build, search
-    index_name: "sindi"
-    create_params: '{"dim":512,"dtype":"sparse","metric_type":"ip","index_param":{"use_reorder": true, "doc_prune_ratio": 0.0, "window_size": 60000}}'
-    search_params: '{"sindi":{"query_prune_ratio": 0.1, "term_prune_ratio": 0, "n_candidate": 30}}'
-    index_path: "/tmp/wholenet-sparse-1m-ip/index/wholenet_index"
-    topk: 10
-    search_mode: "knn"
-    delete_index_after_search: false

benchs/indexes/hgraph-99.yml

@@ -90,14 +90,3 @@ HGRAPH/COHERE/99:
     topk: 10
     search_mode: "knn"
     delete_index_after_search: false
-
-HGRAPH/INTERNET/99:
-    datapath: "/tmp/data/wholenet-sparse-1m-ip.hdf5"
-    type: "build,search" # build, search
-    index_name: "sindi"
-    create_params: '{"dim":512,"dtype":"sparse","metric_type":"ip","index_param":{"use_reorder": false, "doc_prune_ratio": 0.0, "use_quantization": false, "window_size": 60000}}'
-    search_params: '{"sindi":{"query_prune_ratio": 0.0, "term_prune_ratio": 0, "n_candidate": 0}}'
-    index_path: "/tmp/wholenet-sparse-1m-ip/index/wholenet_index"
-    topk: 10
-    search_mode: "knn"
-    delete_index_after_search: false

docs/docs/en/src/indexes/sindi.md

@@ -74,7 +74,7 @@ and `metric_type` **must** be `"ip"`.
 | `dim` | int | — (required) | Maximum number of non-zero elements per sparse vector. *Not* the vocabulary size. |
 | `term_id_limit` | int | `1000000` | Upper bound on term id values (≥ max term id + 1, up to 50 000 000). |
 | `window_size` | int | `50000` | Documents per window (range: 10 000 – 60 000). |
-| `doc_prune_ratio` | float | `0.0` | Fraction of lowest-weight terms dropped per doc at build time (0.0 – 0.9). |
+| `doc_prune_ratio` | float | `0.0` | Fraction of lowest-weight terms dropped per doc at build time (`[0.0, 1.0)`). |
 | `use_quantization` | bool or string | `false` | `false` stores FP32 values, `true` stores SQ8 values, and `"fp16"` stores FP16 values. |
 | `use_reorder` | bool | `false` | Keep a forward store and rescore candidates after coarse SINDI scoring. |
 | `rerank_type` | string | `"fp32"` | Forward-store type used when `use_reorder` is enabled. `fp32` keeps exact values; `dmq8` stores compressed 8-bit DMQ codes. |
@@ -140,8 +140,9 @@ Search-time parameters live under the `sindi` sub-object:
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
 | `n_candidate` | int | `0` | Candidate heap size. When `0`, defaults to `SPARSE_AMPLIFICATION_FACTOR · topk` (500×). If set, must satisfy `1 ≤ n_candidate ≤ SPARSE_AMPLIFICATION_FACTOR · topk`. |
-| `query_prune_ratio` | float | `0.0` | Fraction of lowest-weight query terms skipped (0.0 – 0.9). |
-| `term_prune_ratio` | float | `0.0` | Fraction of term-list entries skipped (0.0 – 0.9). |
+| `query_prune_ratio` | float | `0.0` | Fraction of lowest-weight query terms skipped (`[0.0, 1.0)`). |
+| `term_prune_ratio` | float | `0.0` | Fraction of the lowest-value postings skipped from each term list (`[0.0, 1.0)`). |
+| `term_retain_threshold` | uint64 | `0` | Maximum postings for one term across all windows. A value of `0` disables this limit; positive values allow each window to scan at most `floor(threshold / window_count)` postings. |
 
 SINDI chooses the heap-insertion strategy automatically from the build-time
 `doc_prune_ratio` and search-time `query_prune_ratio`. With the current `0.1`

docs/docs/zh/src/indexes/sindi.md

@@ -69,7 +69,7 @@ auto result = index->KnnSearch(
 | `dim` | int | —（必填） | 单条稀疏向量允许的最大非零项数量，**不是** 词表大小 |
 | `term_id_limit` | int | `1000000` | 词项 ID 的上界（应 ≥ 最大词项 ID + 1，最高 50 000 000） |
 | `window_size` | int | `50000` | 每个窗口容纳的文档数（取值范围 10 000 – 60 000） |
-| `doc_prune_ratio` | float | `0.0` | 构建阶段按文档丢弃权重最低词项的比例（0.0 – 0.9） |
+| `doc_prune_ratio` | float | `0.0` | 构建阶段按文档丢弃权重最低词项的比例，取值范围为 `[0.0, 1.0)` |
 | `use_quantization` | bool 或 string | `false` | `false` 存 FP32，`true` 存 SQ8，`"fp16"` 存 FP16 |
 | `use_reorder` | bool | `false` | 是否保留一份正排存储，在 SINDI 粗排后对候选做精排 |
 | `rerank_type` | string | `"fp32"` | `use_reorder` 开启时使用的正排存储类型。`fp32` 保留精确值；`dmq8` 使用压缩的 8-bit DMQ 编码 |
@@ -130,8 +130,9 @@ auto result = index->KnnSearch(
 | 参数 | 类型 | 默认值 | 说明 |
 |------|------|--------|------|
 | `n_candidate` | int | `0` | 候选堆大小。为 `0` 时自动取 `SPARSE_AMPLIFICATION_FACTOR · topk`（500 倍）；若显式设置，须满足 `1 ≤ n_candidate ≤ SPARSE_AMPLIFICATION_FACTOR · topk` |
-| `query_prune_ratio` | float | `0.0` | 查询时丢弃权重最低查询项的比例（0.0 – 0.9） |
-| `term_prune_ratio` | float | `0.0` | 查询时丢弃倒排表中低权项的比例（0.0 – 0.9） |
+| `query_prune_ratio` | float | `0.0` | 查询时丢弃权重最低查询项的比例，取值范围为 `[0.0, 1.0)` |
+| `term_prune_ratio` | float | `0.0` | 每条倒排链中按 value 丢弃低权 posting 的比例，取值范围为 `[0.0, 1.0)` |
+| `term_retain_threshold` | uint64 | `0` | 单个 term 在所有 window 中最多扫描的 posting 总数；`0` 表示关闭此限制，正数使每个 window 最多扫描 `floor(threshold / window_count)` 个 |
 
 SINDI 会根据构建阶段的 `doc_prune_ratio` 与检索阶段的 `query_prune_ratio`
 自动选择堆插入策略。按当前 `0.1` 阈值，当两个比例都 `<= 0.1` 时，SINDI 使用

examples/cpp/109_index_sindi.cpp

@@ -123,12 +123,16 @@ main(int argc, char** argv) {
      *
      * - sindi: Parameters specific to sparse indexing search:
      *   - query_prune_ratio: Ratio of term pruning for the query (0 = no pruning).
+     *   - term_prune_ratio: Fraction of the lowest-value postings to skip.
+     *   - term_retain_threshold: Per-term posting budget across all index windows.
      *   - n_candidate: Number of candidates for re-ranking. Must be greater than topK.
      *     This parameter is ignored if use_reorder is false in the build parameters.
      */
     auto sindi_search_parameters = R"({
         "sindi": {
             "query_prune_ratio": 0,
+            "term_prune_ratio": 0.1,
+            "term_retain_threshold": 100000,
             "n_candidate": 0
         }
     })";

examples/cpp/403_persistent_streaming_load.cpp

@@ -186,7 +186,6 @@ get_example_config() {
                         "use_reorder": true,
                         "use_quantization": false,
                         "doc_prune_ratio": 0.0,
-                        "term_prune_ratio": 0.0,
                         "window_size": 10000,
                         "term_id_limit": 30001,
                         "avg_doc_term_length": 100

examples/python/110_index_sindi_sq8_dmq_reorder.py

@@ -133,7 +133,8 @@ def sindi_sq8_dmq8_reorder_test():
         {
             "sindi": {
                 "query_prune_ratio": 0.2,  # Prune 20% of least-relevant query terms
-                "term_prune_ratio": 0.1,   # Prune 10% of least-relevant terms from posting lists
+                # Prune the lowest-value 10%, capped by the per-window share of 100,000 postings.
+                "term_prune_ratio": 0.1, "term_retain_threshold": 100000,
                 "n_candidate": 10,         # Keep top 10 candidates after inverted phase for reranking
             }
         }