feat(io): add optional read cache for disk IO

examples/cpp/406_feature_read_cache.cpp

@@ -0,0 +1,144 @@
+// Copyright 2024-present the vsag project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <vsag/vsag.h>
+
+#include <cstdint>
+#include <iostream>
+#include <random>
+#include <string>
+#include <vector>
+
+int
+main(int argc, char** argv) {
+    vsag::init();
+
+    /******************* Prepare Base Dataset *****************/
+    int64_t num_vectors = 1000;
+    int64_t dim = 64;
+    std::vector<int64_t> ids(num_vectors);
+    std::vector<float> datas(num_vectors * dim);
+    std::mt19937 rng(47);
+    std::uniform_real_distribution<float> distrib_real;
+    for (int64_t i = 0; i < num_vectors; ++i) {
+        ids[i] = i;
+    }
+    for (int64_t i = 0; i < dim * num_vectors; ++i) {
+        datas[i] = distrib_real(rng);
+    }
+    auto base = vsag::Dataset::Make();
+    base->NumElements(num_vectors)
+        ->Dim(dim)
+        ->Ids(ids.data())
+        ->Float32Vectors(datas.data())
+        ->Owner(false);
+
+    std::string precise_file_path = argc > 1 ? argv[1] : "vsag_read_cache_precise";
+
+    /******************* Create HGraph Index with ReadCache on precise codes *****************/
+    std::string hgraph_build_parameters = R"(
+    {
+        "dtype": "float32",
+        "metric_type": "l2",
+        "dim": 64,
+        "index_param": {
+            "base_quantization_type": "sq8",
+            "max_degree": 16,
+            "ef_construction": 40,
+            "alpha": 1.2,
+            "use_reorder": true,
+            "precise_quantization_type": "fp32",
+            "base_io_type": "memory_io",
+            "precise_io_type": "async_io",
+            "precise_enable_read_cache": true,
+            "precise_file_path": ")" + precise_file_path +
+                                          R"(",
+            "precise_cache_total_size": 131072
+        }
+    }
+    )";
+    vsag::Resource resource(vsag::Engine::CreateDefaultAllocator(), nullptr);
+    vsag::Engine engine(&resource);
+    vsag::Options::Instance().set_block_size_limit(2 * 1024 * 1024);
+    auto index_result = engine.CreateIndex("hgraph", hgraph_build_parameters);
+    if (not index_result.has_value()) {
+        std::cerr << "Failed to create index: " << index_result.error().message << std::endl;
+        return -1;
+    }
+    auto index = index_result.value();
+
+    /******************* Build HGraph Index *****************/
+    if (auto build_result = index->Build(base); build_result.has_value()) {
+        std::cout << "After Build(), Index HGraph with ReadCache contains: "
+                  << index->GetNumElements() << std::endl;
+    } else {
+        std::cerr << "Failed to build index: " << build_result.error().message << std::endl;
+        return -1;
+    }
+
+    /******************* Serialize to Disk *****************/
+    auto serialize_result = index->Serialize();
+    if (not serialize_result.has_value()) {
+        auto error = serialize_result.error();
+        std::cerr << "Failed to serialize index: " << error.message << std::endl;
+        return -1;
+    }
+
+    /******************* Deserialize from Disk *****************/
+    auto deserialize_result = vsag::Factory::CreateIndex("hgraph", hgraph_build_parameters);
+    if (not deserialize_result.has_value()) {
+        std::cerr << "Failed to create index for deserialization: "
+                  << deserialize_result.error().message << std::endl;
+        return -1;
+    }
+    auto restored_index = deserialize_result.value();
+    if (auto ds_result = restored_index->Deserialize(serialize_result.value());
+        not ds_result.has_value()) {
+        std::cerr << "Failed to deserialize index: " << ds_result.error().message << std::endl;
+        return -1;
+    }
+
+    /******************* Prepare Query Dataset *****************/
+    std::vector<float> query_vector(dim);
+    for (int64_t i = 0; i < dim; ++i) {
+        query_vector[i] = distrib_real(rng);
+    }
+    auto query = vsag::Dataset::Make();
+    query->NumElements(1)->Dim(dim)->Float32Vectors(query_vector.data())->Owner(false);
+
+    /******************* KnnSearch For Restored Index *****************/
+    auto hgraph_search_parameters = R"(
+    {
+        "hgraph": {
+            "ef_search": 40
+        }
+    }
+    )";
+    int64_t topk = 10;
+    auto search_result = restored_index->KnnSearch(query, topk, hgraph_search_parameters);
+    if (not search_result.has_value()) {
+        std::cerr << "Failed to search index: " << search_result.error().message << std::endl;
+        return -1;
+    }
+    auto result = search_result.value();
+
+    /******************* Print Search Result *****************/
+    std::cout << "results: " << std::endl;
+    for (int64_t i = 0; i < result->GetDim(); ++i) {
+        std::cout << result->GetIds()[i] << ": " << result->GetDistances()[i] << std::endl;
+    }
+
+    engine.Shutdown();
+    return 0;
+}

examples/cpp/407_feature_ivf_read_cache.cpp

@@ -0,0 +1,140 @@
+// Copyright 2024-present the vsag project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <vsag/vsag.h>
+
+#include <cstdint>
+#include <iostream>
+#include <random>
+#include <string>
+#include <vector>
+
+int
+main(int argc, char** argv) {
+    vsag::init();
+
+    /******************* Prepare Base Dataset *****************/
+    int64_t num_vectors = 1000;
+    int64_t dim = 64;
+    std::vector<int64_t> ids(num_vectors);
+    std::vector<float> datas(num_vectors * dim);
+    std::mt19937 rng(47);
+    std::uniform_real_distribution<float> distrib_real;
+    for (int64_t i = 0; i < num_vectors; ++i) {
+        ids[i] = i;
+    }
+    for (int64_t i = 0; i < dim * num_vectors; ++i) {
+        datas[i] = distrib_real(rng);
+    }
+    auto base = vsag::Dataset::Make();
+    base->NumElements(num_vectors)
+        ->Dim(dim)
+        ->Ids(ids.data())
+        ->Float32Vectors(datas.data())
+        ->Owner(false);
+
+    std::string precise_file_path = argc > 1 ? argv[1] : "vsag_ivf_read_cache_precise";
+
+    /******************* Create IVF Index with ReadCache on reorder codes *****************/
+    std::string ivf_build_params = R"(
+    {
+        "dtype": "float32",
+        "metric_type": "l2",
+        "dim": 64,
+        "index_param": {
+            "buckets_count": 10,
+            "base_quantization_type": "fp32",
+            "partition_strategy_type": "ivf",
+            "ivf_train_type": "kmeans",
+            "train_sample_count": 800,
+            "use_reorder": true,
+            "precise_quantization_type": "fp32",
+            "base_io_type": "memory_io",
+            "precise_io_type": "async_io",
+            "precise_enable_read_cache": true,
+            "precise_file_path": ")" +
+                                   precise_file_path + R"(",
+            "precise_cache_total_size": 131072
+        }
+    }
+    )";
+    auto index_result = vsag::Factory::CreateIndex("ivf", ivf_build_params);
+    if (not index_result.has_value()) {
+        std::cerr << "Failed to create index: " << index_result.error().message << std::endl;
+        return -1;
+    }
+    auto index = index_result.value();
+
+    /******************* Build IVF Index *****************/
+    if (auto build_result = index->Build(base); build_result.has_value()) {
+        std::cout << "After Build(), Index IVF with reorder ReadCache contains: "
+                  << index->GetNumElements() << std::endl;
+    } else {
+        std::cerr << "Failed to build index: " << build_result.error().message << std::endl;
+        return -1;
+    }
+
+    /******************* Serialize to Disk *****************/
+    auto serialize_result = index->Serialize();
+    if (not serialize_result.has_value()) {
+        auto error = serialize_result.error();
+        std::cerr << "Failed to serialize index: " << error.message << std::endl;
+        return -1;
+    }
+
+    /******************* Deserialize from Disk *****************/
+    auto deserialize_result = vsag::Factory::CreateIndex("ivf", ivf_build_params);
+    if (not deserialize_result.has_value()) {
+        std::cerr << "Failed to create index for deserialization: "
+                  << deserialize_result.error().message << std::endl;
+        return -1;
+    }
+    auto restored_index = deserialize_result.value();
+    if (auto ds_result = restored_index->Deserialize(serialize_result.value());
+        not ds_result.has_value()) {
+        std::cerr << "Failed to deserialize index: " << ds_result.error().message << std::endl;
+        return -1;
+    }
+
+    /******************* Prepare Query Dataset *****************/
+    std::vector<float> query_vector(dim);
+    for (int64_t i = 0; i < dim; ++i) {
+        query_vector[i] = distrib_real(rng);
+    }
+    auto query = vsag::Dataset::Make();
+    query->NumElements(1)->Dim(dim)->Float32Vectors(query_vector.data())->Owner(false);
+
+    /******************* KnnSearch For Restored Index *****************/
+    auto ivf_search_parameters = R"(
+    {
+        "ivf": {
+            "scan_buckets_count": 4
+        }
+    })";
+    int64_t topk = 10;
+    auto search_result = restored_index->KnnSearch(query, topk, ivf_search_parameters);
+    if (not search_result.has_value()) {
+        std::cerr << "Failed to search index: " << search_result.error().message << std::endl;
+        return -1;
+    }
+    auto result = search_result.value();
+
+    /******************* Print Search Result *****************/
+    std::cout << "results: " << std::endl;
+    for (int64_t i = 0; i < result->GetDim(); ++i) {
+        std::cout << result->GetIds()[i] << ": " << result->GetDistances()[i] << std::endl;
+    }
+
+    return 0;
+}

examples/cpp/CMakeLists.txt

@@ -154,6 +154,11 @@ target_link_libraries(324_feature_lazy_hgraph_extra_info vsag)
 
 add_executable(325_feature_uring_io 325_feature_uring_io.cpp)
 target_link_libraries(325_feature_uring_io vsag)
-
 add_executable(324_feature_hgraph_mci_companion 324_feature_hgraph_mci_companion.cpp)
 target_link_libraries(324_feature_hgraph_mci_companion vsag)
+
+add_executable(406_feature_read_cache 406_feature_read_cache.cpp)
+target_link_libraries(406_feature_read_cache vsag)
+
+add_executable(407_feature_ivf_read_cache 407_feature_ivf_read_cache.cpp)
+target_link_libraries(407_feature_ivf_read_cache vsag)

include/vsag/constants.h

@@ -219,6 +219,14 @@ extern const char* const HGRAPH_BASE_PQ_DIM;
 extern const char* const HGRAPH_BASE_FILE_PATH;
 extern const char* const HGRAPH_BASE_DIRECT_READ;
 extern const char* const HGRAPH_BASE_SUPPLEMENT_FILE_PATH;
+extern const char* const HGRAPH_BASE_ENABLE_READ_CACHE;
+extern const char* const HGRAPH_BASE_CACHE_TOTAL_SIZE;
+extern const char* const HGRAPH_GRAPH_ENABLE_READ_CACHE;
+extern const char* const HGRAPH_GRAPH_CACHE_TOTAL_SIZE;
+extern const char* const HGRAPH_PRECISE_ENABLE_READ_CACHE;
+extern const char* const HGRAPH_PRECISE_CACHE_TOTAL_SIZE;
+extern const char* const HGRAPH_RAW_VECTOR_ENABLE_READ_CACHE;
+extern const char* const HGRAPH_RAW_VECTOR_CACHE_TOTAL_SIZE;
 extern const char* const HGRAPH_PRECISE_IO_TYPE;
 extern const char* const HGRAPH_PRECISE_FILE_PATH;
 extern const char* const HGRAPH_PRECISE_DIRECT_READ;
@@ -261,6 +269,10 @@ extern const char* const IVF_BASE_QUANTIZATION_TYPE;
 extern const char* const IVF_BASE_IO_TYPE;
 extern const char* const IVF_BASE_PQ_DIM;
 extern const char* const IVF_BASE_FILE_PATH;
+extern const char* const IVF_BASE_ENABLE_READ_CACHE;
+extern const char* const IVF_BASE_CACHE_TOTAL_SIZE;
+extern const char* const IVF_PRECISE_ENABLE_READ_CACHE;
+extern const char* const IVF_PRECISE_CACHE_TOTAL_SIZE;
 extern const char* const IVF_PRECISE_QUANTIZATION_TYPE;
 extern const char* const IVF_PRECISE_IO_TYPE;
 extern const char* const IVF_PRECISE_FILE_PATH;