diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index c7a6f4e561..b597a1e34f 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -27,6 +27,7 @@ jobs: - 'src/**' - 'include/**' - 'tests/**' + - 'tools/**' - 'CMakeLists.txt' - 'Makefile' - 'extern/**' @@ -165,6 +166,8 @@ jobs: key: build-macos-${{ hashFiles('./CMakeLists.txt') }}-${{ hashFiles('./.circleci/fresh_ci_cache.commit') }} - name: Make Asan run: export CMAKE_GENERATOR="Ninja" CMAKE_EXPORT_COMPILE_COMMANDS=ON; make asan VSAG_ENABLE_EXAMPLES=ON COMPILE_JOBS=3 + - name: Run Eval Tests + run: ./build/tests/unittests "[eval]" build-python-x86: name: Python Build X86 diff --git a/docs/docs/en/src/resources/eval.md b/docs/docs/en/src/resources/eval.md index 04525dd08b..53f2beb310 100644 --- a/docs/docs/en/src/resources/eval.md +++ b/docs/docs/en/src/resources/eval.md @@ -85,7 +85,45 @@ Note: under `global.exporters`, each entry is a **named** exporter (a YAML map), - **Efficiency**: QPS, TPS - **Quality**: average recall and quantile recall (P0/P10/P50/P90...) - **Latency**: average, P50/P95/P99 -- **Resource**: peak memory usage +- **Resource**: sampled process RSS peak and index-owned memory usage + +## Memory Metrics + +Memory monitoring samples the process resident set size (RSS) every 5 ms while the measured phase +is running. The human-readable `memory_peak(build)` and `memory_peak(search)` fields report the +largest sampled RSS increase above the phase baseline, using 1024-based units. A sampled peak can +miss an allocation that exists for less than one sampling interval. + +The phase boundaries are: + +- `build`: from immediately before `Index::Build()` until it returns. Dataset loading and index + serialization are outside the interval. +- `search`: from immediately before the first measured query pass until that pass finishes. Index + deserialization is outside the interval. When a latency/QPS measurement pass is present, RSS + sampling runs alongside that existing first pass instead of adding a separate pass that would + pre-warm later measurements. If there is no latency pass, a memory-only pass runs before recall + so recall bookkeeping is not counted. KNN result statistics are collected afterward, outside + all memory and performance monitor intervals. + +JSON output also includes exact numeric fields for each `` (`build` or `search`): + +| Field | Meaning | +| --- | --- | +| `memory_rss_baseline_bytes()` | Process RSS at the start of the phase | +| `memory_rss_peak_bytes()` | Largest process RSS sampled during the phase | +| `memory_peak_delta_bytes()` | Peak RSS minus baseline RSS, clamped at zero | +| `memory_peak_sample_count()` | Number of successful RSS samples | +| `memory_peak_failed_sample_count()` | Number of failed RSS samples | +| `memory_peak_available()` | Whether a valid baseline and sampler were available | +| `memory_peak_error()` | Sampling error, or an empty string on success | + +`index_memory(B)` reports memory owned by the index according to `Index::GetMemoryUsage()`, while +`memory_detail(B)` contains the available component breakdown. These index-level values are +separate from process RSS, which also includes the dataset, evaluator bookkeeping, allocator +overhead, and other process state. + +On an unsupported platform or when the baseline cannot be read, the human-readable value is +`N/A`, `memory_peak_available()` is `false`, and the error field explains the failure. ## Search Modes diff --git a/docs/docs/zh/src/resources/eval.md b/docs/docs/zh/src/resources/eval.md index f0c51d4103..35db586daa 100644 --- a/docs/docs/zh/src/resources/eval.md +++ b/docs/docs/zh/src/resources/eval.md @@ -80,7 +80,42 @@ eval_case1: - **效率**:QPS、TPS - **效果**:平均召回率、分位召回率(P0/P10/P50/P90...) - **延迟**:平均延迟、P50/P95/P99 延迟 -- **资源**:峰值内存占用 +- **资源**:进程 RSS 采样峰值与索引自身内存占用 + +## 内存指标 + +内存监控在被测阶段运行期间每 5 ms 采样一次进程常驻集大小(RSS)。 +`memory_peak(build)` 和 `memory_peak(search)` 是便于阅读的兼容字段,表示相对于阶段 +基线的最大 RSS 采样增量,单位按 1024 进位。持续时间短于一个采样周期的瞬时分配可能 +不会被捕获。 + +测量边界如下: + +- `build`:从调用 `Index::Build()` 前开始,到该调用返回为止;数据集加载和索引序列化 + 不在测量区间内。 +- `search`:从第一个被测查询批次开始前到该批次完成为止;索引反序列化不在测量区间内。 + 存在延迟/QPS 测量轮次时,RSS 采样与原有的第一个查询轮次同时运行,不再额外执行一个 + 会预热后续指标的内存轮次。没有延迟轮次时,内存专用轮次会先于召回率轮次执行,避免 + 把召回率统计开销计入 RSS。KNN 结果统计在此后独立采集,不属于内存或性能监控区间。 + +JSON 输出还会为每个 ``(`build` 或 `search`)提供以下精确数值字段: + +| 字段 | 含义 | +| --- | --- | +| `memory_rss_baseline_bytes()` | 阶段开始时的进程 RSS | +| `memory_rss_peak_bytes()` | 阶段内采样到的最大进程 RSS | +| `memory_peak_delta_bytes()` | 峰值 RSS 减去基线 RSS,最小为零 | +| `memory_peak_sample_count()` | 成功取得 RSS 的采样次数 | +| `memory_peak_failed_sample_count()` | RSS 采样失败次数 | +| `memory_peak_available()` | 是否取得有效基线并成功启动采样器 | +| `memory_peak_error()` | 采样错误;成功时为空字符串 | + +`index_memory(B)` 通过 `Index::GetMemoryUsage()` 报告索引自身占用, +`memory_detail(B)` 则提供可用的组件明细。这些索引级指标与进程 RSS 分开:进程 RSS +还会包含数据集、评估器自身开销、allocator 开销和进程中的其他状态。 + +在不支持的平台或无法读取基线时,便于阅读的字段为 `N/A`, +`memory_peak_available()` 为 `false`,错误字段会说明失败原因。 ## 搜索模式 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f9205c18b1..fcbbaaf1c4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -15,19 +15,32 @@ # fixtures is already added by top-level CMakeLists.txt before src +find_package (Threads REQUIRED) # unittests add_executable (unittests test_main.cpp + ${PROJECT_SOURCE_DIR}/tools/eval/case/search_pass_runner.cpp + ${PROJECT_SOURCE_DIR}/tools/eval/case/search_pass_runner_test.cpp + ${PROJECT_SOURCE_DIR}/tools/eval/monitor/memory_peak_monitor_test.cpp + ${PROJECT_SOURCE_DIR}/tools/eval/monitor/memory_peak_monitor.cpp + ${PROJECT_SOURCE_DIR}/tools/eval/monitor/monitor.cpp $ $ $ $ ) +target_include_directories (unittests PRIVATE + ${PROJECT_SOURCE_DIR}/tools/eval/case + ${PROJECT_SOURCE_DIR}/tools/eval/monitor +) + target_link_libraries (unittests PRIVATE - Catch2::Catch2 + Catch2::Catch2 + Threads::Threads + nlohmann_json::nlohmann_json simd_test vsag_test algorithm_test factory_test attr_test datacell_test quantizer_test storage_test io_test utils_test impl_test vsag_static diff --git a/tools/eval/CMakeLists.txt b/tools/eval/CMakeLists.txt index 3b87f885a5..dadcda6b3a 100644 --- a/tools/eval/CMakeLists.txt +++ b/tools/eval/CMakeLists.txt @@ -16,6 +16,7 @@ set (eval_srcs case/eval_case.cpp + case/search_pass_runner.cpp case/search_eval_case.cpp case/build_eval_case.cpp exporter/exporter.cpp diff --git a/tools/eval/case/build_eval_case.cpp b/tools/eval/case/build_eval_case.cpp index 3eb243ec46..8d0569288d 100644 --- a/tools/eval/case/build_eval_case.cpp +++ b/tools/eval/case/build_eval_case.cpp @@ -16,6 +16,7 @@ #include "./build_eval_case.h" #include +#include #include #include @@ -25,11 +26,54 @@ namespace vsag::eval { +namespace { + +class StartedMonitorGuard { +public: + explicit StartedMonitorGuard(std::vector& monitors) : monitors_(monitors) { + } + + ~StartedMonitorGuard() { + while (stopped_monitor_count_ < started_monitor_count_) { + const auto monitor_id = started_monitor_count_ - stopped_monitor_count_ - 1; + ++stopped_monitor_count_; + try { + monitors_[monitor_id]->Stop(); + } catch (...) { + } + } + } + + void + StartAll() { + for (auto& monitor : monitors_) { + monitor->Start(); + ++started_monitor_count_; + } + } + + void + StopNext() { + const auto monitor_id = started_monitor_count_ - stopped_monitor_count_ - 1; + monitors_[monitor_id]->Stop(); + ++stopped_monitor_count_; + } + +private: + std::vector& monitors_; + uint64_t started_monitor_count_{0}; + uint64_t stopped_monitor_count_{0}; +}; + +} // namespace + BuildEvalCase::BuildEvalCase(const std::string& dataset_path, const std::string& index_path, vsag::IndexPtr index, - EvalConfig config) - : EvalCase(dataset_path, index_path, index), config_(std::move(config)) { + EvalConfig config, + EvalDatasetPtr dataset) + : EvalCase(dataset_path, index_path, std::move(index), std::move(dataset)), + config_(std::move(config)) { this->init_monitors(); } @@ -68,16 +112,15 @@ BuildEvalCase::do_build() { } else { base->SparseVectors((const SparseVector*)this->dataset_ptr_->GetTrain()); } - for (auto& monitor : monitors_) { - monitor->Start(); - } + StartedMonitorGuard monitor_guard(monitors_); + monitor_guard.StartAll(); auto build_index = index_->Build(base); if (not build_index.has_value()) { throw std::runtime_error(build_index.error().message); } - for (auto& monitor : monitors_) { - monitor->Record(); - monitor->Stop(); + for (auto monitor = monitors_.rbegin(); monitor != monitors_.rend(); ++monitor) { + (*monitor)->Record(); + monitor_guard.StopNext(); } } void @@ -105,6 +148,11 @@ BuildEvalCase::process_result() { result["index_info"] = JsonType::parse(config_.build_param); result["action"] = "build"; result["index"] = config_.index_name; + try { + result["index_memory(B)"] = this->index_->GetMemoryUsage(); + } catch (const std::exception& e) { + logger_->Error(e.what()); + } try { auto detail = this->index_->GetMemoryUsageDetail(); for (const auto& [name, size] : detail) { diff --git a/tools/eval/case/build_eval_case.h b/tools/eval/case/build_eval_case.h index d43267cbe7..6e890c3201 100644 --- a/tools/eval/case/build_eval_case.h +++ b/tools/eval/case/build_eval_case.h @@ -25,7 +25,8 @@ class BuildEvalCase : public EvalCase { BuildEvalCase(const std::string& dataset_path, const std::string& index_path, vsag::IndexPtr index, - EvalConfig config); + EvalConfig config, + EvalDatasetPtr dataset = nullptr); ~BuildEvalCase() override = default; diff --git a/tools/eval/case/build_search_eval_case.h b/tools/eval/case/build_search_eval_case.h index c9f4f2e47f..e7bd6b8220 100644 --- a/tools/eval/case/build_search_eval_case.h +++ b/tools/eval/case/build_search_eval_case.h @@ -25,19 +25,27 @@ class BuildSearchEvalCase : public EvalCase { BuildSearchEvalCase(const std::string& dataset_path, const std::string& index_path, vsag::IndexPtr index, - EvalConfig config) - : EvalCase(dataset_path, index_path, index) { - build_ = EvalCase::MakeInstance(config, "build"); - search_ = EvalCase::MakeInstance(config, "search"); - config_ = std::move(config); + EvalConfig config, + EvalDatasetPtr dataset = nullptr) + : EvalCase(dataset_path, index_path, nullptr, std::move(dataset)), + config_(std::move(config)) { + build_ = std::make_shared( + dataset_path_, index_path_, std::move(index), config_, dataset_ptr_); } ~BuildSearchEvalCase() override = default; JsonType Run() override { - auto build_result = build_->Run(); - auto search_result = search_->Run(); + auto build = std::move(build_); + if (build == nullptr) { + build = EvalCase::MakeInstance(config_, "build", dataset_ptr_); + } + auto build_result = build->Run(); + build.reset(); + + auto search = EvalCase::MakeInstance(config_, "search", dataset_ptr_); + auto search_result = search->Run(); return merge_results(build_result, search_result); } @@ -50,9 +58,8 @@ class BuildSearchEvalCase : public EvalCase { } private: - EvalCasePtr build_; - EvalCasePtr search_; EvalConfig config_; + EvalCasePtr build_; }; } // namespace vsag::eval diff --git a/tools/eval/case/eval_case.cpp b/tools/eval/case/eval_case.cpp index ef99325181..f7a49e6d9b 100644 --- a/tools/eval/case/eval_case.cpp +++ b/tools/eval/case/eval_case.cpp @@ -26,15 +26,23 @@ namespace vsag::eval { -EvalCase::EvalCase(std::string dataset_path, std::string index_path, vsag::IndexPtr index) - : dataset_path_(std::move(dataset_path)), index_path_(std::move(index_path)), index_(index) { - this->dataset_ptr_ = EvalDataset::Load(dataset_path_); +EvalCase::EvalCase(std::string dataset_path, + std::string index_path, + vsag::IndexPtr index, + EvalDatasetPtr dataset) + : dataset_path_(std::move(dataset_path)), + index_path_(std::move(index_path)), + dataset_ptr_(std::move(dataset)), + index_(std::move(index)) { + if (this->dataset_ptr_ == nullptr) { + this->dataset_ptr_ = EvalDataset::Load(dataset_path_); + } this->logger_ = vsag::Options::Instance().logger(); this->basic_info_ = this->dataset_ptr_->GetInfo(); } EvalCasePtr -EvalCase::MakeInstance(const EvalConfig& config, std::string type) { +EvalCase::MakeInstance(const EvalConfig& config, std::string type, EvalDatasetPtr dataset) { auto dataset_path = config.dataset_path; auto index_path = config.index_path; auto index_name = config.index_name; @@ -48,14 +56,16 @@ EvalCase::MakeInstance(const EvalConfig& config, std::string type) { } if (type == "build") { - return std::make_shared(dataset_path, index_path, index.value(), config); + return std::make_shared( + dataset_path, index_path, index.value(), config, std::move(dataset)); } if (type == "search") { - return std::make_shared(dataset_path, index_path, index.value(), config); + return std::make_shared( + dataset_path, index_path, index.value(), config, std::move(dataset)); } if (type == "build,search") { return std::make_shared( - dataset_path, index_path, index.value(), config); + dataset_path, index_path, index.value(), config, std::move(dataset)); } return nullptr; } diff --git a/tools/eval/case/eval_case.h b/tools/eval/case/eval_case.h index 8a7316242f..d3365e828a 100644 --- a/tools/eval/case/eval_case.h +++ b/tools/eval/case/eval_case.h @@ -31,7 +31,9 @@ using EvalCasePtr = std::shared_ptr; class EvalCase { public: static EvalCasePtr - MakeInstance(const EvalConfig& config, std::string type = "none"); + MakeInstance(const EvalConfig& config, + std::string type = "none", + EvalDatasetPtr dataset = nullptr); static void MergeJsonType(const JsonType& input, JsonType& output) { @@ -86,7 +88,10 @@ class EvalCase { } public: - explicit EvalCase(std::string dataset_path, std::string index_path, vsag::IndexPtr index); + explicit EvalCase(std::string dataset_path, + std::string index_path, + vsag::IndexPtr index, + EvalDatasetPtr dataset = nullptr); virtual ~EvalCase() = default; diff --git a/tools/eval/case/search_eval_case.cpp b/tools/eval/case/search_eval_case.cpp index 3a12c02708..b422e2d709 100644 --- a/tools/eval/case/search_eval_case.cpp +++ b/tools/eval/case/search_eval_case.cpp @@ -26,6 +26,7 @@ #include "../monitor/latency_monitor.h" #include "../monitor/memory_peak_monitor.h" #include "../monitor/recall_monitor.h" +#include "search_pass_runner.h" #include "typing.h" #include "vsag/filter.h" #include "vsag_exception.h" @@ -57,8 +58,10 @@ class FilterObj : public vsag::Filter { SearchEvalCase::SearchEvalCase(const std::string& dataset_path, const std::string& index_path, vsag::IndexPtr index, - EvalConfig config) - : EvalCase(dataset_path, index_path, index), config_(std::move(config)) { + EvalConfig config, + EvalDatasetPtr dataset) + : EvalCase(dataset_path, index_path, std::move(index), std::move(dataset)), + config_(std::move(config)) { auto search_mode = config_.search_mode; if (search_mode == "knn") { this->search_type_ = SearchType::KNN; @@ -93,7 +96,8 @@ SearchEvalCase::init_latency_monitor() { if (config_.enable_percent_latency) { latency_monitor->SetMetrics("percent_latency"); } - this->monitors_.emplace_back(std::move(latency_monitor)); + this->latency_monitor_ = latency_monitor; + this->monitors_.emplace_back(this->latency_monitor_); } } @@ -115,8 +119,7 @@ SearchEvalCase::init_recall_monitor() { void SearchEvalCase::init_memory_monitor() { if (config_.enable_memory) { - auto memory_peak_monitor = std::make_shared("search"); - this->monitors_.emplace_back(std::move(memory_peak_monitor)); + this->memory_monitor_ = std::make_shared("search"); } } @@ -152,39 +155,46 @@ SearchEvalCase::deserialize(std::ifstream& infile) { void SearchEvalCase::do_knn_search() { + SearchPassRunner::Run( + this->monitors_, + this->latency_monitor_, + this->memory_monitor_, + [this](const MonitorPtr& monitor) { this->run_knn_search_pass(monitor.get(), false); }, + [this]() { this->run_knn_search_pass(nullptr, true); }); +} + +void +SearchEvalCase::run_knn_search_pass(Monitor* monitor, bool collect_statistics) { uint64_t topk = config_.top_k; auto query_count = this->dataset_ptr_->GetNumberOfQuery(); this->logger_->Debug("query count is " + std::to_string(query_count)); auto min_query = std::max(static_cast(query_count), config_.search_query_count); - for (uint64_t monitor_id = 0; monitor_id < this->monitors_.size(); ++monitor_id) { - auto& monitor = this->monitors_[monitor_id]; - const bool collect_statistics = monitor_id == 0; - monitor->Start(); - omp_set_num_threads(config_.num_threads_searching); + omp_set_num_threads(config_.num_threads_searching); #pragma omp parallel for schedule(dynamic) - for (int64_t id = 0; id < min_query; ++id) { - auto i = id % query_count; - auto query = vsag::Dataset::Make(); - query->NumElements(1)->Dim(this->dataset_ptr_->GetDim())->Owner(false); - const void* query_vector = this->dataset_ptr_->GetOneTest(i); - if (this->dataset_ptr_->GetVectorType() == DENSE_VECTORS) { - if (this->dataset_ptr_->GetTestDataType() == vsag::DATATYPE_FLOAT32) { - query->Float32Vectors((const float*)query_vector); - } else if (this->dataset_ptr_->GetTestDataType() == vsag::DATATYPE_INT8) { - query->Int8Vectors((const int8_t*)query_vector); - } - } else { - query->SparseVectors((const SparseVector*)query_vector); - } - auto result = this->index_->KnnSearch(query, topk, config_.search_param); - if (not result.has_value()) { - std::cerr << "query error: " << result.error().message << std::endl; - exit(-1); - } - if (collect_statistics) { - this->record_statistics(result.value()); + for (int64_t id = 0; id < min_query; ++id) { + auto i = id % query_count; + auto query = vsag::Dataset::Make(); + query->NumElements(1)->Dim(this->dataset_ptr_->GetDim())->Owner(false); + const void* query_vector = this->dataset_ptr_->GetOneTest(i); + if (this->dataset_ptr_->GetVectorType() == DENSE_VECTORS) { + if (this->dataset_ptr_->GetTestDataType() == vsag::DATATYPE_FLOAT32) { + query->Float32Vectors((const float*)query_vector); + } else if (this->dataset_ptr_->GetTestDataType() == vsag::DATATYPE_INT8) { + query->Int8Vectors((const int8_t*)query_vector); } + } else { + query->SparseVectors((const SparseVector*)query_vector); + } + auto result = this->index_->KnnSearch(query, topk, config_.search_param); + if (not result.has_value()) { + std::cerr << "query error: " << result.error().message << std::endl; + exit(-1); + } + if (collect_statistics) { + this->record_statistics(result.value()); + } + if (monitor != nullptr) { const int64_t* neighbors = result.value()->GetIds(); int64_t* ground_truth_neighbors = dataset_ptr_->GetNeighbors(i); auto record = std::make_tuple(neighbors, @@ -194,7 +204,6 @@ SearchEvalCase::do_knn_search() { result.value()->GetDim()); monitor->Record(&record); } - monitor->Stop(); } } @@ -204,6 +213,16 @@ SearchEvalCase::do_range_search() { void SearchEvalCase::do_knn_filter_search() { + SearchPassRunner::Run( + this->monitors_, + this->latency_monitor_, + this->memory_monitor_, + [this](const MonitorPtr& monitor) { this->run_knn_filter_search_pass(monitor.get()); }, + []() {}); +} + +void +SearchEvalCase::run_knn_filter_search_pass(Monitor* monitor) { uint64_t topk = config_.top_k; auto query_count = this->dataset_ptr_->GetNumberOfQuery(); auto train_labels = this->dataset_ptr_->GetTrainLabels(); @@ -216,33 +235,32 @@ SearchEvalCase::do_knn_filter_search() { } this->logger_->Debug("query count is " + std::to_string(query_count)); auto min_query = std::max(query_count, 10000); - for (auto& monitor : this->monitors_) { - monitor->Start(); - for (int64_t id = 0; id < min_query; ++id) { - auto i = id % query_count; - auto query = vsag::Dataset::Make(); - query->NumElements(1)->Dim(this->dataset_ptr_->GetDim())->Owner(false); - const void* query_vector = this->dataset_ptr_->GetOneTest(i); - if (this->dataset_ptr_->GetTestDataType() == vsag::DATATYPE_FLOAT32) { - query->Float32Vectors((const float*)query_vector); - } else if (this->dataset_ptr_->GetTestDataType() == vsag::DATATYPE_INT8) { - query->Int8Vectors((const int8_t*)query_vector); - } - auto test_label = test_labels[i]; - auto filter = std::make_shared( - train_labels, test_label, this->dataset_ptr_->GetValidRatio(test_label)); - auto result = this->index_->KnnSearch(query, topk, config_.search_param, filter); - if (not result.has_value()) { - std::cerr << "query error: " << result.error().message << std::endl; - exit(-1); - } + + for (int64_t id = 0; id < min_query; ++id) { + auto i = id % query_count; + auto query = vsag::Dataset::Make(); + query->NumElements(1)->Dim(this->dataset_ptr_->GetDim())->Owner(false); + const void* query_vector = this->dataset_ptr_->GetOneTest(i); + if (this->dataset_ptr_->GetTestDataType() == vsag::DATATYPE_FLOAT32) { + query->Float32Vectors((const float*)query_vector); + } else if (this->dataset_ptr_->GetTestDataType() == vsag::DATATYPE_INT8) { + query->Int8Vectors((const int8_t*)query_vector); + } + auto test_label = test_labels[i]; + auto filter = std::make_shared( + train_labels, test_label, this->dataset_ptr_->GetValidRatio(test_label)); + auto result = this->index_->KnnSearch(query, topk, config_.search_param, filter); + if (not result.has_value()) { + std::cerr << "query error: " << result.error().message << std::endl; + exit(-1); + } + if (monitor != nullptr) { const int64_t* neighbors = result.value()->GetIds(); int64_t* ground_truth_neighbors = dataset_ptr_->GetNeighbors(i); auto record = std::make_tuple( neighbors, ground_truth_neighbors, dataset_ptr_.get(), query_vector, topk); monitor->Record(&record); } - monitor->Stop(); } } @@ -257,11 +275,19 @@ SearchEvalCase::process_result() { const auto& one_result = monitor->GetResult(); EvalCase::MergeJsonType(one_result, result); } + if (this->memory_monitor_ != nullptr) { + EvalCase::MergeJsonType(this->memory_monitor_->GetResult(), result); + } result["action"] = "search"; result["search_mode"] = config_.search_mode; result["index_info"] = JsonType::parse(config_.build_param); result["search_param"] = config_.search_param; result["index"] = config_.index_name; + try { + result["index_memory(B)"] = this->index_->GetMemoryUsage(); + } catch (const std::exception& e) { + logger_->Error(e.what()); + } try { auto detail = this->index_->GetMemoryUsageDetail(); for (const auto& [name, size] : detail) { diff --git a/tools/eval/case/search_eval_case.h b/tools/eval/case/search_eval_case.h index 27f93ed6dd..06c2d9fb9d 100644 --- a/tools/eval/case/search_eval_case.h +++ b/tools/eval/case/search_eval_case.h @@ -28,7 +28,8 @@ class SearchEvalCase : public EvalCase { SearchEvalCase(const std::string& dataset_path, const std::string& index_path, vsag::IndexPtr index, - EvalConfig config); + EvalConfig config, + EvalDatasetPtr dataset = nullptr); ~SearchEvalCase() override = default; @@ -61,12 +62,18 @@ class SearchEvalCase : public EvalCase { void do_knn_search(); + void + run_knn_search_pass(Monitor* monitor, bool collect_statistics); + void do_range_search(); void do_knn_filter_search(); + void + run_knn_filter_search_pass(Monitor* monitor); + void do_range_filter_search(); @@ -85,6 +92,10 @@ class SearchEvalCase : public EvalCase { private: std::vector monitors_{}; + MonitorPtr latency_monitor_{nullptr}; + + MonitorPtr memory_monitor_{nullptr}; + SearchType search_type_{SearchType::KNN}; EvalConfig config_; diff --git a/tools/eval/case/search_pass_runner.cpp b/tools/eval/case/search_pass_runner.cpp new file mode 100644 index 0000000000..7f586685bb --- /dev/null +++ b/tools/eval/case/search_pass_runner.cpp @@ -0,0 +1,122 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "search_pass_runner.h" + +#include +#include + +namespace vsag::eval { + +namespace { + +class MonitorPassGuard { +public: + explicit MonitorPassGuard(const MonitorPtr& monitor) : monitor_(monitor) { + } + + ~MonitorPassGuard() { + if (started_) { + try { + monitor_->Stop(); + } catch (...) { + } + } + } + + void + Start() { + monitor_->Start(); + started_ = true; + } + + void + Stop() { + monitor_->Stop(); + started_ = false; + } + +private: + const MonitorPtr& monitor_; + bool started_{false}; +}; + +void +run_monitored_pass(const MonitorPtr& monitor, + const MonitorPtr& memory_monitor, + const SearchPassRunner::SearchPass& search_pass) { + MonitorPassGuard monitor_guard(monitor); + monitor_guard.Start(); + + if (memory_monitor != nullptr) { + MonitorPassGuard memory_guard(memory_monitor); + memory_guard.Start(); + search_pass(monitor); + memory_guard.Stop(); + } else { + search_pass(monitor); + } + + monitor_guard.Stop(); +} + +} // namespace + +void +SearchPassRunner::Run(const std::vector& monitors, + const MonitorPtr& latency_monitor, + const MonitorPtr& memory_monitor, + const SearchPass& search_pass, + const StatisticsPass& statistics_pass) { + if (not search_pass) { + throw std::invalid_argument("search pass callback is required"); + } + if (not statistics_pass) { + throw std::invalid_argument("statistics pass callback is required"); + } + if (std::any_of(monitors.begin(), monitors.end(), [](const MonitorPtr& monitor) { + return monitor == nullptr; + })) { + throw std::invalid_argument("search pass monitor must not be null"); + } + if (latency_monitor != nullptr && latency_monitor == memory_monitor) { + throw std::invalid_argument("latency and memory monitors must be distinct"); + } + + const bool has_latency_pass = + latency_monitor != nullptr && + std::find(monitors.begin(), monitors.end(), latency_monitor) != monitors.end(); + bool ran_query_pass = false; + + if (memory_monitor != nullptr && not has_latency_pass) { + MonitorPassGuard memory_guard(memory_monitor); + memory_guard.Start(); + search_pass(nullptr); + ran_query_pass = true; + memory_guard.Stop(); + } + + for (const auto& monitor : monitors) { + const auto shared_memory_monitor = + monitor == latency_monitor ? memory_monitor : MonitorPtr{nullptr}; + run_monitored_pass(monitor, shared_memory_monitor, search_pass); + ran_query_pass = true; + } + + if (ran_query_pass) { + statistics_pass(); + } +} + +} // namespace vsag::eval diff --git a/tools/eval/case/search_pass_runner.h b/tools/eval/case/search_pass_runner.h new file mode 100644 index 0000000000..fdee498456 --- /dev/null +++ b/tools/eval/case/search_pass_runner.h @@ -0,0 +1,37 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "../monitor/monitor.h" + +namespace vsag::eval { + +class SearchPassRunner { +public: + using SearchPass = std::function; + using StatisticsPass = std::function; + + static void + Run(const std::vector& monitors, + const MonitorPtr& latency_monitor, + const MonitorPtr& memory_monitor, + const SearchPass& search_pass, + const StatisticsPass& statistics_pass); +}; + +} // namespace vsag::eval diff --git a/tools/eval/case/search_pass_runner_test.cpp b/tools/eval/case/search_pass_runner_test.cpp new file mode 100644 index 0000000000..5d169af55f --- /dev/null +++ b/tools/eval/case/search_pass_runner_test.cpp @@ -0,0 +1,244 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "search_pass_runner.h" + +#include +#include +#include +#include +#include +#include + +namespace vsag::eval { + +namespace { + +struct trace_state { + std::vector events; + std::vector query_passes; + std::vector memory_active_during_query; + bool memory_active{false}; + bool memory_active_during_statistics{false}; + uint64_t statistics_pass_count{0}; +}; + +class TracingMonitor : public Monitor { +public: + TracingMonitor(std::string name, trace_state& trace, bool is_memory = false) + : Monitor(std::move(name)), trace_(trace), is_memory_(is_memory) { + } + + void + Start() override { + trace_.events.emplace_back(name_ + ".start"); + if (is_memory_) { + trace_.memory_active = true; + } + } + + void + Stop() override { + trace_.events.emplace_back(name_ + ".stop"); + if (is_memory_) { + trace_.memory_active = false; + } + } + + JsonType + GetResult() override { + return {}; + } + + void + Record(void* input = nullptr) override { + static_cast(input); + trace_.events.emplace_back(name_ + ".record"); + } + +private: + trace_state& trace_; + bool is_memory_; +}; + +trace_state +run_standard_schedule(bool enable_memory) { + trace_state trace; + auto latency = std::make_shared("latency", trace); + auto recall = std::make_shared("recall", trace); + MonitorPtr memory = nullptr; + if (enable_memory) { + memory = std::make_shared("memory", trace, true); + } + + SearchPassRunner::Run( + {latency, recall}, + latency, + memory, + [&trace](const MonitorPtr& monitor) { + trace.query_passes.emplace_back(monitor->GetName()); + trace.memory_active_during_query.emplace_back(trace.memory_active); + trace.events.emplace_back("query." + monitor->GetName()); + monitor->Record(); + }, + [&trace]() { + trace.events.emplace_back("statistics"); + trace.memory_active_during_statistics = trace.memory_active; + ++trace.statistics_pass_count; + }); + return trace; +} + +} // namespace + +TEST_CASE("SearchPassRunner keeps ordinary passes stable when memory is toggled", + "[ut][eval][search_pass]") { + const auto without_memory = run_standard_schedule(false); + const auto with_memory = run_standard_schedule(true); + + const std::vector expected_query_passes = {"latency", "recall"}; + const std::vector expected_without_memory_activity = {false, false}; + const std::vector expected_with_memory_activity = {true, false}; + CHECK(without_memory.query_passes == expected_query_passes); + CHECK(with_memory.query_passes == expected_query_passes); + CHECK(without_memory.statistics_pass_count == 1); + CHECK(with_memory.statistics_pass_count == 1); + + CHECK(without_memory.memory_active_during_query == expected_without_memory_activity); + CHECK(with_memory.memory_active_during_query == expected_with_memory_activity); + CHECK_FALSE(without_memory.memory_active_during_statistics); + CHECK_FALSE(with_memory.memory_active_during_statistics); + + const std::vector expected_without_memory = { + "latency.start", + "query.latency", + "latency.record", + "latency.stop", + "recall.start", + "query.recall", + "recall.record", + "recall.stop", + "statistics", + }; + const std::vector expected_with_memory = { + "latency.start", + "memory.start", + "query.latency", + "latency.record", + "memory.stop", + "latency.stop", + "recall.start", + "query.recall", + "recall.record", + "recall.stop", + "statistics", + }; + CHECK(without_memory.events == expected_without_memory); + CHECK(with_memory.events == expected_with_memory); +} + +TEST_CASE("SearchPassRunner uses a memory-only pass when latency is disabled", + "[ut][eval][search_pass]") { + trace_state trace; + auto recall = std::make_shared("recall", trace); + auto memory = std::make_shared("memory", trace, true); + + SearchPassRunner::Run( + {recall}, + nullptr, + memory, + [&trace](const MonitorPtr& monitor) { + const std::string pass_name = monitor == nullptr ? "memory-only" : monitor->GetName(); + trace.query_passes.emplace_back(pass_name); + trace.memory_active_during_query.emplace_back(trace.memory_active); + trace.events.emplace_back("query." + pass_name); + if (monitor != nullptr) { + monitor->Record(); + } + }, + [&trace]() { + trace.events.emplace_back("statistics"); + trace.memory_active_during_statistics = trace.memory_active; + ++trace.statistics_pass_count; + }); + + const std::vector expected_events = { + "memory.start", + "query.memory-only", + "memory.stop", + "recall.start", + "query.recall", + "recall.record", + "recall.stop", + "statistics", + }; + const std::vector expected_query_passes = {"memory-only", "recall"}; + const std::vector expected_memory_activity = {true, false}; + CHECK(trace.events == expected_events); + CHECK(trace.query_passes == expected_query_passes); + CHECK(trace.memory_active_during_query == expected_memory_activity); + CHECK_FALSE(trace.memory_active_during_statistics); + CHECK(trace.statistics_pass_count == 1); +} + +TEST_CASE("SearchPassRunner unwinds memory before the ordinary monitor on failure", + "[ut][eval][search_pass]") { + trace_state trace; + auto latency = std::make_shared("latency", trace); + auto recall = std::make_shared("recall", trace); + auto memory = std::make_shared("memory", trace, true); + + CHECK_THROWS_AS(SearchPassRunner::Run( + {latency, recall}, + latency, + memory, + [&trace](const MonitorPtr& monitor) { + trace.events.emplace_back("query." + monitor->GetName()); + throw std::runtime_error("injected search failure"); + }, + [&trace]() { + trace.events.emplace_back("statistics"); + ++trace.statistics_pass_count; + }), + std::runtime_error); + + const std::vector expected_events = { + "latency.start", + "memory.start", + "query.latency", + "memory.stop", + "latency.stop", + }; + CHECK(trace.events == expected_events); + CHECK_FALSE(trace.memory_active); + CHECK(trace.statistics_pass_count == 0); +} + +TEST_CASE("SearchPassRunner skips statistics when there is no query pass", + "[ut][eval][search_pass]") { + uint64_t search_pass_count = 0; + uint64_t statistics_pass_count = 0; + + SearchPassRunner::Run( + {}, + nullptr, + nullptr, + [&search_pass_count](const MonitorPtr&) { ++search_pass_count; }, + [&statistics_pass_count]() { ++statistics_pass_count; }); + + CHECK(search_pass_count == 0); + CHECK(statistics_pass_count == 0); +} + +} // namespace vsag::eval diff --git a/tools/eval/monitor/memory_peak_monitor.cpp b/tools/eval/monitor/memory_peak_monitor.cpp index 7935bc54d3..4819f76857 100644 --- a/tools/eval/monitor/memory_peak_monitor.cpp +++ b/tools/eval/monitor/memory_peak_monitor.cpp @@ -15,60 +15,277 @@ #include "memory_peak_monitor.h" -#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include + +#if defined(__linux__) +#include +#elif defined(__APPLE__) +#include +#endif namespace vsag::eval { -static std::string -GetProcFileName(pid_t pid) { - return "/proc/" + std::to_string(pid) + "/statm"; +namespace { + +constexpr std::chrono::milliseconds K_DEFAULT_SAMPLE_INTERVAL{5}; + +MemoryPeakMonitor::MemoryReadResult +read_process_memory() { +#if defined(__linux__) + errno = 0; + std::ifstream statm("/proc/self/statm"); + if (not statm.is_open()) { + const auto error_number = errno; + std::string error = "failed to open /proc/self/statm"; + if (error_number != 0) { + error += ": "; + error += std::strerror(error_number); + } + return {false, 0, std::move(error)}; + } + + uint64_t total_pages = 0; + uint64_t resident_pages = 0; + if (not(statm >> total_pages >> resident_pages)) { + return {false, 0, "failed to parse resident pages from /proc/self/statm"}; + } + + errno = 0; + const long page_size = sysconf(_SC_PAGESIZE); + if (page_size <= 0) { + const auto error_number = errno; + std::string error = "sysconf(_SC_PAGESIZE) failed"; + if (error_number != 0) { + error += ": "; + error += std::strerror(error_number); + } + return {false, 0, std::move(error)}; + } + + const auto page_size_bytes = static_cast(page_size); + if (resident_pages > std::numeric_limits::max() / page_size_bytes) { + return {false, 0, "resident memory byte count overflows uint64_t"}; + } + return {true, resident_pages * page_size_bytes, {}}; +#elif defined(__APPLE__) + mach_task_basic_info_data_t memory_info{}; + mach_msg_type_number_t info_count = MACH_TASK_BASIC_INFO_COUNT; + const auto status = task_info(mach_task_self(), + MACH_TASK_BASIC_INFO, + reinterpret_cast(&memory_info), + &info_count); + if (status != KERN_SUCCESS) { + return {false, 0, "task_info failed with status " + std::to_string(status)}; + } + return {true, static_cast(memory_info.resident_size), {}}; +#else + return {false, 0, "resident memory sampling is unavailable on this platform"}; +#endif +} + +std::string +format_bytes(uint64_t bytes) { + constexpr std::array k_units = {"B", "KB", "MB", "GB", "TB"}; + auto size = static_cast(bytes); + uint64_t unit_index = 0; + while (size >= 1024.0L && unit_index + 1 < k_units.size()) { + size /= 1024.0L; + ++unit_index; + } + + std::ostringstream stream; + stream << std::fixed << std::setprecision(2) << size << " " << k_units[unit_index]; + return stream.str(); +} + +} // namespace + +MemoryPeakMonitor::MemoryPeakMonitor(std::string name) + : MemoryPeakMonitor(std::move(name), read_process_memory, K_DEFAULT_SAMPLE_INTERVAL) { +} + +MemoryPeakMonitor::MemoryPeakMonitor(std::string name, + MemoryReader reader, + std::chrono::milliseconds sample_interval) + : Monitor("memory_peak_monitor"), + reader_(std::move(reader)), + sample_interval_(std::max(sample_interval, std::chrono::milliseconds{1})), + process_name_(std::move(name)) { +} + +MemoryPeakMonitor::~MemoryPeakMonitor() { + try { + stop_sampling(); + } catch (...) { + } } -MemoryPeakMonitor::MemoryPeakMonitor(const std::string& name) - : Monitor("memory_peak_monitor"), process_name_(name) { - this->pid_ = getpid(); - this->infile_.open(GetProcFileName(pid_)); - uint64_t val1, val2; - this->infile_ >> val1 >> val2; - this->infile_.clear(); - this->infile_.seekg(0, std::ios::beg); - init_memory_ = val2; +MemoryPeakMonitor::MemoryReadResult +MemoryPeakMonitor::read_memory() const { + if (not reader_) { + return {false, 0, "memory reader is not configured"}; + } + + try { + auto result = reader_(); + if (not result.available && result.error.empty()) { + result.error = "memory reader reported no data"; + } + return result; + } catch (const std::exception& exception) { + return {false, 0, "memory reader failed: " + std::string(exception.what())}; + } catch (...) { + return {false, 0, "memory reader failed with an unknown error"}; + } } void MemoryPeakMonitor::Start() { + Stop(); + + { + std::lock_guard lock(state_mutex_); + available_ = false; + baseline_bytes_ = 0; + absolute_peak_bytes_ = 0; + sample_count_ = 0; + failure_count_ = 0; + last_error_.clear(); + running_ = true; + worker_ready_ = false; + } + + try { + sampling_thread_ = std::thread(&MemoryPeakMonitor::sampling_loop, this); + } catch (const std::exception& exception) { + std::lock_guard lock(state_mutex_); + running_ = false; + failure_count_ = 1; + last_error_ = "failed to start memory sampler: " + std::string(exception.what()); + return; + } + + bool baseline_available = false; + { + std::unique_lock lock(state_mutex_); + stop_condition_.wait(lock, [this]() { return worker_ready_; }); + baseline_available = available_; + } + if (not baseline_available && sampling_thread_.joinable()) { + sampling_thread_.join(); + } } + void MemoryPeakMonitor::Stop() { + stop_sampling(); +} + +void +MemoryPeakMonitor::stop_sampling() { + { + std::lock_guard lock(state_mutex_); + if (not running_) { + return; + } + running_ = false; + } + + stop_condition_.notify_all(); + if (sampling_thread_.joinable()) { + sampling_thread_.join(); + } } + Monitor::JsonType MemoryPeakMonitor::GetResult() { + bool available = false; + uint64_t baseline_bytes = 0; + uint64_t absolute_peak_bytes = 0; + uint64_t sample_count = 0; + uint64_t failure_count = 0; + std::string last_error; + { + std::lock_guard lock(state_mutex_); + available = available_; + baseline_bytes = baseline_bytes_; + absolute_peak_bytes = absolute_peak_bytes_; + sample_count = sample_count_; + failure_count = failure_count_; + last_error = last_error_; + } + + const uint64_t delta_bytes = + absolute_peak_bytes >= baseline_bytes ? absolute_peak_bytes - baseline_bytes : 0; + JsonType result; - std::vector metrics = {"B", "KB", "MB", "GB", "TB"}; - auto size = - static_cast((this->max_memory_ - this->init_memory_) * sysconf(_SC_PAGESIZE)); - size_t i = 0; - while (size >= 1024.0F && i < metrics.size() - 1) { - size /= 1024; - i++; - } - std::ostringstream oss; - oss << std::fixed << std::setprecision(2) << size; - result["memory_peak(" + process_name_ + ")"] = oss.str() + " " + metrics[i]; + result[metric_name("memory_peak")] = available ? format_bytes(delta_bytes) : "N/A"; + result[metric_name("memory_rss_baseline_bytes")] = baseline_bytes; + result[metric_name("memory_rss_peak_bytes")] = absolute_peak_bytes; + result[metric_name("memory_peak_delta_bytes")] = delta_bytes; + result[metric_name("memory_peak_sample_count")] = sample_count; + result[metric_name("memory_peak_failed_sample_count")] = failure_count; + result[metric_name("memory_peak_available")] = available; + result[metric_name("memory_peak_error")] = last_error; return result; } + void -MemoryPeakMonitor::Record(void* input) { - std::lock_guard lock(record_mutex_); +MemoryPeakMonitor::Record(void* /*input*/) { +} - uint64_t val1, val2; - this->infile_ >> val1 >> val2; - this->infile_.clear(); - this->infile_.seekg(0, std::ios::beg); - if (max_memory_ < val2) { - max_memory_ = val2; +void +MemoryPeakMonitor::sample_memory() { + const auto sample = read_memory(); + std::lock_guard lock(state_mutex_); + if (sample.available) { + absolute_peak_bytes_ = std::max(absolute_peak_bytes_, sample.bytes); + ++sample_count_; + return; } + + ++failure_count_; + last_error_ = sample.error; +} + +void +MemoryPeakMonitor::sampling_loop() { + const auto baseline = read_memory(); + std::unique_lock lock(state_mutex_); + available_ = baseline.available; + baseline_bytes_ = baseline.available ? baseline.bytes : 0; + absolute_peak_bytes_ = baseline_bytes_; + sample_count_ = baseline.available ? 1 : 0; + failure_count_ = baseline.available ? 0 : 1; + last_error_ = baseline.error; + running_ = baseline.available; + worker_ready_ = true; + stop_condition_.notify_all(); + if (not running_) { + return; + } + while ( + not stop_condition_.wait_for(lock, sample_interval_, [this]() { return not running_; })) { + lock.unlock(); + sample_memory(); + lock.lock(); + } + lock.unlock(); + sample_memory(); +} + +std::string +MemoryPeakMonitor::metric_name(const std::string& metric) const { + return metric + "(" + process_name_ + ")"; } } // namespace vsag::eval diff --git a/tools/eval/monitor/memory_peak_monitor.h b/tools/eval/monitor/memory_peak_monitor.h index efea6aecbf..d95c2caa69 100644 --- a/tools/eval/monitor/memory_peak_monitor.h +++ b/tools/eval/monitor/memory_peak_monitor.h @@ -15,10 +15,13 @@ #pragma once -#include - #include -#include +#include +#include +#include +#include +#include +#include #include "monitor.h" @@ -26,9 +29,22 @@ namespace vsag::eval { class MemoryPeakMonitor : public Monitor { public: - explicit MemoryPeakMonitor(const std::string& name); + class MemoryReadResult { + public: + bool available{false}; + uint64_t bytes{0}; + std::string error{}; + }; + + using MemoryReader = std::function; + + explicit MemoryPeakMonitor(std::string name); - ~MemoryPeakMonitor() override = default; + MemoryPeakMonitor(std::string name, + MemoryReader reader, + std::chrono::milliseconds sample_interval); + + ~MemoryPeakMonitor() override; void Start() override; @@ -40,16 +56,41 @@ class MemoryPeakMonitor : public Monitor { GetResult() override; void - Record(void* input) override; + Record(void* input = nullptr) override; private: - uint64_t max_memory_{0}; - uint64_t init_memory_{0}; - std::string process_name_{}; + MemoryReadResult + read_memory() const; - pid_t pid_{0}; + void + sample_memory(); - std::ifstream infile_{}; + void + sampling_loop(); + + void + stop_sampling(); + + std::string + metric_name(const std::string& metric) const; + +private: + MemoryReader reader_; + std::chrono::milliseconds sample_interval_; + std::string process_name_; + + mutable std::mutex state_mutex_; + std::condition_variable stop_condition_; + std::thread sampling_thread_; + bool running_{false}; + bool worker_ready_{false}; + + bool available_{false}; + uint64_t baseline_bytes_{0}; + uint64_t absolute_peak_bytes_{0}; + uint64_t sample_count_{0}; + uint64_t failure_count_{0}; + std::string last_error_{"memory monitor has not been started"}; }; } // namespace vsag::eval diff --git a/tools/eval/monitor/memory_peak_monitor_test.cpp b/tools/eval/monitor/memory_peak_monitor_test.cpp new file mode 100644 index 0000000000..776d694728 --- /dev/null +++ b/tools/eval/monitor/memory_peak_monitor_test.cpp @@ -0,0 +1,215 @@ +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "memory_peak_monitor.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace vsag::eval { + +namespace { + +constexpr std::chrono::milliseconds K_LONG_SAMPLE_INTERVAL{60000}; + +std::string +metric_name(const std::string& metric) { + return metric + "(test)"; +} + +std::string +format_delta(uint64_t delta_bytes) { + std::atomic read_count{0}; + MemoryPeakMonitor monitor( + "test", + [&read_count, delta_bytes]() { + const auto call = read_count.fetch_add(1); + return MemoryPeakMonitor::MemoryReadResult{true, call == 0 ? 0 : delta_bytes, {}}; + }, + K_LONG_SAMPLE_INTERVAL); + monitor.Start(); + monitor.Stop(); + return monitor.GetResult()[metric_name("memory_peak")].get(); +} + +} // namespace + +TEST_CASE("MemoryPeakMonitor does not read memory before Start", "[ut][eval][memory_peak]") { + std::atomic read_count{0}; + MemoryPeakMonitor monitor( + "test", + [&read_count]() { + ++read_count; + return MemoryPeakMonitor::MemoryReadResult{true, 1024, {}}; + }, + K_LONG_SAMPLE_INTERVAL); + + const auto result = monitor.GetResult(); + CHECK(read_count == 0); + CHECK(result[metric_name("memory_peak")] == "N/A"); + CHECK(result[metric_name("memory_peak_available")] == false); + CHECK(result[metric_name("memory_peak_error")] == "memory monitor has not been started"); +} + +TEST_CASE("MemoryPeakMonitor clamps a sample below its baseline", "[ut][eval][memory_peak]") { + std::atomic read_count{0}; + MemoryPeakMonitor monitor( + "test", + [&read_count]() { + const auto call = read_count.fetch_add(1); + const uint64_t bytes = call == 0 ? 4096 : 1024; + return MemoryPeakMonitor::MemoryReadResult{true, bytes, {}}; + }, + K_LONG_SAMPLE_INTERVAL); + + monitor.Start(); + monitor.Stop(); + + const auto result = monitor.GetResult(); + CHECK(result[metric_name("memory_peak")] == "0.00 B"); + CHECK(result[metric_name("memory_rss_baseline_bytes")] == 4096); + CHECK(result[metric_name("memory_rss_peak_bytes")] == 4096); + CHECK(result[metric_name("memory_peak_delta_bytes")] == 0); + CHECK(result[metric_name("memory_peak_sample_count")] == 2); + CHECK(result[metric_name("memory_peak_failed_sample_count")] == 0); + CHECK(result[metric_name("memory_peak_available")] == true); +} + +TEST_CASE("MemoryPeakMonitor samples a transient peak in the background", + "[ut][eval][memory_peak]") { + std::mutex reader_mutex; + std::condition_variable reader_condition; + uint64_t read_count = 0; + MemoryPeakMonitor monitor( + "test", + [&]() { + std::lock_guard lock(reader_mutex); + const auto call = read_count++; + if (read_count >= 2) { + reader_condition.notify_all(); + } + const uint64_t bytes = call == 0 ? 100 : (call == 1 ? 300 : 150); + return MemoryPeakMonitor::MemoryReadResult{true, bytes, {}}; + }, + std::chrono::milliseconds{1}); + + monitor.Start(); + { + std::unique_lock lock(reader_mutex); + REQUIRE(reader_condition.wait_for( + lock, std::chrono::seconds{1}, [&]() { return read_count >= 2; })); + } + monitor.Stop(); + + const auto result = monitor.GetResult(); + CHECK(result[metric_name("memory_peak")] == "200.00 B"); + CHECK(result[metric_name("memory_rss_peak_bytes")] == 300); + CHECK(result[metric_name("memory_peak_delta_bytes")] == 200); + CHECK(result[metric_name("memory_peak_sample_count")].get() >= 3); +} + +TEST_CASE("MemoryPeakMonitor reports a baseline read failure", "[ut][eval][memory_peak]") { + std::atomic read_count{0}; + MemoryPeakMonitor monitor( + "test", + [&read_count]() { + ++read_count; + return MemoryPeakMonitor::MemoryReadResult{false, 0, "baseline unavailable"}; + }, + std::chrono::milliseconds{1}); + + monitor.Start(); + monitor.Stop(); + + const auto result = monitor.GetResult(); + CHECK(read_count == 1); + CHECK(result[metric_name("memory_peak")] == "N/A"); + CHECK(result[metric_name("memory_rss_baseline_bytes")] == 0); + CHECK(result[metric_name("memory_rss_peak_bytes")] == 0); + CHECK(result[metric_name("memory_peak_delta_bytes")] == 0); + CHECK(result[metric_name("memory_peak_sample_count")] == 0); + CHECK(result[metric_name("memory_peak_failed_sample_count")] == 1); + CHECK(result[metric_name("memory_peak_available")] == false); + CHECK(result[metric_name("memory_peak_error")] == "baseline unavailable"); +} + +TEST_CASE("MemoryPeakMonitor can restart and Stop is idempotent", "[ut][eval][memory_peak]") { + std::atomic read_count{0}; + MemoryPeakMonitor monitor( + "test", + [&read_count]() { + const auto call = read_count.fetch_add(1); + const uint64_t values[] = {100, 200, 500, 550}; + const uint64_t index = std::min(call, 3); + return MemoryPeakMonitor::MemoryReadResult{true, values[index], {}}; + }, + K_LONG_SAMPLE_INTERVAL); + + monitor.Start(); + for (uint64_t i = 0; i < 10; ++i) { + monitor.Record(); + } + CHECK(read_count == 1); + monitor.Stop(); + monitor.Stop(); + CHECK(read_count == 2); + + monitor.Start(); + monitor.Stop(); + const auto result = monitor.GetResult(); + CHECK(read_count == 4); + CHECK(result[metric_name("memory_rss_baseline_bytes")] == 500); + CHECK(result[metric_name("memory_rss_peak_bytes")] == 550); + CHECK(result[metric_name("memory_peak_delta_bytes")] == 50); + CHECK(result[metric_name("memory_peak_sample_count")] == 2); +} + +TEST_CASE("MemoryPeakMonitor formats binary unit boundaries", "[ut][eval][memory_peak]") { + CHECK(format_delta(1023) == "1023.00 B"); + CHECK(format_delta(1024) == "1.00 KB"); + CHECK(format_delta(uint64_t{1024} * 1024) == "1.00 MB"); + CHECK(format_delta(uint64_t{1024} * 1024 * 1024) == "1.00 GB"); + CHECK(format_delta(uint64_t{1024} * 1024 * 1024 * 1024) == "1.00 TB"); +} + +TEST_CASE("MemoryPeakMonitor default reader reports platform availability", + "[ut][eval][memory_peak]") { + MemoryPeakMonitor monitor("test"); + monitor.Start(); + monitor.Stop(); + + const auto result = monitor.GetResult(); +#if defined(__linux__) || defined(__APPLE__) + CHECK(result[metric_name("memory_peak_available")] == true); + CHECK(result[metric_name("memory_rss_baseline_bytes")].get() > 0); + CHECK(result[metric_name("memory_rss_peak_bytes")].get() > 0); + CHECK(result[metric_name("memory_peak_sample_count")].get() >= 2); + CHECK(result[metric_name("memory_peak_failed_sample_count")] == 0); + CHECK(result[metric_name("memory_peak_error")].get().empty()); +#else + CHECK(result[metric_name("memory_peak")] == "N/A"); + CHECK(result[metric_name("memory_peak_available")] == false); + CHECK(result[metric_name("memory_peak_sample_count")] == 0); + CHECK(result[metric_name("memory_peak_failed_sample_count")] == 1); + CHECK_FALSE(result[metric_name("memory_peak_error")].get().empty()); +#endif +} + +} // namespace vsag::eval