diff --git a/benchmark/locomo/vikingbot/preflight_eval_config.py b/benchmark/locomo/vikingbot/preflight_eval_config.py index d8c0236e28..8ef8a703f8 100644 --- a/benchmark/locomo/vikingbot/preflight_eval_config.py +++ b/benchmark/locomo/vikingbot/preflight_eval_config.py @@ -64,7 +64,9 @@ def _resolve_ov_conf_path() -> Path: return Path(configured_path).expanduser() resolved = resolve_config_path(None, OPENVIKING_CONFIG_ENV, DEFAULT_OV_CONF) - default_path = str(resolved) if resolved is not None else str(Path.home() / ".openviking" / "ov.conf") + default_path = ( + str(resolved) if resolved is not None else str(Path.home() / ".openviking" / "ov.conf") + ) if _is_interactive(): _log_info(f"OpenViking 配置默认路径: {default_path}") diff --git a/benchmark/retrieval/grep/vikingdb_bm25/README.md b/benchmark/retrieval/grep/vikingdb_bm25/README.md new file mode 100644 index 0000000000..fb78244d4c --- /dev/null +++ b/benchmark/retrieval/grep/vikingdb_bm25/README.md @@ -0,0 +1,118 @@ +# VikingDB BM25 Grep Benchmark + +Benchmark suite for evaluating OpenViking's grep retrieval with VikingDB BM25 engine. + +## Directory Structure + +``` +vikingdb_bm25/ +├── ai_wiki.txt # Source text for synthetic data generation +├── effectiveness/ # Retrieval effectiveness (recall/precision/F1) +│ ├── step1_add_resource.py +│ └── step2_quality.py +└── performance/ # Retrieval performance (latency + returned match count at scale) + ├── step0_prepare_data.py + ├── step1_add_resource.py + ├── step2_reindex.py + └── step3_benchmark.py +``` + +## Effectiveness — Retrieval Quality + +Tests whether grep can find **all** matching files in real code repositories. + +**Data source:** Real code repos (download manually, place under `~/.openviking/data/benchmark/`). + +| Step | Script | Description | +|------|--------|-------------| +| 1 | `step1_add_resource.py` | Import code repos (with indexing, single import) | +| 2 | `step2_quality.py` | Compare grep results vs ground truth (fs engine, cached) | + +### Usage + +```bash +# Step 1: Import repos (with VLM/embedding, single import) +cd effectiveness/ +python3 step1_add_resource.py --source ~/.openviking/data/benchmark/OpenViking-main + +# Step 2: Evaluate retrieval quality +# First run MUST use engine=fs in ov.conf to generate ground truth cache: +# 1. Set ov.conf: "grep": {"engine": "fs"} +# 2. Restart server +python3 step2_quality.py --keywords grep reindex SyncHTTPClient + +# Subsequent runs can use any engine (ground truth is read from cache): +# 1. Set ov.conf: "grep": {"engine": "auto", "switch_to_remote_threshold": 0} +# 2. Restart server +python3 step2_quality.py --keywords grep reindex SyncHTTPClient + +# Optional: --regenerate-ground-truth (force recompute, requires engine=fs) +``` + +## Performance — Latency at Scale + +Tests grep speed and returned match count on a large synthetic dataset (default: 200K files). + +**Data source:** Generated from `ai_wiki.txt` with target words injected at known probabilities. + +| Step | Script | Description | +|------|--------|-------------| +| 0 | `step0_prepare_data.py` | Generate synthetic dataset (dir_xxx/wiki_xxx.txt) | +| 1 | `step1_add_resource.py` | Import data (no VLM/embedding, fast) | +| 2 | `step2_reindex.py` | Async reindex via openviking-server (concurrency=16, polling) | +| 3 | `step3_benchmark.py` | Measure latency and returned match count with `node_limit=256` | + +### Target Words + +15 words across 5 probability tiers: + +These word groups are defined in `performance/step0_prepare_data.py` and reused by `performance/step3_benchmark.py`. + +| Probability | Words | Expected hits (per 200K files) | +|-------------|-------|-------------------------------| +| 1% | heliofract, prismcache, fluxkernel | ~2,000 | +| 0.1% | auroracode, kiteshade, glyphvector | ~200 | +| 0.1% | cortexmint, latticewave, spiralsync | ~200 | +| 0.05% | ripplehash, embertrace, novaframe | ~100 | +| 0.01% | zephyrloom, quartzrelay, nebulaindex | ~20 | + +### Usage + +```bash +cd performance/ + +# Step 0: Generate data (default: 200 dirs x 1000 files = 200K files) +python3 step0_prepare_data.py + +# Optional: append more data for scale-out without overwriting existing dirs +python3 step0_prepare_data.py --start-dir 100 --num-dirs 100 + +# Step 1: Import without indexing (fast) +python3 step1_add_resource.py + +# Step 2: Build vector indexes (requires openviking-server running) +python3 step2_reindex.py +# Optional: --concurrency N (default: 16) + +# Step 3: Benchmark — run with different engine configs +# Run A: fs engine +# 1. Set ov.conf: "grep": {"engine": "fs"} +# 2. Restart server +python3 step3_benchmark.py --engine-label fs + +# Run B: auto engine (bm25) +# 1. Set ov.conf: "grep": {"engine": "auto", "switch_to_remote_threshold": 0} +# 2. Restart server +python3 step3_benchmark.py --engine-label auto --compare step3_result_fs.json +``` + +## Key Concepts + +- **Effectiveness** tests compare grep results against ground truth from fs-engine grep (cached locally) +- **Performance** tests compare grep latency and returned match counts between engine configs; no ground truth is generated +- **Effectiveness** imports real repos with indexing in a single step, then evaluates quality +- **Performance** imports synthetic data without indexing, builds vector indexes asynchronously, then benchmarks latency +- **Performance** import/reindex steps support resumable execution via progress files +- Change grep engine via `ov.conf` and restart the server between benchmark runs +- To horizontally scale the synthetic dataset, run Step 0 again with a new `--start-dir`, + then rerun Step 1 and Step 2. diff --git a/benchmark/retrieval/grep/vikingdb_bm25/README_CN.md b/benchmark/retrieval/grep/vikingdb_bm25/README_CN.md new file mode 100644 index 0000000000..30fb446a36 --- /dev/null +++ b/benchmark/retrieval/grep/vikingdb_bm25/README_CN.md @@ -0,0 +1,117 @@ +# VikingDB BM25 Grep 基准测试 + +用于评估 OpenViking grep 检索配合 VikingDB BM25 引擎的基准测试套件。 + +## 目录结构 + +``` +vikingdb_bm25/ +├── ai_wiki.txt # 合成数据生成的原始文本 +├── effectiveness/ # 检索效果测试（召回率/精确率/F1） +│ ├── step1_add_resource.py +│ └── step2_quality.py +└── performance/ # 检索性能测试（延迟 + 大规模返回匹配数） + ├── step0_prepare_data.py + ├── step1_add_resource.py + ├── step2_reindex.py + └── step3_benchmark.py +``` + +## Effectiveness — 检索效果 + +测试 grep 在真实代码仓库中是否能找到**所有**匹配文件。 + +**数据来源：** 真实代码仓库（手动下载，放置于 `~/.openviking/data/benchmark/`）。 + +| 步骤 | 脚本 | 说明 | +|------|------|------| +| 1 | `step1_add_resource.py` | 导入代码仓库（含建索引，一次性导入） | +| 2 | `step2_quality.py` | SDK grep 与 fs 引擎 ground truth 对比（缓存） | + +### 使用方法 + +```bash +# 步骤 1：导入代码仓库（含建索引，一次性导入） +cd effectiveness/ +python3 step1_add_resource.py --source ~/.openviking/data/benchmark/OpenViking-main + +# 步骤 2：评估检索质量 +# 首次运行必须使用 engine=fs 生成 ground truth 缓存： +# 1. 设置 ov.conf: "grep": {"engine": "fs"} +# 2. 重启服务 +python3 step2_quality.py --keywords grep reindex SyncHTTPClient + +# 后续运行可使用任意引擎（ground truth 从缓存读取）： +# 1. 设置 ov.conf: "grep": {"engine": "auto", "switch_to_remote_threshold": 0} +# 2. 重启服务 +python3 step2_quality.py --keywords grep reindex SyncHTTPClient + +# 可选参数：--regenerate-ground-truth （强制重算，需 engine=fs） +``` + +## Performance — 检索延迟 + +在大规模合成数据集（默认 20 万文件）上测试 grep 速度和返回匹配数。 + +**数据来源：** 从 `ai_wiki.txt` 生成，按已知概率注入目标单词。 + +| 步骤 | 脚本 | 说明 | +|------|------|------| +| 0 | `step0_prepare_data.py` | 生成合成数据集（dir_xxx/wiki_xxx.txt） | +| 1 | `step1_add_resource.py` | 导入数据（不建索引，速度快） | +| 2 | `step2_reindex.py` | 通过 openviking-server 异步构建索引（并发=16，轮询） | +| 3 | `step3_benchmark.py` | 使用 `node_limit=256` 测量延迟和返回匹配数 | + +### 目标单词 + +15 个单词，分 5 个概率层级： + +这些词组定义在 `performance/step0_prepare_data.py` 中，并由 `performance/step3_benchmark.py` 复用。 + +| 概率 | 单词 | 预期命中数（每 20 万文件） | +|------|------|---------------------------| +| 1% | heliofract, prismcache, fluxkernel | ~2,000 | +| 0.1% | auroracode, kiteshade, glyphvector | ~200 | +| 0.1% | cortexmint, latticewave, spiralsync | ~200 | +| 0.05% | ripplehash, embertrace, novaframe | ~100 | +| 0.01% | zephyrloom, quartzrelay, nebulaindex | ~20 | + +### 使用方法 + +```bash +cd performance/ + +# 步骤 0：生成数据（默认：200 目录 x 1000 文件 = 20 万文件） +python3 step0_prepare_data.py + +# 可选：追加更多数据，用于水平扩容，不覆盖已有目录 +python3 step0_prepare_data.py --start-dir 100 --num-dirs 100 + +# 步骤 1：导入数据（不建索引，速度快） +python3 step1_add_resource.py + +# 步骤 2：构建向量索引（需 openviking-server 运行中） +python3 step2_reindex.py +# 可选参数：--concurrency N （默认：16） + +# 步骤 3：基准测试 — 用不同引擎配置各跑一次 +# 运行 A：fs 引擎 +# 1. 设置 ov.conf: "grep": {"engine": "fs"} +# 2. 重启服务 +python3 step3_benchmark.py --engine-label fs + +# 运行 B：auto 引擎（bm25） +# 1. 设置 ov.conf: "grep": {"engine": "auto", "switch_to_remote_threshold": 0} +# 2. 重启服务 +python3 step3_benchmark.py --engine-label auto --compare step3_result_fs.json +``` + +## 核心概念 + +- **Effectiveness（效果测试）** 将 grep 结果与 fs 引擎的 ground truth 对比（本地缓存） +- **Performance（性能测试）** 对比不同引擎的延迟和返回匹配数，不生成 ground truth +- **Effectiveness** 直接一次性导入真实代码仓并建索引，然后执行效果评估 +- **Performance** 先导入合成数据（不建索引），再异步建向量索引，最后执行延迟基准测试 +- **Performance** 的导入与 reindex 步骤支持**断点续传**（各有独立进度文件） +- 切换 grep 引擎需修改 `ov.conf` 并重启服务，在不同运行之间对比 +- 如需水平扩展合成数据集，可用新的 `--start-dir` 再运行步骤 0，然后重跑步骤 1 和步骤 2。 diff --git a/benchmark/retrieval/grep/vikingdb_bm25/ai_wiki.txt b/benchmark/retrieval/grep/vikingdb_bm25/ai_wiki.txt new file mode 100644 index 0000000000..8030428061 --- /dev/null +++ b/benchmark/retrieval/grep/vikingdb_bm25/ai_wiki.txt @@ -0,0 +1,369 @@ +"AI" redirects here. For other uses, see AI (disambiguation) and Artificial intelligence (disambiguation). + +Artificial intelligence (AI) is the capability of computational systems to perform tasks typically associated with human intelligence, such as learning, reasoning, problem-solving, perception, and decision-making. It is a field of research in engineering, mathematics and computer science that develops and studies methods and software that enable machines to perceive their environment and use learning and intelligence to take actions that maximize their chances of achieving defined goals.[1] + +High-profile applications of AI include advanced web search engines, chatbots, virtual assistants, autonomous vehicles, and play and analysis in strategy games (e.g., chess and Go). Since the 2020s, generative AI has become widely available to generate images, audio, and videos from text prompts. + +The traditional goals of AI research include learning, reasoning, knowledge representation, planning, natural language processing, and perception, as well as support for robotics.[a] To reach these goals, AI researchers have used techniques including state space search and mathematical optimization, formal logic, artificial neural networks, and methods based on statistics, operations research, and economics.[b] AI also draws upon psychology, linguistics, philosophy, neuroscience, and other fields.[2] Some companies, such as OpenAI, Google DeepMind and Meta, aim to create artificial general intelligence (AGI) – AI that can complete virtually any cognitive task at least as well as a human.[3] + +Artificial intelligence was founded as an academic discipline in 1956,[4] and the field went through multiple cycles of optimism throughout its history,[5][6] followed by periods of disappointment and loss of funding, known as AI winters.[7][8] Funding and interest increased substantially after 2012, when graphics processing units began being used to accelerate neural networks, and deep learning outperformed previous AI techniques.[9] This growth accelerated further after 2017 with the transformer architecture.[10] In the 2020s, an AI boom has coincided with advances in generative AI, which allowed for the creation and modification of media. In addition to AI safety and unintended consequences and harms from the use of AI, ethical concerns, AI's long-term effects, and potential existential risks have prompted discussions of AI regulation. + +Goals +The general problem of simulating (or creating) intelligence has been broken into subproblems. These consist of particular traits or capabilities that researchers expect an intelligent system to display. The traits described below have received the most attention and cover the scope of AI research.[a] + +Reasoning and problem-solving +Early researchers developed algorithms that imitated step-by-step reasoning that humans use when they solve puzzles or make logical deductions.[11] By the late 1980s and 1990s, methods were developed for dealing with uncertain or incomplete information, employing concepts from probability and economics.[12] + +Many of these algorithms are insufficient for solving large reasoning problems because they experience a "combinatorial explosion": They become exponentially slower as the problems grow.[13] Even humans rarely use the step-by-step deduction that early AI research could model. They solve most of their problems using fast, intuitive judgments.[14] Accurate and efficient reasoning is an unsolved problem. + +Knowledge representation + +An ontology represents knowledge as a set of concepts within a domain and the relationships between those concepts. +Knowledge representation and knowledge engineering[15] allow AI programs to answer questions intelligently and make deductions about real-world facts. Formal knowledge representations are used in content-based indexing and retrieval,[16] scene interpretation,[17] clinical decision support,[18] knowledge discovery (mining "interesting" and actionable inferences from large databases),[19] and other areas.[20] + +A knowledge base is a body of knowledge represented in a form that can be used by a program. An ontology is the set of objects, relations, concepts, and properties used by a particular domain of knowledge.[21] Knowledge bases need to represent things such as objects, properties, categories, and relations between objects;[22] situations, events, states, and time;[23] causes and effects;[24] knowledge about knowledge (what we know about what other people know);[25] default reasoning (things that humans assume are true until they are told differently and will remain true even when other facts are changing);[26] and many other aspects and domains of knowledge. + +Among the most difficult problems in knowledge representation are the breadth of commonsense knowledge (the set of atomic facts that the average person knows is enormous);[27] and the sub-symbolic form of most commonsense knowledge (much of what people know is not represented as "facts" or "statements" that they could express verbally).[14] There is also the difficulty of knowledge acquisition, the problem of obtaining knowledge for AI applications.[c] + +Planning and decision-making +An "agent" is any entity (artificial or not) that perceives and takes actions in the world. A rational agent has goals or preferences and takes actions to make them happen.[d][30] In automated planning, the agent has a specific goal.[31] In automated decision-making, the agent has preferences—there are some situations it would prefer to be in, and some situations it is trying to avoid. The decision-making agent assigns a number to each situation (called the "utility") that measures how much the agent prefers it. For each possible action, it can calculate the "expected utility": the utility of all possible outcomes of the action, weighted by the probability that the outcome will occur. It can then choose the action with the maximum expected utility.[32] + +In classical planning, the agent knows exactly what the effect of any action will be.[33] In most real-world problems, however, the agent may not be certain about the situation they are in (it is "unknown" or "unobservable") and it may not know for certain what will happen after each possible action (it is not "deterministic"). It must choose an action by making a probabilistic guess and then reassess the situation to see if the action worked.[34] + +Alongside thorough testing and improvement based on previous decisions, having an explanation for why the agent took certain decisions is a way to build trust, especially when the decisions have to be relied upon.[35] + +In some problems, the agent's preferences may be uncertain, especially if there are other agents or humans involved. These can be learned (e.g., with inverse reinforcement learning), or the agent can seek information to improve its preferences.[36] Information value theory can be used to weigh the value of exploratory or experimental actions.[37] The space of possible future actions and situations is typically intractably large, so the agents must take actions and evaluate situations while being uncertain of what the outcome will be. + +A Markov decision process has a transition model that describes the probability that a particular action will change the state in a particular way and a reward function that supplies the utility of each state and the cost of each action. A policy associates a decision with each possible state. The policy could be calculated (e.g., by iteration), be heuristic, or it can be learned.[38] + +Game theory describes the rational behavior of multiple interacting agents and is used in AI programs that make decisions that involve other agents.[39] + +Learning +Machine learning is the study of programs that can improve their performance on a given task automatically.[40] It has been a part of AI from the beginning.[e] + + +In supervised learning, the training data is labelled with the expected answers, while in unsupervised learning, the model identifies patterns or structures in unlabelled data. +There are several kinds of machine learning. Unsupervised learning analyzes a stream of data and finds patterns and makes predictions without any other guidance.[43] Supervised learning requires labeling the training data with the expected answers, and comes in two main varieties: classification (where the program must learn to predict what category the input belongs in) and regression (where the program must deduce a numeric function based on numeric input).[44] + +In reinforcement learning, the agent is rewarded for good responses and punished for bad ones. The agent learns to choose responses that are classified as "good".[45] Transfer learning is when the knowledge gained from one problem is applied to a new problem.[46] Deep learning is a type of machine learning that runs inputs through biologically inspired artificial neural networks for all of these types of learning.[47] + +Computational learning theory can assess learners by computational complexity, by sample complexity (how much data is required), or by other notions of optimization.[48] + +Natural language processing +Natural language processing (NLP) allows programs to read, write and communicate in human languages.[49] Specific problems include speech recognition, speech synthesis, machine translation, information extraction, information retrieval and question answering.[50] + +Early work, based on Noam Chomsky's generative grammar and semantic networks, had difficulty with word-sense disambiguation[f] unless restricted to small domains called "micro-worlds" (due to the common sense knowledge problem[27]). Margaret Masterman believed that it was meaning and not grammar that was the key to understanding languages, and that thesauri and not dictionaries should be the basis of computational language structure. + +Modern deep learning techniques for NLP include word embedding (representing words, typically as vectors encoding their meaning),[51] transformers (a deep learning architecture using an attention mechanism),[52] and others.[53] In 2019, generative pre-trained transformer (or "GPT") language models began to generate coherent text,[54][55] and by 2023, these models were able to get human-level scores on the bar exam, SAT test, GRE test, and many other real-world applications.[56] + +Perception +Machine perception is the ability to use input from sensors (such as cameras, microphones, wireless signals, active lidar, sonar, radar, and tactile sensors) to deduce aspects of the world. Computer vision is the ability to analyze visual input.[57] + +The field includes speech recognition,[58] image classification,[59] facial recognition, object recognition,[60] object tracking,[61] and robotic perception.[62] + +Social intelligence + +Kismet, a robot head made in the 1990s, is a machine that can recognize and simulate emotions.[63] +Affective computing is a field that comprises systems that recognize, interpret, process, or simulate human feeling, emotion, and mood.[64] For example, some virtual assistants are programmed to speak conversationally or even to banter humorously; it makes them appear more sensitive to the emotional dynamics of human interaction, or to otherwise facilitate human–computer interaction. + +However, this tends to give naïve users an unrealistic conception of the intelligence of existing computer agents.[65] Moderate successes related to affective computing include textual sentiment analysis and, more recently, multimodal sentiment analysis, wherein AI classifies the effects displayed by a videotaped subject.[66] + +General intelligence +A machine with artificial general intelligence would be able to solve a wide variety of problems with breadth and versatility similar to human intelligence.[67] + +Techniques +AI research uses a wide variety of techniques to accomplish the goals above.[b] + +Search and optimization +There are two different kinds of search used in AI: state space search and local search: + +State space search +State space search searches through a tree of possible states to try to find a goal state.[68] For example, planning algorithms search through trees of goals and subgoals, attempting to find a path to a target goal, a process called means-ends analysis.[69] + +Simple exhaustive searches[70] are rarely sufficient for most real-world problems: the search space (the number of places to search) quickly grows to astronomical numbers. The result is a search that is too slow or never completes.[13] "Heuristics" or "rules of thumb" can help prioritize choices that are more likely to reach a goal.[71] + +Adversarial search is used for game-playing programs, such as chess or Go. It searches through a tree of possible moves and countermoves, looking for a winning position.[72] + +Local search + +Illustration of gradient descent for three different starting points; two parameters (represented by the plan coordinates) are adjusted in order to minimize the loss function (the height). +Local search uses mathematical optimization to find a solution to a problem. It begins with some form of guess and refines it incrementally.[73] + +Gradient descent is a type of local search that optimizes a set of numerical parameters by incrementally adjusting them to minimize a loss function. Variants of gradient descent are commonly used to train neural networks,[74] through the backpropagation algorithm. + +Another type of local search is evolutionary computation, which aims to iteratively improve a set of candidate solutions by "mutating" and "recombining" them, selecting only the fittest to survive each generation.[75] + +Distributed search processes can coordinate via swarm intelligence algorithms. Two popular swarm algorithms used in search are particle swarm optimization (inspired by bird flocking) and ant colony optimization (inspired by ant trails).[76] + +Logic +Formal logic is used for reasoning and knowledge representation.[77] Formal logic comes in two main forms: propositional logic (which operates on statements that are true or false and uses logical connectives such as "and", "or", "not" and "implies")[78] and predicate logic (which also operates on objects, predicates and relations and uses quantifiers such as "Every X is a Y" and "There are some Xs that are Ys").[79] + +Deductive reasoning in logic is the process of proving a new statement (conclusion) from other statements that are given and assumed to be true (the premises).[80] Proofs can be structured as proof trees, in which nodes are labelled by sentences, and children nodes are connected to parent nodes by inference rules. + +Given a problem and a set of premises, problem-solving reduces to searching for a proof tree whose root node is labelled by a solution of the problem and whose leaf nodes are labelled by premises or axioms. In the case of Horn clauses, problem-solving search can be performed by reasoning forwards from the premises or backwards from the problem.[81] In the more general case of the clausal form of first-order logic, resolution is a single, axiom-free rule of inference, in which a problem is solved by proving a contradiction from premises that include the negation of the problem to be solved.[82] + +Inference in both Horn clause logic and first-order logic is undecidable, and therefore intractable. However, backward reasoning with Horn clauses, which underpins computation in the logic programming language Prolog, is Turing complete. Moreover, its efficiency is competitive with computation in other symbolic programming languages.[83] + +Fuzzy logic assigns a "degree of truth" between 0 and 1. It can therefore handle propositions that are vague and partially true.[84] + +Non-monotonic logics, including logic programming with negation as failure, are designed to handle default reasoning.[26] Other specialized versions of logic have been developed to describe many complex domains. + +Probabilistic methods for uncertain reasoning + +A simple Bayesian network, with the associated conditional probability tables +Many problems in AI (including reasoning, planning, learning, perception, and robotics) require the agent to operate with incomplete or uncertain information. AI researchers have devised a number of tools to solve these problems using methods from probability theory and economics.[85] Precise mathematical tools have been developed that analyze how an agent can make choices and plan, using decision theory, decision analysis,[86] and information value theory.[87] These tools include models such as Markov decision processes,[88] dynamic decision networks,[89] game theory and mechanism design.[90] + +Bayesian networks[91] are a tool that can be used for reasoning (using the Bayesian inference algorithm),[g][93] learning (using the expectation–maximization algorithm),[h][95] planning (using decision networks)[96] and perception (using dynamic Bayesian networks).[89] + +Probabilistic algorithms can also be used for filtering, prediction, smoothing, and finding explanations for streams of data, thus helping perception systems analyze processes that occur over time (e.g., hidden Markov models or Kalman filters).[89] + + +Expectation–maximization clustering of Old Faithful eruption data starts from a random guess but then successfully converges on an accurate clustering of the two physically distinct modes of eruption. +Classifiers and statistical learning methods +The simplest AI applications can be divided into two types: classifiers (e.g., "if shiny then diamond"), on one hand, and controllers (e.g., "if diamond then pick up"), on the other hand. Classifiers[97] are functions that use pattern matching to determine the closest match. They can be fine-tuned based on chosen examples using supervised learning. Each pattern (also called an "observation") is labeled with a certain predefined class. All the observations combined with their class labels are known as a data set. When a new observation is received, that observation is classified based on previous experience.[44] + +There are many kinds of classifiers in use.[98] The decision tree is the simplest and most widely used symbolic machine learning algorithm.[99] K-nearest neighbor algorithm was the most widely used analogical AI until the mid-1990s, and Kernel methods such as the support vector machine (SVM) displaced k-nearest neighbor in the 1990s.[100] The naive Bayes classifier is reportedly the "most widely used learner"[101] at Google, due in part to its scalability.[102] Neural networks are also used as classifiers.[103] + +Artificial neural networks + +A neural network is an interconnected group of nodes, akin to the vast network of neurons in the human brain. +An artificial neural network is based on a collection of nodes also known as artificial neurons, which loosely model the neurons in a biological brain. It is trained to recognise patterns; once trained, it can recognise those patterns in fresh data. There is an input, at least one hidden layer of nodes and an output. Each node applies a function and once the weight crosses its specified threshold, the data is transmitted to the next layer. A network is typically called a deep neural network if it has at least 2 hidden layers.[103] + +Learning algorithms for neural networks use local search to choose the weights that will get the right output for each input during training. The most common training technique is the backpropagation algorithm.[104] Neural networks learn to model complex relationships between inputs and outputs and find patterns in data. In theory, a neural network can learn any function.[105] + +In feedforward neural networks the signal passes in only one direction.[106] The term perceptron typically refers to a single-layer neural network.[107] In contrast, deep learning uses many layers.[108] Recurrent neural networks (RNNs) feed the output signal back into the input, which allows short-term memories of previous input events. Long short-term memory networks (LSTMs) are recurrent neural networks that better preserve longterm dependencies and are less sensitive to the vanishing gradient problem.[109] Convolutional neural networks (CNNs) use layers of kernels to more efficiently process local patterns. This local processing is especially important in image processing, where the early CNN layers typically identify simple local patterns such as edges and curves, with subsequent layers detecting more complex patterns like textures, and eventually whole objects.[110] + +Deep learning + +Deep learning is a subset of machine learning, which is itself a subset of artificial intelligence.[111] +Deep learning uses several layers of neurons between the network's inputs and outputs.[108] The multiple layers can progressively extract higher-level features from the raw input. For example, in image processing, lower layers may identify edges, while higher layers may identify the concepts relevant to a human such as digits, letters, or faces.[112] + +Deep learning has profoundly improved the performance of programs in many important subfields of artificial intelligence, including computer vision, speech recognition, natural language processing, image classification,[113] and others. The reason that deep learning performs so well in so many applications is not known as of 2021.[114] The sudden success of deep learning in 2012–2015 did not occur because of some new discovery or theoretical breakthrough (deep neural networks and backpropagation had been described by many people, as far back as the 1950s)[i] but because of two factors: the incredible increase in computer power (including the hundred-fold increase in speed by switching to GPUs) and the availability of vast amounts of training data, especially the giant curated datasets used for benchmark testing, such as ImageNet.[j] + +GPT +Generative pre-trained transformers (GPT) are large language models (LLMs) that generate text based on the semantic relationships between words in sentences. Text-based GPT models are pre-trained on a large corpus of text that can be from the Internet. The pretraining consists of predicting the next token (a token being usually a word, subword, or punctuation). Throughout this pretraining, GPT models accumulate knowledge about the world and can then generate human-like text by repeatedly predicting the next token. Typically, a subsequent training phase makes the model more truthful, useful, and harmless, usually with a technique called reinforcement learning from human feedback (RLHF). Current GPT models are prone to generating falsehoods called "hallucinations". These can be reduced with RLHF and quality data, but the problem has been getting worse for reasoning systems.[122] Such systems are used in chatbots, which allow people to ask a question or request a task in simple text.[123][124] + +Current models and services include ChatGPT, Claude, Gemini, Copilot, and Meta AI.[125] Multimodal GPT models can process different types of data (modalities) such as images, videos, sound, and text.[126] + +Hardware and software +Main articles: Programming languages for artificial intelligence and Hardware for artificial intelligence + +Raspberry Pi AI Kit +In the late 2010s, graphics processing units (GPUs) that were increasingly designed with AI-specific enhancements and used with specialized TensorFlow software had replaced previously used central processing unit (CPUs) as the dominant means for large-scale (commercial and academic) machine learning models' training.[127] Specialized programming languages such as Prolog were used in early AI research,[128] but general-purpose programming languages like Python have become predominant.[129] + +The transistor density in integrated circuits has been observed to roughly double every 18 months—a trend known as Moore's law, named after the Intel co-founder Gordon Moore, who first identified it. Improvements in GPUs have been even faster,[130] a trend sometimes called Huang's law,[131] named after Nvidia co-founder and CEO Jensen Huang. + +Applications +Main article: Applications of artificial intelligence + +AI Overviews, an example of AI use on search engines +AI and machine learning technology is used in most of the essential applications of the 2020s, including: + +search engines (such as Google Search) +targeting online advertisements +recommendation systems (offered by Netflix, YouTube or Amazon) driving internet traffic +targeted advertising (AdSense, Facebook) +virtual assistants (such as Siri or Alexa) +autonomous vehicles (including drones, ADAS and self-driving cars) +automatic language translation (Microsoft Translator, Google Translate) +facial recognition (Apple's FaceID or Microsoft's DeepFace and Google's FaceNet) +image labeling (used by Facebook, Apple's Photos and TikTok). +The deployment of AI may be overseen by a chief automation officer (CAO). + +Health and medicine +Main article: Artificial intelligence in healthcare +It has been suggested that AI can overcome discrepancies in funding allocated to different fields of research.[132] + +AlphaFold 2 (2021) demonstrated the ability to approximate, in hours rather than months, the 3D structure of a protein.[133] In 2023, it was reported that AI-guided drug discovery helped find a class of antibiotics capable of killing two different types of drug-resistant bacteria.[134] In 2024, researchers used machine learning to accelerate the search for Parkinson's disease drug treatments. Their aim was to identify compounds that block the clumping, or aggregation, of alpha-synuclein (the protein that characterises Parkinson's disease). They were able to speed up the initial screening process ten-fold and reduce the cost by a thousand-fold.[135][136] + +Gaming +Main article: Artificial intelligence in video games +Game playing programs have been used since the 1950s to demonstrate and test AI's most advanced techniques.[137] Deep Blue became the first computer chess-playing system to beat a reigning world chess champion, Garry Kasparov, on 11 May 1997.[138] In 2011, in a Jeopardy! quiz show exhibition match, IBM's question answering system, Watson, defeated the two greatest Jeopardy! champions, Brad Rutter and Ken Jennings, by a significant margin.[139] In March 2016, AlphaGo won 4 out of 5 games of Go in a match with Go champion Lee Sedol, becoming the first computer Go-playing system to beat a professional Go player without handicaps. Then, in 2017, it defeated Ke Jie, who was the best Go player in the world.[140] Other programs handle imperfect-information games, such as the poker-playing program Pluribus.[141] DeepMind developed increasingly generalistic reinforcement learning models, such as with MuZero, which could be trained to play chess, Go, or Atari games.[142] In 2019, DeepMind's AlphaStar achieved grandmaster level in StarCraft II, a particularly challenging real-time strategy game that involves incomplete knowledge of what happens on the map.[143] In 2021, an AI agent competed in a PlayStation Gran Turismo competition, winning against four of the world's best Gran Turismo drivers using deep reinforcement learning.[144] In 2024, Google DeepMind introduced SIMA, a type of AI capable of autonomously playing nine previously unseen open-world video games by observing screen output, as well as executing short, specific tasks in response to natural language instructions.[145] + +Mathematics +In mathematics, probabilistic large language models are versatile, but can also produce wrong answers in the form of hallucinations. The Alibaba Group developed a version of its Qwen models called Qwen2-Math, that achieved state-of-the-art performance on several mathematical benchmarks, including 84% accuracy on the MATH dataset of competition mathematics problems.[146] In January 2025, Microsoft proposed the technique rStar-Math that leverages Monte Carlo tree search and step-by-step reasoning, enabling a relatively small language model like Qwen-7B to solve 53% of the AIME 2024 and 90% of the MATH benchmark problems.[147] Google DeepMind has developed models for solving mathematical problems: AlphaTensor, AlphaGeometry, AlphaProof and AlphaEvolve.[148][149] + +When natural language is used to describe mathematical problems, converters can transform such prompts into a formal language such as Lean to define mathematical tasks. The experimental model Gemini Deep Think accepts natural language prompts directly and achieved gold medal results in the International Math Olympiad of 2025.[150] + +Topological deep learning integrates various topological approaches. + +Finance +According to Nicolas Firzli, director of the World Pensions & Investments Forum, it may be too early to see the emergence of highly innovative AI-informed financial products and services. He argues that "the deployment of AI tools will simply further automatise things: destroying tens of thousands of jobs in banking, financial planning, and pension advice in the process, but I'm not sure it will unleash a new wave of [e.g., sophisticated] pension innovation."[151] + +Military +Main article: Military applications of artificial intelligence +Various countries are deploying AI military applications.[152] The main applications enhance command and control, communications, sensors, integration and interoperability.[153] Research is targeting intelligence collection and analysis, logistics, cyber operations, information operations, and semiautonomous and autonomous vehicles.[152] AI technologies enable coordination of sensors and effectors, threat detection and identification, marking of enemy positions, target acquisition, coordination and deconfliction of distributed Joint Fires between networked combat vehicles, both human-operated and autonomous.[153] + +AI has been used in military operations in Iraq, Syria, Israel and Ukraine.[152][154][155][156] + +Generative AI +These paragraphs are an excerpt from Generative AI.[edit] +Generative artificial intelligence (GenAI) is a subfield of artificial intelligence (AI) that uses generative models to generate text, images, videos, audio, software code (vibe coding) or other forms of data.[157] These models learn the underlying patterns and structures of their training data, and use them to generate new data[158] in response to input, which often takes the form of natural language prompts.[159][160] + +The prevalence of generative AI tools has increased significantly since the AI boom in the 2020s. This boom was made possible by improvements in deep neural networks, particularly large language models (LLMs), which are based on the transformer architecture. Generative AI applications include chatbots such as ChatGPT, Claude, Copilot, DeepSeek, Google Gemini and Grok; text-to-image models such as DALL-E, Firefly, Stable Diffusion, and Midjourney; and text-to-video models such as Veo, LTX and Sora.[161][162][163] + +Companies in a variety of sectors have used generative AI, including those in software development, healthcare,[164] finance,[165] entertainment,[166] customer service,[167] sales and marketing,[168] art, writing,[169] and product design.[170] + +Agents +Main article: Agentic AI +See also: OpenClaw and CrewAI +AI agents are software entities designed to perceive their environment, make decisions, and take actions autonomously to achieve specific goals. These agents can interact with users, their environment, or other agents. AI agents are used in various applications, including virtual assistants, chatbots, autonomous vehicles, game-playing systems, and industrial robotics. AI agents operate within the constraints of their programming, available computational resources, and hardware limitations. This means they are restricted to performing tasks within their defined scope and have finite memory and processing capabilities. In real-world applications, AI agents often face time constraints for decision-making and action execution. Many AI agents incorporate learning algorithms, enabling them to improve their performance over time through experience or training. Using machine learning, AI agents can adapt to new situations and optimise their behaviour for their designated tasks.[171][172][173] + +Web search +Microsoft introduced Copilot Search in February 2023 under the name Bing Chat. Copilot Search provides AI-generated summaries.[174] + +Google introduced an AI Mode at its Google I/O event on 20 May 2025.[175] + +Sexuality +Applications of AI in this domain include AI-enabled menstruation and fertility trackers that analyze user data to offer predictions,[176] AI-integrated sex toys (e.g., teledildonics),[177] AI-generated sexual education content,[178] and AI agents that simulate sexual and romantic partners (e.g., Replika).[179] AI is also used for the production of non-consensual deepfake pornography, raising significant ethical and legal concerns.[180] + +AI technologies have also been used to attempt to identify online gender-based violence and online sexual grooming of minors.[181][182] + +Other industry-specific tasks +In a 2017 survey, one in five companies reported having incorporated "AI" in some offerings or processes.[183] + +In the field of evacuation and disaster management, AI has been used to investigate patterns in large-scale and small-scale evacuations using historical data from GPS, videos or social media.[184][185][186] + +During the 2024 Indian elections, US$50 million was spent on authorized AI-generated content, notably by creating deepfakes of allied (including sometimes deceased) politicians to better engage with voters, and by translating speeches to various local languages.[187] + +The use of generative AI by law firms for legal research resulted in the creation of the global "AI Hallucination Cases" database, in April 2025, established by HEC Paris and Sciences Po legal data analysis lecturer Damien Charlotin.[188][189] By 2026, judges had issued sanctions and bar associations had issued warnings due to attorney submissions to the courts containing fabricated case law citations hallucinated by AI tools.[190] + +See also: Hallucination (artificial intelligence) § In legal filings +Ethics +Main article: Ethics of artificial intelligence + +Street art in Tel Aviv[191][192] +AI has potential benefits and potential risks.[193] AI may be able to advance science and find solutions for serious problems: Demis Hassabis of DeepMind hopes to "solve intelligence, and then use that to solve everything else".[194] However, as the use of AI has become widespread, several unintended consequences and risks have been identified.[195][196] In-production systems can sometimes not factor ethics and bias into their AI training processes, especially when the AI algorithms are inherently unexplainable in deep learning.[197] + +Risks and harm +Privacy and copyright +Further information: Information privacy and Artificial intelligence and copyright +Machine learning algorithms require large amounts of data. The techniques used to acquire this data have raised concerns about privacy, surveillance and copyright. + +AI-powered devices and services, such as virtual assistants and IoT products, continuously collect personal information, raising concerns about intrusive data gathering and unauthorized access by third parties. The loss of privacy is further exacerbated by AI's ability to process and combine vast amounts of data, potentially leading to a surveillance society where individual activities are constantly monitored and analyzed without adequate safeguards or transparency. + +Sensitive user data collected may include online activity records, geolocation data, video, or audio.[198] For example, in order to build speech recognition algorithms, Amazon has recorded millions of private conversations and allowed temporary workers to listen to and transcribe some of them.[199] Opinions about this widespread surveillance range from those who see it as a necessary evil to those for whom it is clearly unethical and a violation of the right to privacy.[200] + +AI developers argue that this is the only way to deliver valuable applications and have developed several techniques that attempt to preserve privacy while still obtaining the data, such as data aggregation, de-identification and differential privacy.[201] Since 2016, some privacy experts, such as Cynthia Dwork, have begun to view privacy in terms of fairness. Brian Christian wrote that experts have pivoted "from the question of 'what they know' to the question of 'what they're doing with it'."[202] + +Generative AI is often trained on unlicensed copyrighted works, including in domains such as images or computer code; the output is then used under the rationale of "fair use". Experts disagree about how well and under what circumstances this rationale will hold up in courts of law; relevant factors may include "the purpose and character of the use of the copyrighted work" and "the effect upon the potential market for the copyrighted work".[203][204] Website owners can indicate that they do not want their content scraped via a "robots.txt" file.[205] However, some companies will scrape content regardless[206][207] because the robots.txt file has no real authority. In 2023, leading authors (including John Grisham and Jonathan Franzen) sued AI companies for using their work to train generative AI.[208][209] Another discussed approach is to envision a separate sui generis system of protection for creations generated by AI to ensure fair attribution and compensation for human authors.[210] + +Dominance by tech giants +The commercial AI scene is dominated by Big Tech companies such as Alphabet Inc., Amazon, Apple Inc., Meta Platforms, and Microsoft.[211][212][213] Some of these players already own the vast majority of existing cloud infrastructure and computing power from data centers, allowing them to entrench further in the marketplace.[214][215] + +Power needs and environmental impacts +See also: Environmental impacts of artificial intelligence + +Fueled by a growth in AI, data centers' demand for power increased in the 2020s.[216] +Technology companies have built electricity and artificial intelligence infrastructure to facilitate the AI boom of the 2020s. A 2025 report from the consulting firm McKinsey & Company estimated that by 2030, $2.7 trillion would be invested into AI infrastructure and data centers in the US, surpassing World War II's Manhattan Project every month.[217] + +In January 2024, the International Energy Agency (IEA) released Electricity 2024, Analysis and Forecast to 2026.[218] This is the first IEA report to make projections for data centers and power consumption by AI and cryptocurrency. The report states that power demand for these uses might double by 2026, with the additional power consumption equaling that of Japan.[219] + +Power consumption by AI is responsible for an increase in fossil fuel use, and has delayed closings of obsolete, carbon-emitting coal energy facilities. A ChatGPT search involves the use of 10 times the electrical energy as a Google search.[220] + +A 2024 Goldman Sachs Research Paper, AI Data Centers and the Coming US Power Demand Surge, found "US power demand (is) likely to experience growth not seen in a generation...." and forecasts that, by 2030, US data centers will consume 8% of US power, as opposed to 3% in 2022, presaging growth for the electrical power generation industry by a variety of means.[221] Data centers' need for more and more electrical power is such that they might max out the electrical grid. The Big Tech companies counter that AI can be used to maximize the utilization of the grid by all.[222] + +In 2024, The Wall Street Journal reported that big AI companies have begun negotiations with the US nuclear power providers to provide electricity to the data centers. In March 2024 Amazon purchased a Pennsylvania nuclear-powered data center for US$650 million.[223] + +In September 2024, Microsoft announced an agreement with Constellation Energy to re-open the Three Mile Island nuclear power plant to provide Microsoft with 100% of all electric power produced by the plant for 20 years. Reopening the plant, which suffered a partial nuclear meltdown of its Unit 2 reactor in 1979, will require Constellation to get through strict regulatory processes which will include extensive safety scrutiny from the US Nuclear Regulatory Commission. If approved (this will be the first ever US re-commissioning of a nuclear plant), over 835 megawatts of power – enough for 800,000 homes – of energy will be produced. The cost for re-opening and upgrading is estimated at US$1.6 billion and is dependent on tax breaks for nuclear power contained in the 2022 US Inflation Reduction Act.[224] As of 2024, the US government and the state of Michigan have been investing almost US$2 billion to reopen the Palisades Nuclear reactor on Lake Michigan. Closed since 2022, the plant was planned to be reopened in October 2025.[225] + +After the last approval in September 2023, Taiwan suspended the approval of data centers north of Taoyuan with a capacity of more than 5 MW in 2024, due to power supply shortages.[226] Taiwan aims to phase out nuclear power by 2025.[226] + +Singapore imposed a ban on the opening of data centers in 2019 due to electric power, but in 2022, lifted this ban.[226] + +Although most nuclear plants in Japan have been shut down after the 2011 Fukushima nuclear accident, according to an October 2024 Bloomberg article in Japanese, cloud gaming services company Ubitus, in which Nvidia has a stake, is looking for land in Japan near a nuclear power plant for a new data center for generative AI.[227] + +On 1 November 2024, the Federal Energy Regulatory Commission (FERC) rejected an application submitted by Talen Energy for approval to supply some electricity from the nuclear power station Susquehanna to Amazon's data center.[228] According to the Commission Chairman Willie L. Phillips, it is a burden on the electricity grid as well as a significant cost shifting concern to households and other business sectors.[228] + +In 2025, a report prepared by the IEA estimated the greenhouse gas emissions from the energy consumption of AI at 180 million tons. By 2035, these emissions could rise to 300–500 million tonnes depending on what measures will be taken. This is below 1.5% of the energy sector emissions. The emissions reduction potential of AI was estimated at 5% of the energy sector emissions, but rebound effects (for example if people switch from public transport to autonomous cars) can reduce it.[229] + +Misinformation +See also: Content moderation +YouTube, Facebook and others use recommender systems to guide users to more content. These AI programs were given the goal of maximizing user engagement (that is, the only goal was to keep people watching). The AI learned that users tended to choose misinformation, conspiracy theories, and extreme partisan content, and, to keep them watching, the AI recommended more of it. Users also tended to watch more content on the same subject, so the AI led people into filter bubbles where they received multiple versions of the same misinformation.[230] This convinced many users that the misinformation was true, and ultimately undermined trust in institutions, the media and the government.[231] The AI program had correctly learned to maximize its goal, but the result was harmful to society. After the U.S. election in 2016, major technology companies took some steps to mitigate the problem.[232] + +In the early 2020s, generative AI began to create images, audio, and texts that are virtually indistinguishable from real photographs, recordings, or human writing,[233] while realistic AI-generated videos became feasible in the mid-2020s.[234][235][236] It is possible for bad actors to use this technology to create massive amounts of misinformation or propaganda;[237] one such potential malicious use is deepfakes for computational propaganda.[238] AI pioneer and Nobel Prize-winning computer scientist Geoffrey Hinton expressed concern about AI enabling "authoritarian leaders to manipulate their electorates" on a large scale, among other risks.[239] The ability to influence electorates has been proved in at least one study. This same study shows more inaccurate statements from the models when they advocate for candidates of the political right.[240] + +AI researchers at Microsoft, OpenAI, universities and other organisations have suggested using "personhood credentials" as a way to overcome online deception enabled by AI models.[241] + +Algorithmic bias and fairness +Main articles: Algorithmic bias and Fairness (machine learning) +Machine learning applications can be biased[k] if they learn from biased data.[243] The developers may not be aware that the bias exists.[244] Discriminatory behavior by some LLMs can be observed in their output.[245] Bias can be introduced by the way training data is selected and by the way a model is deployed.[246][243] If a biased algorithm is used to make decisions that can seriously harm people (as it can in medicine, finance, recruitment, housing or policing) then the algorithm may cause discrimination.[247] The field of fairness studies how to prevent harms from algorithmic biases. + +On 28 June 2015, Google Photos's new image labeling feature mistakenly identified Jacky Alcine and a friend as "gorillas" because they were black. The system was trained on a dataset that contained very few images of black people,[248] a problem called "sample size disparity".[249] Google "fixed" this problem by preventing the system from labelling anything as a "gorilla". Eight years later, in 2023, Google Photos still could not identify a gorilla, and neither could similar products from Apple, Facebook, Microsoft and Amazon.[250] + +COMPAS is a commercial program widely used by U.S. courts to assess the likelihood of a defendant becoming a recidivist. In 2016, Julia Angwin at ProPublica discovered that COMPAS exhibited racial bias, despite the fact that the program was not told the races of the defendants. Although the error rate for both whites and blacks was calibrated equal at exactly 61%, the errors for each race were different—the system consistently overestimated the chance that a black person would re-offend and would underestimate the chance that a white person would not re-offend.[251] In 2017, several researchers[l] showed that it was mathematically impossible for COMPAS to accommodate all possible measures of fairness when the base rates of re-offense were different for whites and blacks in the data.[253] + +A program can make biased decisions even if the data does not explicitly mention a problematic feature (such as "race" or "gender"). The feature will correlate with other features (like "address", "shopping history" or "first name"), and the program will make the same decisions based on these features as it would on "race" or "gender".[254] Moritz Hardt said "the most robust fact in this research area is that fairness through blindness doesn't work."[255] + +Criticism of COMPAS highlighted that machine learning models are designed to make "predictions" that are only valid if we assume that the future will resemble the past. If they are trained on data that includes the results of racist decisions in the past, machine learning models must predict that racist decisions will be made in the future. If an application then uses these predictions as recommendations, some of these "recommendations" will likely be racist.[256] Thus, machine learning is not well suited to help make decisions in areas where there is hope that the future will be better than the past. It is descriptive rather than prescriptive.[m] + +Bias and unfairness may go undetected because the developers are overwhelmingly white and male: among AI engineers, about 4% are black and 20% are women.[249] + +There are various conflicting definitions and mathematical models of fairness. These notions depend on ethical assumptions, and are influenced by beliefs about society. One broad category is distributive fairness, which focuses on the outcomes, often identifying groups and seeking to compensate for statistical disparities. Representational fairness tries to ensure that AI systems do not reinforce negative stereotypes or render certain groups invisible. Procedural fairness focuses on the decision process rather than the outcome. The most relevant notions of fairness may depend on the context, notably the type of AI application and the stakeholders. The subjectivity in the notions of bias and fairness makes it difficult for companies to operationalize them. Having access to sensitive attributes such as race or gender is also considered by many AI ethicists to be necessary in order to compensate for biases, but it may conflict with anti-discrimination laws.[242] + +At the 2022 ACM Conference on Fairness, Accountability, and Transparency a paper reported that a CLIP‑based (Contrastive Language-Image Pre-training) robotic system reproduced harmful gender‑ and race‑linked stereotypes in a simulated manipulation task. The authors recommended robot‑learning methods which physically manifest such harms be "paused, reworked, or even wound down when appropriate, until outcomes can be proven safe, effective, and just."[258][259][260] + +Lack of transparency +See also: Explainable AI, Algorithmic transparency, and Right to explanation +Many AI systems are so complex that their designers cannot explain how they reach their decisions.[261] Particularly with deep neural networks, in which there are many non-linear relationships between inputs and outputs. But some popular explainability techniques exist.[262] + +It is impossible to be certain that a program is operating correctly if no one knows how exactly it works. There have been many cases where a machine learning program passed rigorous tests, but nevertheless learned something different than what the programmers intended. For example, a system that could identify skin diseases better than medical professionals was found to actually have a strong tendency to classify images with a ruler as "cancerous", because pictures of malignancies typically include a ruler to show the scale.[263] Another machine learning system designed to help effectively allocate medical resources was found to classify patients with asthma as being at "low risk" of dying from pneumonia. Having asthma is actually a severe risk factor, but since the patients having asthma would usually get much more medical care, they were relatively unlikely to die according to the training data. The correlation between asthma and low risk of dying from pneumonia was real, but misleading.[264] + +People who have been harmed by an algorithm's decision have a right to an explanation.[265] Doctors, for example, are expected to clearly and completely explain to their colleagues the reasoning behind any decision they make. Early drafts of the European Union's General Data Protection Regulation in 2016 included an explicit statement that this right exists.[n] Industry experts noted that this is an unsolved problem with no solution in sight. Regulators argued that nevertheless the harm is real: if the problem has no solution, the tools should not be used.[266] + +DARPA established the XAI ("Explainable Artificial Intelligence") program in 2014 to try to solve these problems.[267] + +Several approaches aim to address the transparency problem. SHAP enables to visualise the contribution of each feature to the output.[268] LIME can locally approximate a model's outputs with a simpler, interpretable model.[269] Multitask learning provides a large number of outputs in addition to the target classification. These other outputs can help developers deduce what the network has learned.[270] Deconvolution, DeepDream and other generative methods can allow developers to see what different layers of a deep network for computer vision have learned, and produce output that can suggest what the network is learning.[271] For generative pre-trained transformers, Anthropic developed a technique based on dictionary learning that associates patterns of neuron activations with human-understandable concepts.[272] + +Bad actors and weaponized AI +Main articles: Lethal autonomous weapon, Artificial intelligence arms race, and AI safety +Artificial intelligence provides a number of tools that are useful to bad actors, such as authoritarian governments, terrorists, criminals or rogue states. + +A lethal autonomous weapon is a machine that locates, selects and engages human targets without human supervision.[o] Widely available AI tools can be used by bad actors to develop inexpensive autonomous weapons and, if produced at scale, they are potentially weapons of mass destruction.[274] Even when used in conventional warfare, they currently cannot reliably choose targets and could potentially kill an innocent person.[274] In 2014, 30 nations (including China) supported a ban on autonomous weapons under the United Nations' Convention on Certain Conventional Weapons, however the United States and others disagreed.[275] By 2015, over fifty countries were reported to be researching battlefield robots.[276] + +AI tools make it easier for authoritarian governments to efficiently control their citizens in several ways. Face and voice recognition allow widespread surveillance. Machine learning, operating this data, can classify potential enemies of the state and prevent them from hiding. Recommendation systems can precisely target propaganda and misinformation for maximum effect. Deepfakes and generative AI aid in producing misinformation. Advanced AI can make authoritarian centralized decision-making more competitive than liberal and decentralized systems such as markets. It lowers the cost and difficulty of digital warfare and advanced spyware.[277] All these technologies have been available since 2020 or earlier—AI facial recognition systems are already being used for mass surveillance in China.[278][279] + +There are many other ways in which AI is expected to help bad actors, some of which can not be foreseen. For example, machine-learning AI is able to design tens of thousands of toxic molecules in a matter of hours.[280] + +Technological unemployment +Main articles: Workplace impact of artificial intelligence and Technological unemployment +Economists have frequently highlighted the risks of redundancies from AI, and speculated about unemployment if there is no adequate social policy for full employment.[281] + +In the past, technology has tended to increase rather than reduce total employment, but economists acknowledge that "we're in uncharted territory" with AI.[282] A survey of economists showed disagreement about whether the increasing use of robots and AI will cause a substantial increase in long-term unemployment, but they generally agree that it could be a net benefit if productivity gains are redistributed.[283] Risk estimates vary; for example, in the 2010s, Michael Osborne and Carl Benedikt Frey estimated 47% of U.S. jobs are at "high risk" of potential automation, while an OECD report classified only 9% of U.S. jobs as "high risk".[p][285] The methodology of speculating about future employment levels has been criticised as lacking evidential foundation, and for implying that technology, rather than social policy, creates unemployment, as opposed to redundancies.[281] In April 2023, it was reported that 70% of the jobs for Chinese video game illustrators had been eliminated by generative artificial intelligence.[286][287] Early-career workers showed decreasing employment rates in some AI-exposed occupations.[288] + +Unlike previous waves of automation, many middle-class jobs may be eliminated by artificial intelligence; The Economist stated in 2015 that "the worry that AI could do to white-collar jobs what steam power did to blue-collar ones during the Industrial Revolution" is "worth taking seriously".[289] Jobs at extreme risk range from paralegals to fast food cooks, while job demand is likely to increase for care-related professions ranging from personal healthcare to the clergy.[290] In July 2025, Ford CEO Jim Farley predicted that "artificial intelligence is going to replace literally half of all white-collar workers in the U.S."[291] + +From the early days of the development of artificial intelligence, there have been arguments, for example, those put forward by Joseph Weizenbaum, about whether tasks that can be done by computers actually should be done by them, given the difference between computers and humans, and between quantitative calculation and qualitative, value-based judgement.[292] + +Substitution for human–human interaction +See also: Deaths linked to chatbots +With the increase of loneliness in the early 21st century, AI is sometimes identified as a potential source of relief to this problem. It would be possible, via human-like qualities built into AI products,[293] for individuals to assume that this need can be met by artificial means.[294][295] In some cases, people approach artificial intelligence for companionship when they believe that they would not find acceptance due to feeling outcast.[296] Examples of harm coming to humans from advanced chatbots have been reported in courts in the United States, with AI companies accused of creating products that endanger humans through emotional confusion or deception.[297][298] + +Existential risk +Main article: Existential risk from artificial intelligence +Recent public debates in artificial intelligence have increasingly focused on its broader societal and ethical implications. It has been argued AI will become so powerful that humanity may irreversibly lose control of it. This could, as physicist Stephen Hawking stated, "spell the end of the human race".[299] This scenario has been common in science fiction, when a computer or robot suddenly develops a human-like "self-awareness" (or "sentience" or "consciousness") and becomes a malevolent character.[q] These sci-fi scenarios are misleading in several ways. + +First, AI does not require human-like sentience to be an existential risk. Modern AI programs are given specific goals and use learning and intelligence to achieve them. Philosopher Nick Bostrom argued that if one gives almost any goal to a sufficiently powerful AI, it may choose to destroy humanity to achieve it (he used the example of an automated paperclip factory that destroys the world to get more iron for paperclips).[301] Stuart Russell gives the example of household robot that tries to find a way to kill its owner to prevent it from being unplugged, reasoning that "you can't fetch the coffee if you're dead."[302] In order to be safe for humanity, a superintelligence would have to be genuinely aligned with humanity's morality and values so that it is "fundamentally on our side".[303] + +Second, Yuval Noah Harari argues that AI does not require a robot body or physical control to pose an existential risk. The essential parts of civilization are not physical. Things like ideologies, law, government, money and the economy are built on language; they exist because there are stories that billions of people believe. The current prevalence of misinformation suggests that an AI could use language to convince people to believe anything, even to take actions that are destructive.[304] Geoffrey Hinton said in 2025 that modern AI is particularly "good at persuasion" and getting better all the time. He asks "Suppose you wanted to invade the capital of the US. Do you have to go there and do it yourself? No. You just have to be good at persuasion."[305] + +The opinions amongst experts and industry insiders are mixed, with sizable fractions both concerned and unconcerned by risk from eventual superintelligent AI.[306] Personalities such as Stephen Hawking, Bill Gates, and Elon Musk,[307] as well as AI pioneers such as Geoffrey Hinton, Yoshua Bengio, Stuart Russell, Demis Hassabis, and Sam Altman, have expressed concerns about existential risk from AI. + +In May 2023, Geoffrey Hinton announced his resignation from Google in order to be able to "freely speak out about the risks of AI" without "considering how this impacts Google".[308] He notably mentioned risks of an AI takeover,[309] and stressed that in order to avoid the worst outcomes, establishing safety guidelines will require cooperation among those competing in use of AI.[310] + +In 2023, many leading AI experts endorsed the joint statement that "Mitigating the risk of extinction from AI should be a global priority alongside other societal-scale risks such as pandemics and nuclear war".[311] + +Some other researchers were more optimistic. AI pioneer Jürgen Schmidhuber did not sign the joint statement, emphasising that in 95% of all cases, AI research is about making "human lives longer and healthier and easier."[312] While the tools that are now being used to improve lives can also be used by bad actors, "they can also be used against the bad actors."[313][314] Andrew Ng also argued that "it's a mistake to fall for the doomsday hype on AI—and that regulators who do will only benefit vested interests."[315] Yann LeCun, a Turing Award winner, disagreed with the idea that AI will subordinate humans "simply because they are smarter, let alone destroy [us]",[316] "scoff[ing] at his peers' dystopian scenarios of supercharged misinformation and even, eventually, human extinction." In contrast, he claimed that "intelligent machines will usher in a new renaissance for humanity, a new era of enlightenment."[317] In the early 2010s, experts argued that the risks are too distant in the future to warrant research or that humans will be valuable from the perspective of a superintelligent machine.[318] However, after 2016, the study of current and future risks and possible solutions became a serious area of research.[319] + +Ethical machines and alignment +Main articles: Machine ethics, AI safety, Friendly artificial intelligence, Artificial moral agents, and Human Compatible +See also: Human-AI interaction +Friendly AI are machines that have been designed from the beginning to minimize risks and to make choices that benefit humans. Eliezer Yudkowsky, who coined the term, argues that developing friendly AI should be a higher research priority: it may require a large investment and it must be completed before AI becomes an existential risk.[320] + +Machines with intelligence have the potential to use their intelligence to make ethical decisions. The field of machine ethics provides machines with ethical principles and procedures for resolving ethical dilemmas.[321] The field of machine ethics is also called computational morality,[321] and was founded at an AAAI symposium in 2005.[322] + +Other approaches include Wendell Wallach's "artificial moral agents"[323] and Stuart J. Russell's three principles for developing provably beneficial machines.[324] + +Open source +See also: Open-source artificial intelligence and Lists of open-source artificial intelligence software +Active organizations in the AI open-source community include Hugging Face,[325] Google,[326] EleutherAI and Meta.[327] Various AI models, such as Llama 2, Mistral or Stable Diffusion, have been made open-weight,[328][329] meaning that their architecture and trained parameters (the "weights") are publicly available. Open-weight models can be freely fine-tuned, which allows companies to specialize them with their own data and for their own use-case.[330] Open-weight models are useful for research and innovation but can also be misused. Since they can be fine-tuned, any built-in security measure, such as objecting to harmful requests, can be trained away until it becomes ineffective. Some researchers warn that future AI models may develop dangerous capabilities (such as the potential to drastically facilitate bioterrorism) and that once released on the Internet, they cannot be deleted everywhere if needed. They recommend pre-release audits and cost-benefit analyses.[331] diff --git a/benchmark/retrieval/grep/vikingdb_bm25/effectiveness/step1_add_resource.py b/benchmark/retrieval/grep/vikingdb_bm25/effectiveness/step1_add_resource.py new file mode 100644 index 0000000000..e8e18c08b4 --- /dev/null +++ b/benchmark/retrieval/grep/vikingdb_bm25/effectiveness/step1_add_resource.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Step 1 (Effectiveness): Import real code repos into OpenViking (with indexing). + +Imports the entire source directory as a single resource via +SyncOpenViking.add_resource (wait=True, build_index=True, summarize=True). +add_resource handles recursive traversal internally. + +After import, run step2_quality.py to evaluate retrieval quality. + +Prerequisites: + - Download code repos and place them under the source directory manually. + +Usage: + python3 step1_add_resource.py + python3 step1_add_resource.py --source ~/.openviking/data/benchmark/OpenViking-main +""" + +from __future__ import annotations + +import argparse +import os +import time + +from openviking.sync_client import SyncOpenViking + +DEFAULT_SOURCE = os.path.expanduser("~/.openviking/data/benchmark/OpenViking-main") +BENCHMARK_PARENT = "viking://resources/benchmark/effectiveness" + + +def main(): + parser = argparse.ArgumentParser( + description="Step 1 (Effectiveness): Import real code repos (with indexing)" + ) + parser.add_argument( + "--source", + default=DEFAULT_SOURCE, + help=f"Local directory to import (default: {DEFAULT_SOURCE})", + ) + parser.add_argument( + "--parent", + default=BENCHMARK_PARENT, + help=f"Parent Viking URI (default: {BENCHMARK_PARENT})", + ) + args = parser.parse_args() + + source = os.path.expanduser(args.source) + if not os.path.isdir(source): + print(f"ERROR: Source directory does not exist: {source}") + return + + print("=" * 80) + print("Step 1 (Effectiveness): Import Code Repos (with VLM/embedding)") + print("=" * 80) + print(f" Source: {source}") + print(f" Parent: {args.parent}") + print(" Indexing: ENABLED (build_index=True, summarize=True)") + print() + + client = SyncOpenViking() + client.initialize() + + t0 = time.monotonic() + try: + result = client.add_resource( + path=source, + parent=args.parent, + reason="benchmark effectiveness", + wait=True, + create_parent=True, + build_index=True, + summarize=True, + ) + elapsed = time.monotonic() - t0 + root_uri = result.get("root_uri", "?") + print(f"OK ({elapsed:.1f}s) -> {root_uri}") + print() + print("Import completed successfully.") + print("Next step: run step2_quality.py to evaluate retrieval quality") + except Exception as e: + elapsed = time.monotonic() - t0 + print(f"FAILED ({elapsed:.1f}s): {e}") + + client.close() + + +if __name__ == "__main__": + main() diff --git a/benchmark/retrieval/grep/vikingdb_bm25/effectiveness/step2_quality.py b/benchmark/retrieval/grep/vikingdb_bm25/effectiveness/step2_quality.py new file mode 100644 index 0000000000..55237517ca --- /dev/null +++ b/benchmark/retrieval/grep/vikingdb_bm25/effectiveness/step2_quality.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""Step 2 (Effectiveness): Evaluate retrieval quality for real code repos. + +Compares grep results (current engine) against ground truth from fs-engine grep. +Computes Recall, Precision, F1 per query pattern. + +Ground truth is obtained by running grep with engine=fs (must be configured +in ov.conf on first run). Results are cached locally so subsequent runs +can use a different engine config while still comparing against the same +ground truth. + +Prerequisites: + 1. Run step1_add_resource.py to import repos (with indexing) + 2. First run: set ov.conf grep engine to "fs" and restart server + +Usage: + python3 step2_quality.py --keywords grep reindex SyncHTTPClient +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import time + +from openviking_cli.client.sync_http import SyncHTTPClient + +BASE_URI = "viking://resources/benchmark/effectiveness" +DATA_DIR = os.path.expanduser("~/.openviking/data/benchmark/effectiveness") +GROUND_TRUTH_DIR = os.path.join(DATA_DIR, ".ground_truth") +MISS_DIR = os.path.join(DATA_DIR, ".miss") +RESULT_DIR = os.path.join(DATA_DIR, ".result") + +KEYWORDS: list[str] = [ + # High frequency English + "embedding", + "grep", + # Medium frequency English + "vikingdb", + "reindex", + # Low frequency English + "build_index", + # CamelCase + "SyncHTTPClient", + "MarkdownParser", + "DataDirectoryLocked", + # snake_case + "add_resource", + "process_lock", + # Chinese + "检索", + "向量数据库", +] # Can also be overridden via --keywords + + +def _sanitize_filename(s: str, max_len: int = 40) -> str: + """Make a string safe for use as a filename component. Preserves Unicode.""" + s = re.sub(r'[/\\:*?"<>|\0]', "_", s) + s = s.strip("_ ") + return s[:max_len] + + +def _cache_hash(uri: str, pattern: str) -> str: + """Short hash for cache disambiguation.""" + return hashlib.sha256(uri.encode("utf-8") + pattern.encode("utf-8")).hexdigest()[:8] + + +def build_test_patterns(keywords: list[str] | None = None) -> list[tuple[str, str]]: + kws = keywords if keywords else KEYWORDS + patterns = [] + for kw in kws: + patterns.append((f"keyword: {kw}", kw)) + if len(kws) >= 2: + patterns.append((f"multi 2: {kws[0]}|{kws[1]}", f"{kws[0]}|{kws[1]}")) + patterns.append(("no-match: zzz_nonexistent_quality", "zzz_nonexistent_quality")) + return patterns + + +def run_sdk_grep(client: SyncHTTPClient, uri: str, pattern: str) -> tuple[set[str], float]: + t0 = time.monotonic() + result = client.grep(uri=uri, pattern=pattern, node_limit=100000) + elapsed = time.monotonic() - t0 + uris = set() + if isinstance(result, dict): + for match in result.get("matches", []): + uri_val = match.get("uri", "") + if uri_val: + uris.add(uri_val.rstrip("/")) + return uris, elapsed + + +def _ground_truth_cache_path(pattern: str, uri: str) -> str: + h = _cache_hash(uri, pattern) + safe_pattern = _sanitize_filename(pattern) + return os.path.join(GROUND_TRUTH_DIR, f"eff_{safe_pattern}_{h}.json") + + +def _load_ground_truth_cache(pattern: str, uri: str) -> set[str] | None: + path = _ground_truth_cache_path(pattern, uri) + if not os.path.isfile(path): + # Fallback: try old-style hash-only filename + old_h = hashlib.sha256(uri.encode("utf-8")).hexdigest() + old_h += hashlib.sha256(pattern.encode("utf-8")).hexdigest()[:16] + old_path = os.path.join(GROUND_TRUTH_DIR, f"eff_{old_h[:16]}.json") + if os.path.isfile(old_path): + with open(old_path, encoding="utf-8") as f: + data = json.load(f) + return set(data.get("uris", [])) + return None + with open(path, encoding="utf-8") as f: + data = json.load(f) + return set(data.get("uris", [])) + + +def _save_ground_truth_cache(pattern: str, uri: str, uris: set[str]) -> None: + os.makedirs(GROUND_TRUTH_DIR, exist_ok=True) + path = _ground_truth_cache_path(pattern, uri) + with open(path, "w", encoding="utf-8") as f: + json.dump( + {"pattern": pattern, "uri": uri, "uris": sorted(uris)}, f, indent=2, ensure_ascii=False + ) + + +def compute_ground_truth(client: SyncHTTPClient, uri: str, pattern: str) -> tuple[set[str], float]: + """Compute ground truth via OV grep (fs engine). First run must have engine=fs.""" + cached = _load_ground_truth_cache(pattern, uri) + if cached is not None: + return cached, 0.0 + + truth_uris, elapsed = run_sdk_grep(client, uri, pattern) + _save_ground_truth_cache(pattern, uri, truth_uris) + return truth_uris, elapsed + + +def _miss_cache_path(pattern: str, uri: str) -> str: + h = _cache_hash(uri, pattern) + safe_pattern = _sanitize_filename(pattern) + return os.path.join(MISS_DIR, f"eff_{safe_pattern}_{h}.json") + + +def _save_miss(pattern: str, uri: str, missed_uris: set[str], extra_uris: set[str]) -> None: + """Save miss analysis (FN and FP) to .miss directory.""" + if not missed_uris and not extra_uris: + return + os.makedirs(MISS_DIR, exist_ok=True) + path = _miss_cache_path(pattern, uri) + data: dict = {"pattern": pattern, "uri": uri} + if missed_uris: + data["missed_fn"] = sorted(missed_uris) + data["missed_fn_count"] = len(missed_uris) + if extra_uris: + data["extra_fp"] = sorted(extra_uris) + data["extra_fp_count"] = len(extra_uris) + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def compute_metrics(truth: set[str], predicted: set[str]) -> dict: + if not truth and not predicted: + return {"recall": 1.0, "precision": 1.0, "f1": 1.0, "tp": 0, "fp": 0, "fn": 0} + if not truth: + return {"recall": 0.0, "precision": 0.0, "f1": 0.0, "tp": 0, "fp": len(predicted), "fn": 0} + tp = len(truth & predicted) + fp = len(predicted - truth) + fn = len(truth - predicted) + recall = tp / len(truth) + precision = tp / len(predicted) if predicted else 0.0 + f1 = 2 * recall * precision / (recall + precision) if (recall + precision) > 0 else 0.0 + return {"recall": recall, "precision": precision, "f1": f1, "tp": tp, "fp": fp, "fn": fn} + + +def main(): + import argparse + + parser = argparse.ArgumentParser( + description="Step 2 (Effectiveness): Evaluate retrieval quality" + ) + parser.add_argument( + "--keywords", + nargs="+", + default=None, + help="Keywords to search (e.g. --keywords grep reindex SyncHTTPClient)", + ) + parser.add_argument( + "--regenerate-ground-truth", + action="store_true", + help="Regenerate ground truth cache (requires engine=fs in ov.conf)", + ) + args = parser.parse_args() + + uri = BASE_URI + keywords = args.keywords if args.keywords else KEYWORDS + if not keywords: + print("WARNING: KEYWORDS list is empty. Fill it with real terms before running.") + print(" Use --keywords kw1 kw2 ... or edit step2_quality.py.\n") + + test_patterns = build_test_patterns(keywords) + + print("=" * 110) + print("Effectiveness Evaluation: grep vs ground truth (fs engine)") + print("=" * 110) + print(f"URI: {uri}") + print(f"Patterns: {len(test_patterns)}") + print() + print("NOTE: First run requires ov.conf grep engine=fs to generate ground truth.") + print(" Subsequent runs can use any engine; cached ground truth is reused.") + print() + + client = SyncHTTPClient() + client.initialize() + + # Phase 1: Compute ground truth (needs fs engine on first run) + print("--- Phase 1: Ground truth (fs engine) ---") + ground_truth_map: dict[str, tuple[set[str], float]] = {} + for label, pattern in test_patterns: + if args.regenerate_ground_truth: + cache_path = _ground_truth_cache_path(pattern, uri) + if os.path.isfile(cache_path): + os.remove(cache_path) + truth_uris, gt_elapsed = compute_ground_truth(client, uri, pattern) + ground_truth_map[pattern] = (truth_uris, gt_elapsed) + cached_str = "(cached)" if gt_elapsed == 0.0 else f"({gt_elapsed:.2f}s)" + print(f" {label}: {len(truth_uris)} matches {cached_str}") + + print() + print("--- Phase 2: Evaluate with current engine ---") + + results = [] + try: + for label, pattern in test_patterns: + truth_uris, gt_elapsed = ground_truth_map[pattern] + try: + auto_uris, auto_elapsed = run_sdk_grep(client, uri, pattern) + except Exception as e: + print(f" {label} FAILED: {e}") + results.append( + { + "label": label, + "pattern": pattern, + "error": str(e), + "truth_count": len(truth_uris), + } + ) + continue + + metrics = compute_metrics(truth_uris, auto_uris) + + # Save miss analysis + missed_uris = truth_uris - auto_uris # FN + extra_uris = auto_uris - truth_uris # FP + _save_miss(pattern, uri, missed_uris, extra_uris) + + miss_str = f" missed={len(missed_uris)}" if missed_uris else "" + extra_str = f" extra={len(extra_uris)}" if extra_uris else "" + print( + f" {label}: truth={len(truth_uris)} found={len(auto_uris)} " + f"Recall={metrics['recall']:.4f} Prec={metrics['precision']:.4f} F1={metrics['f1']:.4f}" + f"{miss_str}{extra_str}" + ) + + results.append( + { + "label": label, + "pattern": pattern, + "truth_count": len(truth_uris), + "found_count": len(auto_uris), + "gt_elapsed_s": round(gt_elapsed, 3), + "sdk_elapsed_s": round(auto_elapsed, 3), + **metrics, + } + ) + finally: + client.close() + + # Summary table + print() + print("=" * 120) + print( + f"{'Label':<45} {'Truth':>6} {'Found':>6} {'Recall':>8} {'Prec':>8} {'F1':>8} {'Missed':>8} {'Extra':>8}" + ) + print("-" * 120) + for r in results: + if "error" in r: + print( + f"{r['label']:<45} {r['truth_count']:>6} {'ERR':>6} " + f"{'---':>8} {'---':>8} {'---':>8} {'---':>8} {'---':>8}" + ) + else: + print( + f"{r['label']:<45} {r['truth_count']:>6} {r['found_count']:>6} " + f"{r['recall']:>8.4f} {r['precision']:>8.4f} {r['f1']:>8.4f} {r['fn']:>8} {r['fp']:>8}" + ) + print() + + # Save results to local file + os.makedirs(RESULT_DIR, exist_ok=True) + output_file = os.path.join(RESULT_DIR, "step2_result.json") + with open(output_file, "w", encoding="utf-8") as f: + json.dump( + {"uri": uri, "patterns": len(test_patterns), "results": results}, + f, + indent=2, + ensure_ascii=False, + ) + print(f"Results saved to: {output_file}") + print(f"Miss analysis saved to: {MISS_DIR}/") + print(f"Ground truth cache: {GROUND_TRUTH_DIR}/") + + +if __name__ == "__main__": + main() diff --git a/benchmark/retrieval/grep/vikingdb_bm25/performance/step0_prepare_data.py b/benchmark/retrieval/grep/vikingdb_bm25/performance/step0_prepare_data.py new file mode 100644 index 0000000000..36b72a548d --- /dev/null +++ b/benchmark/retrieval/grep/vikingdb_bm25/performance/step0_prepare_data.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Step 0 (Performance): Prepare synthetic benchmark data for grep testing. + +Reads a source text file (ai_wiki.txt), replicates it across configurable +directories and files, and injects target words at specified probabilities +for retrieval testing. + +Directory layout: + /dir_000/wiki_000.txt ... dir_000/wiki_999.txt + /dir_001/wiki_000.txt ... dir_001/wiki_999.txt + ... + +Each dir_xxx contains 1000 files. Default: 200 directories (200,000 files). + +Target words are injected by replacing a random word in the text. +All target words must NOT exist in the original source text. + +Default target words and probabilities (15 words, 5 tiers): + 1% : heliofract, prismcache, fluxkernel + 0.1% : auroracode, kiteshade, glyphvector + 0.1% : cortexmint, latticewave, spiralsync + 0.05% : ripplehash, embertrace, novaframe + 0.01% : zephyrloom, quartzrelay, nebulaindex + +Usage: + python3 step0_prepare_data.py + python3 step0_prepare_data.py --num-dirs 50 --seed 42 + python3 step0_prepare_data.py --start-dir 100 --num-dirs 100 # append dir_100..dir_199 +""" + +from __future__ import annotations + +import argparse +import os +import random +import re +import time + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_SOURCE = os.path.join(SCRIPT_DIR, "..", "ai_wiki.txt") +DEFAULT_OUTPUT = os.path.expanduser("~/.openviking/data/benchmark/synthetic") +FILES_PER_DIR = 1000 + +TARGET_GROUPS: list[tuple[float, list[str]]] = [ + (0.01, ["heliofract", "prismcache", "fluxkernel"]), + (0.001, ["auroracode", "kiteshade", "glyphvector"]), + (0.001, ["cortexmint", "latticewave", "spiralsync"]), + (0.0005, ["ripplehash", "embertrace", "novaframe"]), + (0.0001, ["zephyrloom", "quartzrelay", "nebulaindex"]), +] + + +def verify_target_words(text: str) -> None: + """Verify that no target word appears in the source text.""" + text_lower = text.lower() + conflicts = [] + for prob, words in TARGET_GROUPS: + for word in words: + if word.lower() in text_lower: + conflicts.append((word, prob)) + if conflicts: + for word, prob in conflicts: + print(f" CONFLICT: target word '{word}' (p={prob}) found in source text!") + raise ValueError( + f"{len(conflicts)} target word(s) already exist in source text. " + "Choose different target words." + ) + + +def inject_word(text: str, target: str) -> str: + """Replace a random word in the text with the target word.""" + words = list(re.finditer(r"\b\w+\b", text)) + if not words: + return text + match = random.choice(words) + return text[: match.start()] + target + text[match.end() :] + + +def generate_dataset( + source_text: str, + output_dir: str, + num_dirs: int, + start_dir: int = 0, + seed: int = 42, +) -> dict: + rng = random.Random(seed + start_dir) + total_files = num_dirs * FILES_PER_DIR + injection_stats = {word: 0 for _prob, words in TARGET_GROUPS for word in words} + + end_dir = start_dir + num_dirs - 1 if num_dirs > 0 else start_dir + print(f" Generating {num_dirs} dirs x {FILES_PER_DIR} files = {total_files} files") + print(f" Directory range: dir_{start_dir:03d} .. dir_{end_dir:03d}") + print(f" Output: {output_dir}") + print() + + t0 = time.monotonic() + + for offset in range(num_dirs): + dir_idx = start_dir + offset + dir_name = f"dir_{dir_idx:03d}" + dir_path = os.path.join(output_dir, dir_name) + os.makedirs(dir_path, exist_ok=True) + + for file_idx in range(FILES_PER_DIR): + file_name = f"wiki_{file_idx:03d}.txt" + file_path = os.path.join(dir_path, file_name) + content = source_text + + for prob, words in TARGET_GROUPS: + for word in words: + if rng.random() < prob: + content = inject_word(content, word) + injection_stats[word] += 1 + + with open(file_path, "w") as f: + f.write(content) + + if (offset + 1) % 10 == 0 or offset == num_dirs - 1: + elapsed = time.monotonic() - t0 + print(f" [{offset + 1}/{num_dirs}] dirs created ({elapsed:.1f}s)") + + elapsed = time.monotonic() - t0 + return { + "total_files": total_files, + "start_dir": start_dir, + "num_dirs": num_dirs, + "files_per_dir": FILES_PER_DIR, + "elapsed_s": round(elapsed, 1), + "injection_stats": injection_stats, + } + + +def main(): + parser = argparse.ArgumentParser( + description="Step 0 (Performance): Prepare synthetic benchmark data" + ) + parser.add_argument( + "--source", default=DEFAULT_SOURCE, help=f"Source text file (default: {DEFAULT_SOURCE})" + ) + parser.add_argument( + "--output", default=DEFAULT_OUTPUT, help=f"Output directory (default: {DEFAULT_OUTPUT})" + ) + parser.add_argument( + "--num-dirs", type=int, default=200, help="Number of directories (default: 200)" + ) + parser.add_argument( + "--start-dir", + type=int, + default=0, + help="Starting directory index for append/scale-out runs (default: 0)", + ) + parser.add_argument("--seed", type=int, default=42, help="Random seed (default: 42)") + args = parser.parse_args() + + source = os.path.normpath(os.path.expanduser(args.source)) + output = os.path.expanduser(args.output) + + print("=" * 80) + print("Step 0 (Performance): Prepare Synthetic Benchmark Data") + print("=" * 80) + print(f" Source: {source}") + print(f" Output: {output}") + print(f" StartDir: {args.start_dir}") + print(f" Dirs: {args.num_dirs}") + print(f" Seed: {args.seed}") + print() + + if not os.path.isfile(source): + print(f"ERROR: Source file not found: {source}") + return + + with open(source) as f: + source_text = f.read() + + print(f" Source text size: {len(source_text):,} chars") + print() + print(" Verifying target words against source text...") + try: + verify_target_words(source_text) + except ValueError as e: + print(f"ERROR: {e}") + return + print(" All target words verified OK.") + print() + print(" Target words and injection probabilities:") + for prob, words in sorted(TARGET_GROUPS, key=lambda item: item[0], reverse=True): + pct = f"{prob * 100:.3f}%" + print(f" {pct:>8s} : {', '.join(words)}") + print() + + summary = generate_dataset( + source_text, output, args.num_dirs, start_dir=args.start_dir, seed=args.seed + ) + + print() + print("=" * 80) + print("Summary:") + print(f" Total files: {summary['total_files']:,}") + print(f" Start dir: {summary['start_dir']}") + print(f" Directories: {summary['num_dirs']}") + print(f" Elapsed: {summary['elapsed_s']}s") + print() + print(" Target word injection counts:") + total_files = summary["total_files"] + for prob, words in sorted(TARGET_GROUPS, key=lambda item: item[0], reverse=True): + for word in words: + actual = summary["injection_stats"][word] + expected = total_files * prob + pct = actual / total_files * 100 if total_files > 0 else 0 + print( + f" {word:<18s} actual={actual:>6d} " + f"expected~{expected:>8.1f} " + f"rate={pct:.3f}% (target={prob * 100:.3f}%)" + ) + + print() + print(f"Data ready at: {output}") + print("Next: run step1_add_resource.py to import into OpenViking") + + +if __name__ == "__main__": + main() diff --git a/benchmark/retrieval/grep/vikingdb_bm25/performance/step1_add_resource.py b/benchmark/retrieval/grep/vikingdb_bm25/performance/step1_add_resource.py new file mode 100644 index 0000000000..3d5a56a08b --- /dev/null +++ b/benchmark/retrieval/grep/vikingdb_bm25/performance/step1_add_resource.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Step 1 (Performance): Import synthetic data into OpenViking WITHOUT indexing. + +Imports each directory recursively via SyncOpenViking.add_resource with +build_index=False and summarize=False, to skip slow VLM/embedding steps. +Progress is saved after each directory for resumability. + +After all imports are done, run step2_reindex.py to build vector indexes, +then step3_benchmark.py to measure performance. + +Usage: + python3 step1_add_resource.py + python3 step1_add_resource.py --source ~/.openviking/data/benchmark/synthetic +""" + +from __future__ import annotations + +import argparse +import os +import time + +from openviking.sync_client import SyncOpenViking + +DEFAULT_SOURCE = os.path.expanduser("~/.openviking/data/benchmark/synthetic") +PROGRESS_FILE = os.path.expanduser("~/.openviking/data/benchmark/.perf-import-progress") +BENCHMARK_PARENT = "viking://resources/benchmark/performance" + + +def load_progress() -> set[str]: + if not os.path.exists(PROGRESS_FILE): + return set() + with open(PROGRESS_FILE) as f: + return {line.strip() for line in f if line.strip()} + + +def save_progress(rel_dir: str) -> None: + os.makedirs(os.path.dirname(PROGRESS_FILE), exist_ok=True) + with open(PROGRESS_FILE, "a") as f: + f.write(rel_dir + "\n") + + +def scan_subdirs_recursive(root: str) -> list[str]: + """Return sorted list of all subdirectory relative paths (deterministic order).""" + result: list[str] = [] + + def _walk(dir_path: str, rel_prefix: str) -> None: + try: + entries = sorted(os.listdir(dir_path)) + except OSError: + return + for name in entries: + if name.startswith("."): + continue + full = os.path.join(dir_path, name) + if not os.path.isdir(full): + continue + rel = f"{rel_prefix}/{name}" if rel_prefix else name + result.append(rel) + _walk(full, rel) + + _walk(root, "") + return result + + +def main(): + parser = argparse.ArgumentParser( + description="Step 1 (Performance): Import synthetic data (no indexing)" + ) + parser.add_argument( + "--source", + default=DEFAULT_SOURCE, + help=f"Local directory to import (default: {DEFAULT_SOURCE})", + ) + parser.add_argument( + "--parent", + default=BENCHMARK_PARENT, + help=f"Parent Viking URI (default: {BENCHMARK_PARENT})", + ) + args = parser.parse_args() + + source = os.path.expanduser(args.source) + if not os.path.isdir(source): + print(f"ERROR: Source directory does not exist: {source}") + return + + print("=" * 80) + print("Step 1 (Performance): Import Synthetic Data (no VLM/embedding)") + print("=" * 80) + print(f" Source: {source}") + print(f" Parent: {args.parent}") + print(f" Progress: {PROGRESS_FILE}") + print(" Indexing: DISABLED (build_index=False, summarize=False)") + print() + + subdirs = scan_subdirs_recursive(source) + total = len(subdirs) + print(f" Total directories to import: {total}") + print() + + if total == 0: + print("No subdirectories found. Nothing to import.") + return + + completed = load_progress() + if completed: + already_done = [d for d in subdirs if d in completed] + print(f" Resuming: {len(already_done)} directories already imported") + print() + + client = SyncOpenViking() + client.initialize() + + results = [] + for i, rel_dir in enumerate(subdirs, 1): + if rel_dir in completed: + print(f" [{i}/{total}] SKIP (already done): {rel_dir}") + continue + + dir_path = os.path.join(source, rel_dir) + parent_rel = os.path.dirname(rel_dir) + parent_uri = f"{args.parent}/{parent_rel}" if parent_rel else args.parent + print(f" [{i}/{total}] Importing: {rel_dir} ...", end="", flush=True) + + t0 = time.monotonic() + try: + result = client.add_resource( + path=dir_path, + parent=parent_uri, + reason=f"benchmark perf: {rel_dir}", + wait=True, + create_parent=True, + build_index=False, + summarize=False, + ) + elapsed = time.monotonic() - t0 + root_uri = result.get("root_uri", "?") + print(f" OK ({elapsed:.1f}s) -> {root_uri}") + save_progress(rel_dir) + results.append({"dir": rel_dir, "status": "ok", "elapsed_s": round(elapsed, 1)}) + except Exception as e: + elapsed = time.monotonic() - t0 + print(f" FAILED ({elapsed:.1f}s): {e}") + results.append( + { + "dir": rel_dir, + "status": "failed", + "elapsed_s": round(elapsed, 1), + "error": str(e)[:500], + } + ) + + client.close() + + print() + print("Summary:") + ok_count = sum(1 for r in results if r["status"] == "ok") + failed_count = sum(1 for r in results if r["status"] == "failed") + skipped_count = sum(1 for d in subdirs if d in completed) + total_done = skipped_count + ok_count + + for r in results: + status = r["status"] + line = f" {status.upper():>7s} {r['dir']} ({r['elapsed_s']}s)" + if status == "failed": + line += f" -- {r.get('error', '')}" + print(line) + + print() + if total_done >= total and failed_count == 0: + print(f"All {total} directories imported successfully (no indexing).") + print("Next step: run step2_reindex.py to build vector indexes") + else: + print( + f" Imported: {ok_count} Failed: {failed_count} " + f"Skipped: {skipped_count} Remaining: {total - total_done}" + ) + if failed_count > 0: + print("Re-run this script to resume from where it left off.") + + +if __name__ == "__main__": + main() diff --git a/benchmark/retrieval/grep/vikingdb_bm25/performance/step2_reindex.py b/benchmark/retrieval/grep/vikingdb_bm25/performance/step2_reindex.py new file mode 100644 index 0000000000..41df30d3be --- /dev/null +++ b/benchmark/retrieval/grep/vikingdb_bm25/performance/step2_reindex.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Step 2 (Performance): Build vector indexes for imported data. + +Submits async reindex tasks for each first-level subdirectory via +SyncHTTPClient.reindex(wait=False), with a concurrency limit of 2 +running tasks. When a task completes, the next one is submitted. +This avoids tree-lock conflicts and prevents resource exhaustion. + +Prerequisites: + 1. Run step1_add_resource.py to import data (without indexing) + 2. Start openviking-server manually + +Usage: + python3 step2_reindex.py +""" + +from __future__ import annotations + +import argparse +import os +import time + +from openviking_cli.client.sync_http import SyncHTTPClient + +DEFAULT_SOURCE = os.path.expanduser("~/.openviking/data/benchmark/synthetic") +PROGRESS_FILE = os.path.expanduser("~/.openviking/data/benchmark/.perf-reindex-progress") +BENCHMARK_PARENT = "viking://resources/benchmark/performance" + +POLL_INTERVAL = 5 # seconds between task status checks +MAX_CONCURRENT = 16 # max running tasks at a time + + +def load_progress() -> set[str]: + if not os.path.exists(PROGRESS_FILE): + return set() + with open(PROGRESS_FILE) as f: + return {line.strip() for line in f if line.strip()} + + +def save_progress(rel_dir: str) -> None: + os.makedirs(os.path.dirname(PROGRESS_FILE), exist_ok=True) + with open(PROGRESS_FILE, "a") as f: + f.write(rel_dir + "\n") + + +def scan_first_level_dirs(root: str) -> list[str]: + """Return sorted list of first-level subdirectory names.""" + try: + entries = sorted(os.listdir(root)) + except OSError: + return [] + return [e for e in entries if not e.startswith(".") and os.path.isdir(os.path.join(root, e))] + + +def main(): + parser = argparse.ArgumentParser( + description="Step 2 (Performance): Build vector indexes via openviking-server" + ) + parser.add_argument( + "--source", + default=DEFAULT_SOURCE, + help=f"Local source directory (must match step1, default: {DEFAULT_SOURCE})", + ) + parser.add_argument( + "--parent", + default=BENCHMARK_PARENT, + help=f"Parent Viking URI (default: {BENCHMARK_PARENT})", + ) + parser.add_argument( + "--concurrency", + type=int, + default=MAX_CONCURRENT, + help=f"Max concurrent reindex tasks (default: {MAX_CONCURRENT})", + ) + args = parser.parse_args() + + source = os.path.expanduser(args.source) + max_concurrent = max(1, args.concurrency) + + print("=" * 80) + print("Step 2 (Performance): Build Vector Indexes (via openviking-server)") + print("=" * 80) + print(f" Source: {source}") + print(f" Parent: {args.parent}") + print(f" Progress: {PROGRESS_FILE}") + print(" Mode: vectors_only (wait=False, async)") + print(f" Concurrency: {max_concurrent}") + print() + print(" Prerequisite: openviking-server must be running!") + print() + + # Scan first-level dirs only + first_level = scan_first_level_dirs(source) + total = len(first_level) + print(f" First-level directories to reindex: {total}") + print() + + if total == 0: + print("No subdirectories found. Run step1_add_resource.py first.") + return + + completed = load_progress() + if completed: + already_done = [d for d in first_level if d in completed] + print(f" Resuming: {len(already_done)} directories already reindexed") + print() + + client = SyncHTTPClient() + client.initialize() + + # Build work queue (skip already completed) + work_queue: list[str] = [name for name in first_level if name not in completed] + skipped_count = len(first_level) - len(work_queue) + + # running: task_id -> (name, submit_time) + running: dict[str, tuple[str, float]] = {} + results: list[dict] = [] + + def _submit_next() -> bool: + """Submit the next item from work_queue if slot available. Returns True if submitted.""" + if not work_queue or len(running) >= max_concurrent: + return False + name = work_queue.pop(0) + dir_uri = f"{args.parent}/{name}" + idx = total - len(work_queue) + print(f" [{idx}/{total}] Submitting: {name} ...", end="", flush=True) + try: + result = client.reindex(uri=dir_uri, mode="vectors_only", wait=False) + task_id = result.get("task_id", "") + if task_id: + print(f" task_id={task_id[:8]}...") + running[task_id] = (name, time.monotonic()) + else: + print(" completed synchronously") + save_progress(name) + results.append({"dir": name, "status": "ok", "elapsed_s": 0.0}) + return True + except Exception as e: + print(f" FAILED: {e}") + results.append({"dir": name, "status": "failed", "error": str(e)[:500]}) + return True + + # Fill initial slots + while len(running) < max_concurrent and work_queue: + _submit_next() + + if not running and not results: + client.close() + _print_summary(results, skipped_count, first_level) + return + + # Poll loop: check running tasks, submit new ones as slots free up + print() + print(f" Running {len(running)} tasks, {len(work_queue)} queued") + print() + + while running: + done_ids = [] + for task_id, (name, submit_time) in list(running.items()): + try: + task_info = client.get_task(task_id) + except Exception: + continue + if task_info is None: + continue + status = task_info.get("status", "") + if status in ("completed", "failed"): + elapsed = time.monotonic() - submit_time + if status == "completed": + print(f" DONE {name} ({elapsed:.1f}s)") + save_progress(name) + results.append({"dir": name, "status": "ok", "elapsed_s": round(elapsed, 1)}) + else: + error = task_info.get("error", "unknown error") + print(f" FAIL {name} ({elapsed:.1f}s): {error}") + results.append( + { + "dir": name, + "status": "failed", + "elapsed_s": round(elapsed, 1), + "error": error, + } + ) + done_ids.append(task_id) + + for tid in done_ids: + del running[tid] + + # Fill freed slots + while len(running) < max_concurrent and work_queue: + _submit_next() + + if running: + time.sleep(POLL_INTERVAL) + + client.close() + _print_summary(results, skipped_count, first_level) + + +def _print_summary(results: list[dict], skipped_count: int, all_dirs: list[str]) -> None: + print() + print("Summary:") + ok_count = sum(1 for r in results if r.get("status") == "ok") + failed_count = sum(1 for r in results if r.get("status") == "failed") + total_done = skipped_count + ok_count + + for r in results: + status = r.get("status", "unknown") + line = f" {status.upper():>7s} {r.get('dir', '?')}" + if "elapsed_s" in r: + line += f" ({r['elapsed_s']}s)" + if status == "failed": + line += f" -- {r.get('error', '')}" + print(line) + + print() + total = len(all_dirs) + if total_done >= total and failed_count == 0: + print(f"All {total} directories reindexed successfully.") + print("Next step: run step3_benchmark.py to measure performance") + else: + print( + f" Reindexed: {ok_count} Failed: {failed_count} " + f"Skipped: {skipped_count} Remaining: {total - total_done}" + ) + if failed_count > 0: + print("Re-run this script to resume from where it left off.") + + +if __name__ == "__main__": + main() diff --git a/benchmark/retrieval/grep/vikingdb_bm25/performance/step3_benchmark.py b/benchmark/retrieval/grep/vikingdb_bm25/performance/step3_benchmark.py new file mode 100644 index 0000000000..c756f325ca --- /dev/null +++ b/benchmark/retrieval/grep/vikingdb_bm25/performance/step3_benchmark.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +"""Step 3 (Performance): Benchmark grep latency and match count. + +Runs grep queries against the synthetic dataset, measuring latency and +returned match count with a fixed node_limit. + +Run twice with different ov.conf engine settings to compare: + 1. Set ov.conf: "grep": {"engine": "fs"}, restart, then: + python3 step3_benchmark.py --engine-label fs + 2. Set ov.conf: "grep": {"engine": "auto", "switch_to_remote_threshold": 0}, restart, then: + python3 step3_benchmark.py --engine-label auto --compare step3_result_fs.json + +Results are saved to step3_result_{engine_label}.json. +""" + +from __future__ import annotations + +import argparse +import json +import os +import time + +from openviking_cli.client.sync_http import SyncHTTPClient + +BASE_URI = "viking://resources/benchmark/performance" +SYNTHETIC_DIR = os.path.expanduser("~/.openviking/data/benchmark/synthetic") + +# Same target words as step0_prepare_data.py +TARGET_GROUPS: list[tuple[float, list[str]]] = [ + (0.01, ["heliofract", "prismcache", "fluxkernel"]), + (0.001, ["auroracode", "kiteshade", "glyphvector"]), + (0.001, ["cortexmint", "latticewave", "spiralsync"]), + (0.0005, ["ripplehash", "embertrace", "novaframe"]), + (0.0001, ["zephyrloom", "quartzrelay", "nebulaindex"]), +] + +RUNS = 3 +WARMUP = 1 +GREP_NODE_LIMIT = 256 + + +def _format_probability(probability: float) -> str: + return f"{probability * 100:.3f}%" + + +def count_local_files() -> int: + """Count total .txt files in the synthetic dataset.""" + count = 0 + if not os.path.isdir(SYNTHETIC_DIR): + return 0 + for _root, _dirs, files in os.walk(SYNTHETIC_DIR): + for f in files: + if f.endswith(".txt"): + count += 1 + return count + + +def run_grep(client: SyncHTTPClient, pattern: str, uri: str) -> tuple[float, int, set[str]]: + start = time.monotonic() + result = client.grep(uri=uri, pattern=pattern, node_limit=GREP_NODE_LIMIT) + elapsed = time.monotonic() - start + match_uris: set[str] = set() + if isinstance(result, dict): + for match in result.get("matches", []): + uri_val = match.get("uri", "") + if uri_val: + match_uris.add(uri_val.rstrip("/")) + return elapsed, len(match_uris), match_uris + + +def benchmark_engine(client: SyncHTTPClient, total_files: int) -> list[dict]: + results = [] + + for prob, words in sorted(TARGET_GROUPS, key=lambda item: item[0], reverse=True): + for word in words: + expected = int(total_files * prob) + label = f"{word} (p={_format_probability(prob)}, expect~{expected})" + + print(f" {label} ...", end=" ", flush=True) + + # Warmup + for _ in range(WARMUP): + try: + run_grep(client, word, BASE_URI) + except Exception: + pass + + # Benchmark runs + times = [] + match_count = 0 + failed = False + for _ in range(RUNS): + try: + elapsed, matches, _ = run_grep(client, word, BASE_URI) + times.append(elapsed) + match_count = matches + except Exception as e: + failed = True + print(f"FAILED ({e})") + break + + if failed: + results.append({"label": label, "word": word, "probability": prob, "error": True}) + else: + avg_ms = sum(times) / len(times) * 1000 + min_ms = min(times) * 1000 + max_ms = max(times) * 1000 + print(f"avg={avg_ms:.1f}ms matches={match_count} expected~{expected}") + results.append( + { + "label": label, + "word": word, + "probability": prob, + "avg_ms": round(avg_ms, 1), + "min_ms": round(min_ms, 1), + "max_ms": round(max_ms, 1), + "matches": match_count, + "expected_approx": expected, + } + ) + + # No-match test + label = "no-match: zzz_nonexistent_perf" + print(f" {label} ...", end=" ", flush=True) + for _ in range(WARMUP): + try: + run_grep(client, "zzz_nonexistent_perf", BASE_URI) + except Exception: + pass + times = [] + match_count = 0 + failed = False + for _ in range(RUNS): + try: + elapsed, matches, _ = run_grep(client, "zzz_nonexistent_perf", BASE_URI) + times.append(elapsed) + match_count = matches + except Exception as e: + failed = True + print(f"FAILED ({e})") + break + if failed: + results.append({"label": label, "word": "zzz_nonexistent_perf", "error": True}) + else: + avg_ms = sum(times) / len(times) * 1000 + min_ms = min(times) * 1000 + max_ms = max(times) * 1000 + print(f"avg={avg_ms:.1f}ms matches={match_count}") + results.append( + { + "label": label, + "word": "zzz_nonexistent_perf", + "avg_ms": round(avg_ms, 1), + "min_ms": round(min_ms, 1), + "max_ms": round(max_ms, 1), + "matches": match_count, + } + ) + + return results + + +def print_comparison( + current_label: str, current: list[dict], compare_label: str, compare: list[dict] +): + compare_by_word = {} + for r in compare: + if "error" not in r and "word" in r: + compare_by_word[r["word"]] = r + + print() + print("=" * 120) + print(f" Comparison: {compare_label} vs {current_label}") + print("=" * 120) + print( + f"{'Word':<20} {'Prob':>8} {compare_label + '(ms)':>14} {current_label + '(ms)':>14} {'speedup':>10} {'Cmp matches':>12} {'Cur matches':>12}" + ) + print("-" * 120) + + for r in current: + if "error" in r: + print(f"{r.get('word', '?'):<20} {'ERR':>8} {'ERR':>14} {'ERR':>14} {'---':>10}") + continue + word = r["word"] + cur_ms = r["avg_ms"] + cmp = compare_by_word.get(word) + if not cmp: + print( + f"{word:<20} {_format_probability(r.get('probability', 0)):>8} {'N/A':>14} {cur_ms:>14.1f} {'---':>10}" + ) + continue + cmp_ms = cmp["avg_ms"] + speedup = cmp_ms / cur_ms if cur_ms > 0 else float("inf") + speedup_str = f"{speedup:.1f}x" + print( + f"{word:<20} {_format_probability(r.get('probability', 0)):>8} " + f"{cmp_ms:>14.1f} {cur_ms:>14.1f} {speedup_str:>10} " + f"{cmp.get('matches', '?'):>12} {r.get('matches', '?'):>12}" + ) + print() + + +def main(): + parser = argparse.ArgumentParser(description="Step 3 (Performance): Benchmark grep") + parser.add_argument( + "--engine-label", + required=True, + help="Label for this engine config (e.g. fs, auto). Used in output filename.", + ) + parser.add_argument( + "--compare", + default=None, + help="Path to a previous step3_result_*.json for comparison", + ) + args = parser.parse_args() + + total_files = count_local_files() + + client = SyncHTTPClient(timeout=3600) + client.initialize() + + print("=" * 80) + print(f"Step 3 (Performance): Grep Benchmark — engine={args.engine_label}") + print("=" * 80) + print(f" URI: {BASE_URI}") + print(f" Total files: {total_files:,}") + print(f" Grep limit: {GREP_NODE_LIMIT}") + print(f" Runs per test: {RUNS} (warmup: {WARMUP})") + print() + print("Ensure ov.conf has the desired grep config and the server is restarted.") + print() + + try: + results = benchmark_engine(client, total_files) + finally: + client.close() + + output_file = f"step3_result_{args.engine_label}.json" + with open(output_file, "w", encoding="utf-8") as f: + json.dump( + { + "engine_label": args.engine_label, + "total_files": total_files, + "grep_node_limit": GREP_NODE_LIMIT, + "results": results, + }, + f, + indent=2, + ensure_ascii=False, + ) + print(f"\nResults saved to {output_file}") + + print() + print( + f"{'Word':<20} {'Prob':>8} {'Avg(ms)':>10} {'Min(ms)':>10} {'Max(ms)':>10} {'Matches':>10} {'Expect~':>10}" + ) + print("-" * 96) + for r in results: + if "error" in r: + print(f"{r.get('word', '?'):<20} {'FAILED':>10}") + else: + print( + f"{r['word']:<20} {_format_probability(r.get('probability', 0)):>8} " + f"{r['avg_ms']:>10.1f} {r['min_ms']:>10.1f} " + f"{r['max_ms']:>10.1f} {r['matches']:>10} " + f"{r.get('expected_approx', '?'):>10}" + ) + print() + + if args.compare: + if not os.path.isfile(args.compare): + print(f"Warning: compare file not found: {args.compare}") + else: + with open(args.compare) as f: + prev = json.load(f) + prev_label = prev.get("engine_label", "previous") + prev_results = prev.get("results", []) + print_comparison(args.engine_label, results, prev_label, prev_results) + + +if __name__ == "__main__": + main() diff --git a/bot/tests/test_openviking_api_key_type.py b/bot/tests/test_openviking_api_key_type.py index 26c63b7292..0d303653cd 100644 --- a/bot/tests/test_openviking_api_key_type.py +++ b/bot/tests/test_openviking_api_key_type.py @@ -423,9 +423,7 @@ def test_validate_openviking_auth_exits_for_auth_mode_mismatch(monkeypatch, caps assert "user-key" not in captured.err -def test_validate_openviking_auth_warns_for_api_key_mode_without_user_key( - monkeypatch, capsys -): +def test_validate_openviking_auth_warns_for_api_key_mode_without_user_key(monkeypatch, capsys): config = SimpleNamespace( ov_server=SimpleNamespace( mode="remote", @@ -516,9 +514,7 @@ def _fake_probe(_server_url, _path, *, headers=None): assert captured.err == "" -def test_validate_openviking_auth_uses_effective_auth_mode_not_legacy_mode( - monkeypatch, capsys -): +def test_validate_openviking_auth_uses_effective_auth_mode_not_legacy_mode(monkeypatch, capsys): config = SimpleNamespace( ov_server=SimpleNamespace( effective_auth_mode="dev", @@ -1737,9 +1733,7 @@ def test_tool_context_syncs_legacy_memory_user_alias(): @pytest.mark.asyncio -async def test_viking_memory_context_keeps_legacy_users_separate_from_peers( - monkeypatch, tmp_path -): +async def test_viking_memory_context_keeps_legacy_users_separate_from_peers(monkeypatch, tmp_path): calls = [] class _FakeClient: @@ -1923,8 +1917,7 @@ async def find(self, *, query, target_uri, context_type=None, limit): "memories": [ { "uri": ( - f"viking://user/peers/{self.actor_peer_id}/" - "memories/events/e1.md" + f"viking://user/peers/{self.actor_peer_id}/memories/events/e1.md" ), "score": 0.9, } @@ -1974,9 +1967,7 @@ async def _fake_create(**kwargs): @pytest.mark.asyncio -async def test_viking_memory_type_quota_groups_with_event_summaries_and_uris( - monkeypatch, tmp_path -): +async def test_viking_memory_type_quota_groups_with_event_summaries_and_uris(monkeypatch, tmp_path): clients = [] base_uri = "viking://user/default/peers/sender-1/memories" @@ -1988,8 +1979,7 @@ def __init__(self): f"{base_uri}/events/e2.md": ( "Summary: long event summary\n" "2023-01-01 (Sunday) ChatLog:\n" - "full long event details " - + ("x" * 800) + "full long event details " + ("x" * 800) ), f"{base_uri}/events/e3.md": "legacy event without summary", f"{base_uri}/entities/en1.md": "short entity", @@ -2092,9 +2082,7 @@ async def _fake_create(**_kwargs): @pytest.mark.asyncio -async def test_viking_memory_type_quota_continues_after_oversized_entity( - monkeypatch, tmp_path -): +async def test_viking_memory_type_quota_continues_after_oversized_entity(monkeypatch, tmp_path): base_uri = "viking://user/default/peers/sender-1/memories" class _FakeClient: @@ -2136,17 +2124,15 @@ async def _fake_create(**_kwargs): ) assert '' in result - assert 'viking://user/default/peers/sender-1/memories/entities/long.md' in result + assert "viking://user/default/peers/sender-1/memories/entities/long.md" in result assert '' in result - assert 'viking://user/default/peers/sender-1/memories/entities/short.md' in result + assert "viking://user/default/peers/sender-1/memories/entities/short.md" in result assert "short fact" in result assert "long fact " not in result @pytest.mark.asyncio -async def test_viking_memory_type_quota_does_not_overflow_preference_budget( - monkeypatch, tmp_path -): +async def test_viking_memory_type_quota_does_not_overflow_preference_budget(monkeypatch, tmp_path): base_uri = "viking://user/default/peers/sender-1/memories" class _FakeClient: @@ -2194,9 +2180,7 @@ async def _fake_create(**_kwargs): @pytest.mark.asyncio -async def test_viking_memory_context_returns_empty_after_profile_filter( - monkeypatch, tmp_path -): +async def test_viking_memory_context_returns_empty_after_profile_filter(monkeypatch, tmp_path): class _FakeClient: async def search_memory(self, **_kwargs): return [ @@ -2305,9 +2289,7 @@ def should_sender_fanout(self): async def search(self, query, target_uri=None, limit=20, user_id=None): calls.append((target_uri, user_id)) return { - "memories": [ - {"uri": "viking://user/peers/sender-0/memories/a.md", "score": 0.9} - ] + "memories": [{"uri": "viking://user/peers/sender-0/memories/a.md", "score": 0.9}] } async def close(self): @@ -2356,7 +2338,9 @@ def _memory_target_uri(self, _user_id=None): def build_current_memory_target_uris(self, *, peer_ids=None, include_self=True): uris = ["viking://user/memories/"] if include_self else [] - uris.extend(f"viking://user/default/peers/{peer_id}/memories/" for peer_id in peer_ids or []) + uris.extend( + f"viking://user/default/peers/{peer_id}/memories/" for peer_id in peer_ids or [] + ) return uris async def grep(self, uri, pattern, case_insensitive=False, user_id=None): @@ -2391,7 +2375,9 @@ def _memory_target_uri(self, _user_id=None): def build_current_memory_target_uris(self, *, peer_ids=None, include_self=True): uris = ["viking://user/memories/"] if include_self else [] - uris.extend(f"viking://user/default/peers/{peer_id}/memories/" for peer_id in peer_ids or []) + uris.extend( + f"viking://user/default/peers/{peer_id}/memories/" for peer_id in peer_ids or [] + ) return uris async def list_resources(self, path=None, recursive=False): @@ -2425,7 +2411,9 @@ def _memory_target_uri(self, _user_id=None): def build_current_memory_target_uris(self, *, peer_ids=None, include_self=True): uris = ["viking://user/memories/"] if include_self else [] - uris.extend(f"viking://user/default/peers/{peer_id}/memories/" for peer_id in peer_ids or []) + uris.extend( + f"viking://user/default/peers/{peer_id}/memories/" for peer_id in peer_ids or [] + ) return uris async def glob(self, pattern, uri="viking://"): @@ -2462,8 +2450,7 @@ def _memory_target_uri(self, _user_id=None): def build_current_memory_target_uris(self, *, peer_ids=None, include_self=True): uris = ["viking://user/admin/memories/"] if include_self else [] uris.extend( - f"viking://user/admin/peers/{peer_id}/memories/" - for peer_id in peer_ids or [] + f"viking://user/admin/peers/{peer_id}/memories/" for peer_id in peer_ids or [] ) return uris diff --git a/bot/vikingbot/agent/memory.py b/bot/vikingbot/agent/memory.py index c0697c7757..79e0853122 100644 --- a/bot/vikingbot/agent/memory.py +++ b/bot/vikingbot/agent/memory.py @@ -17,9 +17,7 @@ _TYPE_QUOTA_EVENT_CHAR_RATIO = 0.75 _TYPE_QUOTA_PREFERENCE_FULL_LIMIT = 1 _MEMORY_TYPE_DESCRIPTIONS = { - "events": ( - "Event memories. The URI path includes the event date." - ), + "events": ("Event memories. The URI path includes the event date."), "entities": ( "Entity and topic memories. Use them for stable facts, attributes, " "relationships, and background about people, hobbies, places, or concepts." @@ -185,9 +183,7 @@ def _select_type_quota_memories( if quota <= 0: continue type_memories = [ - memory - for memory in memories - if cls._infer_memory_type(memory) == memory_type + memory for memory in memories if cls._infer_memory_type(memory) == memory_type ][:quota] selected.extend( cls._with_recall_metadata(memory, memory_type, rank) @@ -320,9 +316,7 @@ async def _parse_viking_memory( memory_type = self._infer_memory_type(memory) or "other" should_try_full = idx <= full_limit if use_type_budgets: - should_try_full = ( - memory_type in type_char_budgets - ) or ( + should_try_full = (memory_type in type_char_budgets) or ( memory_type == "preferences" and preference_full_count < max(0, preference_full_limit) ) @@ -621,9 +615,7 @@ async def read_memory_content(uri: str, level: str = "read") -> str: type_char_budgets=( self._type_quota_char_budgets(recall_max_chars) if use_type_quota else None ), - preference_full_limit=( - _TYPE_QUOTA_PREFERENCE_FULL_LIMIT if use_type_quota else 0 - ), + preference_full_limit=(_TYPE_QUOTA_PREFERENCE_FULL_LIMIT if use_type_quota else 0), include_uri_entries=True, read_content=read_memory_content, ) diff --git a/bot/vikingbot/agent/tools/ov_file.py b/bot/vikingbot/agent/tools/ov_file.py index ea7cbf7386..5d467c6bbd 100644 --- a/bot/vikingbot/agent/tools/ov_file.py +++ b/bot/vikingbot/agent/tools/ov_file.py @@ -182,7 +182,11 @@ def parameters(self) -> dict[str, Any]: } async def execute( - self, tool_context: "ToolContext", uri: str = "viking://", recursive: bool = False, **kwargs: Any + self, + tool_context: "ToolContext", + uri: str = "viking://", + recursive: bool = False, + **kwargs: Any, ) -> str: client = None try: @@ -430,9 +434,7 @@ async def execute( "viking://resources/", self._current_memory_uri(client), self._current_skill_uri(client), - *self._peer_memory_uris( - client, tool_context, peer_ids=peer_ids - ), + *self._peer_memory_uris(client, tool_context, peer_ids=peer_ids), ] ) else: @@ -808,7 +810,9 @@ async def execute( timestamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) session_id = f"{source_session_id}__memory_commit__{timestamp}__{commit_seq:04d}" result = await client.commit(session_id, messages, peer_id=tool_context.sender_id) - session_id = result.get("session_id", session_id) if isinstance(result, dict) else session_id + session_id = ( + result.get("session_id", session_id) if isinstance(result, dict) else session_id + ) commit_result = result.get("commit", {}) if isinstance(result, dict) else {} archive_uri = commit_result.get("archive_uri") memory_diff_uri = f"{archive_uri}/memory_diff.json" if archive_uri else None diff --git a/bot/vikingbot/channels/feishu.py b/bot/vikingbot/channels/feishu.py index fdafc85517..22ed24305e 100644 --- a/bot/vikingbot/channels/feishu.py +++ b/bot/vikingbot/channels/feishu.py @@ -778,7 +778,7 @@ async def _parse_message_content( except Exception as e: logger.warning(f"Failed to process {msg_type} message: {e}") - elif msg_type =="interactive": + elif msg_type == "interactive": content = message.content else: content = MSG_TYPE_MAP.get(msg_type, f"[{msg_type}]") diff --git a/bot/vikingbot/config/loader.py b/bot/vikingbot/config/loader.py index 20ef334bd9..7d394c63a8 100644 --- a/bot/vikingbot/config/loader.py +++ b/bot/vikingbot/config/loader.py @@ -9,12 +9,12 @@ import httpx from loguru import logger + from openviking.server.config import ( ServerConfig, get_server_url_from_server_data, ) from openviking_cli.utils.config.ovcli_config import load_ovcli_config - from vikingbot.config.schema import Config CONFIG_PATH = None @@ -186,11 +186,7 @@ def _merge_current_ov_server_config(bot_data: dict, server_data: dict) -> str: bot_data["api_key_type"] = api_key_type server_root_api_key = str(server_data.get("root_api_key") or "").strip() - if ( - api_key_type == "root" - and server_auth_mode == "trusted" - and server_root_api_key - ): + if api_key_type == "root" and server_auth_mode == "trusted" and server_root_api_key: bot_data["api_key"] = server_root_api_key effective_auth_mode = _bot_auth_mode_from_api_key_type(api_key_type, server_auth_mode) diff --git a/bot/vikingbot/openviking_mount/ov_server.py b/bot/vikingbot/openviking_mount/ov_server.py index 57ae73bc1f..77950cc5a7 100644 --- a/bot/vikingbot/openviking_mount/ov_server.py +++ b/bot/vikingbot/openviking_mount/ov_server.py @@ -413,11 +413,7 @@ def build_current_memory_target_uris( uris.append(self._memory_target_uri(None)) normalized_peer_ids = self._dedupe_strings( - [ - pid - for pid in (self._peer_id(peer_id) for peer_id in (peer_ids or [])) - if pid - ] + [pid for pid in (self._peer_id(peer_id) for peer_id in (peer_ids or [])) if pid] ) for peer_id in normalized_peer_ids: try: @@ -450,11 +446,7 @@ def build_memory_search_target_uris( [str(user_id).strip() for user_id in (user_ids or []) if str(user_id).strip()] ) normalized_peer_ids = self._dedupe_strings( - [ - pid - for pid in (self._peer_id(peer_id) for peer_id in (peer_ids or [])) - if pid - ] + [pid for pid in (self._peer_id(peer_id) for peer_id in (peer_ids or [])) if pid] ) effective_owner_user_id = self._effective_user_id(owner_user_id) if owner_user_id else None @@ -479,7 +471,9 @@ def build_memory_search_target_uris( try: target_uris.append(self._current_peer_memory_target_uri(peer_id)) except ValueError as exc: - logger.warning(f"Skip invalid current peer memory target peer_id={peer_id}: {exc}") + logger.warning( + f"Skip invalid current peer memory target peer_id={peer_id}: {exc}" + ) if not target_uris: target_uris.append(self._memory_target_uri(None)) @@ -506,10 +500,7 @@ def build_memory_search_targets( owner_user_id=owner_user_id, peer_ids=peer_ids, ) - return [ - (target_uri, self._owner_user_id_for_uri(target_uri)) - for target_uri in target_uris - ] + return [(target_uri, self._owner_user_id_for_uri(target_uri)) for target_uri in target_uris] def _skill_memory_uri(self, skill_name: str, user_id: Optional[str] = None) -> str: return f"{self._memory_target_uri(user_id)}skills/{skill_name}.md" @@ -706,11 +697,7 @@ def _extract_memories(result: Any) -> list[Any]: ] normalized_peer_ids = self._dedupe_strings( - [ - pid - for pid in (self._peer_id(peer_value) for peer_value in (peer_ids or [])) - if pid - ] + [pid for pid in (self._peer_id(peer_value) for peer_value in (peer_ids or [])) if pid] ) effective_owner_user_id = self._effective_user_id(owner_user_id) if owner_user_id else None @@ -1100,6 +1087,7 @@ async def account_test(): print(res) + if __name__ == "__main__": asyncio.run(main_test()) # asyncio.run(account_test()) diff --git a/bot/vikingbot/providers/vlm_adapter.py b/bot/vikingbot/providers/vlm_adapter.py index 5f91f31bd1..b3e8b585cb 100644 --- a/bot/vikingbot/providers/vlm_adapter.py +++ b/bot/vikingbot/providers/vlm_adapter.py @@ -242,9 +242,7 @@ def _parse_usage(cls, raw_usage: Any) -> dict[str, int]: ) cached = cls._usage_value(prompt_details, "cached_tokens") if prompt_details else 0 reasoning = ( - cls._usage_value(completion_details, "reasoning_tokens") - if completion_details - else 0 + cls._usage_value(completion_details, "reasoning_tokens") if completion_details else 0 ) if cached: usage["cache_read_input_tokens"] = cached diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md index 59a83e69de..d92edbf967 100644 --- a/docs/en/guides/01-configuration.md +++ b/docs/en/guides/01-configuration.md @@ -880,6 +880,24 @@ Retrieval ranking configuration for final search scores. Keep `hotness_alpha` at `0.0` when you need scores to reflect pure vector similarity. Set it above `0.0` only when frequently accessed or recently updated contexts should receive a ranking boost. +### grep + +Grep engine configuration for content pattern search. These settings are server-side only and cannot be overridden per-request. + +```json +{ + "grep": { + "engine": "auto", + "switch_to_remote_threshold": 10000 + } +} +``` + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `engine` | str | Search engine mode: `"auto"` uses VikingDB BM25 recall when available and falls back to local filesystem search; `"fs"` forces local filesystem search only. | `"auto"` | +| `switch_to_remote_threshold` | int | L2 record count threshold to switch to VikingDB BM25 recall. When the number of L2 files under the search scope exceeds this threshold, VikingDB BM25 is used for phase-1 recall; otherwise local filesystem search is used. Set to `0` to always use VikingDB BM25. Must be ≥ 0. | `10000` | + ### storage Storage configuration for context data, including file storage (RAGFS) and vector database storage (VectorDB). @@ -1289,7 +1307,7 @@ openviking-server --config /path/to/ov.conf ### ov.conf -The config sections documented above (embedding, vlm, rerank, storage) all belong to `ov.conf`. SDK embedded mode and server share this file. +The config sections documented above (embedding, vlm, rerank, retrieval, grep, storage) all belong to `ov.conf`. SDK embedded mode and server share this file. For memory-related settings, add a `memory` section in `ov.conf`: diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md index 93b32e9ed1..152f80f3b2 100644 --- a/docs/zh/guides/01-configuration.md +++ b/docs/zh/guides/01-configuration.md @@ -851,6 +851,24 @@ AST 提取支持：Python、JavaScript/TypeScript、Rust、Go、Java、C/C++。 如果需要分数严格反映向量相似度，保持 `hotness_alpha` 为 `0.0`。只有当希望高频访问或最近更新的上下文获得排序提升时，才将它设置为大于 `0.0`。 +### grep + +Grep 引擎配置，用于内容模式搜索。这些设置为服务端配置，不支持请求级别覆盖。 + +```json +{ + "grep": { + "engine": "auto", + "switch_to_remote_threshold": 10000 + } +} +``` + +| 参数 | 类型 | 说明 | 默认值 | +|------|------|------|--------| +| `engine` | str | 搜索引擎模式：`"auto"` 在可用时使用 VikingDB BM25 召回，不可用时回退到本地文件系统搜索；`"fs"` 强制仅使用本地文件系统搜索。 | `"auto"` | +| `switch_to_remote_threshold` | int | 切换到 VikingDB BM25 召回的 L2 记录数阈值。当搜索范围内的 L2 文件数超过此阈值时，使用 VikingDB BM25 进行第一阶段召回；否则使用本地文件系统搜索。设为 `0` 表示始终使用 VikingDB BM25。必须 ≥ 0。 | `10000` | + ### storage 用于存储上下文数据 ，包括文件存储（RAGFS）和向量库存储（VectorDB）。 diff --git a/examples/openclaw-plugin/tests/e2e/test-cjk-token-estimation.py b/examples/openclaw-plugin/tests/e2e/test-cjk-token-estimation.py index 57d9c663ae..3777722e9f 100644 --- a/examples/openclaw-plugin/tests/e2e/test-cjk-token-estimation.py +++ b/examples/openclaw-plugin/tests/e2e/test-cjk-token-estimation.py @@ -19,7 +19,6 @@ import requests - DEFAULT_GATEWAY_URL = os.environ.get("OPENCLAW_GATEWAY_URL", "http://127.0.0.1:19830") DEFAULT_OPENVIKING_URL = os.environ.get("OPENVIKING_BASE_URL", "http://127.0.0.1:2948") DEFAULT_CJK_REPEAT = 120 @@ -182,7 +181,9 @@ def find_ov_context_with_marker( if not session_id or session_id.startswith("memory-store-"): continue try: - ctx = ov_get(openviking_url, f"/api/v1/sessions/{session_id}/context?token_budget=128000") + ctx = ov_get( + openviking_url, f"/api/v1/sessions/{session_id}/context?token_budget=128000" + ) except requests.RequestException: continue if marker in flatten_message_text(ctx): diff --git a/examples/ov.conf.example b/examples/ov.conf.example index a86d31f3a9..58f806f365 100644 --- a/examples/ov.conf.example +++ b/examples/ov.conf.example @@ -150,6 +150,7 @@ "threshold": 0.1, }, "retrieval": {"hotness_alpha": 0.0, "score_propagation_alpha": 1.0}, + "grep": {"engine": "auto", "switch_to_remote_threshold": 1000}, "auto_generate_l0": true, "auto_generate_l1": true, "default_search_mode": "thinking", diff --git a/openviking/async_client.py b/openviking/async_client.py index 1e3f373b2c..2276174cc4 100644 --- a/openviking/async_client.py +++ b/openviking/async_client.py @@ -559,6 +559,7 @@ async def grep( case_insensitive: bool = False, node_limit: Optional[int] = None, exclude_uri: Optional[str] = None, + level_limit: int = 5, ) -> Dict: """Content search""" await self._ensure_initialized() @@ -568,6 +569,7 @@ async def grep( case_insensitive=case_insensitive, node_limit=node_limit, exclude_uri=exclude_uri, + level_limit=level_limit, ) async def glob(self, pattern: str, uri: str = "viking://") -> Dict: diff --git a/openviking/core/context.py b/openviking/core/context.py index 8fb92e246f..9ffb16b783 100644 --- a/openviking/core/context.py +++ b/openviking/core/context.py @@ -41,13 +41,20 @@ class ContextLevel(int, Enum): class Vectorize: text: str = "" + full_text: str = "" # Full content for BM25 (not embedding-truncated) # images: list of image references (data URIs or URLs) for multimodal embedding images: List[str] = [] # video: str = "" # audio: str = "" - def __init__(self, text: str = "", images: Optional[List[str]] = None): + def __init__( + self, + text: str = "", + full_text: str = "", + images: Optional[List[str]] = None, + ): self.text = text + self.full_text = full_text self.images = list(images) if images else [] diff --git a/openviking/models/embedder/volcengine_embedders.py b/openviking/models/embedder/volcengine_embedders.py index 6d609856af..df4c5b18b4 100644 --- a/openviking/models/embedder/volcengine_embedders.py +++ b/openviking/models/embedder/volcengine_embedders.py @@ -209,9 +209,7 @@ def _get_async_client(self): lambda: volcenginesdkarkruntime.AsyncArk(**self._ark_kwargs) ) - async def embed_async( - self, content: "EmbeddingInput", is_query: bool = False - ) -> EmbedResult: + async def embed_async(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult: client = self._get_async_client() async def _embed_call() -> EmbedResult: @@ -387,9 +385,7 @@ def _get_async_client(self): lambda: volcenginesdkarkruntime.AsyncArk(**self._ark_kwargs) ) - async def embed_async( - self, content: "EmbeddingInput", is_query: bool = False - ) -> EmbedResult: + async def embed_async(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult: client = self._get_async_client() async def _embed_call() -> EmbedResult: @@ -578,9 +574,7 @@ def _get_async_client(self): lambda: volcenginesdkarkruntime.AsyncArk(**self._ark_kwargs) ) - async def embed_async( - self, content: "EmbeddingInput", is_query: bool = False - ) -> EmbedResult: + async def embed_async(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult: client = self._get_async_client() async def _embed_call() -> EmbedResult: diff --git a/openviking/models/vlm/backends/codex_responses_adapter.py b/openviking/models/vlm/backends/codex_responses_adapter.py index 43e94ac19b..1b4febf902 100644 --- a/openviking/models/vlm/backends/codex_responses_adapter.py +++ b/openviking/models/vlm/backends/codex_responses_adapter.py @@ -210,7 +210,9 @@ def _build_final_response_from_stream_events( return SimpleNamespace( output=fallback_output, - usage=getattr(completed_response, "usage", None) if completed_response is not None else None, + usage=getattr(completed_response, "usage", None) + if completed_response is not None + else None, ) diff --git a/openviking/parse/image_rewrite.py b/openviking/parse/image_rewrite.py index cdc297e3ba..366f10b622 100644 --- a/openviking/parse/image_rewrite.py +++ b/openviking/parse/image_rewrite.py @@ -30,9 +30,7 @@ # HTML embeds, common in markdown for sizing control. Shared # with the parser so ingestion and rewriting see the same references. -HTML_IMG_PATTERN = re.compile( - r"""(]*?src=["'])([^"']+)(["'][^>]*>)""", re.IGNORECASE -) +HTML_IMG_PATTERN = re.compile(r"""(]*?src=["'])([^"']+)(["'][^>]*>)""", re.IGNORECASE) _FENCE_PATTERN = re.compile(r"^(\s{0,3})(`{3,}|~{3,})") _LIST_ITEM_PATTERN = re.compile(r"^(\s{0,3})([-*+]|\d{1,9}[.)])(\s+)") @@ -103,7 +101,12 @@ def _protected_ranges(content: str): if in_fence: ranges.append((start, end)) m = _FENCE_PATTERN.match(line_content) - if m and m.group(2)[0] == fence_char and len(m.group(2)) >= fence_len and stripped == m.group(2): + if ( + m + and m.group(2)[0] == fence_char + and len(m.group(2)) >= fence_len + and stripped == m.group(2) + ): in_fence = False prev_blank = is_blank continue @@ -255,8 +258,7 @@ async def rewrite_image_uris( try: entries = await viking_fs.ls(md_dir, ctx=ctx) available_images = { - e["name"] for e in entries - if not e.get("isDir") and not e["name"].startswith(".") + e["name"] for e in entries if not e.get("isDir") and not e["name"].startswith(".") } except Exception: logger.debug(f"[image_rewrite] Failed to list directory {md_dir}") @@ -267,25 +269,29 @@ async def rewrite_image_uris( logger.warning(f"[image_rewrite] Failed to read {md_uri}, skipping") continue - new_content, rewrite_count = _rewrite_content(content, md_dir, available_images, path_to_image_name) + new_content, rewrite_count = _rewrite_content( + content, md_dir, available_images, path_to_image_name + ) if rewrite_count > 0: try: await viking_fs.write_file(md_uri, new_content, ctx=ctx) files_processed += 1 references_rewritten += rewrite_count - logger.debug( - f"[image_rewrite] Rewrote {rewrite_count} image ref(s) in {md_uri}" - ) + logger.debug(f"[image_rewrite] Rewrote {rewrite_count} image ref(s) in {md_uri}") except Exception: logger.warning(f"[image_rewrite] Failed to write {md_uri}") # Clean up mapping sidecars — no longer needed after rewrite for map_dir in mappings_by_dir: try: - await viking_fs.rm(f"{map_dir}/{IMAGE_MAPPINGS_FILENAME}", ctx=ctx, lock_handle=lock_handle) + await viking_fs.rm( + f"{map_dir}/{IMAGE_MAPPINGS_FILENAME}", ctx=ctx, lock_handle=lock_handle + ) except Exception as e: - logger.warning(f"[image_rewrite] Failed to delete {map_dir}/{IMAGE_MAPPINGS_FILENAME}: {e}") + logger.warning( + f"[image_rewrite] Failed to delete {map_dir}/{IMAGE_MAPPINGS_FILENAME}: {e}" + ) logger.info( f"[image_rewrite] Processed {len(md_uris)} .md files, " diff --git a/openviking/parse/parsers/code/ast/code_tools.py b/openviking/parse/parsers/code/ast/code_tools.py index f43b5b20cd..963ec81afb 100644 --- a/openviking/parse/parsers/code/ast/code_tools.py +++ b/openviking/parse/parsers/code/ast/code_tools.py @@ -159,9 +159,7 @@ def search_symbols(query: str, files: List[Tuple[str, str]]) -> str: return "\n".join(out) -def _resolve_symbol( - skeleton: CodeSkeleton, symbol: str -) -> Optional[Tuple[str, int, int]]: +def _resolve_symbol(skeleton: CodeSkeleton, symbol: str) -> Optional[Tuple[str, int, int]]: """Find a symbol by 'foo' (bare) or 'Foo.bar' (qualified). Case sensitive. Search priority for bare names (no dot): diff --git a/openviking/parse/parsers/code/ast/extractor.py b/openviking/parse/parsers/code/ast/extractor.py index 127794ebee..323302bea0 100644 --- a/openviking/parse/parsers/code/ast/extractor.py +++ b/openviking/parse/parsers/code/ast/extractor.py @@ -116,9 +116,7 @@ def extract(self, file_name: str, content: str) -> Optional[CodeSkeleton]: try: return extractor.extract(file_name, content) except Exception as e: - logger.warning( - "AST extraction failed for '%s' (language: %s): %s", file_name, lang, e - ) + logger.warning("AST extraction failed for '%s' (language: %s): %s", file_name, lang, e) return None def extract_skeleton( diff --git a/openviking/parse/parsers/markdown.py b/openviking/parse/parsers/markdown.py index 71a43f55ed..02f929eae7 100644 --- a/openviking/parse/parsers/markdown.py +++ b/openviking/parse/parsers/markdown.py @@ -341,9 +341,7 @@ async def _compute_layout( content, frontmatter = self._extract_frontmatter(content) if frontmatter: meta["frontmatter"] = frontmatter - logger.debug( - f"[MarkdownParser] Extracted frontmatter: {list(frontmatter.keys())}" - ) + logger.debug(f"[MarkdownParser] Extracted frontmatter: {list(frontmatter.keys())}") explicit_name = kwargs.get("resource_name") if not explicit_name and kwargs.get("source_name"): @@ -601,7 +599,6 @@ async def _ingest_local_images( seen_paths = set() for path_str in image_refs: - # Skip remote URIs if self._is_remote_uri(path_str): continue @@ -637,7 +634,9 @@ async def _ingest_local_images( image_bytes = await asyncio.to_thread(resolved_path.read_bytes) # Validate pixel size and file size; skip non-compliant images - if not await asyncio.to_thread(self._is_valid_image, image_bytes, resolved_path): + if not await asyncio.to_thread( + self._is_valid_image, image_bytes, resolved_path + ): continue # Get filename and deduplicate @@ -657,7 +656,9 @@ async def _ingest_local_images( logger.warning(f"[MarkdownParser] Failed to ingest image {resolved_path}: {e}") if file_mappings: - rel_md_path = md_uri[len(root_prefix) + 1 :] if md_uri.startswith(root_prefix) else md_uri + rel_md_path = ( + md_uri[len(root_prefix) + 1 :] if md_uri.startswith(root_prefix) else md_uri + ) mappings[rel_md_path] = file_mappings # Write a single mapping file at the root directory for rewrite_image_uris @@ -696,9 +697,7 @@ def _resolve_image_path( # Reject absolute paths: they can point anywhere on the host if path.is_absolute(): - logger.warning( - f"[MarkdownParser] Rejected absolute image path: {path_str}" - ) + logger.warning(f"[MarkdownParser] Rejected absolute image path: {path_str}") return None # Build the list of allowed roots to confine resolution to. @@ -768,9 +767,7 @@ def _is_valid_image(self, image_bytes: bytes, source_path: Path) -> bool: """ # File size check (local file path limit: 10 MB) if len(image_bytes) > self.IMAGE_MAX_FILE_BYTES: - logger.warning( - f"[MarkdownParser] Image exceeds 10MB, skipping: {source_path}" - ) + logger.warning(f"[MarkdownParser] Image exceeds 10MB, skipping: {source_path}") return False # Pixel size check @@ -940,20 +937,15 @@ def _to_rel(new_disk: str, keep_suffix: bool) -> str: rel for rel, text in layout.items() if any( - _gh_slug(h) == anchor - for h in re.findall(r"^#{1,6}\s+(.+)$", text, re.M) + _gh_slug(h) == anchor for h in re.findall(r"^#{1,6}\s+(.+)$", text, re.M) ) ] if len(hits) == 1: - return _to_rel( - os.path.join(target_parent, hits[0]), keep_suffix=True - ) + return _to_rel(os.path.join(target_parent, hits[0]), keep_suffix=True) # B) Single-file document (a bare file, or a directory holding exactly one # file) → the anchor/query lives in that one file; keep the suffix. if len(layout) == 1: - return _to_rel( - os.path.join(target_parent, next(iter(layout))), keep_suffix=True - ) + return _to_rel(os.path.join(target_parent, next(iter(layout))), keep_suffix=True) # C) Single bare file with no suffix to place (e.g. a future small .md kept as # a file) → point at the file itself (empty suffix ⇒ no trailing slash). @@ -979,9 +971,7 @@ async def _ingest_will_handle_image(self, link: str) -> bool: if resolved is not None: try: image_bytes = await asyncio.to_thread(resolved.read_bytes) - handled = await asyncio.to_thread( - self._is_valid_image, image_bytes, resolved - ) + handled = await asyncio.to_thread(self._is_valid_image, image_bytes, resolved) except Exception: handled = False cache[link] = handled diff --git a/openviking/parse/parsers/media/naming.py b/openviking/parse/parsers/media/naming.py index cdc9978ba1..f94c9e2897 100644 --- a/openviking/parse/parsers/media/naming.py +++ b/openviking/parse/parsers/media/naming.py @@ -44,7 +44,9 @@ def resolve_media_names(file_path: Path, ext: str, **kwargs: Any) -> Tuple[str, # a filename-like value ("photo.png") does not double its extension, # while a name that merely contains a dot ("meeting.v1") is preserved. candidate = Path(explicit_name) - display_stem = candidate.stem if candidate.suffix.lower() in MEDIA_EXTENSIONS else explicit_name + display_stem = ( + candidate.stem if candidate.suffix.lower() in MEDIA_EXTENSIONS else explicit_name + ) path_stem = display_stem.replace(" ", "_") original_filename = f"{path_stem}{ext}" else: diff --git a/openviking/server/app.py b/openviking/server/app.py index c61bd59365..c978d19d08 100644 --- a/openviking/server/app.py +++ b/openviking/server/app.py @@ -104,8 +104,7 @@ async def _initialize_auth_plugin( plugin_cls = registry.get(effective_auth_mode) if plugin_cls is None: logger.error( - "Unknown auth_mode: %r. No auth plugin registered. " - "Registered modes: %s.", + "Unknown auth_mode: %r. No auth plugin registered. Registered modes: %s.", effective_auth_mode, ", ".join(registry.list_modes()), ) diff --git a/openviking/server/auth/plugin.py b/openviking/server/auth/plugin.py index ba77898f19..f483336f39 100644 --- a/openviking/server/auth/plugin.py +++ b/openviking/server/auth/plugin.py @@ -5,13 +5,11 @@ from __future__ import annotations import abc -import sys from typing import TYPE_CHECKING, ClassVar, Optional from fastapi import Request from openviking.server.identity import ResolvedIdentity -from openviking_cli.exceptions import UnauthenticatedError if TYPE_CHECKING: from openviking.server.config import ServerConfig diff --git a/openviking/server/auth/plugins/api_key.py b/openviking/server/auth/plugins/api_key.py index da2ceae3cb..3acc05c5db 100644 --- a/openviking/server/auth/plugins/api_key.py +++ b/openviking/server/auth/plugins/api_key.py @@ -121,24 +121,16 @@ async def _try_resolve_oauth_token( record = await provider.load_access_token(api_key) if record is None: - raise UnauthenticatedError( - "OAuth access token is invalid, expired, or revoked" - ) + raise UnauthenticatedError("OAuth access token is invalid, expired, or revoked") import hmac api_key_manager = getattr(request.app.state, "api_key_manager", None) recorded_fp = record.authorizing_key_fp current_fp: Optional[str] = None - if api_key_manager is not None and hasattr( - api_key_manager, "get_user_key_fingerprint" - ): - current_fp = api_key_manager.get_user_key_fingerprint( - record.account_id, record.user_id - ) - if not recorded_fp or not current_fp or not hmac.compare_digest( - recorded_fp, current_fp - ): + if api_key_manager is not None and hasattr(api_key_manager, "get_user_key_fingerprint"): + current_fp = api_key_manager.get_user_key_fingerprint(record.account_id, record.user_id) + if not recorded_fp or not current_fp or not hmac.compare_digest(recorded_fp, current_fp): raise UnauthenticatedError( "OAuth token's authorizing API key has been rotated or revoked; " "please re-authorize the client." @@ -149,9 +141,9 @@ async def _try_resolve_oauth_token( # Role downgrade protection if api_key_manager is not None and hasattr(api_key_manager, "get_user_role"): try: - current_role = Role(api_key_manager.get_user_role( - record.account_id, record.user_id - )) + current_role = Role( + api_key_manager.get_user_role(record.account_id, record.user_id) + ) except Exception: raise UnauthenticatedError( "OAuth token validation failed: unable to verify user's current role; " diff --git a/openviking/server/auth/plugins/dev.py b/openviking/server/auth/plugins/dev.py index 661353dd85..ca73d5f0c9 100644 --- a/openviking/server/auth/plugins/dev.py +++ b/openviking/server/auth/plugins/dev.py @@ -11,7 +11,6 @@ from openviking.server.auth.plugin import AuthPlugin from openviking.server.identity import ResolvedIdentity, Role -from openviking_cli.exceptions import InvalidArgumentError _LOCALHOST_HOSTS = {"127.0.0.1", "localhost", "::1"} diff --git a/openviking/server/auth/plugins/trusted.py b/openviking/server/auth/plugins/trusted.py index af50e5b006..b8da67c151 100644 --- a/openviking/server/auth/plugins/trusted.py +++ b/openviking/server/auth/plugins/trusted.py @@ -127,13 +127,9 @@ async def resolve_identity( if _trusted_request_requires_explicit_identity(request.url.path): missing_fields = [] if not effective_account_id: - missing_fields.append( - "X-OpenViking-Account or explicit account_id in the URL" - ) + missing_fields.append("X-OpenViking-Account or explicit account_id in the URL") if not effective_user_id: - missing_fields.append( - "X-OpenViking-User or explicit user_id in the URL" - ) + missing_fields.append("X-OpenViking-User or explicit user_id in the URL") if missing_fields: raise InvalidArgumentError( "Trusted mode requests must include " + " and ".join(missing_fields) + "." @@ -142,9 +138,7 @@ async def resolve_identity( api_key_manager = getattr(request.app.state, "api_key_manager", None) trusted_role = Role.USER if api_key_manager and effective_account_id and effective_user_id: - looked_up_role = api_key_manager.get_user_role( - effective_account_id, effective_user_id - ) + looked_up_role = api_key_manager.get_user_role(effective_account_id, effective_user_id) if looked_up_role is not None: trusted_role = looked_up_role @@ -214,6 +208,4 @@ def get_request_context_checks( "Trusted mode requests must include X-OpenViking-Account." ) if not identity.user_id: - raise InvalidArgumentError( - "Trusted mode requests must include X-OpenViking-User." - ) + raise InvalidArgumentError("Trusted mode requests must include X-OpenViking-User.") diff --git a/openviking/server/auth/registry.py b/openviking/server/auth/registry.py index 78c7b89587..bf095ca845 100644 --- a/openviking/server/auth/registry.py +++ b/openviking/server/auth/registry.py @@ -45,14 +45,10 @@ class MyAuthPlugin(AuthPlugin): """ mode = plugin_class.auth_mode if not mode: - raise ValueError( - f"AuthPlugin {plugin_class.__name__} must define 'auth_mode'" - ) + raise ValueError(f"AuthPlugin {plugin_class.__name__} must define 'auth_mode'") if mode in self._plugins: existing = self._plugins[mode].__name__ - raise ValueError( - f"Auth mode {mode!r} is already registered by {existing}" - ) + raise ValueError(f"Auth mode {mode!r} is already registered by {existing}") self._plugins[mode] = plugin_class logger.info("Registered auth plugin: %s (%s)", mode, plugin_class.__name__) return plugin_class diff --git a/openviking/server/config.py b/openviking/server/config.py index 27fc87423c..3457849df2 100644 --- a/openviking/server/config.py +++ b/openviking/server/config.py @@ -7,11 +7,10 @@ from pydantic import BaseModel, Field, ValidationError -from openviking.server.identity import AuthMode -from openviking_cli.utils import get_logger - # Import auth plugin registry for config validation from openviking.server.auth.registry import get_registry +from openviking.server.identity import AuthMode +from openviking_cli.utils import get_logger from openviking_cli.utils.config.config_loader import ( load_json_config, resolve_config_path, @@ -351,7 +350,8 @@ def validate_server_config(config: ServerConfig) -> None: # Ensure built-in plugins are registered before validation. # If a non-built-in plugin has already claimed a built-in mode name, # log a security warning and forcefully override it. - from openviking.server.auth.plugins import DevAuthPlugin, ApiKeyAuthPlugin, TrustedAuthPlugin + from openviking.server.auth.plugins import ApiKeyAuthPlugin, DevAuthPlugin, TrustedAuthPlugin + registry = get_registry() _BUILTIN_PLUGINS = { "dev": DevAuthPlugin, @@ -375,8 +375,7 @@ def validate_server_config(config: ServerConfig) -> None: plugin_cls = registry.get(effective_auth_mode) if plugin_cls is None: logger.error( - "Unknown auth_mode: %r. No auth plugin registered for this mode. " - "Registered modes: %s.", + "Unknown auth_mode: %r. No auth plugin registered for this mode. Registered modes: %s.", effective_auth_mode, ", ".join(registry.list_modes()), ) diff --git a/openviking/server/mcp_endpoint.py b/openviking/server/mcp_endpoint.py index e6ce66b5c3..eddca0385e 100644 --- a/openviking/server/mcp_endpoint.py +++ b/openviking/server/mcp_endpoint.py @@ -791,6 +791,7 @@ async def forget(uri: str, recursive: bool = False) -> str: # -- code navigation ------------------------------------------------------- + def _require_viking_uri(uri: str) -> Optional[str]: """Return error message if uri is not a viking:// URI, else None.""" if not isinstance(uri, str) or not uri.startswith("viking://"): diff --git a/openviking/server/oauth/provider.py b/openviking/server/oauth/provider.py index cb8055c3e6..3251102b6f 100644 --- a/openviking/server/oauth/provider.py +++ b/openviking/server/oauth/provider.py @@ -281,7 +281,9 @@ async def exchange_refresh_token( if self._role_resolver is not None: try: token_role = Role(refresh_token.role) - current_role = Role(self._role_resolver(refresh_token.account_id, refresh_token.user_id)) + current_role = Role( + self._role_resolver(refresh_token.account_id, refresh_token.user_id) + ) except (ValueError, Exception): # noqa: BLE001 token_role = None current_role = None diff --git a/openviking/server/routers/code.py b/openviking/server/routers/code.py index a361f0a2c7..a0e228d03a 100644 --- a/openviking/server/routers/code.py +++ b/openviking/server/routers/code.py @@ -54,9 +54,9 @@ async def code_outline_endpoint( service = get_service() content = await service.fs.read(request.uri, ctx=_ctx) if not isinstance(content, str): - return Response( - status="ok", result=f"Error: {request.uri} is not text" - ).model_dump(exclude_none=True) + return Response(status="ok", result=f"Error: {request.uri} is not text").model_dump( + exclude_none=True + ) return Response(status="ok", result=outline_file(content, request.uri)).model_dump( exclude_none=True ) @@ -119,9 +119,9 @@ async def code_expand_endpoint( service = get_service() content = await service.fs.read(request.uri, ctx=_ctx) if not isinstance(content, str): - return Response( - status="ok", result=f"Error: {request.uri} is not text" - ).model_dump(exclude_none=True) + return Response(status="ok", result=f"Error: {request.uri} is not text").model_dump( + exclude_none=True + ) return Response( status="ok", result=expand_symbol(content, request.uri, request.symbol) ).model_dump(exclude_none=True) diff --git a/openviking/server/routers/search.py b/openviking/server/routers/search.py index 0e80c27d33..8c1f06196d 100644 --- a/openviking/server/routers/search.py +++ b/openviking/server/routers/search.py @@ -176,7 +176,7 @@ class GrepRequest(BaseModel): pattern: str case_insensitive: bool = False node_limit: Optional[int] = None - level_limit: int = 5 + level_limit: int = 10 class GlobRequest(BaseModel): diff --git a/openviking/service/core.py b/openviking/service/core.py index b170675b70..14281dd429 100644 --- a/openviking/service/core.py +++ b/openviking/service/core.py @@ -318,6 +318,7 @@ async def initialize(self) -> None: rerank_config=config.rerank, vector_store=self._vikingdb_manager, retrieval_config=config.retrieval, + grep_config=config.grep, enable_recorder=enable_recorder, encryptor=self._encryptor, ) diff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.py index 1bbcca7d28..92995a21e9 100644 --- a/openviking/service/fs_service.py +++ b/openviking/service/fs_service.py @@ -456,7 +456,7 @@ async def grep( exclude_uri: Optional[str] = None, case_insensitive: bool = False, node_limit: Optional[int] = None, - level_limit: int = 5, + level_limit: int = 10, ) -> Dict: """Content search.""" viking_fs = self._ensure_initialized() diff --git a/openviking/service/reindex_executor.py b/openviking/service/reindex_executor.py index cbec3e912d..070258e8ea 100644 --- a/openviking/service/reindex_executor.py +++ b/openviking/service/reindex_executor.py @@ -691,12 +691,15 @@ async def _reindex_resource_vectors_from_entries( counters.warnings.append(f"No vector source found for {file_uri}") continue abstract = self._prefer_non_empty(summary, vector_text) + # Read full file content for BM25 content field (not embedding-truncated) + full_text = await self._safe_read_text(file_uri, ctx=ctx) or vector_text try: await self._upsert_context( uri=file_uri, parent_uri=parent_uri, abstract=abstract, vector_text=vector_text, + full_text=full_text, is_leaf=True, context_type=context_type_for_uri(file_uri), level=ContextLevel.DETAIL, @@ -1239,6 +1242,7 @@ async def _upsert_context( parent_uri: str, abstract: str, vector_text: str, + full_text: str = "", is_leaf: bool, context_type: str, level: ContextLevel, @@ -1268,7 +1272,7 @@ async def _upsert_context( owner_space=owner_space_for_uri(uri, ctx), meta=merged_meta, ) - context.set_vectorize(Vectorize(text=vector_text)) + context.set_vectorize(Vectorize(text=vector_text, full_text=full_text or vector_text)) msg = EmbeddingMsgConverter.from_context(context) if msg is None: raise OpenVikingError( diff --git a/openviking/session/memory/utils/json_parser.py b/openviking/session/memory/utils/json_parser.py index a5ac934089..1f51cd2628 100644 --- a/openviking/session/memory/utils/json_parser.py +++ b/openviking/session/memory/utils/json_parser.py @@ -429,8 +429,10 @@ def parse_json_with_stability( if isinstance(parsed_data, list) and len(parsed_data) > 0: parsed_data = parsed_data[0] tracer.info("Extracted first item from list response") - elif isinstance(parsed_data, list) and len(parsed_data) == 0 and getattr( - model_class, "_allow_empty_list_response", False + elif ( + isinstance(parsed_data, list) + and len(parsed_data) == 0 + and getattr(model_class, "_allow_empty_list_response", False) ): # The operations model opts in (via _allow_empty_list_response) to treating a # bare `[]` as a valid "no operations" outcome: every field is default_factory=list, diff --git a/openviking/session/tool_result_synopsis.py b/openviking/session/tool_result_synopsis.py index af5724ebb3..5109039958 100644 --- a/openviking/session/tool_result_synopsis.py +++ b/openviking/session/tool_result_synopsis.py @@ -37,6 +37,8 @@ _CODE_SYMBOL_LIMIT = 24 _CODE_IMPORT_LINE_CHARS = 180 _CODE_SYMBOL_LINE_CHARS = 200 + + @dataclass(frozen=True) class ToolResultSynopsis: kind: ToolResultKind @@ -95,9 +97,7 @@ def _looks_binary(content: str) -> bool: return False if "\x00" in content: return True - control_chars = sum( - 1 for ch in content[:1000] if ord(ch) < 32 and ch not in {"\n", "\r", "\t"} - ) + control_chars = sum(1 for ch in content[:1000] if ord(ch) < 32 and ch not in {"\n", "\r", "\t"}) return control_chars / max(1, min(len(content), 1000)) > 0.05 @@ -204,7 +204,9 @@ def _summarize_yaml(value: Any) -> ToolResultSynopsis: def _summarize_xml(root: ET.Element) -> ToolResultSynopsis: child_counts = Counter(child.tag for child in list(root)) structure = [f"root: {root.tag}", f"attributes: {len(root.attrib)}"] - structure.extend(f"{tag}: {count}" for tag, count in child_counts.most_common(_XML_CHILD_TAG_LIMIT)) + structure.extend( + f"{tag}: {count}" for tag, count in child_counts.most_common(_XML_CHILD_TAG_LIMIT) + ) return ToolResultSynopsis( kind="xml", title="XML output", @@ -326,8 +328,7 @@ def _summarize_text(content: str, preview_chars: int) -> ToolResultSynopsis: if len(stripped) <= 1: continue if not ( - re.match(r"^#{1,6}\s+", stripped) - or re.match(r"^[A-Z0-9][A-Z0-9\s:_-]{6,}$", stripped) + re.match(r"^#{1,6}\s+", stripped) or re.match(r"^[A-Z0-9][A-Z0-9\s:_-]{6,}$", stripped) ): continue header = _normalize_text_for_line(stripped, 160) @@ -341,7 +342,9 @@ def _summarize_text(content: str, preview_chars: int) -> ToolResultSynopsis: word_count = len(normalized.split()) if normalized else 0 excerpt_chars = _TEXT_EXCERPT_CHARS first = _normalize_text_for_line(content[:excerpt_chars], excerpt_chars) - last = _normalize_text_for_line(content[-excerpt_chars:], excerpt_chars) if excerpt_chars else "" + last = ( + _normalize_text_for_line(content[-excerpt_chars:], excerpt_chars) if excerpt_chars else "" + ) return ToolResultSynopsis( kind="text", title="Text output", @@ -384,7 +387,7 @@ def generate_tool_result_synopsis( structure=[], notable_items=[], sample=_head_tail_sample(content, preview_chars), - ) + ) stripped = content.strip() lower_mime = (mime_type or "").lower() @@ -480,5 +483,7 @@ def render_tool_result_stub( body.append( f"- Use openviking_tool_result_read with ref={ref} and offset/limit to inspect raw output." ) - body.append("- Use openviking_tool_result_list to discover other externalized outputs in this session.") + body.append( + "- Use openviking_tool_result_list to discover other externalized outputs in this session." + ) return "\n".join(header) + "\n\n" + "\n".join(body) diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py index e8789df887..57976ea67b 100644 --- a/openviking/storage/collection_schemas.py +++ b/openviking/storage/collection_schemas.py @@ -41,6 +41,31 @@ logger = get_logger(__name__) EMBEDDING_META_MARKER = "\n\n[openviking.embedding]\n" +# Minimum OV version that supports content field + FullText config for grep bm25 +_FULLTEXT_MIN_VERSION = "0.3.18" + + +def _parse_version(v: str) -> tuple: + """Parse a semver-like string into a comparable tuple of ints. + + Only the first 3 numeric segments are used (e.g. "0.3.18.dev23" → (0, 3, 18)). + Non-numeric suffixes like ".dev23", ".rc1", "+local" are ignored. + """ + try: + parts = v.split(".") + numeric = [] + for p in parts: + # Stop at first non-numeric segment (e.g. "dev23", "rc1") + try: + numeric.append(int(p)) + except ValueError: + break + if len(numeric) == 3: + break + return tuple(numeric) if numeric else (0, 0, 0) + except (ValueError, AttributeError): + return (0, 0, 0) + @dataclass class RequestQueueStats: @@ -104,6 +129,7 @@ def context_collection( {"FieldName": "tags", "FieldType": "string"}, {"FieldName": "search_tags", "FieldType": "list"}, {"FieldName": "abstract", "FieldType": "string"}, + {"FieldName": "content", "FieldType": "text"}, {"FieldName": "account_id", "FieldType": "string"}, {"FieldName": "owner_user_id", "FieldType": "string"}, ] @@ -131,6 +157,12 @@ def context_collection( "Description": description or "Unified context collection", "Fields": fields, "ScalarIndex": scalar_index, + "FullText": [ + { + "Field": "content", + "Analyzer": {"Tokenizer": "standard", "StopWordsFilters": ["symbol"]}, + }, + ], } @@ -179,11 +211,14 @@ def _build_embedding_metadata(config: "OpenVikingConfig") -> Dict[str, Any]: except Exception: model_identity = model + from openviking import __version__ + return { "provider": provider, "model": model, "dimension": dimension, "model_identity": model_identity, + "schema_version": __version__, } @@ -263,6 +298,27 @@ async def init_context_collection(storage) -> bool: base_description, existing_embedding_meta = _decode_collection_description( existing_meta.get("Description") ) + + # Schema compatibility check: warn if collection was created by older OV version + if existing_embedding_meta: + existing_schema_version = existing_embedding_meta.get("schema_version", "0.0.0") + if _parse_version(existing_schema_version) < _parse_version(_FULLTEXT_MIN_VERSION): + fields = existing_meta.get("Fields", []) + has_content = any( + f.get("FieldName") == "content" and f.get("FieldType") == "text" for f in fields + ) + fulltext = existing_meta.get("FullText") or [] + has_content_fulltext = any(ft.get("Field") == "content" for ft in fulltext) + if not (has_content and has_content_fulltext): + logger.warning( + "Collection schema is outdated (created by OV %s, requires >= %s). " + "Missing 'content' field or FullText config. " + "grep engine=auto will fall back to fs. " + "Recreate the collection to enable vikingdb-based grep.", + existing_schema_version, + _FULLTEXT_MIN_VERSION, + ) + if existing_embedding_meta == embedding_meta: return False diff --git a/openviking/storage/observers/filesystem_observer.py b/openviking/storage/observers/filesystem_observer.py index ba392dbcd6..a7079f08f5 100644 --- a/openviking/storage/observers/filesystem_observer.py +++ b/openviking/storage/observers/filesystem_observer.py @@ -38,6 +38,7 @@ def _get_collector(): Lazy import to avoid circular dependencies. """ from openviking.server.dependencies import get_service + service = get_service() return service.debug.observer @@ -95,13 +96,15 @@ def _format_stats_table(self, stats_data: dict) -> str: max_us = op_stats.get("max_time_us", 0) if count > 0: - table_data.append({ - "Operation": op_name, - "Count": count, - "Avg (ms)": f"{avg_us / 1000:.3f}", - "Min (ms)": f"{min_us / 1000:.3f}", - "Max (ms)": f"{max_us / 1000:.3f}", - }) + table_data.append( + { + "Operation": op_name, + "Count": count, + "Avg (ms)": f"{avg_us / 1000:.3f}", + "Min (ms)": f"{min_us / 1000:.3f}", + "Max (ms)": f"{max_us / 1000:.3f}", + } + ) if table_data: result.append(tabulate(table_data, headers="keys", tablefmt="pretty")) diff --git a/openviking/storage/ovpack/vectors.py b/openviking/storage/ovpack/vectors.py index 5fc6c546d5..caa3335f44 100644 --- a/openviking/storage/ovpack/vectors.py +++ b/openviking/storage/ovpack/vectors.py @@ -291,6 +291,8 @@ async def _upsert_vector_snapshot_record( } if not payload.get("abstract"): payload["abstract"] = str(record.get("text") or "") + if "content" not in payload: + payload["content"] = str(record.get("text") or "") try: await vector_store.upsert(payload, ctx=ctx) diff --git a/openviking/storage/queuefs/embedding_msg_converter.py b/openviking/storage/queuefs/embedding_msg_converter.py index 9cac93025d..15cc54493e 100644 --- a/openviking/storage/queuefs/embedding_msg_converter.py +++ b/openviking/storage/queuefs/embedding_msg_converter.py @@ -68,6 +68,11 @@ def from_context(context: Context) -> EmbeddingMsg: resolved_level = int(resolved_level.value) context_data["level"] = int(resolved_level) + # Store full content in content field for bm25 full-text search. + # Use full_text (raw file content) when available; fall back to vectorization_text. + full_content = context.vectorize.full_text or vectorization_text + context_data["content"] = full_content + if vectorization_images: # Multimodal message: combine text (if any) and image references into the # multimodal embedding input format. Image-aware embedders consume this list; diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.py index 9f35c93e7f..dd5d13e9d2 100644 --- a/openviking/storage/queuefs/semantic_dag.py +++ b/openviking/storage/queuefs/semantic_dag.py @@ -4,9 +4,9 @@ import asyncio import threading -from weakref import WeakKeyDictionary from dataclasses import dataclass, field from typing import ClassVar, Dict, List, Optional, Set +from weakref import WeakKeyDictionary from openviking.server.identity import RequestContext from openviking.storage.queuefs.semantic_sidecar import write_semantic_sidecars diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py index 7548c2477c..e5b6723dec 100644 --- a/openviking/storage/queuefs/semantic_processor.py +++ b/openviking/storage/queuefs/semantic_processor.py @@ -988,9 +988,15 @@ async def _rewrite_target_image_uris( d = d.rsplit("/", 1)[0] for rel in candidate_rels: - src_mapping = f"{root_prefix}/{rel}/{mapping_name}" if rel else f"{root_prefix}/{mapping_name}" + src_mapping = ( + f"{root_prefix}/{rel}/{mapping_name}" + if rel + else f"{root_prefix}/{mapping_name}" + ) target_mapping = ( - f"{target_prefix}/{rel}/{mapping_name}" if rel else f"{target_prefix}/{mapping_name}" + f"{target_prefix}/{rel}/{mapping_name}" + if rel + else f"{target_prefix}/{mapping_name}" ) try: await viking_fs.stat(target_mapping, ctx=ctx) @@ -1031,7 +1037,7 @@ async def _generate_text_summary( file_name: str, llm_sem: asyncio.Semaphore, ctx: Optional[RequestContext] = None, - ) -> Dict[str, str]: + ) -> Dict[str, Any]: """Generate summary for a single text file (code, documentation, or other text).""" viking_fs = get_viking_fs() vlm = get_openviking_config().vlm @@ -1039,12 +1045,14 @@ async def _generate_text_summary( content = await viking_fs.read_file(file_path, ctx=active_ctx) if isinstance(content, bytes): - # Try to decode with error handling for text files - try: - content = content.decode("utf-8") - except UnicodeDecodeError: - logger.warning(f"Failed to decode file as UTF-8, skipping: {file_path}") - return {"name": file_name, "summary": ""} + from openviking.utils.embedding_utils import _decode_text_bytes + + content = _decode_text_bytes(content) + + full_content = content or "" + + def result(summary: str) -> Dict[str, Any]: + return {"name": file_name, "summary": summary, "content": full_content} # Limit content length max_chars = get_openviking_config().semantic.max_file_content_chars @@ -1054,7 +1062,7 @@ async def _generate_text_summary( # Generate summary if not vlm.is_available(): logger.warning("VLM not available, using empty summary") - return {"name": file_name, "summary": ""} + return result("") from openviking.session.memory.utils.language import resolve_output_language @@ -1076,7 +1084,7 @@ async def _generate_text_summary( if len(skeleton_text) > max_skeleton_chars: skeleton_text = skeleton_text[:max_skeleton_chars] if code_mode == "ast": - return {"name": file_name, "summary": skeleton_text} + return result(skeleton_text) else: # ast_llm prompt = render_prompt( "semantic.code_ast_summary", @@ -1089,7 +1097,7 @@ async def _generate_text_summary( async with llm_sem: with bind_telemetry_stage("resource_summarize"): summary = await vlm.get_completion_async(prompt) - return {"name": file_name, "summary": summary.strip()} + return result(summary.strip()) if skeleton_text is None: logger.info("AST unsupported language, fallback to LLM: %s", file_path) else: @@ -1103,7 +1111,7 @@ async def _generate_text_summary( async with llm_sem: with bind_telemetry_stage("resource_summarize"): summary = await vlm.get_completion_async(prompt) - return {"name": file_name, "summary": summary.strip()} + return result(summary.strip()) elif file_type == FILE_TYPE_DOCUMENTATION: prompt_id = "semantic.document_summary" @@ -1118,21 +1126,22 @@ async def _generate_text_summary( async with llm_sem: with bind_telemetry_stage("resource_summarize"): summary = await vlm.get_completion_async(prompt) - return {"name": file_name, "summary": summary.strip()} + return result(summary.strip()) async def _generate_single_file_summary( self, file_path: str, llm_sem: Optional[asyncio.Semaphore] = None, ctx: Optional[RequestContext] = None, - ) -> Dict[str, str]: + ) -> Dict[str, Any]: """Generate summary for a single file. Args: file_path: File path Returns: - {"name": file_name, "summary": summary_content} + {"name": file_name, "summary": summary_content}; text files also carry + decoded "content" so vectorization can avoid re-reading the same file. """ file_name = file_path.split("/")[-1] llm_sem = llm_sem or asyncio.Semaphore(self.max_concurrent_llm) diff --git a/openviking/storage/vectordb/collection/http_collection.py b/openviking/storage/vectordb/collection/http_collection.py index 29516e909d..1405e7e7b9 100644 --- a/openviking/storage/vectordb/collection/http_collection.py +++ b/openviking/storage/vectordb/collection/http_collection.py @@ -6,6 +6,7 @@ import requests +import openviking from openviking.storage.vectordb.collection.collection import Collection, ICollection from openviking.storage.vectordb.collection.result import ( AggregateResult, @@ -18,7 +19,10 @@ # Default request timeout (seconds) DEFAULT_TIMEOUT = 30 -headers = {"Content-Type": "application/json"} +headers = { + "Content-Type": "application/json", + "User-Agent": f"openviking/{openviking.__version__}", +} def get_or_create_http_collection( @@ -42,6 +46,10 @@ def get_or_create_http_collection( url = "http://{}:{}/CreateVikingdbCollection".format(host, port) if "Fields" in meta_data: meta_data["Fields"] = json.dumps(meta_data["Fields"]) + if "FullText" in meta_data: + meta_data["FullText"] = json.dumps(meta_data["FullText"]) + if "ScalarIndex" in meta_data: + meta_data["ScalarIndex"] = json.dumps(meta_data["ScalarIndex"]) response = requests.post(url, headers=headers, json=meta_data, timeout=DEFAULT_TIMEOUT) # logger.info(f"CreateVikingdbCollection response: {response.text}") if response.status_code == 200: @@ -551,20 +559,22 @@ def search_by_keywords( output_fields: Optional[List[str]] = None, ) -> SearchResult: url = self.url_prefix + "api/vikingdb/data/search/keywords" + payload = { + "project": self.project_name, + "collection_name": self.collection_name, + "index_name": index_name, + "keywords": json.dumps(keywords) if keywords else None, + "query": query, + "filter": json.dumps(filters) if filters else None, + "output_fields": json.dumps(output_fields) if output_fields else None, + "limit": limit, + "offset": offset, + } + payload = {k: v for k, v in payload.items() if v is not None} response = requests.post( url, headers=headers, - json={ - "project": self.project_name, - "collection_name": self.collection_name, - "index_name": index_name, - "keywords": json.dumps(keywords) if keywords else None, - "query": query, - "filter": json.dumps(filters) if filters else None, - "output_fields": json.dumps(output_fields) if output_fields else None, - "limit": limit, - "offset": offset, - }, + json=payload, timeout=DEFAULT_TIMEOUT, ) # logger.info(f"SearchByKeywords response: {response.text}") diff --git a/openviking/storage/vectordb/collection/vikingdb_clients.py b/openviking/storage/vectordb/collection/vikingdb_clients.py index 4f1d25520f..295cdaab0a 100644 --- a/openviking/storage/vectordb/collection/vikingdb_clients.py +++ b/openviking/storage/vectordb/collection/vikingdb_clients.py @@ -5,6 +5,7 @@ import requests +import openviking from openviking_cli.utils.logger import default_logger as logger # Default request timeout (seconds) @@ -82,6 +83,7 @@ def do_req( headers = { "Accept": "application/json", "Content-Type": "application/json", + "User-Agent": f"openviking/{openviking.__version__}", } headers.update(self.headers) diff --git a/openviking/storage/vectordb/collection/vikingdb_collection.py b/openviking/storage/vectordb/collection/vikingdb_collection.py index 2bab31a8ef..ee865a6bba 100644 --- a/openviking/storage/vectordb/collection/vikingdb_collection.py +++ b/openviking/storage/vectordb/collection/vikingdb_collection.py @@ -347,6 +347,7 @@ def search_by_keywords( "limit": limit, "offset": offset, } + data = {k: v for k, v in data.items() if v is not None} resp_data = self._data_post(path, data) return self._parse_search_result(resp_data) diff --git a/openviking/storage/vectordb/collection/volcengine_api_key_collection.py b/openviking/storage/vectordb/collection/volcengine_api_key_collection.py index a0d0dfe239..f6250ba663 100644 --- a/openviking/storage/vectordb/collection/volcengine_api_key_collection.py +++ b/openviking/storage/vectordb/collection/volcengine_api_key_collection.py @@ -380,6 +380,7 @@ def search_by_keywords( "limit": limit, "offset": offset, } + data = {k: v for k, v in data.items() if v is not None} resp_data = self._data_post(path, data) return self._parse_search_result(resp_data) diff --git a/openviking/storage/vectordb/collection/volcengine_clients.py b/openviking/storage/vectordb/collection/volcengine_clients.py index d4e3dd0844..e68382a693 100644 --- a/openviking/storage/vectordb/collection/volcengine_clients.py +++ b/openviking/storage/vectordb/collection/volcengine_clients.py @@ -7,6 +7,8 @@ from volcengine.base.Request import Request from volcengine.Credentials import Credentials +import openviking + # Default request timeout (seconds) DEFAULT_TIMEOUT = 30 @@ -47,6 +49,7 @@ def prepare_request(self, method, params=None, data=None): "Accept": "application/json", "Content-Type": "application/json", "Host": self.host, + "User-Agent": f"openviking/{openviking.__version__}", } r.set_headers(mheaders) if params: @@ -111,6 +114,7 @@ def prepare_request(self, method, path, params=None, data=None): "Accept": "application/json", "Content-Type": "application/json", "Host": self.host, + "User-Agent": f"openviking/{openviking.__version__}", } r.set_headers(mheaders) if params: @@ -170,6 +174,7 @@ def prepare_request(self, method, path, params=None, data=None): "Content-Type": "application/json", "Host": self.host, "Authorization": f"Bearer {self.api_key}", + "User-Agent": f"openviking/{openviking.__version__}", } r.set_headers(mheaders) if params: diff --git a/openviking/storage/vectordb/collection/volcengine_collection.py b/openviking/storage/vectordb/collection/volcengine_collection.py index 853381aef6..a06d202d6f 100644 --- a/openviking/storage/vectordb/collection/volcengine_collection.py +++ b/openviking/storage/vectordb/collection/volcengine_collection.py @@ -582,6 +582,7 @@ def search_by_keywords( "limit": limit, "offset": offset, } + data = {k: v for k, v in data.items() if v is not None} resp_data = self._data_post(path, data) return self._parse_search_result(resp_data) diff --git a/openviking/storage/vectordb/utils/validation.py b/openviking/storage/vectordb/utils/validation.py index 887d059589..90864eac73 100644 --- a/openviking/storage/vectordb/utils/validation.py +++ b/openviking/storage/vectordb/utils/validation.py @@ -152,6 +152,9 @@ class CollectionMetaConfig(BaseModel): ProjectName: Optional[str] = None Description: Optional[str] = Field(None, max_length=65535) Vectorize: Optional[VectorizeConfig] = None + FullText: Optional[List[dict]] = ( + None # e.g. [{"Field": "content", "Analyzer": {"Tokenizer": "standard"}}] + ) # Internal fields _FieldsCount: Optional[int] = None diff --git a/openviking/storage/vectordb_adapters/base.py b/openviking/storage/vectordb_adapters/base.py index a09f1771a4..c128433327 100644 --- a/openviking/storage/vectordb_adapters/base.py +++ b/openviking/storage/vectordb_adapters/base.py @@ -4,6 +4,7 @@ from __future__ import annotations +import json import math import uuid from abc import ABC, abstractmethod @@ -30,6 +31,29 @@ logger = get_logger(__name__) +# --------------------------------------------------------------------------- +# VikingDB text field byte limit +# --------------------------------------------------------------------------- +# VikingDB rejects upsert when any text field exceeds this byte length. +# Truncation is applied at a valid UTF-8 character boundary so that +# multi-byte sequences are never split in the middle. +VIKINGDB_TEXT_FIELD_BYTE_LIMIT: int = 65535 + + +def _truncate_text_field(text: str, byte_limit: int = VIKINGDB_TEXT_FIELD_BYTE_LIMIT) -> str: + """Truncate *text* so its UTF-8 encoding does not exceed *byte_limit*. + + Walks backwards from *byte_limit* to find the nearest valid UTF-8 lead + byte, ensuring no multi-byte character is split. + """ + encoded = text.encode("utf-8") + if len(encoded) <= byte_limit: + return text + cut = byte_limit + while cut > 0 and (encoded[cut] & 0xC0) == 0x80: + cut -= 1 + return encoded[:cut].decode("utf-8") + def _parse_url(url: str) -> tuple[str, int]: normalized = url @@ -68,6 +92,13 @@ class CollectionAdapter(ABC): mode: str _URI_FIELD_NAMES = {"uri", "parent_uri"} + # Text fields subject to byte-limit truncation before upsert. + _TRUNCATABLE_TEXT_FIELDS: tuple[str, ...] = ("content", "abstract") + + # Per-backend byte limit for text fields. ``None`` means no truncation. + # Subclasses backed by VikingDB should set this to ``VIKINGDB_TEXT_FIELD_BYTE_LIMIT``. + _TEXT_FIELD_BYTE_LIMIT: int | None = None + def __init__(self, collection_name: str, index_name: str = DEFAULT_INDEX_NAME): self._collection_name = collection_name self._index_name = index_name @@ -218,6 +249,11 @@ def _normalize_record_for_write(self, record: Dict[str, Any]) -> Dict[str, Any]: for key in self._URI_FIELD_NAMES: if key in normalized: normalized[key] = self._encode_uri_field_value(normalized[key]) + if self._TEXT_FIELD_BYTE_LIMIT is not None: + for field in self._TRUNCATABLE_TEXT_FIELDS: + value = normalized.get(field) + if isinstance(value, str): + normalized[field] = _truncate_text_field(value, self._TEXT_FIELD_BYTE_LIMIT) return normalized @staticmethod @@ -550,6 +586,47 @@ def count(self, filter: Optional[Dict[str, Any] | FilterExpr] = None) -> int: return 0 + def search_by_keywords( + self, + keywords: Optional[list[str]] = None, + query: Optional[str] = None, + limit: int = 10, + offset: int = 0, + filter: Optional[Dict[str, Any] | FilterExpr] = None, + output_fields: Optional[list[str]] = None, + ) -> list[Dict[str, Any]]: + coll = self.get_collection() + compiled_filter = self._compile_filter(filter) + logger.debug( + "search_by_keywords: keywords=%s query=%s limit=%s offset=%s filter=%s output_fields=%s", + keywords, + query, + limit, + offset, + json.dumps(compiled_filter, ensure_ascii=False), + output_fields, + ) + result = coll.search_by_keywords( + index_name=self._index_name, + keywords=keywords, + query=query, + limit=limit, + offset=offset, + filters=compiled_filter, + output_fields=output_fields, + ) + records: list[Dict[str, Any]] = [] + for item in result.data: + record = dict(item.fields) if item.fields else {} + record["id"] = item.id + raw_score = item.score if item.score is not None else 0.0 + if not math.isfinite(raw_score): + raw_score = 0.0 + record["_score"] = raw_score + record = self._normalize_record_for_read(record) + records.append(record) + return records + def clear(self) -> bool: self.get_collection().delete_all_data() return True diff --git a/openviking/storage/vectordb_adapters/http_adapter.py b/openviking/storage/vectordb_adapters/http_adapter.py index 03c5ca82e1..33a0da4c8f 100644 --- a/openviking/storage/vectordb_adapters/http_adapter.py +++ b/openviking/storage/vectordb_adapters/http_adapter.py @@ -13,13 +13,19 @@ list_vikingdb_collections, ) -from .base import CollectionAdapter, _normalize_collection_names, _parse_url +from .base import ( + VIKINGDB_TEXT_FIELD_BYTE_LIMIT, + CollectionAdapter, + _normalize_collection_names, + _parse_url, +) class HttpCollectionAdapter(CollectionAdapter): """Adapter for remote HTTP vectordb project.""" _DATA_BATCH_SIZE = 100 + _TEXT_FIELD_BYTE_LIMIT = VIKINGDB_TEXT_FIELD_BYTE_LIMIT def __init__( self, diff --git a/openviking/storage/vectordb_adapters/opengauss_adapter.py b/openviking/storage/vectordb_adapters/opengauss_adapter.py index a4c0bbd705..21b4913a80 100644 --- a/openviking/storage/vectordb_adapters/opengauss_adapter.py +++ b/openviking/storage/vectordb_adapters/opengauss_adapter.py @@ -191,7 +191,9 @@ def _sparse_dot(left: Optional[Dict[str, float]], right: Optional[Dict[str, floa class OpenGaussIndex(IIndex): """Metadata-only logical index facade for openGauss.""" - def __init__(self, collection: "OpenGaussCollection", index_name: str, meta: Dict[str, Any]) -> None: + def __init__( + self, collection: "OpenGaussCollection", index_name: str, meta: Dict[str, Any] + ) -> None: super().__init__(meta=meta) self._collection = collection self._index_name = index_name @@ -216,7 +218,9 @@ def search( def aggregate(self, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: raise NotImplementedError("OpenGaussIndex.aggregate is not exposed via raw index interface") - def update(self, scalar_index: Optional[Dict[str, Any]] = None, description: Optional[str] = None): + def update( + self, scalar_index: Optional[Dict[str, Any]] = None, description: Optional[str] = None + ): self._collection.update_index( index_name=self._index_name, scalar_index=scalar_index, @@ -487,7 +491,7 @@ def _function_table_name(self, table_name: str) -> str: def _ensure_columns_for_record(self, record: Dict[str, Any]) -> None: existing = set(self._all_columns()) - for field_name, value in record.items(): + for field_name in record: if field_name in existing: continue if field_name == "id": @@ -529,14 +533,24 @@ def _select_columns( if field == self._sparse_vector_name and not include_sparse: continue wanted.append(field) - if include_vector and self._dense_vector_name in columns and self._dense_vector_name not in wanted: + if ( + include_vector + and self._dense_vector_name in columns + and self._dense_vector_name not in wanted + ): wanted.append(self._dense_vector_name) - if include_sparse and self._sparse_vector_name in columns and self._sparse_vector_name not in wanted: + if ( + include_sparse + and self._sparse_vector_name in columns + and self._sparse_vector_name not in wanted + ): wanted.append(self._sparse_vector_name) return wanted - def _row_to_payload(self, row: Sequence[Any], columns: Sequence[str]) -> Tuple[Any, Dict[str, Any]]: - record = dict(zip(columns, row)) + def _row_to_payload( + self, row: Sequence[Any], columns: Sequence[str] + ) -> Tuple[Any, Dict[str, Any]]: + record = dict(zip(columns, row, strict=False)) record_id = record.pop("id", "") record.pop(self._dense_vector_name, None) sparse = record.pop(self._sparse_vector_name, None) @@ -745,7 +759,9 @@ def update_index( ): meta = self.get_index_meta_data(index_name) or {"IndexName": index_name} if scalar_index is not None: - meta["ScalarIndex"] = list(scalar_index.keys()) if isinstance(scalar_index, dict) else list(scalar_index) + meta["ScalarIndex"] = ( + list(scalar_index.keys()) if isinstance(scalar_index, dict) else list(scalar_index) + ) if description is not None: meta["Description"] = description for field_name in meta.get("ScalarIndex", []) or []: @@ -784,7 +800,9 @@ def drop_index(self, index_name: str): pg_index_name = _safe_identifier(self._table_name, index_name, "vec", prefix="idx") self._execute(f"DROP INDEX IF EXISTS {self._index_ref(pg_index_name)}") for field_name in meta.get("ScalarIndex", []) or []: - scalar_index_name = _safe_identifier(self._table_name, index_name, field_name, prefix="idx") + scalar_index_name = _safe_identifier( + self._table_name, index_name, field_name, prefix="idx" + ) self._execute(f"DROP INDEX IF EXISTS {self._index_ref(scalar_index_name)}") self._delete_index_meta(index_name) @@ -807,7 +825,9 @@ def search_by_vector( return SearchResult() fetch_limit = max(limit + offset, limit) if dense_vector is None: - return self._search_by_sparse(sparse_vector, fetch_limit, offset, limit, filters, output_fields) + return self._search_by_sparse( + sparse_vector, fetch_limit, offset, limit, filters, output_fields + ) columns = self._select_columns(output_fields, include_sparse=bool(sparse_vector)) where_sql, params = self._where_sql(filters) @@ -821,7 +841,9 @@ def search_by_vector( f"ORDER BY {_quote_ident(self._dense_vector_name)} {operator} %s::vector " "LIMIT %s OFFSET %s" ) - rows = self._execute(sql, [vector_text] + params + [vector_text, fetch_limit, 0], fetch=True) + rows = self._execute( + sql, [vector_text] + params + [vector_text, fetch_limit, 0], fetch=True + ) scored_items: List[SearchItemResult] = [] for row in rows: record_id, payload = self._row_to_payload(row[:-1], columns) @@ -853,12 +875,16 @@ def _search_by_sparse( f"SELECT {', '.join(_quote_ident(col) for col in columns)} " f"FROM {self._table_ref()}{where_sql} LIMIT %s" ) - rows = self._execute(sql, params + [max(fetch_limit, self.DEFAULT_SPARSE_SCAN_LIMIT)], fetch=True) + rows = self._execute( + sql, params + [max(fetch_limit, self.DEFAULT_SPARSE_SCAN_LIMIT)], fetch=True + ) items = [] for row in rows: record_id, payload = self._row_to_payload(row, columns) sparse_payload = payload.pop(self._sparse_vector_name, None) - score = _sparse_dot(sparse_vector, sparse_payload if isinstance(sparse_payload, dict) else None) + score = _sparse_dot( + sparse_vector, sparse_payload if isinstance(sparse_payload, dict) else None + ) if score > 0: items.append(SearchItemResult(id=record_id, fields=payload, score=score)) items.sort(key=lambda item: item.score or 0.0, reverse=True) @@ -918,7 +944,7 @@ def search_by_id( ) if not rows: return SearchResult() - row_payload = dict(zip(columns, rows[0])) + row_payload = dict(zip(columns, rows[0], strict=False)) dense_vector = row_payload.get(self._dense_vector_name) sparse_raw = row_payload.get(self._sparse_vector_name) sparse_vector = None @@ -1010,7 +1036,11 @@ def search_by_scalar( items = [] for row in rows: record_id, payload = self._row_to_payload(row, columns) - score = payload.pop(field, None) if output_fields and field not in output_fields else payload.get(field) + score = ( + payload.pop(field, None) + if output_fields and field not in output_fields + else payload.get(field) + ) items.append( SearchItemResult( id=record_id, @@ -1056,7 +1086,9 @@ def upsert_data(self, data_list: List[Dict[str, Any]], ttl=0): def _upsert_row(self, columns: List[str], values: List[Any]) -> None: id_index = columns.index("id") update_columns = [column for column in columns if column != "id"] - update_values = [value for column, value in zip(columns, values) if column != "id"] + update_values = [ + value for column, value in zip(columns, values, strict=False) if column != "id" + ] set_parts = [ f"{_quote_ident(column)} = {'%s::vector' if column == self._dense_vector_name else '%s'}" for column in update_columns @@ -1071,8 +1103,7 @@ def _upsert_row(self, columns: List[str], values: List[Any]) -> None: try: if update_columns: cur.execute( - f"UPDATE {self._table_ref()} " - f"SET {', '.join(set_parts)} WHERE id = %s", + f"UPDATE {self._table_ref()} SET {', '.join(set_parts)} WHERE id = %s", update_values + [values[id_index]], ) if not update_columns or cur.rowcount == 0: @@ -1091,8 +1122,7 @@ def _upsert_row(self, columns: List[str], values: List[Any]) -> None: conn = self._ensure_connection() cur = conn.cursor() cur.execute( - f"UPDATE {self._table_ref()} " - f"SET {', '.join(set_parts)} WHERE id = %s", + f"UPDATE {self._table_ref()} SET {', '.join(set_parts)} WHERE id = %s", update_values + [values[id_index]], ) conn.commit() @@ -1154,7 +1184,9 @@ def aggregate_data( params, fetch=True, ) - return AggregateResult(agg={"_total": int(rows[0][0]) if rows else 0}, op=op, field=None) + return AggregateResult( + agg={"_total": int(rows[0][0]) if rows else 0}, op=op, field=None + ) rows = self._execute( f"SELECT {_quote_ident(field)}, COUNT(*) FROM {self._table_ref()}" f"{where_sql} GROUP BY {_quote_ident(field)}", @@ -1239,14 +1271,18 @@ def from_config(cls, config: Any): index_name=config.index_name or "default", distance_metric=config.distance_metric or "cosine", dense_vector_name=str( - getattr(cfg, "dense_vector_name", None) or params.get("dense_vector_name") or "vector" + getattr(cfg, "dense_vector_name", None) + or params.get("dense_vector_name") + or "vector" ), sparse_vector_name=str( getattr(cfg, "sparse_vector_name", None) or params.get("sparse_vector_name") or "sparse_vector" ), - connect_timeout=int(getattr(cfg, "connect_timeout", None) or params.get("connect_timeout") or 10), + connect_timeout=int( + getattr(cfg, "connect_timeout", None) or params.get("connect_timeout") or 10 + ), mode=str(getattr(cfg, "mode", None) or params.get("mode") or "standalone"), shard_count=int(getattr(cfg, "shard_count", None) or params.get("shard_count") or 32), ) @@ -1320,7 +1356,9 @@ def _try_make_reference_table(self, table_name: str) -> None: return cur = conn.cursor() try: - cur.execute("SELECT create_reference_table(%s)", [self._function_table_name(table_name)]) + cur.execute( + "SELECT create_reference_table(%s)", [self._function_table_name(table_name)] + ) conn.commit() except Exception as exc: conn.rollback() @@ -1481,7 +1519,9 @@ def _normalize_record_for_write(self, record: Dict[str, Any]) -> Dict[str, Any]: normalized["uri"] = normalized_uri normalized["parent_uri"] = self._compute_parent_uri(normalized_uri) normalized["scope_roots"] = self._compute_scope_roots(normalized_uri) - normalized["uri_depth"] = len([part for part in normalized_uri.strip("/").split("/") if part]) + normalized["uri_depth"] = len( + [part for part in normalized_uri.strip("/").split("/") if part] + ) return normalized def _normalize_record_for_read(self, record: Dict[str, Any]) -> Dict[str, Any]: @@ -1518,7 +1558,9 @@ def _compile_filter(self, expr: FilterExpr | Dict[str, Any] | None) -> Dict[str, if isinstance(expr, In): values = list(expr.values) if expr.field in self._URI_FIELD_NAMES: - values = [self._normalize_path(self._encode_uri_field_value(value)) for value in values] + values = [ + self._normalize_path(self._encode_uri_field_value(value)) for value in values + ] return {"op": "must", "field": expr.field, "conds": values} if isinstance(expr, Range): payload: Dict[str, Any] = {"op": "range", "field": expr.field} @@ -1548,5 +1590,7 @@ def _compile_filter(self, expr: FilterExpr | Dict[str, Any] | None) -> Dict[str, return {"op": "must", "field": "parent_uri", "conds": [encoded_path]} if expr.depth == -1: return {"op": "must", "field": "scope_roots", "conds": [encoded_path]} - raise ValueError(f"OpenGauss adapter only supports PathScope depth 0/1/-1, got {expr.depth}") + raise ValueError( + f"OpenGauss adapter only supports PathScope depth 0/1/-1, got {expr.depth}" + ) raise TypeError(f"Unsupported filter expr type: {type(expr)!r}") diff --git a/openviking/storage/vectordb_adapters/qdrant_adapter.py b/openviking/storage/vectordb_adapters/qdrant_adapter.py index 40ee4fff47..743d7f06f8 100644 --- a/openviking/storage/vectordb_adapters/qdrant_adapter.py +++ b/openviking/storage/vectordb_adapters/qdrant_adapter.py @@ -62,17 +62,15 @@ def __init__( def from_config(cls, config: Any): cfg = getattr(config, "qdrant", None) params = dict(getattr(config, "custom_params", {}) or {}) - url = ( - getattr(cfg, "url", None) - or getattr(config, "url", None) - or params.get("url") - ) + url = getattr(cfg, "url", None) or getattr(config, "url", None) or params.get("url") if not url: raise ValueError("Qdrant backend requires qdrant.url or url") return cls( url=str(url).strip().rstrip("/"), api_key=getattr(cfg, "api_key", None) or params.get("api_key"), - timeout_seconds=int(getattr(cfg, "timeout_seconds", None) or params.get("timeout_seconds") or 10), + timeout_seconds=int( + getattr(cfg, "timeout_seconds", None) or params.get("timeout_seconds") or 10 + ), project_name=config.project_name or "default", collection_name=config.name or "context", index_name=config.index_name or "default", @@ -132,7 +130,9 @@ def _sanitize_scalar_index_fields( fields_meta: list[dict[str, Any]], ) -> list[str]: text_fields = self.DEFAULT_TEXT_INDEX_FIELDS if self._enable_text_index else [] - merged = list(dict.fromkeys(list(scalar_index_fields) + self.INTERNAL_PATH_FIELDS + text_fields)) + merged = list( + dict.fromkeys(list(scalar_index_fields) + self.INTERNAL_PATH_FIELDS + text_fields) + ) return merged def _build_default_index_meta( @@ -201,7 +201,9 @@ def _normalize_record_for_write(self, record: Dict[str, Any]) -> Dict[str, Any]: normalized["uri"] = normalized_uri normalized["parent_uri"] = self._compute_parent_uri(normalized_uri) normalized["scope_roots"] = self._compute_scope_roots(normalized_uri) - normalized["uri_depth"] = len([part for part in normalized_uri.strip("/").split("/") if part]) + normalized["uri_depth"] = len( + [part for part in normalized_uri.strip("/").split("/") if part] + ) return normalized def _normalize_record_for_read(self, record: Dict[str, Any]) -> Dict[str, Any]: @@ -266,17 +268,27 @@ def _compile_legacy_dict_filter(self, payload: Dict[str, Any]) -> Dict[str, Any] if op == "and": return self._must_clause( - *(self._compile_legacy_dict_filter(cond) for cond in payload.get("conds", []) if cond) + *( + self._compile_legacy_dict_filter(cond) + for cond in payload.get("conds", []) + if cond + ) ) if op == "or": return self._should_clause( - *(self._compile_legacy_dict_filter(cond) for cond in payload.get("conds", []) if cond) + *( + self._compile_legacy_dict_filter(cond) + for cond in payload.get("conds", []) + if cond + ) ) if op == "must": field = payload.get("field") values = payload.get("conds", []) or [] if field in self._URI_FIELD_NAMES: - values = [self._normalize_path(self._encode_uri_field_value(value)) for value in values] + values = [ + self._normalize_path(self._encode_uri_field_value(value)) for value in values + ] if len(values) <= 1: value = values[0] if values else None return {"must": [self._match_condition(field, value)]} if value is not None else {} @@ -285,7 +297,9 @@ def _compile_legacy_dict_filter(self, payload: Dict[str, Any]) -> Dict[str, Any] field = payload.get("field") values = payload.get("conds", []) or [] if field in self._URI_FIELD_NAMES: - values = [self._normalize_path(self._encode_uri_field_value(value)) for value in values] + values = [ + self._normalize_path(self._encode_uri_field_value(value)) for value in values + ] condition = ( self._match_any_condition(field, list(values)) if len(values) > 1 @@ -307,7 +321,9 @@ def _compile_legacy_dict_filter(self, payload: Dict[str, Any]) -> Dict[str, Any] branches.append({"must": [self._range_condition(field, {"gt": payload["lte"]})]}) return self._should_clause(*branches) if op == "contains": - return {"must": [self._text_condition(payload.get("field"), payload.get("substring", ""))]} + return { + "must": [self._text_condition(payload.get("field"), payload.get("substring", ""))] + } if op == "prefix": field = payload.get("field") prefix = payload.get("prefix", "") @@ -340,7 +356,9 @@ def _compile_filter(self, expr: FilterExpr | Dict[str, Any] | None) -> Dict[str, if isinstance(expr, In): values = list(expr.values) if expr.field in self._URI_FIELD_NAMES: - values = [self._normalize_path(self._encode_uri_field_value(value)) for value in values] + values = [ + self._normalize_path(self._encode_uri_field_value(value)) for value in values + ] if len(values) == 1: return {"must": [self._match_condition(expr.field, values[0])]} return {"must": [self._match_any_condition(expr.field, values)]} @@ -372,5 +390,7 @@ def _compile_filter(self, expr: FilterExpr | Dict[str, Any] | None) -> Dict[str, return {"must": [self._match_condition("parent_uri", encoded_path)]} if expr.depth == -1: return {"must": [self._match_condition("scope_roots", encoded_path)]} - raise ValueError(f"Qdrant adapter only supports PathScope depth 0/1/-1, got {expr.depth}") + raise ValueError( + f"Qdrant adapter only supports PathScope depth 0/1/-1, got {expr.depth}" + ) raise TypeError(f"Unsupported filter expr type: {type(expr)!r}") diff --git a/openviking/storage/vectordb_adapters/vikingdb_private_adapter.py b/openviking/storage/vectordb_adapters/vikingdb_private_adapter.py index d6908c62ea..76ae56e071 100644 --- a/openviking/storage/vectordb_adapters/vikingdb_private_adapter.py +++ b/openviking/storage/vectordb_adapters/vikingdb_private_adapter.py @@ -10,13 +10,14 @@ from openviking.storage.vectordb.collection.vikingdb_clients import VIKINGDB_APIS, VikingDBClient from openviking.storage.vectordb.collection.vikingdb_collection import VikingDBCollection -from .base import CollectionAdapter +from .base import VIKINGDB_TEXT_FIELD_BYTE_LIMIT, CollectionAdapter class VikingDBPrivateCollectionAdapter(CollectionAdapter): """Adapter for private VikingDB deployment.""" _DATA_BATCH_SIZE = 100 + _TEXT_FIELD_BYTE_LIMIT = VIKINGDB_TEXT_FIELD_BYTE_LIMIT def __init__( self, diff --git a/openviking/storage/vectordb_adapters/volcengine_adapter.py b/openviking/storage/vectordb_adapters/volcengine_adapter.py index 9128b50979..747f686199 100644 --- a/openviking/storage/vectordb_adapters/volcengine_adapter.py +++ b/openviking/storage/vectordb_adapters/volcengine_adapter.py @@ -15,13 +15,14 @@ get_or_create_volcengine_collection, ) -from .base import CollectionAdapter +from .base import VIKINGDB_TEXT_FIELD_BYTE_LIMIT, CollectionAdapter class VolcengineCollectionAdapter(CollectionAdapter): """Adapter for Volcengine-hosted VikingDB.""" _DATA_BATCH_SIZE = 100 + _TEXT_FIELD_BYTE_LIMIT = VIKINGDB_TEXT_FIELD_BYTE_LIMIT def __init__( self, diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py index d67aff52fb..3f34bc9ae6 100644 --- a/openviking/storage/viking_fs.py +++ b/openviking/storage/viking_fs.py @@ -18,6 +18,7 @@ import json import os import re +import time from contextlib import contextmanager from dataclasses import dataclass, field from datetime import datetime, timezone @@ -58,13 +59,14 @@ PermissionDeniedError, ) from openviking_cli.session.user_id import UserIdentifier +from openviking_cli.utils.config.grep_config import GrepEngine from openviking_cli.utils.logger import get_logger from openviking_cli.utils.uri import VikingURI if TYPE_CHECKING: from openviking.storage.transaction.lock_handle import LockHandle from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend - from openviking_cli.utils.config import RerankConfig, RetrievalConfig + from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig logger = get_logger(__name__) @@ -126,6 +128,7 @@ def _get_abstract_worker_count() -> int: _ABSTRACT_WORKER_COUNT = _get_abstract_worker_count() +_DEFAULT_GREP_FILE_CONCURRENCY = 32 # ========== Dataclass ========== @@ -164,6 +167,7 @@ def init_viking_fs( rerank_config: Optional["RerankConfig"] = None, vector_store: Optional["VikingVectorIndexBackend"] = None, retrieval_config: Optional["RetrievalConfig"] = None, + grep_config: Optional["GrepConfig"] = None, timeout: int = 10, enable_recorder: bool = False, encryptor: Optional[Any] = None, @@ -172,10 +176,10 @@ def init_viking_fs( Args: agfs: Pre-initialized AGFS client (HTTP or Binding) - agfs_config: AGFS configuration object for backend settings query_embedder: Embedder instance rerank_config: Rerank configuration retrieval_config: Retrieval ranking configuration + grep_config: Grep engine configuration vector_store: Vector store instance enable_recorder: Whether to enable IO recording encryptor: FileEncryptor instance for encryption/decryption @@ -188,6 +192,7 @@ def init_viking_fs( rerank_config=rerank_config, vector_store=vector_store, retrieval_config=retrieval_config, + grep_config=grep_config, encryptor=encryptor, ) @@ -260,6 +265,7 @@ def __init__( rerank_config: Optional["RerankConfig"] = None, vector_store: Optional["VikingVectorIndexBackend"] = None, retrieval_config: Optional["RetrievalConfig"] = None, + grep_config: Optional["GrepConfig"] = None, timeout: int = 10, encryptor: Optional[Any] = None, ): @@ -269,7 +275,11 @@ def __init__( self.rerank_config = rerank_config self.vector_store = vector_store self.retrieval_config = retrieval_config + self.grep_config = grep_config self._encryptor = encryptor + self._count_cache: Dict[str, tuple] = {} # cache_key → (count, timestamp) + self._count_cache_max_size = 1024 + self._fulltext_available: Optional[bool] = None # cached result of _collection_has_fulltext self._bound_ctx: contextvars.ContextVar[Optional[RequestContext]] = contextvars.ContextVar( "vikingfs_bound_ctx", default=None ) @@ -749,13 +759,17 @@ async def grep( exclude_uri: Optional[str] = None, case_insensitive: bool = False, node_limit: Optional[int] = None, - level_limit: int = 5, + level_limit: int = 10, ctx: Optional[RequestContext] = None, ) -> Dict: """Content search by pattern or keywords. + Optimized implementation that uses agfs native grep when possible. The ragfs layer greps transparently over encrypted and plaintext files (it decrypts via account_id when an encryption layer is configured). + Falls back to VikingFS layer implementation if native grep is unavailable. + When engine="auto" and vikingdb is available with sufficient data, + uses vikingdb bm25 recall + local fs precise matching. Args: uri: Viking URI @@ -765,14 +779,156 @@ async def grep( node_limit: Maximum number of results to return level_limit: Maximum depth level to traverse (default: 5) ctx: Request context + Internal bm25 recall limit is auto-adapted from node_limit as + min(node_limit * 5, 100000); when node_limit is unset, use 100000. Returns: Dict with matches, count, match_count, files_scanned """ self._ensure_access(uri, ctx) - await self.stat(uri, ctx=ctx) + # Skip vector_store.count() — the count field is not needed for grep, + # and avoiding it saves one VikingDB API call. + await self.stat(uri, ctx=ctx, skip_count=True) + + # Read engine and threshold from grep_config (ov.conf) + engine = self.grep_config.engine if self.grep_config else "auto" + switch_to_remote_threshold = ( + self.grep_config.switch_to_remote_threshold if self.grep_config else 1000 + ) + + resolved_engine = await self._resolve_grep_engine( + engine, uri, ctx, switch_to_remote_threshold + ) + + if resolved_engine == "fs": + return await self._grep_fs( + uri=uri, + pattern=pattern, + exclude_uri=exclude_uri, + case_insensitive=case_insensitive, + node_limit=node_limit, + level_limit=level_limit, + ctx=ctx, + ) + else: # "vikingdb_then_fs" + return await self._grep_vikingdb_then_fs( + uri=uri, + pattern=pattern, + exclude_uri=exclude_uri, + case_insensitive=case_insensitive, + node_limit=node_limit, + level_limit=level_limit, + ctx=ctx, + ) + + async def _resolve_grep_engine( + self, engine: GrepEngine, uri: str, ctx, switch_to_remote_threshold: int = 1000 + ) -> str: + """Resolve the actual grep engine to use.""" + if engine == "fs": + return "fs" + + # auto mode: check vikingdb availability + vector_store = self._get_vector_store() + if not vector_store: + return "fs" + + backend_type = getattr(vector_store, "_backend_type", "unknown") + if backend_type not in ("volcengine", "vikingdb"): + return "fs" + + # Check collection has content field and FullText config + if not await self._collection_has_fulltext(vector_store, ctx): + return "fs" - return await self._grep_with_agfs( + # switch_to_remote_threshold=0 means always use vikingdb + if switch_to_remote_threshold == 0: + return "vikingdb_then_fs" + + # Check data volume threshold + try: + count = await self._get_cached_count(uri, ctx) + if count < switch_to_remote_threshold: + return "fs" + except Exception: + logger.debug( + "grep engine=auto: count() check failed, falling back to fs", exc_info=True + ) + return "fs" + + return "vikingdb_then_fs" + + async def _collection_has_fulltext(self, vector_store, ctx) -> bool: + """Check if collection has content field and FullText config. + + Result is cached on the VikingFS instance since collection schema + does not change at runtime. + """ + if self._fulltext_available is not None: + return self._fulltext_available + try: + meta = None + if hasattr(vector_store, "get_collection_meta"): + meta = await vector_store.get_collection_meta(ctx=ctx) + if not meta: + self._fulltext_available = False + return False + fields = meta.get("Fields", []) + has_content = any( + f.get("FieldName") == "content" and f.get("FieldType") == "text" for f in fields + ) + fulltext = meta.get("FullText") or [] + has_content_fulltext = any(ft.get("Field") == "content" for ft in fulltext) + result = has_content and has_content_fulltext + self._fulltext_available = result + return result + except Exception: + logger.debug( + "Failed to check collection fulltext config, assuming no fulltext", exc_info=True + ) + return False + + async def _get_cached_count(self, uri: str, ctx) -> int: + """Get cached count of records for a URI (TTL=1h).""" + _COUNT_CACHE_TTL = 3600 + vector_store = self._get_vector_store() + + # Include account_id in cache key for multi-tenant safety + account_id = getattr(ctx, "account_id", None) if ctx else None + cache_key = f"{account_id}:{uri}" if account_id else uri + + now = time.time() + cached = self._count_cache.get(cache_key) + if cached and (now - cached[1]) < _COUNT_CACHE_TTL: + return cached[0] + + count = await vector_store.count(filter=PathScope("uri", uri, depth=-1), ctx=ctx) + # Evict oldest entries if cache exceeds max size + if len(self._count_cache) >= self._count_cache_max_size: + oldest_keys = sorted(self._count_cache, key=lambda k: self._count_cache[k][1]) + for k in oldest_keys[: len(oldest_keys) // 2]: + del self._count_cache[k] + self._count_cache[cache_key] = (count, now) + return count + + async def _grep_fs( + self, uri, pattern, exclude_uri, case_insensitive, node_limit, level_limit, ctx + ): + """Filesystem grep path: prefer native agfs grep and fall back if unavailable.""" + try: + return await self._grep_with_agfs( + uri=uri, + pattern=pattern, + exclude_uri=exclude_uri, + case_insensitive=case_insensitive, + node_limit=node_limit, + level_limit=level_limit, + ctx=ctx, + ) + except (AttributeError, AGFSNotSupportedError, NotImplementedError) as e: + logger.debug(f"agfs grep unavailable, falling back to VikingFS implementation: {e}") + + return await self._grep_encrypted( uri=uri, pattern=pattern, exclude_uri=exclude_uri, @@ -782,6 +938,108 @@ async def grep( ctx=ctx, ) + async def _grep_vikingdb_then_fs( + self, + uri, + pattern, + exclude_uri, + case_insensitive, + node_limit, + level_limit, + ctx, + ): + """VikingDB bm25 recall + local fs precise matching.""" + vector_store = self._get_vector_store() + + # Split regex alternation (e.g. "error|warning|fail") and join as a + # single query string for bm25 search. VikingDB's standard tokenizer + # will handle the tokenization of the query string. + query = " ".join(kw.strip() for kw in pattern.split("|") if kw.strip()) + filter_expr = PathScope("uri", uri, depth=level_limit) + + # Auto-adapt bm25 recall limit: recall up to 5x requested matches + # while capping at VikingDB's max limit. If node_limit is unset, + # use the maximum limit to avoid truncation. + remote_return_limit = min(node_limit * 5, 100000) if node_limit else 100000 + + # Step 1: vikingdb recall candidate files + try: + result = await vector_store.search_by_keywords( + query=query, + limit=remote_return_limit, + filter=filter_expr, + output_fields=["uri"], + ctx=ctx, + ) + except Exception as e: + logger.warning(f"grep vikingdb step failed, falling back to fs: {e}") + return await self._grep_fs( + uri=uri, + pattern=pattern, + exclude_uri=exclude_uri, + case_insensitive=case_insensitive, + node_limit=node_limit, + level_limit=level_limit, + ctx=ctx, + ) + + candidate_uris = [r["uri"] for r in result if r.get("uri")] + if exclude_uri: + candidate_uris = [u for u in candidate_uris if not u.startswith(exclude_uri)] + if not candidate_uris: + # BM25 returned no candidates — the index confirms no matching content + return {"matches": [], "count": 0, "match_count": 0, "files_scanned": 0} + + # Step 2: local fs precise matching on candidate files + return await self._grep_in_files( + candidate_uris, + pattern, + case_insensitive, + node_limit, + ctx, + ) + + async def _grep_in_files( + self, + file_uris: List[str], + pattern: str, + case_insensitive: bool, + node_limit: Optional[int], + ctx: Optional[RequestContext], + ) -> Dict: + """Execute regex matching in specified file list (vikingdb_then_fs Step 2).""" + flags = re.IGNORECASE if case_insensitive else 0 + compiled = re.compile(pattern, flags) + + results = [] + files_scanned = 0 + + for file_uri in file_uris: + files_scanned += 1 + try: + content_bytes = await self.read(file_uri, ctx=ctx) + content = content_bytes.decode("utf-8", errors="replace") + except Exception: + continue + + for line_no, line in enumerate(content.splitlines(), 1): + if compiled.search(line): + results.append({"uri": file_uri, "line": line_no, "content": line}) + if node_limit and len(results) >= node_limit: + return { + "matches": results, + "count": len(results), + "match_count": len(results), + "files_scanned": files_scanned, + } + + return { + "matches": results, + "count": len(results), + "match_count": len(results), + "files_scanned": files_scanned, + } + async def _grep_with_agfs( self, uri: str, @@ -789,7 +1047,7 @@ async def _grep_with_agfs( exclude_uri: Optional[str] = None, case_insensitive: bool = False, node_limit: Optional[int] = None, - level_limit: int = 5, + level_limit: int = 10, ctx: Optional[RequestContext] = None, ) -> Dict: """Grep using agfs native implementation. @@ -883,6 +1141,166 @@ async def _grep_with_agfs( "files_scanned": files_scanned, } + async def _grep_encrypted( + self, + uri: str, + pattern: str, + exclude_uri: Optional[str] = None, + case_insensitive: bool = False, + node_limit: Optional[int] = None, + level_limit: int = 10, + ctx: Optional[RequestContext] = None, + ) -> Dict: + """Grep implementation for encrypted files. + + This implementation decrypts files at VikingFS layer before matching. + Used when encryption is enabled or when agfs.grep is not available. + + Args: + uri: Viking URI + pattern: Regular expression pattern to search for + exclude_uri: Optional URI prefix to exclude from search + case_insensitive: Whether to perform case-insensitive matching + node_limit: Maximum number of results to return + level_limit: Maximum depth level to traverse (default: 5) + ctx: Request context + + Returns: + Dict with matches, count, match_count, files_scanned + """ + flags = re.IGNORECASE if case_insensitive else 0 + compiled_pattern = re.compile(pattern, flags) + excluded_prefix = None + if exclude_uri: + excluded_prefix = self._normalize_uri(exclude_uri).rstrip("/") + self._ensure_access(excluded_prefix, ctx) + file_uris = await self._collect_grep_files( + uri, + excluded_prefix=excluded_prefix, + level_limit=level_limit, + ctx=ctx, + ) + results, files_scanned = await self._grep_files_parallel( + file_uris, + compiled_pattern=compiled_pattern, + node_limit=node_limit, + ctx=ctx, + ) + + return { + "matches": results, + "count": len(results), + "match_count": len(results), + "files_scanned": files_scanned, + } + + async def _collect_grep_files( + self, + uri: str, + excluded_prefix: Optional[str], + level_limit: int, + ctx: Optional[RequestContext] = None, + ) -> List[str]: + file_uris: List[str] = [] + + async def search_recursive(current_uri: str, current_depth: int) -> None: + if current_depth > level_limit: + return + + normalized_current_uri = self._normalize_uri(current_uri) + if excluded_prefix and ( + normalized_current_uri == excluded_prefix + or normalized_current_uri.startswith(excluded_prefix + "/") + ): + logger.debug(f"Skipping excluded uri during grep: {normalized_current_uri}") + return + + try: + entries = await self.ls(normalized_current_uri, ctx=ctx) + except Exception: + return + + for entry in entries: + entry_uri = f"{normalized_current_uri.rstrip('/')}/{entry['name']}" + if excluded_prefix and ( + entry_uri == excluded_prefix or entry_uri.startswith(excluded_prefix + "/") + ): + logger.debug(f"Skipping excluded uri during grep: {entry_uri}") + continue + + if entry.get("isDir"): + await search_recursive(entry_uri, current_depth + 1) + else: + file_uris.append(entry_uri) + + normalized_uri = self._normalize_uri(uri) + if excluded_prefix and ( + normalized_uri == excluded_prefix or normalized_uri.startswith(excluded_prefix + "/") + ): + logger.debug(f"Skipping excluded uri during grep: {normalized_uri}") + return file_uris + try: + root_stat = await self.stat(normalized_uri, ctx=ctx) + except Exception: + return file_uris + if not root_stat.get("isDir", False): + file_uris.append(normalized_uri) + return file_uris + + await search_recursive(uri, 0) + return file_uris + + async def _grep_files_parallel( + self, + file_uris: List[str], + compiled_pattern: re.Pattern, + node_limit: Optional[int], + ctx: Optional[RequestContext] = None, + ) -> tuple[List[Dict[str, Any]], int]: + results: List[Dict[str, Any]] = [] + files_scanned = 0 + for start in range(0, len(file_uris), _DEFAULT_GREP_FILE_CONCURRENCY): + batch_uris = file_uris[start : start + _DEFAULT_GREP_FILE_CONCURRENCY] + batch_jobs = [ + self._grep_single_file(entry_uri, compiled_pattern, ctx) for entry_uri in batch_uris + ] + batch_results = await asyncio.gather(*batch_jobs) + for matches, scanned_count in batch_results: + files_scanned += scanned_count + for match in matches: + results.append(match) + if node_limit and len(results) >= node_limit: + return results, files_scanned + + return results, files_scanned + + async def _grep_single_file( + self, + entry_uri: str, + compiled_pattern: re.Pattern, + ctx: Optional[RequestContext] = None, + ) -> tuple[List[Dict[str, Any]], int]: + try: + content = await self.read(entry_uri, ctx=ctx) + if isinstance(content, bytes): + content = content.decode("utf-8", errors="replace") + + matches: List[Dict[str, Any]] = [] + lines = content.split("\n") + for line_num, line in enumerate(lines, 1): + if compiled_pattern.search(line): + matches.append( + { + "line": line_num, + "uri": entry_uri, + "content": line, + } + ) + return matches, 1 + except Exception as e: + logger.debug(f"Failed to grep {entry_uri}: {e}") + return [], 1 + def _resolve_grep_match_agfs_path(self, base_path: str, match_file: str) -> str: """Resolve a grep match path (relative to query root) into a full AGFS path.""" if match_file == ".": @@ -895,7 +1313,9 @@ def _calculate_grep_match_depth(self, match_file: str) -> int: return 0 return len([part for part in match_file.split("/") if part]) - async def stat(self, uri: str, ctx: Optional[RequestContext] = None) -> Dict[str, Any]: + async def stat( + self, uri: str, ctx: Optional[RequestContext] = None, skip_count: bool = False + ) -> Dict[str, Any]: """ File/directory information. @@ -909,6 +1329,13 @@ async def stat(self, uri: str, ctx: Optional[RequestContext] = None) -> Dict[str count (int): For directories, the number of nodes in the vector index under this directory (including subdirectories). For files, this field is not included. + + Args: + uri: Viking URI + ctx: Request context + skip_count: If True, skip the vector_store.count() call for directories. + Use this when the count field is not needed (e.g. in grep) to avoid + an extra VikingDB API call. """ self._ensure_access(uri, ctx) real_ctx = self._ctx_or_default(ctx) @@ -942,7 +1369,7 @@ async def stat(self, uri: str, ctx: Optional[RequestContext] = None) -> Dict[str if isinstance(result, dict): result["isLocked"] = await self._is_path_locked_async(path) # Add count for directories if vector store available - if result.get("isDir", False): + if not skip_count and result.get("isDir", False): try: vector_store = self._get_vector_store() if vector_store: diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.py index 13440e3462..69d54b0ee4 100644 --- a/openviking/storage/viking_vector_index_backend.py +++ b/openviking/storage/viking_vector_index_backend.py @@ -70,9 +70,16 @@ "id", "uri", "level", + "name", + "description", + "tags", + "abstract", + "content", "account_id", ] +VIKINGDB_CONTENT_MAX_SIZE = 64 * 1024 + class _AsyncVectorAdapter: """Thread-offloaded facade for sync vector adapters.""" @@ -157,7 +164,24 @@ def _prepare_upsert_payload(self, data: Dict[str, Any]) -> Dict[str, Any]: """Drop runtime-only or stale legacy fields before writing back to the current schema.""" payload = {k: v for k, v in data.items() if v is not None} filtered = self._filter_known_fields(payload) - return {k: v for k, v in filtered.items() if v is not None} + result = {k: v for k, v in filtered.items() if v is not None} + + # Ensure text fields required by the schema are present (even if empty). + # VikingDB requires all schema-defined fields in upsert data. + try: + coll = self._get_collection() + meta = self._get_meta_data(coll) + for field in meta.get("Fields", []): + if field.get("FieldType") == "text" and field.get("FieldName") not in result: + result[field["FieldName"]] = "" + except Exception: + pass + + content = result.get("content") + if isinstance(content, (str, bytes)): + result["content"] = content[:VIKINGDB_CONTENT_MAX_SIZE] + + return result @staticmethod def _is_not_found_error(exc: Exception) -> bool: @@ -556,6 +580,38 @@ async def optimize(self) -> bool: logger.info("Optimization requested") return True + async def search_by_keywords( + self, + keywords: Optional[List[str]] = None, + query: Optional[str] = None, + limit: int = 10, + offset: int = 0, + filter: Optional[Dict[str, Any] | FilterExpr] = None, + output_fields: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + try: + if self._bound_account_id: + account_filter = Eq("account_id", self._bound_account_id) + if filter: + if isinstance(filter, dict): + filter = RawDSL(filter) + filter = And([account_filter, filter]) + else: + filter = account_filter + + return await asyncio.to_thread( + self._adapter.search_by_keywords, + keywords=keywords, + query=query, + limit=limit, + offset=offset, + filter=filter, + output_fields=output_fields, + ) + except Exception as e: + logger.error("Error searching by keywords: %s", e) + return [] + async def close(self) -> None: try: await self._async_adapter.call("close") @@ -609,6 +665,7 @@ def __init__(self, config: Optional[VectorDBBackendConfig]): init_cpp_logging() self._config = config + self._backend_type = config.backend # expose for engine resolution self.vector_dim = config.dimension self.distance_metric = config.distance_metric self.sparse_weight = config.sparse_weight @@ -694,8 +751,16 @@ async def collection_exists_bound(self) -> bool: async def get_collection_info(self) -> Optional[Dict[str, Any]]: return await self._get_default_backend().get_collection_info() - async def get_collection_meta(self) -> Optional[Dict[str, Any]]: - return await self._get_default_backend().get_collection_meta() + async def get_collection_meta( + self, + *, + ctx: Optional[RequestContext] = None, + ) -> Optional[Dict[str, Any]]: + if ctx: + backend = self._get_backend_for_context(ctx) + else: + backend = self._get_default_backend() + return await backend.get_collection_meta() async def update_collection_description(self, description: str) -> bool: return await self._get_default_backend().update_collection_description(description) @@ -966,6 +1031,30 @@ async def count( backend = self._get_default_backend() return await backend.count(filter=filter) + async def search_by_keywords( + self, + keywords: Optional[List[str]] = None, + query: Optional[str] = None, + limit: int = 10, + offset: int = 0, + filter: Optional[Dict[str, Any] | FilterExpr] = None, + output_fields: Optional[List[str]] = None, + *, + ctx: Optional[RequestContext] = None, + ) -> List[Dict[str, Any]]: + if ctx: + backend = self._get_backend_for_context(ctx) + else: + backend = self._get_default_backend() + return await backend.search_by_keywords( + keywords=keywords, + query=query, + limit=limit, + offset=offset, + filter=filter, + output_fields=output_fields, + ) + async def clear(self, *, ctx: Optional[RequestContext] = None) -> bool: if ctx: backend = self._get_backend_for_context(ctx) diff --git a/openviking/sync_client.py b/openviking/sync_client.py index fdbdd1cbd0..a7d60e0720 100644 --- a/openviking/sync_client.py +++ b/openviking/sync_client.py @@ -462,10 +462,18 @@ def grep( case_insensitive: bool = False, node_limit: Optional[int] = None, exclude_uri: Optional[str] = None, + level_limit: int = 5, ) -> Dict: """Content search""" return run_async( - self._async_client.grep(uri, pattern, case_insensitive, node_limit, exclude_uri) + self._async_client.grep( + uri, + pattern, + case_insensitive, + node_limit, + exclude_uri, + level_limit, + ) ) def glob(self, pattern: str, uri: str = "viking://") -> Dict: diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py index 5228255412..24c5fb1e8e 100644 --- a/openviking/utils/embedding_utils.py +++ b/openviking/utils/embedding_utils.py @@ -11,6 +11,8 @@ from datetime import datetime, timezone from typing import Any, Dict, Optional +from charset_normalizer import from_bytes + from openviking.core.context import Context, ContextLevel, ResourceContentType, Vectorize from openviking.core.namespace import context_type_for_uri, is_session_uri, owner_space_for_uri from openviking.server.identity import RequestContext @@ -232,6 +234,73 @@ async def _build_image_data_uri( return None +def _coerce_text_file_content(raw: Any) -> str: + """Coerce known text-file content returned by VikingFS into str.""" + if isinstance(raw, bytes): + return _decode_text_bytes(raw) + return raw or "" + + +def _looks_like_binary_bytes(raw: bytes) -> bool: + """Conservative binary check for unknown file bytes.""" + if not raw: + return False + if b"\x00" in raw[:4096]: + return True + + allowed_controls = {9, 10, 12, 13} + sample = raw[:4096] + control_count = sum(byte < 32 and byte not in allowed_controls for byte in sample) + return control_count / len(sample) > 0.3 + + +def _decode_text_bytes(raw: bytes) -> str: + """Decode file bytes for BM25 content. + + Prefer UTF-8. If UTF-8 fails, reject binary-looking bytes, then try charset + sniffing. Return an empty string when no text encoding can be recognized. + """ + if not raw: + return "" + + try: + return raw.decode("utf-8") + except UnicodeDecodeError: + pass + + if _looks_like_binary_bytes(raw): + return "" + + best = from_bytes(raw).best() + if best is None: + return "" + + return str(best) + + +def _decode_unknown_file_bytes(raw: bytes) -> str: + """Decode unknown file bytes with the shared text-byte decoding strategy.""" + return _decode_text_bytes(raw) + + +async def _read_unknown_file_text_for_fulltext( + file_path: str, + viking_fs, + ctx: Optional[RequestContext], +) -> str: + """Best-effort raw file text for BM25/full-text indexing. + + This text is intentionally separate from the embedding input. Unknown file + types may still embed their generated summary, while grep/BM25 should index + the original text when the file can be read as text-like content. + """ + try: + return _decode_unknown_file_bytes(await viking_fs.read_file_bytes(file_path, ctx=ctx)) + except Exception as e: + logger.debug(f"Failed to read full-text content for {file_path}: {e}") + return "" + + async def vectorize_directory_meta( uri: str, abstract: str, @@ -276,7 +345,7 @@ async def vectorize_directory_meta( account_id=ctx.account_id, owner_space=owner_space, ) - context_abstract.set_vectorize(Vectorize(text=abstract)) + context_abstract.set_vectorize(Vectorize(text=abstract, full_text=abstract)) msg_abstract = EmbeddingMsgConverter.from_context(context_abstract) _apply_scalar_overrides( msg_abstract, @@ -309,7 +378,7 @@ async def vectorize_directory_meta( account_id=ctx.account_id, owner_space=owner_space, ) - context_overview.set_vectorize(Vectorize(text=overview)) + context_overview.set_vectorize(Vectorize(text=overview, full_text=overview)) msg_overview = EmbeddingMsgConverter.from_context(context_overview) _apply_scalar_overrides( msg_overview, @@ -338,7 +407,7 @@ async def vectorize_directory_meta( async def vectorize_file( file_path: str, - summary_dict: Dict[str, str], + summary_dict: Dict[str, Any], parent_uri: str, context_type: str = "resource", ctx: Optional[RequestContext] = None, @@ -367,6 +436,10 @@ async def vectorize_file( file_name = summary_dict.get("name") or os.path.basename(file_path) summary = summary_dict.get("summary", "") + has_reusable_content = "content" in summary_dict + reusable_content = ( + _coerce_text_file_content(summary_dict.get("content")) if has_reusable_content else "" + ) created_at, updated_at = await _resolve_context_timestamps( file_path, @@ -399,33 +472,42 @@ async def vectorize_file( logger.warning( f"Unsupported file type for {file_path}, falling back to summary for vectorization" ) - context.set_vectorize(Vectorize(text=summary)) + full_content = ( + reusable_content + if has_reusable_content + else await _read_unknown_file_text_for_fulltext(file_path, viking_fs, ctx) + ) + context.set_vectorize(Vectorize(text=summary, full_text=full_content or summary)) else: logger.warning( f"Unsupported file type for {file_path} and no summary available, skipping vectorization" ) return elif content_type == ResourceContentType.TEXT: - if summary and effective_text_source in {"summary_first", "summary_only"}: - context.set_vectorize(Vectorize(text=summary)) + # Known text files use VikingFS' text read path once, then reuse that + # content for BM25 regardless of whether embedding uses summary or raw text. + try: + content = ( + reusable_content + if has_reusable_content + else _coerce_text_file_content(await viking_fs.read_file(file_path, ctx=ctx)) + ) + except Exception as e: + logger.warning( + f"Failed to read file content for {file_path}, falling back to summary: {e}" + ) + if summary: + context.set_vectorize(Vectorize(text=summary, full_text=summary)) + else: + logger.warning(f"No summary available for {file_path}, skipping vectorization") + return else: - # Read raw file content; embedders apply their own input guard. - try: - content = await viking_fs.read_file(file_path, ctx=ctx) - if isinstance(content, bytes): - content = content.decode("utf-8", errors="replace") - context.set_vectorize(Vectorize(text=content)) - except Exception as e: - logger.warning( - f"Failed to read file content for {file_path}, falling back to summary: {e}" - ) - if summary: - context.set_vectorize(Vectorize(text=summary)) - else: - logger.warning( - f"No summary available for {file_path}, skipping vectorization" - ) - return + if summary and effective_text_source in {"summary_first", "summary_only"}: + # Use summary for vectorization, but reuse the single raw text read for BM25. + context.set_vectorize(Vectorize(text=summary, full_text=content or summary)) + else: + # Embedders apply their own input guard. + context.set_vectorize(Vectorize(text=content, full_text=content)) elif content_type == ResourceContentType.IMAGE and image_vectorization in { "image_only", "image_and_summary", @@ -445,7 +527,7 @@ async def vectorize_file( return elif summary: # For non-text files, use summary - context.set_vectorize(Vectorize(text=summary)) + context.set_vectorize(Vectorize(text=summary, full_text=summary)) else: logger.debug(f"Skipping file {file_path} (no text content or summary)") return diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.py index 67671f5336..f2ebf9e79c 100644 --- a/openviking/utils/resource_processor.py +++ b/openviking/utils/resource_processor.py @@ -305,7 +305,9 @@ async def _set_stage(stage: str) -> None: ) if not target_preexisting: await viking_fs.persist_temp_tree(temp_uri, root_uri, ctx=ctx) - await rewrite_image_uris(root_uri, ctx=ctx, lock_handle=resource_lock.handle) + await rewrite_image_uris( + root_uri, ctx=ctx, lock_handle=resource_lock.handle + ) await viking_fs.delete_temp(parse_result.temp_dir_path, ctx=ctx) temp_uri = root_uri source_committed = True diff --git a/openviking_cli/client/_http_compat.py b/openviking_cli/client/_http_compat.py index 7d63768587..fd2ba8e814 100644 --- a/openviking_cli/client/_http_compat.py +++ b/openviking_cli/client/_http_compat.py @@ -4,7 +4,7 @@ from __future__ import annotations -from typing import Any, Dict +from typing import Any, Dict, List, Optional, Union from openviking_cli._sdk_import import import_openviking_sdk from openviking_cli.exceptions import ( @@ -54,6 +54,38 @@ } +class _CompatHTTPObserver: + def __init__(self, client: "AsyncHTTPClient"): + self._client = client + + @property + def queue(self) -> Dict[str, Any]: + from openviking_cli.utils import run_async + + return run_async(self._client._get_queue_status()) + + @property + def vikingdb(self) -> Dict[str, Any]: + from openviking_cli.utils import run_async + + return run_async(self._client._get_vikingdb_status()) + + @property + def models(self) -> Dict[str, Any]: + from openviking_cli.utils import run_async + + return run_async(self._client._get_models_status()) + + @property + def system(self) -> Dict[str, Any]: + from openviking_cli.utils import run_async + + return run_async(self._client._get_system_status()) + + def is_healthy(self) -> bool: + return self.system.get("is_healthy", False) + + def _raise_legacy_exception(error: Dict[str, Any]) -> None: code = error.get("code", "UNKNOWN") message = error.get("message", "Unknown error") @@ -86,9 +118,145 @@ def _raise_legacy_exception(error: Dict[str, Any]) -> None: class AsyncHTTPClient(import_openviking_sdk().AsyncHTTPClient): + def __init__(self, *args, **kwargs): + sdk = import_openviking_sdk() + legacy_agent_id = kwargs.get("agent_id") + if legacy_agent_id is None and kwargs.get("actor_peer_id") is None: + try: + cli_config = sdk.config.load_ovcli_config() + except ValueError: + cli_config = None + if cli_config is not None: + legacy_agent_id = getattr(cli_config, "agent_id", None) + super().__init__(*args, **kwargs) + self._legacy_agent_id = legacy_agent_id + def _raise_exception(self, error: Dict[str, Any]) -> None: _raise_legacy_exception(error) + async def initialize(self) -> None: + headers: Dict[str, str] = {} + if self._api_key: + headers["X-API-Key"] = self._api_key + if self._account: + headers["X-OpenViking-Account"] = self._account + if self._user_id: + headers["X-OpenViking-User"] = self._user_id + if self._actor_peer_id: + headers["X-OpenViking-Actor-Peer"] = self._actor_peer_id + headers.update(self._extra_headers) + + from openviking_cli.client import http as legacy_http_module + + self._http = legacy_http_module.httpx.AsyncClient( + base_url=self._url, + headers=headers, + timeout=self._timeout, + params={"profile": "1"} if self._profile_enabled else None, + ) + self._observer = _CompatHTTPObserver(self) + + @staticmethod + def _normalize_context_type(value: Optional[Any]) -> Optional[Any]: + if isinstance(value, list): + return [getattr(item, "value", item) for item in value] + return getattr(value, "value", value) + + def _attach_legacy_agent_id(self, payload: Dict[str, Any]) -> None: + if self._legacy_agent_id: + payload["agent_id"] = self._legacy_agent_id + + async def find( + self, + query: str, + target_uri: Union[str, List[str]] = "", + limit: int = 10, + node_limit: Optional[int] = None, + score_threshold: Optional[float] = None, + filter: Optional[Dict[str, Any]] = None, + context_type: Optional[Any] = None, + tags: Optional[List[str]] = None, + telemetry: Any = False, + ) -> Dict[str, Any]: + actual_limit = node_limit if node_limit is not None else limit + payload = { + "query": query, + "target_uri": self._normalize_target_uri(target_uri), + "limit": actual_limit, + "score_threshold": score_threshold, + "filter": filter, + "context_type": self._normalize_context_type(context_type), + "telemetry": telemetry, + } + if tags is not None: + payload["tags"] = tags + self._attach_legacy_agent_id(payload) + response = await self._http.post("/api/v1/search/find", json=payload) + return self._handle_response_data(response).get("result", {}) + + async def search( + self, + query: str, + target_uri: Union[str, List[str]] = "", + session: Optional[Any] = None, + session_id: Optional[str] = None, + limit: int = 10, + node_limit: Optional[int] = None, + score_threshold: Optional[float] = None, + filter: Optional[Dict[str, Any]] = None, + context_type: Optional[Any] = None, + tags: Optional[List[str]] = None, + telemetry: Any = False, + ) -> Dict[str, Any]: + actual_limit = node_limit if node_limit is not None else limit + sid = session_id or (session.session_id if session else None) + payload = { + "query": query, + "target_uri": self._normalize_target_uri(target_uri), + "session_id": sid, + "limit": actual_limit, + "score_threshold": score_threshold, + "filter": filter, + "context_type": self._normalize_context_type(context_type), + "telemetry": telemetry, + } + if tags is not None: + payload["tags"] = tags + self._attach_legacy_agent_id(payload) + response = await self._http.post("/api/v1/search/search", json=payload) + return self._handle_response_data(response).get("result", {}) + + async def add_message( + self, + session_id: str, + role: str, + content: str | None = None, + parts: list[dict] | None = None, + created_at: str | None = None, + peer_id: str | None = None, + telemetry: Any = False, + ) -> Dict[str, Any]: + if self._legacy_agent_id and peer_id is not None: + raise InvalidArgumentError("peer_id cannot be used with legacy agent_id") + payload: Dict[str, Any] = {"role": role} + if parts is not None: + payload["parts"] = parts + elif content is not None: + payload["content"] = content + else: + raise ValueError("Either content or parts must be provided") + if created_at is not None: + payload["created_at"] = created_at + if peer_id is not None: + payload["peer_id"] = peer_id + if self._legacy_agent_id and role == "assistant": + payload["agent_id"] = self._legacy_agent_id + if telemetry is not False: + payload["telemetry"] = telemetry + session_path = self._path_segment(session_id) + response = await self._http.post(f"/api/v1/sessions/{session_path}/messages", json=payload) + return self._handle_response_data(response).get("result", {}) + class SyncHTTPClient(import_openviking_sdk().SyncHTTPClient): def __init__(self, *args, **kwargs): diff --git a/openviking_cli/client/base.py b/openviking_cli/client/base.py index d23a30de73..7b844cb5c9 100644 --- a/openviking_cli/client/base.py +++ b/openviking_cli/client/base.py @@ -218,6 +218,7 @@ async def grep( case_insensitive: bool = False, exclude_uri: Optional[str] = None, node_limit: Optional[int] = None, + level_limit: int = 5, ) -> Dict[str, Any]: """Content search with pattern.""" ... diff --git a/openviking_cli/doctor.py b/openviking_cli/doctor.py index cb86c83857..fa45d02b19 100644 --- a/openviking_cli/doctor.py +++ b/openviking_cli/doctor.py @@ -542,8 +542,7 @@ def check_vikingbot() -> CheckResult: return ( "warn", "bot.ov_server.api_key_type=root without root API key", - "Configure bot.ov_server.api_key with a root API key for trusted " - "OpenViking access", + "Configure bot.ov_server.api_key with a root API key for trusted OpenViking access", ) return "pass", "bot.ov_server configured for trusted OpenViking auth", None diff --git a/openviking_cli/utils/config/__init__.py b/openviking_cli/utils/config/__init__.py index 81c17e564b..31ea7202cf 100644 --- a/openviking_cli/utils/config/__init__.py +++ b/openviking_cli/utils/config/__init__.py @@ -1,5 +1,6 @@ # Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. # SPDX-License-Identifier: AGPL-3.0 +from . import embedding_config from .agfs_config import AGFSConfig from .config_loader import ( load_json_config, @@ -49,8 +50,8 @@ OPENVIKING_WRITE_CHECK_URI_ENV, SYSTEM_CONFIG_DIR, ) -from . import embedding_config from .embedding_config import EmbeddingConfig +from .grep_config import GrepConfig, GrepEngine from .log_config import LogConfig from .open_viking_config import ( OpenVikingConfig, @@ -130,6 +131,8 @@ "OpenVikingConfig", "OpenVikingConfigSingleton", "OVCLIConfig", + "GrepConfig", + "GrepEngine", "RerankConfig", "RetrievalConfig", "StorageConfig", diff --git a/openviking_cli/utils/config/grep_config.py b/openviking_cli/utils/config/grep_config.py new file mode 100644 index 0000000000..75c12c82d1 --- /dev/null +++ b/openviking_cli/utils/config/grep_config.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 + +from typing import Literal + +from pydantic import BaseModel, Field + +# Grep engine mode type alias — import this instead of repeating Literal["auto", "fs"] +GrepEngine = Literal["auto", "fs"] + + +class GrepConfig(BaseModel): + """Configuration for grep engine behavior.""" + + engine: GrepEngine = Field( + default="auto", + description=( + "Search engine mode: 'auto' uses vikingdb bm25 recall when available, " + "'fs' forces local filesystem search." + ), + ) + + switch_to_remote_threshold: int = Field( + default=10000, + ge=0, + description=( + "L2 record count threshold to switch to vikingdb; 0 means always use vikingdb." + ), + ) + + model_config = {"extra": "forbid"} diff --git a/openviking_cli/utils/config/open_viking_config.py b/openviking_cli/utils/config/open_viking_config.py index fe5a30a975..de72d14216 100644 --- a/openviking_cli/utils/config/open_viking_config.py +++ b/openviking_cli/utils/config/open_viking_config.py @@ -21,6 +21,7 @@ ) from .embedding_config import EmbeddingConfig from .encryption_config import EncryptionConfig +from .grep_config import GrepConfig from .log_config import LogConfig from .memory_config import MemoryConfig from .oauth_config import OAuthConfig @@ -135,6 +136,11 @@ class OpenVikingConfig(BaseModel): description="Retrieval ranking configuration", ) + grep: GrepConfig = Field( + default_factory=GrepConfig, + description="Grep engine configuration", + ) + # Encryption configuration encryption: EncryptionConfig = Field( default_factory=EncryptionConfig, description="Encryption configuration" diff --git a/openviking_cli/utils/config/vectordb_config.py b/openviking_cli/utils/config/vectordb_config.py index 3c7aae3138..87773553cd 100644 --- a/openviking_cli/utils/config/vectordb_config.py +++ b/openviking_cli/utils/config/vectordb_config.py @@ -105,7 +105,9 @@ class OpenGaussConfig(BaseModel): ) connect_timeout: int = Field(default=10, description="Database connection timeout in seconds") dense_vector_name: str = Field(default="vector", description="Dense vector column name") - sparse_vector_name: str = Field(default="sparse_vector", description="Sparse vector JSON column name") + sparse_vector_name: str = Field( + default="sparse_vector", description="Sparse vector JSON column name" + ) model_config = {"extra": "forbid", "populate_by_name": True} diff --git a/sdk/python/openviking_sdk/config.py b/sdk/python/openviking_sdk/config.py index 51affa2686..b36ee52270 100644 --- a/sdk/python/openviking_sdk/config.py +++ b/sdk/python/openviking_sdk/config.py @@ -123,7 +123,10 @@ def load_ovcli_config(config_path: Optional[str] = None) -> Optional[OVCLIConfig } unknown_keys = sorted(set(data) - allowed_keys) if unknown_keys: - raise ValueError(f"Unknown field 'ovcli.{unknown_keys[0]}'") + message = f"Unknown field 'ovcli.{unknown_keys[0]}'" + if unknown_keys[0] == "ur": + message += ". Did you mean 'ovcli.url'?" + raise ValueError(message) extra_header_alias = data.get("extra_header") extra_headers_value = data.get("extra_headers", extra_header_alias) diff --git a/tests/cli/test_doctor.py b/tests/cli/test_doctor.py index aad6819836..8e716dfebd 100644 --- a/tests/cli/test_doctor.py +++ b/tests/cli/test_doctor.py @@ -741,7 +741,7 @@ def test_pass_with_user_api_key(self, tmp_path: Path): "api_key": "user-key", "api_key_type": "user", } - } + }, } ) ) @@ -766,9 +766,7 @@ def test_pass_dev_mode_without_bot_ov_server(self, tmp_path: Path): def test_pass_dev_mode_with_ignored_bot_api_key(self, tmp_path: Path): config = tmp_path / "ov.conf" - config.write_text( - json.dumps({"bot": {"ov_server": {"api_key": "stale-key"}}}) - ) + config.write_text(json.dumps({"bot": {"ov_server": {"api_key": "stale-key"}}})) with patch("openviking_cli.doctor._find_config", return_value=config): status, detail, fix = check_vikingbot() diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 654dfc02f2..8f98d3d92c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -84,6 +84,7 @@ def _skip_when_qdrant_unavailable(request): if not _qdrant_available(): pytest.skip(f"Qdrant not available at {QDRANT_URL}") + # (model_name, default_dimension, token_limit) GEMINI_MODELS = [ ("gemini-embedding-2-preview", 3072, 8192), diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.py index 2639931523..8a50aa9351 100644 --- a/tests/misc/test_config_validation.py +++ b/tests/misc/test_config_validation.py @@ -14,8 +14,8 @@ create_agfs_client, mount_agfs_backend, ) -from openviking_cli.utils.config.consts import OPENVIKING_CONFIG_ENV from openviking_cli.utils.config.agfs_config import AGFSConfig, S3Config +from openviking_cli.utils.config.consts import OPENVIKING_CONFIG_ENV from openviking_cli.utils.config.embedding_config import EmbeddingConfig, EmbeddingModelConfig from openviking_cli.utils.config.vectordb_config import VectorDBBackendConfig, VolcengineConfig from openviking_cli.utils.config.vlm_config import VLMConfig diff --git a/tests/misc/test_ragfs_python_manifest_isolation.py b/tests/misc/test_ragfs_python_manifest_isolation.py index 6b9152946e..794213f426 100644 --- a/tests/misc/test_ragfs_python_manifest_isolation.py +++ b/tests/misc/test_ragfs_python_manifest_isolation.py @@ -1,7 +1,6 @@ import re from pathlib import Path - ROOT = Path(__file__).resolve().parents[2] @@ -16,7 +15,9 @@ def _array_items(text: str, key: str) -> set[str]: def _section(text: str, name: str) -> str: - match = re.search(rf"^\[{re.escape(name)}\]\s*$(.*?)(?=^\[|\Z)", text, flags=re.DOTALL | re.MULTILINE) + match = re.search( + rf"^\[{re.escape(name)}\]\s*$(.*?)(?=^\[|\Z)", text, flags=re.DOTALL | re.MULTILINE + ) assert match is not None, f"[{name}] section not found" return match.group(1) diff --git a/tests/parse/test_code_tools.py b/tests/parse/test_code_tools.py index 36a98c47f7..83ea06790f 100644 --- a/tests/parse/test_code_tools.py +++ b/tests/parse/test_code_tools.py @@ -10,7 +10,6 @@ ) from openviking.parse.parsers.code.ast.extractor import get_extractor - PY_SAMPLE = '''"""Module top doc.""" import os @@ -281,14 +280,14 @@ def test_outline_unsupported_language(self): # --------------------------------------------------------------------------- -SECOND_FILE = '''def greet(): +SECOND_FILE = """def greet(): pass class Other: def helper(self): pass -''' +""" class TestSearchSymbols: diff --git a/tests/parse/test_document_parser_threading.py b/tests/parse/test_document_parser_threading.py index 4a93050d81..c230daccab 100644 --- a/tests/parse/test_document_parser_threading.py +++ b/tests/parse/test_document_parser_threading.py @@ -80,9 +80,7 @@ def convert(path: Path, docx_module, resource_name=None, storage=None) -> str: @pytest.mark.asyncio -async def test_word_parser_forwards_original_name_to_markdown( - monkeypatch, tmp_path: Path -): +async def test_word_parser_forwards_original_name_to_markdown(monkeypatch, tmp_path: Path): # On single-file upload the on-disk path is a temp name (upload_.docx) # and the user's original filename arrives via source_name. WordParser must # forward that name to MarkdownParser explicitly; otherwise parse_content can @@ -91,9 +89,7 @@ async def test_word_parser_forwards_original_name_to_markdown( seen = _stub_markdown_parse(parser) _patch_to_thread(monkeypatch, word) monkeypatch.setitem(sys.modules, "docx", SimpleNamespace()) - monkeypatch.setattr( - parser, "_convert_to_markdown", lambda *args, **kwargs: "# converted docx" - ) + monkeypatch.setattr(parser, "_convert_to_markdown", lambda *args, **kwargs: "# converted docx") upload = tmp_path / "upload_abc123.docx" upload.write_bytes(b"placeholder") diff --git a/tests/parse/test_markdown_link_rewrite.py b/tests/parse/test_markdown_link_rewrite.py index f9ca49e619..9270b41932 100644 --- a/tests/parse/test_markdown_link_rewrite.py +++ b/tests/parse/test_markdown_link_rewrite.py @@ -37,7 +37,8 @@ async def _rewrite(self, parser, kb, content, section_subpath=""): async def test_md_target_becomes_directory(self, tmp_path: Path): kb = self._make_tree(tmp_path) out = await self._rewrite( - self._parser(), kb, + self._parser(), + kb, "见 [x](./目录甲/目录乙/目录丙/文档.md)", ) assert out == "见 [x](../目录甲/目录乙/目录丙/文档/)" @@ -45,7 +46,8 @@ async def test_md_target_becomes_directory(self, tmp_path: Path): async def test_nonempty_subpath_adds_one_more_parent(self, tmp_path: Path): kb = self._make_tree(tmp_path) out = await self._rewrite( - self._parser(), kb, + self._parser(), + kb, "[x](./目录甲/目录乙/目录丙/文档.md)", section_subpath="二、示例小节", ) @@ -62,7 +64,13 @@ async def test_image_not_ingestable_depth_adjusted(self, tmp_path: Path): async def test_external_anchor_absolute_unchanged(self, tmp_path: Path): kb = self._make_tree(tmp_path) p = self._parser() - for link in ("https://x.com/a", "viking://resources/x", "#sec", "/abs/p.md", "mailto:a@b.c"): + for link in ( + "https://x.com/a", + "viking://resources/x", + "#sec", + "/abs/p.md", + "mailto:a@b.c", + ): content = f"[t]({link})" assert await self._rewrite(p, kb, content) == content @@ -81,7 +89,8 @@ async def test_directory_target_depth_adjusted(self, tmp_path: Path): async def test_sibling_md_without_dot_prefix(self, tmp_path: Path): kb = self._make_tree(tmp_path) out = await self._rewrite( - self._parser(), kb, + self._parser(), + kb, "[x](目录甲/目录乙/目录丙/文档.md)", ) assert out == "[x](../目录甲/目录乙/目录丙/文档/)" @@ -97,7 +106,8 @@ async def test_fragment_kept_for_small_file(self, tmp_path: Path): # still resolves: point at the file and keep the fragment. kb = self._make_tree(tmp_path) out = await self._rewrite( - self._parser(), kb, + self._parser(), + kb, "[x](./目录甲/目录乙/目录丙/文档.md#流程)", ) assert out == "[x](../目录甲/目录乙/目录丙/文档/文档.md#流程)" @@ -105,7 +115,8 @@ async def test_fragment_kept_for_small_file(self, tmp_path: Path): async def test_query_suffix_kept_for_small_file(self, tmp_path: Path): kb = self._make_tree(tmp_path) out = await self._rewrite( - self._parser(), kb, + self._parser(), + kb, "[x](./目录甲/目录乙/目录丙/文档.md?v=1)", ) assert out == "[x](../目录甲/目录乙/目录丙/文档/文档.md?v=1)" @@ -121,7 +132,8 @@ async def test_large_file_anchor_located(self, tmp_path: Path): ) big.write_text(body, encoding="utf-8") out = await self._rewrite( - self._parser(), kb, + self._parser(), + kb, "[x](./目录甲/目录乙/目录丙/big.md#第3章-排查)", ) assert out.startswith("[x](../目录甲/目录乙/目录丙/big/") @@ -133,7 +145,8 @@ async def test_large_file_unlocatable_anchor_falls_back_to_dir(self, tmp_path: P big = kb / "目录甲" / "目录乙" / "目录丙" / "big.md" big.write_text("# 大文档\n\n" + ("这是一段较长的正文内容。" * 1200), encoding="utf-8") out = await self._rewrite( - self._parser(), kb, + self._parser(), + kb, "[x](./目录甲/目录乙/目录丙/big.md#不存在的章节)", ) assert out == "[x](../目录甲/目录乙/目录丙/big/)" @@ -141,7 +154,8 @@ async def test_large_file_unlocatable_anchor_falls_back_to_dir(self, tmp_path: P async def test_multiple_links_on_one_line(self, tmp_path: Path): kb = self._make_tree(tmp_path) out = await self._rewrite( - self._parser(), kb, + self._parser(), + kb, "a [1](./目录甲/目录乙/目录丙/文档.md) b ![p](./img/a.png)", ) assert out == ( @@ -164,9 +178,15 @@ async def fake_bare_layout(_path): # 模拟未来：目标入库为单个裸文 # 无 suffix → 文件本身（无尾斜杠），而非 文档/ 目录 assert await self._rewrite(p, kb, f"[x]({base})") == "[x](../目录甲/目录乙/目录丙/文档.md)" # ?query → 文件 + 保留查询串 - assert await self._rewrite(p, kb, f"[x]({base}?v=1)") == "[x](../目录甲/目录乙/目录丙/文档.md?v=1)" + assert ( + await self._rewrite(p, kb, f"[x]({base}?v=1)") + == "[x](../目录甲/目录乙/目录丙/文档.md?v=1)" + ) # #anchor → 文件 + 保留锚点（裸单文件内任意锚点仍有效） - assert await self._rewrite(p, kb, f"[x]({base}#任意)") == "[x](../目录甲/目录乙/目录丙/文档.md#任意)" + assert ( + await self._rewrite(p, kb, f"[x]({base}#任意)") + == "[x](../目录甲/目录乙/目录丙/文档.md#任意)" + ) class TestSectionSubpath: @@ -244,10 +264,10 @@ async def ls(self, uri, node_limit=None, show_all_hidden=False, **kw): children = {} for key in list(self.files.keys()) + self.dirs: if key.startswith(prefix): - rest = key[len(prefix):] + rest = key[len(prefix) :] if rest: child_name = rest.split("/")[0] - is_deeper = "/" in rest[len(child_name):] + is_deeper = "/" in rest[len(child_name) :] child_full = f"{prefix}{child_name}" is_dir = children.get(child_name, False) or is_deeper or child_full in self.dirs children[child_name] = is_dir @@ -258,11 +278,14 @@ async def ls(self, uri, node_limit=None, show_all_hidden=False, **kw): if not children[name] and name.startswith(".") and not show_all_hidden: continue child_uri = f"{uri.rstrip('/')}/{name}" - result.append({ - "name": name, "uri": child_uri, - "isDir": children[name], - "type": "directory" if children[name] else "file", - }) + result.append( + { + "name": name, + "uri": child_uri, + "isDir": children[name], + "type": "directory" if children[name] else "file", + } + ) return result async def move_file(self, from_uri, to_uri): @@ -292,9 +315,7 @@ class TestComputeLayoutPurity: async def test_compute_layout_plans_sections_without_touching_vikingfs(self, tmp_path: Path): # A multi-section document large enough to split into several section files. src = tmp_path / "big.md" - body = "".join( - f"## 第{i}章\n\n" + ("正文内容。" * 400) + "\n\n" for i in range(1, 4) - ) + body = "".join(f"## 第{i}章\n\n" + ("正文内容。" * 400) + "\n\n" for i in range(1, 4)) src.write_text(body, encoding="utf-8") fake = FakeVikingFS() @@ -320,9 +341,7 @@ async def test_parse_content_rewrites_link_when_enabled(self, tmp_path: Path): tgt.mkdir(parents=True) (tgt / "文档.md").write_text("# 目标\n\n内容", encoding="utf-8") src = kb / "文档.md" - src.write_text( - "见 [x](./目录甲/目录乙/目录丙/文档.md)", encoding="utf-8" - ) + src.write_text("见 [x](./目录甲/目录乙/目录丙/文档.md)", encoding="utf-8") fake = FakeVikingFS() with patch.object(BaseParser, "_get_viking_fs", return_value=fake): @@ -340,9 +359,7 @@ async def test_parse_content_no_rewrite_when_disabled(self, tmp_path: Path): tgt.mkdir(parents=True) (tgt / "文档.md").write_text("# 目标\n\n内容", encoding="utf-8") src = kb / "文档.md" - src.write_text( - "见 [x](./目录甲/目录乙/目录丙/文档.md)", encoding="utf-8" - ) + src.write_text("见 [x](./目录甲/目录乙/目录丙/文档.md)", encoding="utf-8") fake = FakeVikingFS() with patch.object(BaseParser, "_get_viking_fs", return_value=fake): @@ -360,9 +377,7 @@ async def test_no_rewrite_without_import_root(self, tmp_path: Path): tgt.mkdir(parents=True) (tgt / "文档.md").write_text("# 目标\n\n内容", encoding="utf-8") src = kb / "文档.md" - src.write_text( - "见 [x](./目录甲/目录乙/目录丙/文档.md)", encoding="utf-8" - ) + src.write_text("见 [x](./目录甲/目录乙/目录丙/文档.md)", encoding="utf-8") fake = FakeVikingFS() with patch.object(BaseParser, "_get_viking_fs", return_value=fake): @@ -379,9 +394,7 @@ async def test_directory_ingest_rewrites_cross_file_link(self, tmp_path: Path): tgt = kb / "目录甲" / "目录乙" / "目录丙" tgt.mkdir(parents=True) (tgt / "文档.md").write_text("# 目标\n\n内容", encoding="utf-8") - (kb / "文档.md").write_text( - "见 [x](./目录甲/目录乙/目录丙/文档.md)", encoding="utf-8" - ) + (kb / "文档.md").write_text("见 [x](./目录甲/目录乙/目录丙/文档.md)", encoding="utf-8") fake = FakeVikingFS() with patch.object(BaseParser, "_get_viking_fs", return_value=fake): @@ -473,9 +486,7 @@ async def test_html_img_ingested_and_left_for_rewrite(self, tmp_path: Path): (kb / "img").mkdir(parents=True) _write_valid_png(kb / "img" / "photo.png") src = kb / "page.md" - src.write_text( - '', encoding="utf-8" - ) + src.write_text('', encoding="utf-8") fake = FakeVikingFS() with patch.object(BaseParser, "_get_viking_fs", return_value=fake): @@ -486,11 +497,7 @@ async def test_html_img_ingested_and_left_for_rewrite(self, tmp_path: Path): md = [_decode(c) for u, c in fake.files.items() if u.endswith(".md")] assert md and '' in md[0], fake.files assert any(u.endswith("/photo.png") for u in fake.files), fake.files - mapping = [ - _decode(c) - for u, c in fake.files.items() - if u.endswith(".image_mappings.json") - ] + mapping = [_decode(c) for u, c in fake.files.items() if u.endswith(".image_mappings.json")] assert mapping and "./img/photo.png" in mapping[0], fake.files async def test_html_img_outside_import_root_depth_adjusted(self, tmp_path: Path): @@ -513,9 +520,7 @@ async def test_html_img_outside_import_root_depth_adjusted(self, tmp_path: Path) md = [_decode(c) for u, c in fake.files.items() if u.endswith(".md")] assert md and '' in md[0], fake.files - async def test_directory_ingest_image_within_import_root_becomes_viking( - self, tmp_path: Path - ): + async def test_directory_ingest_image_within_import_root_becomes_viking(self, tmp_path: Path): # Directory ingest passes the import root as allowed_media_dirs, so an # image outside the md's own dir but inside the ingested tree IS taken: # copied next to the section and rewritten to a viking:// URI. @@ -526,8 +531,7 @@ async def test_directory_ingest_image_within_import_root_becomes_viking( (kb / "guides").mkdir() _write_valid_png(kb / "images" / "photo.png") (kb / "guides" / "page.md").write_text( - "![p](../images/photo.png)\n\n" - '', + '![p](../images/photo.png)\n\n', encoding="utf-8", ) @@ -542,7 +546,7 @@ async def test_directory_ingest_image_within_import_root_becomes_viking( src_prefix = doc_dirs[0]["uri"].rstrip("/") + "/" for u in list(fake.files): if u.startswith(src_prefix): - fake.files[f"{root}/{u[len(src_prefix):]}"] = fake.files[u] + fake.files[f"{root}/{u[len(src_prefix) :]}"] = fake.files[u] import openviking.parse.image_rewrite as image_rewrite_mod @@ -600,9 +604,7 @@ async def test_nested_mapping_consumed(self): stats = await rewrite_image_uris(root, lock_handle=None) assert stats == {"files_processed": 1, "references_rewritten": 1} - assert _decode(fake.files[f"{root}/index/index.md"]) == ( - f"![p]({root}/index/logo.png)" - ) + assert _decode(fake.files[f"{root}/index/index.md"]) == (f"![p]({root}/index/logo.png)") assert f"{root}/index/.image_mappings.json" not in fake.files async def test_split_doc_mapping_keys_resolved(self): @@ -694,16 +696,17 @@ async def test_sync_path_carries_nested_mappings_to_target(self): } processor = SemanticProcessor.__new__(SemanticProcessor) - with patch.object(sp_mod, "get_viking_fs", return_value=fake), patch.object( - __import__("openviking.parse.image_rewrite", fromlist=["x"]), - "get_viking_fs", - return_value=fake, + with ( + patch.object(sp_mod, "get_viking_fs", return_value=fake), + patch.object( + __import__("openviking.parse.image_rewrite", fromlist=["x"]), + "get_viking_fs", + return_value=fake, + ), ): await processor._rewrite_target_image_uris(root, target) - assert _decode(fake.files[f"{target}/index/index.md"]) == ( - f"![p]({target}/index/logo.png)" - ) + assert _decode(fake.files[f"{target}/index/index.md"]) == (f"![p]({target}/index/logo.png)") async def test_directory_ingest_images_become_viking_uris(self, tmp_path: Path): # End to end: directory ingest -> persist -> rewrite. Every md referencing @@ -730,7 +733,7 @@ async def test_directory_ingest_images_become_viking_uris(self, tmp_path: Path): src_prefix = doc_dirs[0]["uri"].rstrip("/") + "/" for u in list(fake.files): if u.startswith(src_prefix): - fake.files[f"{root}/{u[len(src_prefix):]}"] = fake.files[u] + fake.files[f"{root}/{u[len(src_prefix) :]}"] = fake.files[u] with self._patched(fake): stats = await rewrite_image_uris(root, lock_handle=None) @@ -741,6 +744,5 @@ async def test_directory_ingest_images_become_viking_uris(self, tmp_path: Path): assert f"![logo]({root}/index/logo.png)" in index_md, index_md assert f"![logo2]({root}/guide/logo.png)" in guide_md, guide_md assert not any( - u.endswith(".image_mappings.json") and u.startswith(root) - for u in fake.files + u.endswith(".image_mappings.json") and u.startswith(root) for u in fake.files ), fake.files diff --git a/tests/parse/test_media_resource_name.py b/tests/parse/test_media_resource_name.py index 1e9be92a31..bd189d716f 100644 --- a/tests/parse/test_media_resource_name.py +++ b/tests/parse/test_media_resource_name.py @@ -42,31 +42,51 @@ def _fake_viking_fs() -> MagicMock: def test_resolve_uses_resource_name(): fp = Path("upload_0123456789abcdef.png") - assert resolve_media_names(fp, ".png", resource_name="vacation") == ("vacation", "vacation", "vacation.png") + assert resolve_media_names(fp, ".png", resource_name="vacation") == ( + "vacation", + "vacation", + "vacation.png", + ) def test_resolve_resource_name_filename_is_not_double_extended(): # A filename-like resource_name is reduced to its stem ("My Holiday.png" -> # "My_Holiday.png", not "My_Holiday.png.png"). fp = Path("upload_x.png") - assert resolve_media_names(fp, ".png", resource_name="My Holiday.png") == ("My Holiday", "My_Holiday", "My_Holiday.png") + assert resolve_media_names(fp, ".png", resource_name="My Holiday.png") == ( + "My Holiday", + "My_Holiday", + "My_Holiday.png", + ) def test_resolve_source_name_stem_when_no_resource_name(): fp = Path("upload_x.png") - assert resolve_media_names(fp, ".png", source_name="My Holiday.png") == ("My Holiday", "My_Holiday", "My_Holiday.png") + assert resolve_media_names(fp, ".png", source_name="My Holiday.png") == ( + "My Holiday", + "My_Holiday", + "My_Holiday.png", + ) def test_resolve_preserves_non_media_dotted_name(): # A name that merely contains a dot (not a media extension) is kept intact — # only a real media extension is stripped, so "meeting.v1" is not truncated. fp = Path("upload_x.png") - assert resolve_media_names(fp, ".png", resource_name="meeting.v1") == ("meeting.v1", "meeting.v1", "meeting.v1.png") + assert resolve_media_names(fp, ".png", resource_name="meeting.v1") == ( + "meeting.v1", + "meeting.v1", + "meeting.v1.png", + ) def test_resolve_empty_resource_name_falls_through_to_source_name(): fp = Path("upload_x.png") - assert resolve_media_names(fp, ".png", resource_name="", source_name="clip.png") == ("clip", "clip", "clip.png") + assert resolve_media_names(fp, ".png", resource_name="", source_name="clip.png") == ( + "clip", + "clip", + "clip.png", + ) def test_resolve_falls_back_to_temp_name_when_no_caller_name(): diff --git a/tests/server/conftest.py b/tests/server/conftest.py index a2ea735768..5a092d97c9 100644 --- a/tests/server/conftest.py +++ b/tests/server/conftest.py @@ -154,9 +154,9 @@ async def service(temp_dir: Path, monkeypatch): @pytest_asyncio.fixture(scope="function") async def app(service: OpenVikingService): """Create FastAPI app with pre-initialized service (no auth).""" - from openviking.server.dependencies import set_service from openviking.server.auth.plugins import DevAuthPlugin from openviking.server.auth.registry import get_registry + from openviking.server.dependencies import set_service config = ServerConfig() fastapi_app = create_app(config=config, service=service) diff --git a/tests/server/oauth/test_storage.py b/tests/server/oauth/test_storage.py index 443dbabdc3..c7d9685e8c 100644 --- a/tests/server/oauth/test_storage.py +++ b/tests/server/oauth/test_storage.py @@ -138,9 +138,7 @@ async def test_auth_code_concurrent_consume_race(store): """Two coroutines racing to consume the same code — exactly one wins.""" code = "race-code" await _insert_code(store, code) - results = await asyncio.gather( - store.consume_auth_code(code), store.consume_auth_code(code) - ) + results = await asyncio.gather(store.consume_auth_code(code), store.consume_auth_code(code)) winners = [r for r in results if r is not None] assert len(winners) == 1 diff --git a/tests/server/test_admin_api.py b/tests/server/test_admin_api.py index ffb2254998..1079d92435 100644 --- a/tests/server/test_admin_api.py +++ b/tests/server/test_admin_api.py @@ -81,9 +81,9 @@ async def initialize_user_directories(self, ctx): def _build_lightweight_admin_test_app() -> FastAPI: - from openviking.server.routers import admin as admin_router from openviking.server.auth.plugins import ApiKeyAuthPlugin from openviking.server.auth.registry import get_registry + from openviking.server.routers import admin as admin_router app = FastAPI() app.state.config = ServerConfig(root_api_key=ROOT_KEY) diff --git a/tests/server/test_api_code.py b/tests/server/test_api_code.py index 614b72391f..9dabebabb3 100644 --- a/tests/server/test_api_code.py +++ b/tests/server/test_api_code.py @@ -2,8 +2,6 @@ # SPDX-License-Identifier: AGPL-3.0 """Tests for /api/v1/code/* endpoints.""" -import pytest - from openviking_cli.exceptions import PermissionDeniedError PY_SAMPLE = '''"""Module top doc.""" @@ -54,9 +52,7 @@ async def fake_read(uri, ctx=None, **_): monkeypatch.setattr(service.fs, "read", fake_read) - resp = await client.post( - "/api/v1/code/outline", json={"uri": "viking://resources/x.py"} - ) + resp = await client.post("/api/v1/code/outline", json={"uri": "viking://resources/x.py"}) assert resp.status_code == 403 body = resp.json() assert body["status"] == "error" @@ -80,9 +76,7 @@ async def fake_read(uri, ctx=None, **_): monkeypatch.setattr(service.fs, "read", fake_read) - resp = await client.post( - "/api/v1/code/outline", json={"uri": "viking://resources/x.py"} - ) + resp = await client.post("/api/v1/code/outline", json={"uri": "viking://resources/x.py"}) assert resp.status_code == 200 assert "is not text" in resp.json()["result"] @@ -116,18 +110,14 @@ async def fake_read(uri, ctx=None, **_): assert "viking://r/a.py" in body["result"] async def test_invalid_uri(self, client): - resp = await client.post( - "/api/v1/code/search", json={"uri": "/tmp/dir", "query": "foo"} - ) + resp = await client.post("/api/v1/code/search", json={"uri": "/tmp/dir", "query": "foo"}) assert resp.status_code == 200 body = resp.json() assert body["result"].startswith("Error:") assert "viking://" in body["result"] async def test_empty_query(self, client): - resp = await client.post( - "/api/v1/code/search", json={"uri": "viking://r", "query": ""} - ) + resp = await client.post("/api/v1/code/search", json={"uri": "viking://r", "query": ""}) assert resp.status_code == 200 assert resp.json()["result"] == "Error: empty query" @@ -137,9 +127,7 @@ async def fake_ls(uri, ctx=None, recursive=False, output=None, **_): monkeypatch.setattr(service.fs, "ls", fake_ls) - resp = await client.post( - "/api/v1/code/search", json={"uri": "viking://r", "query": "foo"} - ) + resp = await client.post("/api/v1/code/search", json={"uri": "viking://r", "query": "foo"}) assert resp.status_code == 200 assert "No supported source files" in resp.json()["result"] @@ -149,9 +137,7 @@ async def fake_ls(uri, ctx=None, recursive=False, output=None, **_): monkeypatch.setattr(service.fs, "ls", fake_ls) - resp = await client.post( - "/api/v1/code/search", json={"uri": "viking://r", "query": "foo"} - ) + resp = await client.post("/api/v1/code/search", json={"uri": "viking://r", "query": "foo"}) assert resp.status_code == 403 body = resp.json() assert body["status"] == "error" @@ -252,9 +238,7 @@ async def fake_read(uri, ctx=None, **_): assert "def greet" in resp.json()["result"] async def test_invalid_uri(self, client): - resp = await client.post( - "/api/v1/code/expand", json={"uri": "/tmp/x.py", "symbol": "foo"} - ) + resp = await client.post("/api/v1/code/expand", json={"uri": "/tmp/x.py", "symbol": "foo"}) assert resp.status_code == 200 body = resp.json() assert body["result"].startswith("Error:") diff --git a/tests/server/test_auth.py b/tests/server/test_auth.py index ad4375ea1c..20da833453 100644 --- a/tests/server/test_auth.py +++ b/tests/server/test_auth.py @@ -90,7 +90,6 @@ def _make_request( ) -> Request: """Create a minimal Starlette request for auth dependency tests.""" # Ensure built-in plugins are registered - from openviking.server.auth.plugins import DevAuthPlugin, ApiKeyAuthPlugin, TrustedAuthPlugin raw_headers = [] for key, value in (headers or {}).items(): @@ -129,7 +128,6 @@ def _build_auth_http_test_app( the test focused on request auth behavior and the structured HTTP error body. """ # Ensure built-in plugins are registered - from openviking.server.auth.plugins import DevAuthPlugin, ApiKeyAuthPlugin, TrustedAuthPlugin app = FastAPI() # When auth is disabled and mode is the default api_key, fall back to dev mode diff --git a/tests/service/test_reindex_placeholder.py b/tests/service/test_reindex_placeholder.py index 260ee8eea3..68804afc1f 100644 --- a/tests/service/test_reindex_placeholder.py +++ b/tests/service/test_reindex_placeholder.py @@ -27,7 +27,9 @@ def test_real_overview_sentinel_is_detected(): def test_sentinel_detection_is_uri_agnostic(): for uri in ("viking://a/b/", "viking://x%20y/z", "viking://"): - assert _is_not_ready_sentinel(f"# {uri} [Directory abstract is not ready]", _ABSTRACT_NOT_READY_SUFFIX) + assert _is_not_ready_sentinel( + f"# {uri} [Directory abstract is not ready]", _ABSTRACT_NOT_READY_SUFFIX + ) def test_content_mentioning_phrase_mid_body_is_preserved(): @@ -42,7 +44,9 @@ def test_authored_content_ending_with_marker_is_preserved(): def test_real_content_preserved(): - assert not _is_not_ready_sentinel("# Project\n\nReal abstract body.", _ABSTRACT_NOT_READY_SUFFIX) + assert not _is_not_ready_sentinel( + "# Project\n\nReal abstract body.", _ABSTRACT_NOT_READY_SUFFIX + ) def test_empty_is_not_a_sentinel(): diff --git a/tests/session/memory/test_json_stability.py b/tests/session/memory/test_json_stability.py index f31b36aa5e..ca9a6150de 100644 --- a/tests/session/memory/test_json_stability.py +++ b/tests/session/memory/test_json_stability.py @@ -5,7 +5,6 @@ """ import json -import logging from typing import List, Optional from unittest.mock import patch @@ -190,7 +189,7 @@ class OperationsLike(BaseModel): OperationsLike._allow_empty_list_response = True - data, error = parse_json_with_stability('[]', model_class=OperationsLike) + data, error = parse_json_with_stability("[]", model_class=OperationsLike) assert error is None assert data.tags == [] @@ -205,7 +204,7 @@ def is_empty(self) -> bool: return not self.tags for model in (self.TestModel, HasIsEmptyButNotOptedIn): - data, error = parse_json_with_stability('[]', model_class=model) + data, error = parse_json_with_stability("[]", model_class=model) assert data is None assert error is not None @@ -392,7 +391,9 @@ class PreferenceOperations(BaseModel): } ) - with patch("openviking.session.memory.utils.json_parser.logger.exception") as mock_exception: + with patch( + "openviking.session.memory.utils.json_parser.logger.exception" + ) as mock_exception: data, error = parse_json_with_stability(content, model_class=PreferenceOperations) assert error is None diff --git a/tests/session/memory/test_memory_isolation_handler.py b/tests/session/memory/test_memory_isolation_handler.py index ebe8ba518c..19cd661ed0 100644 --- a/tests/session/memory/test_memory_isolation_handler.py +++ b/tests/session/memory/test_memory_isolation_handler.py @@ -522,8 +522,6 @@ def test_calculate_memory_uris_missing_peer_id_falls_back_to_first_peer_when_sel uris = handler.calculate_memory_uris(schema, operation, extract_ctx) - assert uris == [ - "viking://user/support_bot/peers/web-visitor-bob/memories/preferences" - ] + assert uris == ["viking://user/support_bot/peers/web-visitor-bob/memories/preferences"] assert operation.memory_fields["user_id"] == "support_bot" assert operation.memory_fields["peer_id"] == "web-visitor-bob" diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.py index c158321e39..4efac92215 100644 --- a/tests/storage/test_collection_schemas.py +++ b/tests/storage/test_collection_schemas.py @@ -18,7 +18,6 @@ _build_embedding_metadata, init_context_collection, ) -from openviking.storage.errors import EmbeddingRebuildRequiredError from openviking.storage.expr import Eq from openviking.storage.queuefs.embedding_msg import EmbeddingMsg from openviking.storage.vectordb import engine as vectordb_engine @@ -199,7 +198,10 @@ async def update_collection_description(self, description): @pytest.mark.asyncio -async def test_init_context_collection_rejects_mismatched_nonempty_collection(monkeypatch): +async def test_init_context_collection_warns_on_mismatched_nonempty_collection(monkeypatch): + """When embedding metadata mismatches for a non-empty collection, the function + logs a warning and returns False (does not raise).""" + class _FakeStorage: async def create_collection(self, name, schema): del name, schema @@ -227,8 +229,8 @@ async def update_collection_description(self, description): # pragma: no cover lambda: config, ) - with pytest.raises(EmbeddingRebuildRequiredError, match="Rebuild is required"): - await init_context_collection(_FakeStorage()) + result = await init_context_collection(_FakeStorage()) + assert result is False def test_build_embedding_metadata_hashes_resolved_local_model_path(tmp_path): @@ -794,6 +796,53 @@ def upsert(self, data): } +@pytest.mark.asyncio +async def test_single_account_backend_truncates_content_only_at_vector_write(): + captured = {} + full_content = "x" * (64 * 1024 + 17) + + class _Collection: + def get_meta_data(self): + return { + "Fields": [ + {"FieldName": "id"}, + {"FieldName": "uri"}, + {"FieldName": "abstract"}, + {"FieldName": "content", "FieldType": "text"}, + {"FieldName": "account_id"}, + ] + } + + class _Adapter: + mode = "local" + + def get_collection(self): + return _Collection() + + def upsert(self, data): + captured["data"] = dict(data) + return [data["id"]] + + backend = _SingleAccountBackend( + config=VectorDBBackendConfig(backend="local", name="context", dimension=2), + bound_account_id="acc1", + shared_adapter=_Adapter(), + ) + source_data = { + "id": "rec-large", + "uri": "viking://resources/large.txt", + "abstract": "sample", + "content": full_content, + "account_id": "acc1", + } + + record_id = await backend.upsert(source_data) + + assert record_id == "rec-large" + assert source_data["content"] == full_content + assert captured["data"]["content"] == full_content[: 64 * 1024] + + @pytest.mark.asyncio async def test_single_account_backend_collection_exists_runs_in_threadpool(monkeypatch): called = {} diff --git a/tests/storage/test_embedding_msg_converter_tenant.py b/tests/storage/test_embedding_msg_converter_tenant.py index 558ddc68d9..61759548bb 100644 --- a/tests/storage/test_embedding_msg_converter_tenant.py +++ b/tests/storage/test_embedding_msg_converter_tenant.py @@ -5,7 +5,7 @@ import pytest -from openviking.core.context import Context +from openviking.core.context import Context, Vectorize from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter from openviking_cli.session.user_id import UserIdentifier @@ -45,3 +45,15 @@ def test_embedding_msg_converter_backfills_account_and_owner_fields( expected_owner_user_id(user) if callable(expected_owner_user_id) else expected_owner_user_id ) assert msg.context_data["owner_user_id"] == expected_user + + +def test_embedding_msg_converter_preserves_full_content_without_vikingdb_truncation(): + full_content = "x" * (64 * 1024 + 17) + context = Context(uri="viking://resources/large.txt", abstract="short embedding text") + context.set_vectorize(Vectorize(text="short embedding text", full_text=full_content)) + + msg = EmbeddingMsgConverter.from_context(context) + + assert msg is not None + assert msg.message == "short embedding text" + assert msg.context_data["content"] == full_content diff --git a/tests/storage/test_ingest_ls_node_limit.py b/tests/storage/test_ingest_ls_node_limit.py index b4c8d4401d..bfbf6c7142 100644 --- a/tests/storage/test_ingest_ls_node_limit.py +++ b/tests/storage/test_ingest_ls_node_limit.py @@ -33,9 +33,7 @@ class _TruncatingVikingFS: TARGET = "viking://resources/wixqa" def __init__(self, n_children: int): - self._children = [ - {"name": f"doc{i:05d}", "isDir": True} for i in range(n_children) - ] + self._children = [{"name": f"doc{i:05d}", "isDir": True} for i in range(n_children)] self.moved: list[tuple[str, str]] = [] async def exists(self, uri, ctx=None): @@ -86,9 +84,7 @@ class _TruncatingLsFS: def __init__(self, dir_uri: str, n_children: int): self._dir_uri = dir_uri - self._children = [ - {"name": f"doc{i:05d}", "isDir": True} for i in range(n_children) - ] + self._children = [{"name": f"doc{i:05d}", "isDir": True} for i in range(n_children)] async def ls(self, uri, node_limit=1000, ctx=None): entries = self._children if uri == self._dir_uri else [] diff --git a/tests/storage/test_local_collection_update.py b/tests/storage/test_local_collection_update.py index 11a3fbdd27..9ebca720c1 100644 --- a/tests/storage/test_local_collection_update.py +++ b/tests/storage/test_local_collection_update.py @@ -36,9 +36,10 @@ def _build_local_collection( ], ids_not_exist=[key for key in primary_keys if key not in existing_by_id], ) - collection._write_data_list = lambda data_list, ttl=0: captured.append( - {"data_list": data_list, "ttl": ttl} - ) or UpsertDataResult(ids=[row["id"] for row in data_list]) + collection._write_data_list = lambda data_list, ttl=0: ( + captured.append({"data_list": data_list, "ttl": ttl}) + or UpsertDataResult(ids=[row["id"] for row in data_list]) + ) return collection diff --git a/tests/storage/test_opengauss_adapter.py b/tests/storage/test_opengauss_adapter.py index b25c0139e5..c831ac7f8d 100644 --- a/tests/storage/test_opengauss_adapter.py +++ b/tests/storage/test_opengauss_adapter.py @@ -223,7 +223,9 @@ def test_drop_index_removes_vector_and_scalar_indexes(): "IndexName": index_name, "ScalarIndex": ["uri", "account_id"], } - collection._delete_index_meta = lambda index_name: statements.append(("delete_meta", index_name)) + collection._delete_index_meta = lambda index_name: statements.append( + ("delete_meta", index_name) + ) collection._execute = lambda sql, params=None, fetch=False: statements.append(sql) collection.drop_index("default") @@ -260,7 +262,9 @@ def test_create_index_normalizes_metadata_to_hnsw(): collection = object.__new__(OpenGaussCollection) saved = {} - collection._create_vector_index = lambda index_name, distance, meta: saved.setdefault("vector_meta", meta) + collection._create_vector_index = lambda index_name, distance, meta: saved.setdefault( + "vector_meta", meta + ) collection._create_scalar_index = lambda index_name, field_name: None collection._save_index_meta = lambda index_name, meta: saved.setdefault("saved_meta", meta) @@ -285,7 +289,9 @@ def fail_vector_index(index_name, distance, meta): raise RuntimeError("boom") collection._create_vector_index = fail_vector_index - collection._create_scalar_index = lambda index_name, field_name: saved.setdefault("scalar", field_name) + collection._create_scalar_index = lambda index_name, field_name: saved.setdefault( + "scalar", field_name + ) collection._save_index_meta = lambda index_name, meta: saved.setdefault("saved_meta", meta) with pytest.raises(RuntimeError, match="boom"): diff --git a/tests/storage/test_rebuild_schema.py b/tests/storage/test_rebuild_schema.py index df40ef57b5..aee7a157cf 100644 --- a/tests/storage/test_rebuild_schema.py +++ b/tests/storage/test_rebuild_schema.py @@ -3,8 +3,20 @@ from openviking.storage.collection_schemas import CollectionSchemas -def test_context_collection_does_not_contain_rebuild_content_snapshot_field(): +def test_context_collection_contains_content_field_for_fulltext(): schema = CollectionSchemas.context_collection("ctx", 8) field_names = {field["FieldName"] for field in schema["Fields"]} - assert "content" not in field_names + # content field is required for VikingDB FullText (bm25) search + assert "content" in field_names + # embedding_content is not a schema field assert "embedding_content" not in field_names + # FullText config must reference the content field + fulltext_cfg = schema.get("FullText", []) + fulltext_fields = [ft["Field"] for ft in fulltext_cfg] + assert "content" in fulltext_fields + + # Analyzer config must include tokenizer + stopwords filter + content_cfg = next(ft for ft in fulltext_cfg if ft.get("Field") == "content") + analyzer = content_cfg.get("Analyzer") or {} + assert analyzer.get("Tokenizer") == "standard" + assert analyzer.get("StopWordsFilters") == ["symbol"] diff --git a/tests/storage/test_semantic_processor_language.py b/tests/storage/test_semantic_processor_language.py index fd3b00f918..d06909f973 100644 --- a/tests/storage/test_semantic_processor_language.py +++ b/tests/storage/test_semantic_processor_language.py @@ -123,10 +123,7 @@ def test_detect_language_english_with_single_korean_char(self): assert language == "en" def test_detect_language_italian(self): - text = ( - "Questo documento descrive le preferenze dell utente " - "e il progetto da completare." - ) + text = "Questo documento descrive le preferenze dell utente e il progetto da completare." language = _detect_language_from_text(text, fallback_language="it") assert language == "it" @@ -143,9 +140,15 @@ def test_strong_italian_text_can_override_system_fallback(self): [ ("project document user data model profile", "en"), ("Ce document décrit les préférences de l utilisateur et le projet à terminer.", "fr"), - ("Este documento describe las preferencias del usuario y el proyecto para completar.", "es"), + ( + "Este documento describe las preferencias del usuario y el proyecto para completar.", + "es", + ), ("Dieses Dokument beschreibt die Präferenzen der Benutzer und das Projekt.", "de"), - ("Este documento descreve as preferências do usuário e o projeto para completar.", "pt"), + ( + "Este documento descreve as preferências do usuário e o projeto para completar.", + "pt", + ), ], ) def test_detect_latin_language_conservatively(self, text, expected): @@ -357,15 +360,19 @@ async def test_e2e_code_output_language( mock_viking_fs = self._create_mock_viking_fs(content) mock_config = self._create_mock_config(mock_vlm) - with patch.dict( - os.environ, - {"LC_ALL": self._LANGUAGE_LOCALE[expected_lang]}, - ), patch( - "openviking.storage.queuefs.semantic_processor.get_viking_fs", - return_value=mock_viking_fs, - ), patch( - "openviking.storage.queuefs.semantic_processor.get_openviking_config", - return_value=mock_config, + with ( + patch.dict( + os.environ, + {"LC_ALL": self._LANGUAGE_LOCALE[expected_lang]}, + ), + patch( + "openviking.storage.queuefs.semantic_processor.get_viking_fs", + return_value=mock_viking_fs, + ), + patch( + "openviking.storage.queuefs.semantic_processor.get_openviking_config", + return_value=mock_config, + ), ): processor = SemanticProcessor() processor._current_ctx = MagicMock() @@ -384,6 +391,7 @@ async def test_e2e_code_output_language( assert _verify_content_language(result["summary"], expected_lang), ( f"{file_name}: Content language mismatch. Expected {expected_lang}, got: {result['summary']}" ) + assert result["content"] == content @pytest.mark.asyncio @pytest.mark.parametrize( @@ -401,15 +409,19 @@ async def test_e2e_russian_arabic_output_language(self, content, file_name, expe mock_viking_fs = self._create_mock_viking_fs(content) mock_config = self._create_mock_config(mock_vlm) - with patch.dict( - os.environ, - {"LC_ALL": self._LANGUAGE_LOCALE[expected_lang]}, - ), patch( - "openviking.storage.queuefs.semantic_processor.get_viking_fs", - return_value=mock_viking_fs, - ), patch( - "openviking.storage.queuefs.semantic_processor.get_openviking_config", - return_value=mock_config, + with ( + patch.dict( + os.environ, + {"LC_ALL": self._LANGUAGE_LOCALE[expected_lang]}, + ), + patch( + "openviking.storage.queuefs.semantic_processor.get_viking_fs", + return_value=mock_viking_fs, + ), + patch( + "openviking.storage.queuefs.semantic_processor.get_openviking_config", + return_value=mock_config, + ), ): processor = SemanticProcessor() processor._current_ctx = MagicMock() @@ -517,12 +529,16 @@ def test_non_english_locale_hint_wins_over_timezone(self): assert _resolve_system_fallback_language("en") == "ja" def test_local_timezone_hint_used_when_tz_env_absent(self): - with patch.dict(os.environ, {}, clear=True), patch( - "openviking.session.memory.utils.language.locale.getlocale", - return_value=("C", "UTF-8"), - ), patch( - "openviking.session.memory.utils.language.os.path.realpath", - return_value="/usr/share/zoneinfo.default/Asia/Shanghai", + with ( + patch.dict(os.environ, {}, clear=True), + patch( + "openviking.session.memory.utils.language.locale.getlocale", + return_value=("C", "UTF-8"), + ), + patch( + "openviking.session.memory.utils.language.os.path.realpath", + return_value="/usr/share/zoneinfo.default/Asia/Shanghai", + ), ): assert _resolve_system_fallback_language("en") == "zh-CN" diff --git a/tests/storage/test_viking_fs_grep.py b/tests/storage/test_viking_fs_grep.py index ee74a9b310..11c7f10e6d 100644 --- a/tests/storage/test_viking_fs_grep.py +++ b/tests/storage/test_viking_fs_grep.py @@ -1,15 +1,28 @@ # Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. # SPDX-License-Identifier: AGPL-3.0 +import time + import pytest -from openviking.storage.viking_fs import VikingFS +import openviking.storage.viking_fs as viking_fs_module +from openviking.storage.viking_fs import _DEFAULT_GREP_FILE_CONCURRENCY, VikingFS +from openviking_cli.utils.config.grep_config import GrepConfig class _DummyAgfs: pass +class _DummyVectorStore: + def __init__(self): + self.calls = [] + + async def search_by_keywords(self, **kwargs): + self.calls.append(kwargs) + return [] + + @pytest.fixture def fs(monkeypatch): viking_fs = VikingFS(agfs=_DummyAgfs()) @@ -27,10 +40,225 @@ def fs(monkeypatch): return viking_fs -async def _fake_stat(uri, ctx=None): +async def _fake_stat(uri, ctx=None, skip_count=False): return {"name": uri.rsplit("/", 1)[-1], "isDir": True} +def test_grep_config_default_switch_to_remote_threshold_is_10000(): + assert GrepConfig().switch_to_remote_threshold == 10000 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("node_limit", "expected_remote_limit"), + [ + (7, 35), + (None, 100000), + (50000, 100000), + ], +) +async def test_grep_vikingdb_auto_remote_limit_uses_five_times_node_limit( + monkeypatch, node_limit, expected_remote_limit +): + fs = VikingFS(agfs=_DummyAgfs()) + vector_store = _DummyVectorStore() + monkeypatch.setattr(fs, "_get_vector_store", lambda: vector_store) + + result = await fs._grep_vikingdb_then_fs( + uri="viking://resources", + pattern="needle", + exclude_uri=None, + case_insensitive=False, + node_limit=node_limit, + level_limit=10, + ctx=None, + ) + + assert result == {"matches": [], "count": 0, "match_count": 0, "files_scanned": 0} + assert vector_store.calls[0]["limit"] == expected_remote_limit + + +@pytest.mark.asyncio +async def test_grep_preserves_dfs_order_and_node_limit(monkeypatch): + fs = VikingFS(agfs=_DummyAgfs()) + + async def fake_stat(uri, ctx=None, skip_count=False): + return {"isDir": True} + + async def fake_ls(uri, ctx=None, **kwargs): + entries = { + "viking://resources": [ + {"name": "dir_a", "isDir": True}, + {"name": "dir_b", "isDir": True}, + ], + "viking://resources/dir_a": [ + {"name": "a1.md", "isDir": False}, + {"name": "a2.md", "isDir": False}, + ], + "viking://resources/dir_b": [ + {"name": "b1.md", "isDir": False}, + ], + } + return entries.get(uri, []) + + def fake_agfs_read(path, offset=0, size=-1): + contents = { + "/resources/dir_a/a1.md": "match a1 line1\nskip\nmatch a1 line3", + "/resources/dir_a/a2.md": "match a2 line1", + "/resources/dir_b/b1.md": "match b1 line1", + } + return contents[path].encode() + + monkeypatch.setattr(fs, "stat", fake_stat) + monkeypatch.setattr(fs, "ls", fake_ls) + monkeypatch.setattr( + fs, + "_uri_to_path", + lambda uri, ctx=None: uri.replace("viking://", "/"), + ) + monkeypatch.setattr(fs.agfs, "read", fake_agfs_read, raising=False) + + result = await fs.grep("viking://resources", pattern="match", node_limit=3) + + assert result["count"] == 3 + assert result["files_scanned"] == 2 + assert result["matches"] == [ + { + "line": 1, + "uri": "viking://resources/dir_a/a1.md", + "content": "match a1 line1", + }, + { + "line": 3, + "uri": "viking://resources/dir_a/a1.md", + "content": "match a1 line3", + }, + { + "line": 1, + "uri": "viking://resources/dir_a/a2.md", + "content": "match a2 line1", + }, + ] + + +@pytest.mark.asyncio +async def test_grep_parallel_reads_respect_concurrency_limit(monkeypatch): + fs = VikingFS(agfs=_DummyAgfs()) + + async def fake_stat(uri, ctx=None, skip_count=False): + return {"isDir": True} + + async def fake_ls(uri, ctx=None, **kwargs): + entries = { + "viking://resources": [{"name": f"file{i}.md", "isDir": False} for i in range(12)] + } + return entries.get(uri, []) + + active_reads = 0 + max_active_reads = 0 + + def fake_agfs_read(path, offset=0, size=-1): + nonlocal active_reads, max_active_reads + active_reads += 1 + max_active_reads = max(max_active_reads, active_reads) + time.sleep(0.01) + active_reads -= 1 + return f"match from {path}".encode() + + monkeypatch.setattr(fs, "stat", fake_stat) + monkeypatch.setattr(fs, "ls", fake_ls) + monkeypatch.setattr( + fs, + "_uri_to_path", + lambda uri, ctx=None: uri.replace("viking://", "/"), + ) + monkeypatch.setattr(fs.agfs, "read", fake_agfs_read, raising=False) + + result = await fs.grep("viking://resources", pattern="match") + + assert result["count"] == 12 + assert result["files_scanned"] == 12 + assert max_active_reads > 1 + assert max_active_reads <= min(12, _DEFAULT_GREP_FILE_CONCURRENCY) + + +@pytest.mark.asyncio +async def test_grep_parallel_reads_work_with_blocking_agfs_read(monkeypatch): + fs = VikingFS(agfs=_DummyAgfs()) + + async def fake_stat(uri, ctx=None, skip_count=False): + return {"isDir": True} + + async def fake_ls(uri, ctx=None, **kwargs): + if uri == "viking://resources": + return [{"name": f"file{i}.md", "isDir": False} for i in range(8)] + return [] + + def fake_agfs_read(path, offset=0, size=-1): + time.sleep(0.05) + return f"match from {path}".encode() + + monkeypatch.setattr(fs, "stat", fake_stat) + monkeypatch.setattr(fs, "ls", fake_ls) + monkeypatch.setattr( + fs, + "_uri_to_path", + lambda uri, ctx=None: uri.replace("viking://", "/"), + ) + monkeypatch.setattr(fs.agfs, "read", fake_agfs_read, raising=False) + + started = time.perf_counter() + result = await fs.grep("viking://resources", pattern="match") + elapsed = time.perf_counter() - started + + assert result["count"] == 8 + assert result["files_scanned"] == 8 + assert elapsed < 0.30 + + +@pytest.mark.asyncio +async def test_grep_stops_scheduling_later_batches_after_node_limit(monkeypatch): + fs = VikingFS(agfs=_DummyAgfs()) + + async def fake_stat(uri, ctx=None, skip_count=False): + return {"isDir": True} + + async def fake_ls(uri, ctx=None, **kwargs): + if uri == "viking://resources": + return [{"name": f"file{i}.md", "isDir": False} for i in range(6)] + return [] + + read_paths = [] + + def fake_agfs_read(path, offset=0, size=-1): + read_paths.append(path) + contents = { + "/resources/file0.md": "match file0 line1\nmatch file0 line2", + "/resources/file1.md": "match file1 line1", + "/resources/file2.md": "match file2 line1", + "/resources/file3.md": "match file3 line1", + "/resources/file4.md": "match file4 line1", + "/resources/file5.md": "match file5 line1", + } + return contents[path].encode() + + monkeypatch.setattr(fs, "stat", fake_stat) + monkeypatch.setattr(fs, "ls", fake_ls) + monkeypatch.setattr( + fs, + "_uri_to_path", + lambda uri, ctx=None: uri.replace("viking://", "/"), + ) + monkeypatch.setattr(fs.agfs, "read", fake_agfs_read, raising=False) + monkeypatch.setattr(viking_fs_module, "_DEFAULT_GREP_FILE_CONCURRENCY", 2) + + result = await fs.grep("viking://resources", pattern="match", node_limit=2) + + assert result["count"] == 2 + assert result["files_scanned"] == 1 + assert read_paths == ["/resources/file0.md", "/resources/file1.md"] + + @pytest.mark.asyncio async def test_grep_delegates_to_agfs_with_expected_filters(monkeypatch, fs): calls = [] diff --git a/tests/unit/test_locomo_peer_wiring.py b/tests/unit/test_locomo_peer_wiring.py index d70c86a059..3f1f4a22ed 100644 --- a/tests/unit/test_locomo_peer_wiring.py +++ b/tests/unit/test_locomo_peer_wiring.py @@ -67,8 +67,6 @@ def test_build_session_messages_non_group_uses_sample_peer_and_prefixes_speaker( assert messages[1]["text"] == "Bob: Hello Alice" - - @pytest.mark.asyncio async def test_viking_ingest_uses_message_peer_id(monkeypatch): calls = [] diff --git a/tests/unit/test_namespace_uri_classification.py b/tests/unit/test_namespace_uri_classification.py index 8be3124419..7f8312532b 100644 --- a/tests/unit/test_namespace_uri_classification.py +++ b/tests/unit/test_namespace_uri_classification.py @@ -146,4 +146,3 @@ def test_current_user_short_content_roots_are_canonicalized_from_content_segment assert is_content_namespace_root_uri("viking://resources", ctx) assert is_content_root_uri("viking://resources", ctx, kind="resource") assert not is_content_namespace_root_uri("viking://user/resources/docs", ctx) - diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.py index 5389379636..4613b6e202 100644 --- a/tests/unit/test_vectorize_file_strategy.py +++ b/tests/unit/test_vectorize_file_strategy.py @@ -27,10 +27,19 @@ def get_queue(self, _name): class DummyFS: def __init__(self, content): self.content = content + self.read_file_calls = 0 + self.read_file_bytes_calls = 0 async def read_file(self, _path, ctx=None): + self.read_file_calls += 1 return self.content + async def read_file_bytes(self, _path, ctx=None): + self.read_file_bytes_calls += 1 + if isinstance(self.content, bytes): + return self.content + return str(self.content).encode("utf-8") + async def exists(self, _path, ctx=None): return False @@ -45,6 +54,9 @@ class DummyUser: def user_space_name(self): return "default" + def to_dict(self): + return {"account_id": self.account_id, "user_id": self.user_id} + class DummyReq: def __init__(self): @@ -82,6 +94,266 @@ async def test_vectorize_file_uses_summary_first(monkeypatch): assert queue.items[0].get_vectorization_text() == "short summary" +@pytest.mark.asyncio +async def test_vectorize_unknown_text_file_embeds_summary_but_indexes_raw_content(monkeypatch): + queue = DummyQueue() + raw_makefile = "build:\n\tcargo build --locked\n" + monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue)) + monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: DummyFS(raw_makefile)) + monkeypatch.setattr( + embedding_utils, + "get_openviking_config", + lambda: types.SimpleNamespace( + embedding=types.SimpleNamespace(text_source="summary_first", max_input_tokens=1000) + ), + ) + + await embedding_utils.vectorize_file( + file_path="viking://user/default/resources/Makefile", + summary_dict={"name": "Makefile", "summary": "VLM generated build file summary"}, + parent_uri="viking://user/default/resources", + ctx=DummyReq(), + ) + + assert len(queue.items) == 1 + msg = queue.items[0] + assert msg.message == "VLM generated build file summary" + assert msg.context_data["content"] == raw_makefile + + +@pytest.mark.asyncio +async def test_vectorize_unknown_text_file_sniffs_non_utf8_raw_content(monkeypatch): + queue = DummyQueue() + raw_content = ( + "# 构建脚本\n" + "目标: 编译项目\n" + "说明: 这是一个中文 Makefile 内容，用于测试编码探测。\n" + "命令: cargo build --locked\n" + ) + fs = DummyFS(raw_content.encode("gb18030")) + monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue)) + monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: fs) + monkeypatch.setattr( + embedding_utils, + "get_openviking_config", + lambda: types.SimpleNamespace( + embedding=types.SimpleNamespace(text_source="summary_first", max_input_tokens=1000) + ), + ) + + await embedding_utils.vectorize_file( + file_path="viking://user/default/resources/Makefile", + summary_dict={"name": "Makefile", "summary": "VLM generated build file summary"}, + parent_uri="viking://user/default/resources", + ctx=DummyReq(), + ) + + assert len(queue.items) == 1 + msg = queue.items[0] + assert msg.message == "VLM generated build file summary" + assert msg.context_data["content"] == raw_content + assert fs.read_file_bytes_calls == 1 + assert fs.read_file_calls == 0 + + +@pytest.mark.asyncio +async def test_vectorize_unknown_file_reuses_summary_content_without_reread(monkeypatch): + queue = DummyQueue() + raw_content = "build:\n\tcargo build --locked\n" + fs = DummyFS("should not be read") + monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue)) + monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: fs) + monkeypatch.setattr( + embedding_utils, + "get_openviking_config", + lambda: types.SimpleNamespace( + embedding=types.SimpleNamespace(text_source="summary_first", max_input_tokens=1000) + ), + ) + + await embedding_utils.vectorize_file( + file_path="viking://user/default/resources/Makefile", + summary_dict={ + "name": "Makefile", + "summary": "VLM generated build file summary", + "content": raw_content, + }, + parent_uri="viking://user/default/resources", + ctx=DummyReq(), + ) + + assert len(queue.items) == 1 + msg = queue.items[0] + assert msg.message == "VLM generated build file summary" + assert msg.context_data["content"] == raw_content + assert fs.read_file_bytes_calls == 0 + assert fs.read_file_calls == 0 + + +@pytest.mark.asyncio +async def test_vectorize_unknown_binary_file_falls_back_to_summary(monkeypatch): + queue = DummyQueue() + summary = "VLM generated binary file summary" + binary_content = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01" + fs = DummyFS(binary_content) + monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue)) + monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: fs) + monkeypatch.setattr( + embedding_utils, + "get_openviking_config", + lambda: types.SimpleNamespace( + embedding=types.SimpleNamespace(text_source="summary_first", max_input_tokens=1000) + ), + ) + + await embedding_utils.vectorize_file( + file_path="viking://user/default/resources/model.weights", + summary_dict={"name": "model.weights", "summary": summary}, + parent_uri="viking://user/default/resources", + ctx=DummyReq(), + ) + + assert len(queue.items) == 1 + msg = queue.items[0] + assert msg.message == summary + assert msg.context_data["content"] == summary + assert fs.read_file_bytes_calls == 1 + assert fs.read_file_calls == 0 + + +@pytest.mark.asyncio +async def test_vectorize_unknown_unrecognizable_encoding_falls_back_to_summary(monkeypatch): + queue = DummyQueue() + summary = "VLM generated unknown file summary" + fs = DummyFS(b"\xff\xfe\xfd") + monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue)) + monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: fs) + monkeypatch.setattr( + embedding_utils, + "get_openviking_config", + lambda: types.SimpleNamespace( + embedding=types.SimpleNamespace(text_source="summary_first", max_input_tokens=1000) + ), + ) + monkeypatch.setattr( + embedding_utils, + "from_bytes", + lambda _raw: types.SimpleNamespace(best=lambda: None), + ) + + await embedding_utils.vectorize_file( + file_path="viking://user/default/resources/unknown.data", + summary_dict={"name": "unknown.data", "summary": summary}, + parent_uri="viking://user/default/resources", + ctx=DummyReq(), + ) + + assert len(queue.items) == 1 + msg = queue.items[0] + assert msg.message == summary + assert msg.context_data["content"] == summary + assert fs.read_file_bytes_calls == 1 + assert fs.read_file_calls == 0 + + +@pytest.mark.asyncio +async def test_vectorize_text_summary_first_reuses_single_file_read(monkeypatch): + queue = DummyQueue() + fs = DummyFS("# README\nraw text for bm25\n") + monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue)) + monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: fs) + monkeypatch.setattr( + embedding_utils, + "get_openviking_config", + lambda: types.SimpleNamespace( + embedding=types.SimpleNamespace(text_source="summary_first", max_input_tokens=1000) + ), + ) + + await embedding_utils.vectorize_file( + file_path="viking://user/default/resources/README.md", + summary_dict={"name": "README.md", "summary": "summary for embedding"}, + parent_uri="viking://user/default/resources", + ctx=DummyReq(), + ) + + assert len(queue.items) == 1 + msg = queue.items[0] + assert msg.message == "summary for embedding" + assert msg.context_data["content"] == "# README\nraw text for bm25\n" + assert fs.read_file_calls == 1 + assert fs.read_file_bytes_calls == 0 + + +@pytest.mark.asyncio +async def test_vectorize_text_file_reuses_summary_content_without_reread(monkeypatch): + queue = DummyQueue() + raw_content = "# README\nraw text already read during summary\n" + fs = DummyFS("should not be read") + monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue)) + monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: fs) + monkeypatch.setattr( + embedding_utils, + "get_openviking_config", + lambda: types.SimpleNamespace( + embedding=types.SimpleNamespace(text_source="summary_first", max_input_tokens=1000) + ), + ) + + await embedding_utils.vectorize_file( + file_path="viking://user/default/resources/README.md", + summary_dict={ + "name": "README.md", + "summary": "summary for embedding", + "content": raw_content, + }, + parent_uri="viking://user/default/resources", + ctx=DummyReq(), + ) + + assert len(queue.items) == 1 + msg = queue.items[0] + assert msg.message == "summary for embedding" + assert msg.context_data["content"] == raw_content + assert fs.read_file_calls == 0 + assert fs.read_file_bytes_calls == 0 + + +@pytest.mark.asyncio +async def test_vectorize_text_bytes_sniffs_non_utf8_content(monkeypatch): + queue = DummyQueue() + raw_content = ( + "# 说明文档\n" + "目标: 验证已知 TEXT 文件的 bytes 内容也会进行编码探测。\n" + "说明: 这是一个中文 README 内容，用于测试 GB18030 编码识别。\n" + "命令: openviking benchmark run\n" + ) + fs = DummyFS(raw_content.encode("gb18030")) + monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue)) + monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: fs) + monkeypatch.setattr( + embedding_utils, + "get_openviking_config", + lambda: types.SimpleNamespace( + embedding=types.SimpleNamespace(text_source="summary_first", max_input_tokens=1000) + ), + ) + + await embedding_utils.vectorize_file( + file_path="viking://user/default/resources/README.md", + summary_dict={"name": "README.md", "summary": "summary for embedding"}, + parent_uri="viking://user/default/resources", + ctx=DummyReq(), + ) + + assert len(queue.items) == 1 + msg = queue.items[0] + assert msg.message == "summary for embedding" + assert msg.context_data["content"] == raw_content + assert fs.read_file_calls == 1 + assert fs.read_file_bytes_calls == 0 + + @pytest.mark.asyncio async def test_vectorize_file_preserves_content_until_embedder_input_guard(monkeypatch): queue = DummyQueue() diff --git a/uv.lock b/uv.lock index fd1d9c03a1..fda63669c8 100644 --- a/uv.lock +++ b/uv.lock @@ -911,62 +911,59 @@ wheels = [ [[package]] name = "cryptography" -version = "46.0.5" +version = "49.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/60/04/ee2a9e8542e4fa2773b81771ff8349ff19cdd56b7258a0cc442639052edb/cryptography-46.0.5.tar.gz", hash = "sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d", size = 750064, upload-time = "2026-02-10T19:18:38.255Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/81/b0bb27f2ba931a65409c6b8a8b358a7f03c0e46eceacddff55f7c84b1f3b/cryptography-46.0.5-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad", size = 7176289, upload-time = "2026-02-10T19:17:08.274Z" }, - { url = "https://files.pythonhosted.org/packages/ff/9e/6b4397a3e3d15123de3b1806ef342522393d50736c13b20ec4c9ea6693a6/cryptography-46.0.5-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b", size = 4275637, upload-time = "2026-02-10T19:17:10.53Z" }, - { url = "https://files.pythonhosted.org/packages/63/e7/471ab61099a3920b0c77852ea3f0ea611c9702f651600397ac567848b897/cryptography-46.0.5-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b", size = 4424742, upload-time = "2026-02-10T19:17:12.388Z" }, - { url = "https://files.pythonhosted.org/packages/37/53/a18500f270342d66bf7e4d9f091114e31e5ee9e7375a5aba2e85a91e0044/cryptography-46.0.5-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263", size = 4277528, upload-time = "2026-02-10T19:17:13.853Z" }, - { url = "https://files.pythonhosted.org/packages/22/29/c2e812ebc38c57b40e7c583895e73c8c5adb4d1e4a0cc4c5a4fdab2b1acc/cryptography-46.0.5-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d", size = 4947993, upload-time = "2026-02-10T19:17:15.618Z" }, - { url = "https://files.pythonhosted.org/packages/6b/e7/237155ae19a9023de7e30ec64e5d99a9431a567407ac21170a046d22a5a3/cryptography-46.0.5-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed", size = 4456855, upload-time = "2026-02-10T19:17:17.221Z" }, - { url = "https://files.pythonhosted.org/packages/2d/87/fc628a7ad85b81206738abbd213b07702bcbdada1dd43f72236ef3cffbb5/cryptography-46.0.5-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2", size = 3984635, upload-time = "2026-02-10T19:17:18.792Z" }, - { url = "https://files.pythonhosted.org/packages/84/29/65b55622bde135aedf4565dc509d99b560ee4095e56989e815f8fd2aa910/cryptography-46.0.5-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2", size = 4277038, upload-time = "2026-02-10T19:17:20.256Z" }, - { url = "https://files.pythonhosted.org/packages/bc/36/45e76c68d7311432741faf1fbf7fac8a196a0a735ca21f504c75d37e2558/cryptography-46.0.5-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0", size = 4912181, upload-time = "2026-02-10T19:17:21.825Z" }, - { url = "https://files.pythonhosted.org/packages/6d/1a/c1ba8fead184d6e3d5afcf03d569acac5ad063f3ac9fb7258af158f7e378/cryptography-46.0.5-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731", size = 4456482, upload-time = "2026-02-10T19:17:25.133Z" }, - { url = "https://files.pythonhosted.org/packages/f9/e5/3fb22e37f66827ced3b902cf895e6a6bc1d095b5b26be26bd13c441fdf19/cryptography-46.0.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82", size = 4405497, upload-time = "2026-02-10T19:17:26.66Z" }, - { url = "https://files.pythonhosted.org/packages/1a/df/9d58bb32b1121a8a2f27383fabae4d63080c7ca60b9b5c88be742be04ee7/cryptography-46.0.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1", size = 4667819, upload-time = "2026-02-10T19:17:28.569Z" }, - { url = "https://files.pythonhosted.org/packages/ea/ed/325d2a490c5e94038cdb0117da9397ece1f11201f425c4e9c57fe5b9f08b/cryptography-46.0.5-cp311-abi3-win32.whl", hash = "sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48", size = 3028230, upload-time = "2026-02-10T19:17:30.518Z" }, - { url = "https://files.pythonhosted.org/packages/e9/5a/ac0f49e48063ab4255d9e3b79f5def51697fce1a95ea1370f03dc9db76f6/cryptography-46.0.5-cp311-abi3-win_amd64.whl", hash = "sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4", size = 3480909, upload-time = "2026-02-10T19:17:32.083Z" }, - { url = "https://files.pythonhosted.org/packages/00/13/3d278bfa7a15a96b9dc22db5a12ad1e48a9eb3d40e1827ef66a5df75d0d0/cryptography-46.0.5-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2", size = 7119287, upload-time = "2026-02-10T19:17:33.801Z" }, - { url = "https://files.pythonhosted.org/packages/67/c8/581a6702e14f0898a0848105cbefd20c058099e2c2d22ef4e476dfec75d7/cryptography-46.0.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678", size = 4265728, upload-time = "2026-02-10T19:17:35.569Z" }, - { url = "https://files.pythonhosted.org/packages/dd/4a/ba1a65ce8fc65435e5a849558379896c957870dd64fecea97b1ad5f46a37/cryptography-46.0.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87", size = 4408287, upload-time = "2026-02-10T19:17:36.938Z" }, - { url = "https://files.pythonhosted.org/packages/f8/67/8ffdbf7b65ed1ac224d1c2df3943553766914a8ca718747ee3871da6107e/cryptography-46.0.5-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee", size = 4270291, upload-time = "2026-02-10T19:17:38.748Z" }, - { url = "https://files.pythonhosted.org/packages/f8/e5/f52377ee93bc2f2bba55a41a886fd208c15276ffbd2569f2ddc89d50e2c5/cryptography-46.0.5-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981", size = 4927539, upload-time = "2026-02-10T19:17:40.241Z" }, - { url = "https://files.pythonhosted.org/packages/3b/02/cfe39181b02419bbbbcf3abdd16c1c5c8541f03ca8bda240debc467d5a12/cryptography-46.0.5-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9", size = 4442199, upload-time = "2026-02-10T19:17:41.789Z" }, - { url = "https://files.pythonhosted.org/packages/c0/96/2fcaeb4873e536cf71421a388a6c11b5bc846e986b2b069c79363dc1648e/cryptography-46.0.5-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648", size = 3960131, upload-time = "2026-02-10T19:17:43.379Z" }, - { url = "https://files.pythonhosted.org/packages/d8/d2/b27631f401ddd644e94c5cf33c9a4069f72011821cf3dc7309546b0642a0/cryptography-46.0.5-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4", size = 4270072, upload-time = "2026-02-10T19:17:45.481Z" }, - { url = "https://files.pythonhosted.org/packages/f4/a7/60d32b0370dae0b4ebe55ffa10e8599a2a59935b5ece1b9f06edb73abdeb/cryptography-46.0.5-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0", size = 4892170, upload-time = "2026-02-10T19:17:46.997Z" }, - { url = "https://files.pythonhosted.org/packages/d2/b9/cf73ddf8ef1164330eb0b199a589103c363afa0cf794218c24d524a58eab/cryptography-46.0.5-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663", size = 4441741, upload-time = "2026-02-10T19:17:48.661Z" }, - { url = "https://files.pythonhosted.org/packages/5f/eb/eee00b28c84c726fe8fa0158c65afe312d9c3b78d9d01daf700f1f6e37ff/cryptography-46.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826", size = 4396728, upload-time = "2026-02-10T19:17:50.058Z" }, - { url = "https://files.pythonhosted.org/packages/65/f4/6bc1a9ed5aef7145045114b75b77c2a8261b4d38717bd8dea111a63c3442/cryptography-46.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d", size = 4652001, upload-time = "2026-02-10T19:17:51.54Z" }, - { url = "https://files.pythonhosted.org/packages/86/ef/5d00ef966ddd71ac2e6951d278884a84a40ffbd88948ef0e294b214ae9e4/cryptography-46.0.5-cp314-cp314t-win32.whl", hash = "sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a", size = 3003637, upload-time = "2026-02-10T19:17:52.997Z" }, - { url = "https://files.pythonhosted.org/packages/b7/57/f3f4160123da6d098db78350fdfd9705057aad21de7388eacb2401dceab9/cryptography-46.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4", size = 3469487, upload-time = "2026-02-10T19:17:54.549Z" }, - { url = "https://files.pythonhosted.org/packages/e2/fa/a66aa722105ad6a458bebd64086ca2b72cdd361fed31763d20390f6f1389/cryptography-46.0.5-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31", size = 7170514, upload-time = "2026-02-10T19:17:56.267Z" }, - { url = "https://files.pythonhosted.org/packages/0f/04/c85bdeab78c8bc77b701bf0d9bdcf514c044e18a46dcff330df5448631b0/cryptography-46.0.5-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18", size = 4275349, upload-time = "2026-02-10T19:17:58.419Z" }, - { url = "https://files.pythonhosted.org/packages/5c/32/9b87132a2f91ee7f5223b091dc963055503e9b442c98fc0b8a5ca765fab0/cryptography-46.0.5-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235", size = 4420667, upload-time = "2026-02-10T19:18:00.619Z" }, - { url = "https://files.pythonhosted.org/packages/a1/a6/a7cb7010bec4b7c5692ca6f024150371b295ee1c108bdc1c400e4c44562b/cryptography-46.0.5-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a", size = 4276980, upload-time = "2026-02-10T19:18:02.379Z" }, - { url = "https://files.pythonhosted.org/packages/8e/7c/c4f45e0eeff9b91e3f12dbd0e165fcf2a38847288fcfd889deea99fb7b6d/cryptography-46.0.5-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76", size = 4939143, upload-time = "2026-02-10T19:18:03.964Z" }, - { url = "https://files.pythonhosted.org/packages/37/19/e1b8f964a834eddb44fa1b9a9976f4e414cbb7aa62809b6760c8803d22d1/cryptography-46.0.5-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614", size = 4453674, upload-time = "2026-02-10T19:18:05.588Z" }, - { url = "https://files.pythonhosted.org/packages/db/ed/db15d3956f65264ca204625597c410d420e26530c4e2943e05a0d2f24d51/cryptography-46.0.5-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229", size = 3978801, upload-time = "2026-02-10T19:18:07.167Z" }, - { url = "https://files.pythonhosted.org/packages/41/e2/df40a31d82df0a70a0daf69791f91dbb70e47644c58581d654879b382d11/cryptography-46.0.5-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1", size = 4276755, upload-time = "2026-02-10T19:18:09.813Z" }, - { url = "https://files.pythonhosted.org/packages/33/45/726809d1176959f4a896b86907b98ff4391a8aa29c0aaaf9450a8a10630e/cryptography-46.0.5-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d", size = 4901539, upload-time = "2026-02-10T19:18:11.263Z" }, - { url = "https://files.pythonhosted.org/packages/99/0f/a3076874e9c88ecb2ecc31382f6e7c21b428ede6f55aafa1aa272613e3cd/cryptography-46.0.5-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c", size = 4452794, upload-time = "2026-02-10T19:18:12.914Z" }, - { url = "https://files.pythonhosted.org/packages/02/ef/ffeb542d3683d24194a38f66ca17c0a4b8bf10631feef44a7ef64e631b1a/cryptography-46.0.5-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4", size = 4404160, upload-time = "2026-02-10T19:18:14.375Z" }, - { url = "https://files.pythonhosted.org/packages/96/93/682d2b43c1d5f1406ed048f377c0fc9fc8f7b0447a478d5c65ab3d3a66eb/cryptography-46.0.5-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9", size = 4667123, upload-time = "2026-02-10T19:18:15.886Z" }, - { url = "https://files.pythonhosted.org/packages/45/2d/9c5f2926cb5300a8eefc3f4f0b3f3df39db7f7ce40c8365444c49363cbda/cryptography-46.0.5-cp38-abi3-win32.whl", hash = "sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72", size = 3010220, upload-time = "2026-02-10T19:18:17.361Z" }, - { url = "https://files.pythonhosted.org/packages/48/ef/0c2f4a8e31018a986949d34a01115dd057bf536905dca38897bacd21fac3/cryptography-46.0.5-cp38-abi3-win_amd64.whl", hash = "sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595", size = 3467050, upload-time = "2026-02-10T19:18:18.899Z" }, - { url = "https://files.pythonhosted.org/packages/eb/dd/2d9fdb07cebdf3d51179730afb7d5e576153c6744c3ff8fded23030c204e/cryptography-46.0.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c", size = 3476964, upload-time = "2026-02-10T19:18:20.687Z" }, - { url = "https://files.pythonhosted.org/packages/e9/6f/6cc6cc9955caa6eaf83660b0da2b077c7fe8ff9950a3c5e45d605038d439/cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a", size = 4218321, upload-time = "2026-02-10T19:18:22.349Z" }, - { url = "https://files.pythonhosted.org/packages/3e/5d/c4da701939eeee699566a6c1367427ab91a8b7088cc2328c09dbee940415/cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356", size = 4381786, upload-time = "2026-02-10T19:18:24.529Z" }, - { url = "https://files.pythonhosted.org/packages/ac/97/a538654732974a94ff96c1db621fa464f455c02d4bb7d2652f4edc21d600/cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da", size = 4217990, upload-time = "2026-02-10T19:18:25.957Z" }, - { url = "https://files.pythonhosted.org/packages/ae/11/7e500d2dd3ba891197b9efd2da5454b74336d64a7cc419aa7327ab74e5f6/cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257", size = 4381252, upload-time = "2026-02-10T19:18:27.496Z" }, - { url = "https://files.pythonhosted.org/packages/bc/58/6b3d24e6b9bc474a2dcdee65dfd1f008867015408a271562e4b690561a4d/cryptography-46.0.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7", size = 3407605, upload-time = "2026-02-10T19:18:29.233Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/1f/99/d1c90d6041656cc6ee229dc99cd67fd0cd5aec3c5f7d72fffc27cc750054/cryptography-49.0.0.tar.gz", hash = "sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493", size = 854345, upload-time = "2026-06-12T20:02:30.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/22/adf66990e63584a68dfb50c24f48a125c07b1699899381c8151e63ed458c/cryptography-49.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db", size = 4032100, upload-time = "2026-06-12T20:02:32.143Z" }, + { url = "https://files.pythonhosted.org/packages/09/41/3797cfaf69cae04a13ee78ebd83f0678d9c02b4779d21ce24445326f1a69/cryptography-49.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db", size = 4692978, upload-time = "2026-06-12T20:01:21.305Z" }, + { url = "https://files.pythonhosted.org/packages/e6/8b/43011f7ebe515a8aa20d61f290a326cd890c2e738e16e59eaff8d9c3a412/cryptography-49.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325", size = 4716422, upload-time = "2026-06-12T20:01:48.566Z" }, + { url = "https://files.pythonhosted.org/packages/4a/91/01ce7303a4579e6d3a6abef01bd322848e9ea7a219adcabc5048b9033571/cryptography-49.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2", size = 4700503, upload-time = "2026-06-12T20:02:47.091Z" }, + { url = "https://files.pythonhosted.org/packages/62/99/a2c95cf8293f07491e9e27c20cc4dcd18176d944e674679adeb1d0173fd6/cryptography-49.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b", size = 5309779, upload-time = "2026-06-12T20:02:08.987Z" }, + { url = "https://files.pythonhosted.org/packages/20/2c/0622f20ff02b2ef32558733443805dc82fd4c275be01b2d19d14676f3a1b/cryptography-49.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6", size = 4749683, upload-time = "2026-06-12T20:02:03.335Z" }, + { url = "https://files.pythonhosted.org/packages/a3/5b/c5246635d5fd3b64e0d45ae10e99fd32fe9676a79915ccfe5a61ba9af1a5/cryptography-49.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c", size = 4337874, upload-time = "2026-06-12T20:02:54.323Z" }, + { url = "https://files.pythonhosted.org/packages/6d/88/05563c7fe2e914e87d1a536d06fe83e66b4e1d95cb593e05aea375531da8/cryptography-49.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7", size = 4700283, upload-time = "2026-06-12T20:01:34.822Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b6/d7696e4e890d6ae1469935164c9e5215c557671cb78d6e3f458ccceaa632/cryptography-49.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68", size = 5265844, upload-time = "2026-06-12T20:01:24.09Z" }, + { url = "https://files.pythonhosted.org/packages/a9/3c/f3ad17eecc1a57b0ba236dc01f90e783c51f4a2f35f64777cc4f47a184b2/cryptography-49.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9", size = 4749290, upload-time = "2026-06-12T20:01:30.848Z" }, + { url = "https://files.pythonhosted.org/packages/4f/01/339573cf1023163a400b0b5d16f6d507de413b9f60be6fd1b77feeaf6737/cryptography-49.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f", size = 4834612, upload-time = "2026-06-12T20:01:29.246Z" }, + { url = "https://files.pythonhosted.org/packages/71/fd/577302e213a1be9468f92d1afef66fcf1ef83d516819d9992ca547f592bd/cryptography-49.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459", size = 4980804, upload-time = "2026-06-12T20:01:42.853Z" }, + { url = "https://files.pythonhosted.org/packages/1f/09/f42b1d190c5ba75f72062a387f8030d1d75f6ab035788f1d9c4b01de6525/cryptography-49.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e", size = 3810026, upload-time = "2026-06-12T20:02:39.262Z" }, + { url = "https://files.pythonhosted.org/packages/ec/9e/db72b3ae7fc9cfad53e630e56c6ae83b9b6ff0bf3718ffb8012d20b3aabf/cryptography-49.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:73a205dce83953d131a4aa1e0fd917a2fd1c5b1eef251e9d7152efefcbf5caf7", size = 4013892, upload-time = "2026-06-12T20:02:10.735Z" }, + { url = "https://files.pythonhosted.org/packages/86/12/c48a424f38db03027be9f7ed5c7dc5de9933dbee992865f98b13727a009d/cryptography-49.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:196ecd6a36e4e9aa10270393bb98d8df88fccee0bf1e5128b91ae4eb4375896d", size = 4678835, upload-time = "2026-06-12T20:02:48.743Z" }, + { url = "https://files.pythonhosted.org/packages/68/28/8a3ad4653662c93fc44dc4e5d8fd374c25c42e07b34bbfbadf49cf57a5a8/cryptography-49.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7abcee80084cda3f7691f3eb1ce480d8df49cec637b429aa35986c1de71738aa", size = 4697239, upload-time = "2026-06-12T20:02:56.03Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b2/2193fc74f81aee4f9b62733133b73b5176718932ed8f2e4b03fa040480a6/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:4ae387c9cb68ea569ca17e490d66d8142b81c3cc814bf179974b7d146e490bbb", size = 4685593, upload-time = "2026-06-12T20:02:50.666Z" }, + { url = "https://files.pythonhosted.org/packages/47/f1/1d3eaa243bfc5de4a187b22aa8c048b3e4980bfbe830ac46e6bac2e66947/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:f37d847238971164fdbc68ade6f6574aecc9c0af714190e2083429ff68f4ce9d", size = 5289961, upload-time = "2026-06-12T20:01:46.468Z" }, + { url = "https://files.pythonhosted.org/packages/58/39/2d51306721330c486495853eda1c567880ff036de15a14c4b74f399934af/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c2bc30226390d60ea19d9f82b19db005fe0452154a23c1c410c12ea801e43561", size = 4731145, upload-time = "2026-06-12T20:02:16.832Z" }, + { url = "https://files.pythonhosted.org/packages/17/50/983e838c7fd0d87fd8c969bcdd328edaf5f756e38df5281637424c155873/cryptography-49.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:07cab27cc7b7e0fd28e5e26bb9eeedde5c135c868b46de4a27845abe94af6122", size = 4321719, upload-time = "2026-06-12T20:02:52.611Z" }, + { url = "https://files.pythonhosted.org/packages/a7/f5/8f571d7e27c55bce9f76f026143bcb1e040a4233149ecca0bea5fa5dd5f7/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:b20133d204d2bb56ba047642199603876c872026ca53e79c35b83772ab2cc505", size = 4685209, upload-time = "2026-06-12T20:02:07.282Z" }, + { url = "https://files.pythonhosted.org/packages/e7/84/0e27016a6fc5a0886f797018b26aa42f40c09a82332bff77822a451deaaa/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:b970c6da94d5bb18629db453d14f2a1300f6bf59b61e9b82377931ef95504866", size = 5246285, upload-time = "2026-06-12T20:01:32.439Z" }, + { url = "https://files.pythonhosted.org/packages/11/2d/5e1fb307cb5931881516b464c98774b3f2c36b5d4bb9a2830253cf553cad/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d8ecde755e2e91bf773fc94e8c9d730cd7f2007004cb492263a794ec3899a1c8", size = 4730441, upload-time = "2026-06-12T20:02:01.469Z" }, + { url = "https://files.pythonhosted.org/packages/e4/c0/bff5a02ee731d207d6a1ed51732549d8c53d2bc8da1d10ec6f2844201d68/cryptography-49.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e3fb64c420688e5319ae25113a354015abbd8dffbfbc41781a1ea66fc7622ac3", size = 4815869, upload-time = "2026-06-12T20:01:36.574Z" }, + { url = "https://files.pythonhosted.org/packages/b9/26/814681d14248d95d73d5c3eea0c39a94eb8302df966f670a2c60de90974b/cryptography-49.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32703d93296f5c1f4b53349ad3a250c2cae0fdecd3a3dd5d47e616d8d616af27", size = 4960948, upload-time = "2026-06-12T20:02:18.688Z" }, + { url = "https://files.pythonhosted.org/packages/4c/fe/93ecac273d3738939d023612ad12cca9a3740a5345d69fda04134c43fd96/cryptography-49.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:33cd0565932807baddb67b96dbee92f2c374b5c89dee09fd74079aeb8c8dba61", size = 3799153, upload-time = "2026-06-12T20:01:39.059Z" }, + { url = "https://files.pythonhosted.org/packages/19/2a/5bb823f5bedcf80718cea7fbc95ec5515cca3769633c4b01a32be7f30e7c/cryptography-49.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a", size = 4025947, upload-time = "2026-06-12T20:01:25.745Z" }, + { url = "https://files.pythonhosted.org/packages/3d/df/40577043ca124e17012f408ddddaeb213b856336ac82ddb3bc915f39e29f/cryptography-49.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4", size = 4692429, upload-time = "2026-06-12T20:01:53.628Z" }, + { url = "https://files.pythonhosted.org/packages/2c/99/2d13299eb3dd27b02dcfaafcc91d6b5cb3329f7cbd6d8f51921acd566c1a/cryptography-49.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18", size = 4700968, upload-time = "2026-06-12T20:02:45.383Z" }, + { url = "https://files.pythonhosted.org/packages/a5/4d/9c0cd02f95e2602dd5e563da149ee0830abef3537be8b34dc56281ebe27a/cryptography-49.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69", size = 4697758, upload-time = "2026-06-12T20:01:41.13Z" }, + { url = "https://files.pythonhosted.org/packages/24/01/186c825898477d77e2324d5360fefe622ff1d8d1963ec0554e2cada8ec77/cryptography-49.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64", size = 5298863, upload-time = "2026-06-12T20:02:24.579Z" }, + { url = "https://files.pythonhosted.org/packages/b8/7b/62cbbab75d0659865bf0273790031544a0b16c8072d258f9428dcd8190dc/cryptography-49.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21", size = 4735983, upload-time = "2026-06-12T20:01:50.14Z" }, + { url = "https://files.pythonhosted.org/packages/6c/72/3e798c064bc39e471008075d0f9bc9daf77a80879c092e4a8e170c585ed4/cryptography-49.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9", size = 4334173, upload-time = "2026-06-12T20:01:44.743Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ee/6fca21d1ac73e06f8bef71940abfd4d2f6472b4bca284d770f32bd4086f6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc", size = 4697298, upload-time = "2026-06-12T20:02:20.918Z" }, + { url = "https://files.pythonhosted.org/packages/67/d0/a5fcd3515f0bae49a7b6d0413cc1bdccdcc1fc0047037a0d480642cdc5d6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8", size = 5254338, upload-time = "2026-06-12T20:02:22.737Z" }, + { url = "https://files.pythonhosted.org/packages/a0/84/84fe36f19caf857d61cb7fc9c63035a47ffabd84ea12d1d393148efa3615/cryptography-49.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36", size = 4735650, upload-time = "2026-06-12T20:02:41.389Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a0/db537264e234f7273a73ec020873d6d6b39dfd8a53db78b550ca8320440e/cryptography-49.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e", size = 4834820, upload-time = "2026-06-12T20:01:51.847Z" }, + { url = "https://files.pythonhosted.org/packages/93/77/8df9eb486495979bccecd1062e2eaf435250e84437040295b57d09048b0b/cryptography-49.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b", size = 4967968, upload-time = "2026-06-12T20:02:12.524Z" }, + { url = "https://files.pythonhosted.org/packages/c2/e6/f60198ea8d9dfa15fff9ed4ca02ce362f6eadd9ba757dcc50634c4257b63/cryptography-49.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001", size = 3785547, upload-time = "2026-06-12T20:02:26.847Z" }, + { url = "https://files.pythonhosted.org/packages/63/d3/4a83af35d65e3fad632c926fad684c193ea4398569ccb0bbbc7fe8f5dc9a/cryptography-49.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc1e275c2f1d97b1a6450b8b0ea3ebfa6e087a611c2b26cb2404d48588abab7b", size = 3993685, upload-time = "2026-06-12T20:02:14.883Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a7/f9dac0ab7f80368c56993a7bf638ef9935f825c91902798481fac0898138/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c83782480a4a9da4d0feb51950131ba32e12e70813848b3343f6e18c28a66838", size = 4676239, upload-time = "2026-06-12T20:02:28.793Z" }, + { url = "https://files.pythonhosted.org/packages/d7/70/2ba3769dd0ae167e2f33dfa9592d45db6ff9a61d62ca1a5b3d1bdd09068f/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b39efa323140595abd3ecca8529d321ae50f55f3aa3ba9cc81ea56a6011953d5", size = 4715584, upload-time = "2026-06-12T20:01:27.495Z" }, + { url = "https://files.pythonhosted.org/packages/94/64/2923570ac1c0bd3a737aa366ac3abbbbde273042308b8cde95e2364a6e6a/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:b47db11c2c3525083296069b98ac5221907455e989ae0c2e3008bde851921615", size = 4675885, upload-time = "2026-06-12T20:01:55.49Z" }, + { url = "https://files.pythonhosted.org/packages/ab/f8/614dc7e051418cfe53d55173c1e24c6b0085e89996fe90508c2fdf769aef/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:084ef1af862eb07ec46d25f68689f2102a9fc0e05ce7b80f14f5fe51e4eef0f6", size = 4715449, upload-time = "2026-06-12T20:02:05.469Z" }, + { url = "https://files.pythonhosted.org/packages/aa/50/a9caea39ad19c431c1a3f8a31114df65b260cdfe67786b6c7e7c040c4c44/cryptography-49.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6", size = 3783731, upload-time = "2026-06-12T20:02:43.319Z" }, ] [[package]] @@ -1559,7 +1556,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/3f/9859f655d11901e7b2996c6e3d33e0caa9a1d4572c3bc61ed0faa64b2f4c/greenlet-3.3.2-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9bc885b89709d901859cf95179ec9f6bb67a3d2bb1f0e88456461bd4b7f8fd0d", size = 277747, upload-time = "2026-02-20T20:16:21.325Z" }, { url = "https://files.pythonhosted.org/packages/fb/07/cb284a8b5c6498dbd7cba35d31380bb123d7dceaa7907f606c8ff5993cbf/greenlet-3.3.2-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b568183cf65b94919be4438dc28416b234b678c608cafac8874dfeeb2a9bbe13", size = 579202, upload-time = "2026-02-20T20:47:28.955Z" }, { url = "https://files.pythonhosted.org/packages/ed/45/67922992b3a152f726163b19f890a85129a992f39607a2a53155de3448b8/greenlet-3.3.2-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:527fec58dc9f90efd594b9b700662ed3fb2493c2122067ac9c740d98080a620e", size = 590620, upload-time = "2026-02-20T20:55:55.581Z" }, - { url = "https://files.pythonhosted.org/packages/03/5f/6e2a7d80c353587751ef3d44bb947f0565ec008a2e0927821c007e96d3a7/greenlet-3.3.2-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:508c7f01f1791fbc8e011bd508f6794cb95397fdb198a46cb6635eb5b78d85a7", size = 602132, upload-time = "2026-02-20T21:02:43.261Z" }, { url = "https://files.pythonhosted.org/packages/ad/55/9f1ebb5a825215fadcc0f7d5073f6e79e3007e3282b14b22d6aba7ca6cb8/greenlet-3.3.2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad0c8917dd42a819fe77e6bdfcb84e3379c0de956469301d9fd36427a1ca501f", size = 591729, upload-time = "2026-02-20T20:20:58.395Z" }, { url = "https://files.pythonhosted.org/packages/24/b4/21f5455773d37f94b866eb3cf5caed88d6cea6dd2c6e1f9c34f463cba3ec/greenlet-3.3.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:97245cc10e5515dbc8c3104b2928f7f02b6813002770cfaffaf9a6e0fc2b94ef", size = 1551946, upload-time = "2026-02-20T20:49:31.102Z" }, { url = "https://files.pythonhosted.org/packages/00/68/91f061a926abead128fe1a87f0b453ccf07368666bd59ffa46016627a930/greenlet-3.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8c1fdd7d1b309ff0da81d60a9688a8bd044ac4e18b250320a96fc68d31c209ca", size = 1618494, upload-time = "2026-02-20T20:21:06.541Z" }, @@ -1567,7 +1563,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f3/47/16400cb42d18d7a6bb46f0626852c1718612e35dcb0dffa16bbaffdf5dd2/greenlet-3.3.2-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:c56692189a7d1c7606cb794be0a8381470d95c57ce5be03fb3d0ef57c7853b86", size = 278890, upload-time = "2026-02-20T20:19:39.263Z" }, { url = "https://files.pythonhosted.org/packages/a3/90/42762b77a5b6aa96cd8c0e80612663d39211e8ae8a6cd47c7f1249a66262/greenlet-3.3.2-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ebd458fa8285960f382841da585e02201b53a5ec2bac6b156fc623b5ce4499f", size = 581120, upload-time = "2026-02-20T20:47:30.161Z" }, { url = "https://files.pythonhosted.org/packages/bf/6f/f3d64f4fa0a9c7b5c5b3c810ff1df614540d5aa7d519261b53fba55d4df9/greenlet-3.3.2-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a443358b33c4ec7b05b79a7c8b466f5d275025e750298be7340f8fc63dff2a55", size = 594363, upload-time = "2026-02-20T20:55:56.965Z" }, - { url = "https://files.pythonhosted.org/packages/9c/8b/1430a04657735a3f23116c2e0d5eb10220928846e4537a938a41b350bed6/greenlet-3.3.2-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4375a58e49522698d3e70cc0b801c19433021b5c37686f7ce9c65b0d5c8677d2", size = 605046, upload-time = "2026-02-20T21:02:45.234Z" }, { url = "https://files.pythonhosted.org/packages/72/83/3e06a52aca8128bdd4dcd67e932b809e76a96ab8c232a8b025b2850264c5/greenlet-3.3.2-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e2cd90d413acbf5e77ae41e5d3c9b3ac1d011a756d7284d7f3f2b806bbd6358", size = 594156, upload-time = "2026-02-20T20:20:59.955Z" }, { url = "https://files.pythonhosted.org/packages/70/79/0de5e62b873e08fe3cef7dbe84e5c4bc0e8ed0c7ff131bccb8405cd107c8/greenlet-3.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:442b6057453c8cb29b4fb36a2ac689382fc71112273726e2423f7f17dc73bf99", size = 1554649, upload-time = "2026-02-20T20:49:32.293Z" }, { url = "https://files.pythonhosted.org/packages/5a/00/32d30dee8389dc36d42170a9c66217757289e2afb0de59a3565260f38373/greenlet-3.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:45abe8eb6339518180d5a7fa47fa01945414d7cca5ecb745346fc6a87d2750be", size = 1619472, upload-time = "2026-02-20T20:21:07.966Z" }, @@ -1576,7 +1571,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/ab/1608e5a7578e62113506740b88066bf09888322a311cff602105e619bd87/greenlet-3.3.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:ac8d61d4343b799d1e526db579833d72f23759c71e07181c2d2944e429eb09cd", size = 280358, upload-time = "2026-02-20T20:17:43.971Z" }, { url = "https://files.pythonhosted.org/packages/a5/23/0eae412a4ade4e6623ff7626e38998cb9b11e9ff1ebacaa021e4e108ec15/greenlet-3.3.2-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ceec72030dae6ac0c8ed7591b96b70410a8be370b6a477b1dbc072856ad02bd", size = 601217, upload-time = "2026-02-20T20:47:31.462Z" }, { url = "https://files.pythonhosted.org/packages/f8/16/5b1678a9c07098ecb9ab2dd159fafaf12e963293e61ee8d10ecb55273e5e/greenlet-3.3.2-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a2a5be83a45ce6188c045bcc44b0ee037d6a518978de9a5d97438548b953a1ac", size = 611792, upload-time = "2026-02-20T20:55:58.423Z" }, - { url = "https://files.pythonhosted.org/packages/5c/c5/cc09412a29e43406eba18d61c70baa936e299bc27e074e2be3806ed29098/greenlet-3.3.2-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ae9e21c84035c490506c17002f5c8ab25f980205c3e61ddb3a2a2a2e6c411fcb", size = 626250, upload-time = "2026-02-20T21:02:46.596Z" }, { url = "https://files.pythonhosted.org/packages/50/1f/5155f55bd71cabd03765a4aac9ac446be129895271f73872c36ebd4b04b6/greenlet-3.3.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43e99d1749147ac21dde49b99c9abffcbc1e2d55c67501465ef0930d6e78e070", size = 613875, upload-time = "2026-02-20T20:21:01.102Z" }, { url = "https://files.pythonhosted.org/packages/fc/dd/845f249c3fcd69e32df80cdab059b4be8b766ef5830a3d0aa9d6cad55beb/greenlet-3.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4c956a19350e2c37f2c48b336a3afb4bff120b36076d9d7fb68cb44e05d95b79", size = 1571467, upload-time = "2026-02-20T20:49:33.495Z" }, { url = "https://files.pythonhosted.org/packages/2a/50/2649fe21fcc2b56659a452868e695634722a6655ba245d9f77f5656010bf/greenlet-3.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6c6f8ba97d17a1e7d664151284cb3315fc5f8353e75221ed4324f84eb162b395", size = 1640001, upload-time = "2026-02-20T20:21:09.154Z" }, @@ -1585,7 +1579,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ac/48/f8b875fa7dea7dd9b33245e37f065af59df6a25af2f9561efa8d822fde51/greenlet-3.3.2-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:aa6ac98bdfd716a749b84d4034486863fd81c3abde9aa3cf8eff9127981a4ae4", size = 279120, upload-time = "2026-02-20T20:19:01.9Z" }, { url = "https://files.pythonhosted.org/packages/49/8d/9771d03e7a8b1ee456511961e1b97a6d77ae1dea4a34a5b98eee706689d3/greenlet-3.3.2-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab0c7e7901a00bc0a7284907273dc165b32e0d109a6713babd04471327ff7986", size = 603238, upload-time = "2026-02-20T20:47:32.873Z" }, { url = "https://files.pythonhosted.org/packages/59/0e/4223c2bbb63cd5c97f28ffb2a8aee71bdfb30b323c35d409450f51b91e3e/greenlet-3.3.2-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d248d8c23c67d2291ffd47af766e2a3aa9fa1c6703155c099feb11f526c63a92", size = 614219, upload-time = "2026-02-20T20:55:59.817Z" }, - { url = "https://files.pythonhosted.org/packages/94/2b/4d012a69759ac9d77210b8bfb128bc621125f5b20fc398bce3940d036b1c/greenlet-3.3.2-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ccd21bb86944ca9be6d967cf7691e658e43417782bce90b5d2faeda0ff78a7dd", size = 628268, upload-time = "2026-02-20T21:02:48.024Z" }, { url = "https://files.pythonhosted.org/packages/7a/34/259b28ea7a2a0c904b11cd36c79b8cef8019b26ee5dbe24e73b469dea347/greenlet-3.3.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b6997d360a4e6a4e936c0f9625b1c20416b8a0ea18a8e19cabbefc712e7397ab", size = 616774, upload-time = "2026-02-20T20:21:02.454Z" }, { url = "https://files.pythonhosted.org/packages/0a/03/996c2d1689d486a6e199cb0f1cf9e4aa940c500e01bdf201299d7d61fa69/greenlet-3.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:64970c33a50551c7c50491671265d8954046cb6e8e2999aacdd60e439b70418a", size = 1571277, upload-time = "2026-02-20T20:49:34.795Z" }, { url = "https://files.pythonhosted.org/packages/d9/c4/2570fc07f34a39f2caf0bf9f24b0a1a0a47bc2e8e465b2c2424821389dfc/greenlet-3.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1a9172f5bf6bd88e6ba5a84e0a68afeac9dc7b6b412b245dd64f52d83c81e55b", size = 1640455, upload-time = "2026-02-20T20:21:10.261Z" }, @@ -1594,7 +1587,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/ae/8bffcbd373b57a5992cd077cbe8858fff39110480a9d50697091faea6f39/greenlet-3.3.2-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8d1658d7291f9859beed69a776c10822a0a799bc4bfe1bd4272bb60e62507dab", size = 279650, upload-time = "2026-02-20T20:18:00.783Z" }, { url = "https://files.pythonhosted.org/packages/d1/c0/45f93f348fa49abf32ac8439938726c480bd96b2a3c6f4d949ec0124b69f/greenlet-3.3.2-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18cb1b7337bca281915b3c5d5ae19f4e76d35e1df80f4ad3c1a7be91fadf1082", size = 650295, upload-time = "2026-02-20T20:47:34.036Z" }, { url = "https://files.pythonhosted.org/packages/b3/de/dd7589b3f2b8372069ab3e4763ea5329940fc7ad9dcd3e272a37516d7c9b/greenlet-3.3.2-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2e47408e8ce1c6f1ceea0dffcdf6ebb85cc09e55c7af407c99f1112016e45e9", size = 662163, upload-time = "2026-02-20T20:56:01.295Z" }, - { url = "https://files.pythonhosted.org/packages/cd/ac/85804f74f1ccea31ba518dcc8ee6f14c79f73fe36fa1beba38930806df09/greenlet-3.3.2-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3cb43ce200f59483eb82949bf1835a99cf43d7571e900d7c8d5c62cdf25d2f9", size = 675371, upload-time = "2026-02-20T21:02:49.664Z" }, { url = "https://files.pythonhosted.org/packages/d2/d8/09bfa816572a4d83bccd6750df1926f79158b1c36c5f73786e26dbe4ee38/greenlet-3.3.2-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63d10328839d1973e5ba35e98cccbca71b232b14051fd957b6f8b6e8e80d0506", size = 664160, upload-time = "2026-02-20T20:21:04.015Z" }, { url = "https://files.pythonhosted.org/packages/48/cf/56832f0c8255d27f6c35d41b5ec91168d74ec721d85f01a12131eec6b93c/greenlet-3.3.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8e4ab3cfb02993c8cc248ea73d7dae6cec0253e9afa311c9b37e603ca9fad2ce", size = 1619181, upload-time = "2026-02-20T20:49:36.052Z" }, { url = "https://files.pythonhosted.org/packages/0a/23/b90b60a4aabb4cec0796e55f25ffbfb579a907c3898cd2905c8918acaa16/greenlet-3.3.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:94ad81f0fd3c0c0681a018a976e5c2bd2ca2d9d94895f23e7bb1af4e8af4e2d5", size = 1687713, upload-time = "2026-02-20T20:21:11.684Z" }, @@ -1603,7 +1595,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/98/6d/8f2ef704e614bcf58ed43cfb8d87afa1c285e98194ab2cfad351bf04f81e/greenlet-3.3.2-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:e26e72bec7ab387ac80caa7496e0f908ff954f31065b0ffc1f8ecb1338b11b54", size = 286617, upload-time = "2026-02-20T20:19:29.856Z" }, { url = "https://files.pythonhosted.org/packages/5e/0d/93894161d307c6ea237a43988f27eba0947b360b99ac5239ad3fe09f0b47/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b466dff7a4ffda6ca975979bab80bdadde979e29fc947ac3be4451428d8b0e4", size = 655189, upload-time = "2026-02-20T20:47:35.742Z" }, { url = "https://files.pythonhosted.org/packages/f5/2c/d2d506ebd8abcb57386ec4f7ba20f4030cbe56eae541bc6fd6ef399c0b41/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b8bddc5b73c9720bea487b3bffdb1840fe4e3656fba3bd40aa1489e9f37877ff", size = 658225, upload-time = "2026-02-20T20:56:02.527Z" }, - { url = "https://files.pythonhosted.org/packages/d1/67/8197b7e7e602150938049d8e7f30de1660cfb87e4c8ee349b42b67bdb2e1/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:59b3e2c40f6706b05a9cd299c836c6aa2378cabe25d021acd80f13abf81181cf", size = 666581, upload-time = "2026-02-20T21:02:51.526Z" }, { url = "https://files.pythonhosted.org/packages/8e/30/3a09155fbf728673a1dea713572d2d31159f824a37c22da82127056c44e4/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b26b0f4428b871a751968285a1ac9648944cea09807177ac639b030bddebcea4", size = 657907, upload-time = "2026-02-20T20:21:05.259Z" }, { url = "https://files.pythonhosted.org/packages/f3/fd/d05a4b7acd0154ed758797f0a43b4c0962a843bedfe980115e842c5b2d08/greenlet-3.3.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1fb39a11ee2e4d94be9a76671482be9398560955c9e568550de0224e41104727", size = 1618857, upload-time = "2026-02-20T20:49:37.309Z" }, { url = "https://files.pythonhosted.org/packages/6f/e1/50ee92a5db521de8f35075b5eff060dd43d39ebd46c2181a2042f7070385/greenlet-3.3.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:20154044d9085151bc309e7689d6f7ba10027f8f5a8c0676ad398b951913d89e", size = 1680010, upload-time = "2026-02-20T20:21:13.427Z" }, @@ -3528,6 +3519,7 @@ dependencies = [ { name = "opentelemetry-exporter-otlp-proto-http" }, { name = "opentelemetry-instrumentation-asyncio" }, { name = "opentelemetry-sdk" }, + { name = "openviking-sdk" }, { name = "pathspec" }, { name = "pdfminer-six" }, { name = "pdfplumber" }, @@ -3735,7 +3727,7 @@ requires-dist = [ { name = "build", marker = "extra == 'build'" }, { name = "cmake", marker = "extra == 'build'", specifier = ">=3.15" }, { name = "croniter", marker = "extra == 'bot'", specifier = ">=2.0.0" }, - { name = "cryptography", specifier = ">=42.0.0" }, + { name = "cryptography", specifier = ">=48.0.1" }, { name = "datasets", marker = "extra == 'benchmark'", specifier = ">=2.0.0" }, { name = "datasets", marker = "extra == 'eval'", specifier = ">=2.0.0" }, { name = "datasets", marker = "extra == 'test'", specifier = ">=2.0.0" }, @@ -3785,6 +3777,7 @@ requires-dist = [ { name = "opentelemetry-instrumentation-asyncio", specifier = ">=0.61b0" }, { name = "opentelemetry-sdk", specifier = ">=1.14" }, { name = "openviking", extras = ["bot", "bot-dingtalk", "bot-feishu", "bot-fuse", "bot-langfuse", "bot-opencode", "bot-qq", "bot-sandbox", "bot-slack", "bot-telegram"], marker = "extra == 'bot-full'" }, + { name = "openviking-sdk", specifier = ">=0.1.1" }, { name = "pandas", marker = "extra == 'benchmark'", specifier = ">=2.0.0" }, { name = "pandas", marker = "extra == 'eval'", specifier = ">=2.0.0" }, { name = "pandas", marker = "extra == 'test'", specifier = ">=2.0.0" }, @@ -3804,7 +3797,7 @@ requires-dist = [ { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=4.0.0" }, { name = "pytest-xdist", marker = "extra == 'test'", specifier = ">=3.5.0" }, { name = "python-docx", specifier = ">=1.0.0" }, - { name = "python-multipart", specifier = ">=0.0.27" }, + { name = "python-multipart", specifier = ">=0.0.31" }, { name = "python-pptx", specifier = ">=1.0.0" }, { name = "python-socketio", marker = "extra == 'bot'", specifier = ">=5.11.0" }, { name = "python-socks", extras = ["asyncio"], marker = "extra == 'bot'", specifier = ">=2.4.0" }, @@ -3856,6 +3849,18 @@ provides-extras = ["test", "opengauss", "dev", "doc", "eval", "gemini", "gemini- [package.metadata.requires-dev] dev = [{ name = "pytest", specifier = ">=9.0.2" }] +[[package]] +name = "openviking-sdk" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/d2/1638f2d9592a87e2350a5d663db46b2bab8fff57b79f688a3ed63da069a1/openviking_sdk-0.1.2.tar.gz", hash = "sha256:673370c7df89fa7f7c6708e05251450e1cb736c5b94d1ba87ff18cd40d51eff8", size = 26700, upload-time = "2026-06-22T06:12:06.241Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/c7/f21f7899a8902bbc9b2f8bfc6f60eb0b64addf44fc71b76f0c72241183f6/openviking_sdk-0.1.2-py3-none-any.whl", hash = "sha256:9e4c719d0f3f84dd686ffce45b80e8730c815ce6e4da94b94416307c679caa5f", size = 16904, upload-time = "2026-06-22T06:12:04.866Z" }, +] + [[package]] name = "orjson" version = "3.11.7" @@ -5008,11 +5013,11 @@ wheels = [ [[package]] name = "python-multipart" -version = "0.0.27" +version = "0.0.32" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/69/9b/f23807317a113dc36e74e75eb265a02dd1a4d9082abc3c1064acd22997c4/python_multipart-0.0.27.tar.gz", hash = "sha256:9870a6a8c5a20a5bf4f07c017bd1489006ff8836cff097b6933355ee2b49b602", size = 44043, upload-time = "2026-04-27T10:51:26.649Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/42/55c32bb9b12693c092ad250a0e82edb5b31ddeda6eb772de5f308b3804ad/python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e", size = 46881, upload-time = "2026-06-04T16:18:58.647Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/99/78/4126abcbdbd3c559d43e0db7f7b9173fc6befe45d39a2856cc0b8ec2a5a6/python_multipart-0.0.27-py3-none-any.whl", hash = "sha256:6fccfad17a27334bd0193681b369f476eda3409f17381a2d65aa7df3f7275645", size = 29254, upload-time = "2026-04-27T10:51:24.997Z" }, + { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, ] [[package]]