From 42b06998f1ca3ebf04095ebe59b1df0b4c52ba98 Mon Sep 17 00:00:00 2001 From: Kokoro2336 <2529677678@qq.com> Date: Mon, 13 Jul 2026 23:32:14 +0800 Subject: [PATCH 1/6] metric(qwen3): report prefix-cache query/hit Signed-off-by: Kokoro2336 <2529677678@qq.com> --- docs/index.md | 2 +- .../subsystems/frontend/prometheus-metrics.md | 61 +++++++++++++++++-- openinfer-engine/src/engine.rs | 10 +++ openinfer-glm52/src/scheduler/load.rs | 1 + openinfer-qwen3/src/executor.rs | 10 +++ openinfer-qwen3/src/scheduler.rs | 56 +++++++++++++++-- openinfer-qwen3/src/scheduler/effects.rs | 8 +++ openinfer-qwen3/src/scheduler/plan.rs | 1 + openinfer-qwen3/src/scheduler/resolve.rs | 1 + openinfer-qwen3/src/scheduler/tests.rs | 44 ++++++++++++- openinfer-sim/tests/frontend_e2e.rs | 4 ++ openinfer-vllm-frontend/src/bridge.rs | 19 +++++- openinfer-vllm-frontend/src/bridge/tests.rs | 22 +++++-- 13 files changed, 223 insertions(+), 16 deletions(-) diff --git a/docs/index.md b/docs/index.md index be0107d9..e05158ec 100644 --- a/docs/index.md +++ b/docs/index.md @@ -141,7 +141,7 @@ Organized by domain (model line / subsystem / playbook / lesson) instead of by l | `subsystems/frontend/simulated-inference-engine.md` | CPU-only simulated model crate for vLLM/OpenAI frontend and `vllm bench serve` validation without CUDA, real model weights, or real-model performance claims. | | `subsystems/frontend/cpu-profiling-baseline.md` | Frontend CPU profiling baseline using `openinfer-sim` with fixed TTFT=5ms/TPOT=12ms: 200 req / concurrency=16 shows ~150ms TTFT overhead (no dominant hotspot), heap allocation ~10%, stream polling ~7.5%, IPC ~1%; reproducible benchmark command and perf evidence documented. | | `subsystems/frontend/startup-time.md` | Qwen3-4B warm startup-to-ready 3.25s → ~1.45s: frontend tokenizer load runs concurrently with the engine load (HTTP still binds only after the engine registers), and the source safetensors mmap is kept alive to dodge ~0.4s of munmap stalling the next cudaMalloc. | -| `subsystems/frontend/prometheus-metrics.md` | `/metrics` request histograms work for every model; Qwen3, Qwen3.5, and GLM5.2 schedulers also publish running/waiting/KV engine gauges through `LoadSnapshot` watches. | +| `subsystems/frontend/prometheus-metrics.md` | `/metrics` request histograms come from bridge events/PrefillStats; scheduler stats ride `LoadSnapshot`. Qwen3 now reports real prefix-cache query/hit counters via monotonic totals differenced by the bridge, plus running/waiting/KV gauges. | | `subsystems/frontend/dashboards/README.md` | Grafana 10.4-validated dashboard for OpenInfer's live `/metrics` surface: HTTP traffic, request outcomes, scheduler/KV state, token throughput, and request latency. | ## subsystems / correctness diff --git a/docs/subsystems/frontend/prometheus-metrics.md b/docs/subsystems/frontend/prometheus-metrics.md index e17df01c..164d2ded 100644 --- a/docs/subsystems/frontend/prometheus-metrics.md +++ b/docs/subsystems/frontend/prometheus-metrics.md @@ -1,6 +1,6 @@ # Prometheus /metrics via the vLLM frontend -**TL;DR:** `/metrics` exposes request histograms for every model and engine gauges for schedulers that publish `LoadSnapshot`: Qwen3 and Qwen3.5 use one logical engine, while GLM5.2 EP8/DP8 uses eight rank-local engines and GLM5.2 TP8 uses one logical engine. The bridge forwards each partition's stats under the same identity the vLLM frontend uses for least-load routing. +**TL;DR:** `/metrics` exposes request histograms for every model and engine stats for schedulers that publish `LoadSnapshot`: Qwen3 reports real prefix-cache query/hit counters plus gauges, while GLM5.2 reports rank-local gauges. Monotonic scheduler totals are differenced by the bridge so coalesced watch updates remain delta-safe. Last touched: 2026-07 @@ -9,7 +9,7 @@ Last touched: 2026-07 Two independent paths feed the upstream Prometheus registry (`vllm-metrics`, served by `vllm-server` at `/metrics` with its HTTP middleware counters): 1. **Per-request path (works for every model crate).** The bridge stamps each request's first output with `Queued`/`Scheduled` timestamps and `PrefillStats` (prompt/computed/cached token split). The upstream `RequestMetricsTracker` turns those into `time_to_first_token_seconds`, `inter_token_latency_seconds`, `request_queue_time_seconds`, `prompt_tokens_total`, `generation_tokens_total`, `request_success_total`, `prompt_tokens_by_source_total`, … unconditionally — `disable_log_stats` only gates the periodic *text* logger, not Prometheus. -2. **Engine-gauge path (needs one `LoadSnapshot` watch per scheduler partition).** The scheduler publishes `LoadSnapshot { kv_used_blocks, kv_total_blocks, num_running_reqs, num_waiting_reqs }` at scheduler boundaries; one bridge identity per partition forwards its snapshot as a stats-only `RequestBatchOutputs`. The enclosing `engine_index` is both the routing identity and the Prometheus `engine` label. Watches coalesce to ≤1 message per scheduler step, and the scheduler's final idle publish settles the gauges back to 0. +2. **Engine-stats path (needs one `LoadSnapshot` watch per scheduler partition).** The scheduler publishes occupancy gauges at scheduler boundaries; Qwen3 also includes monotonic `prefix_cache_queries_total` / `prefix_cache_hits_total` token counts. One bridge identity per partition forwards a stats-only `RequestBatchOutputs`. The bridge differences the prefix totals against its previous observation before filling vLLM's interval-valued `SchedulerStats`, so a watch update that coalesces several steps loses no increments and Prometheus `inc_by` never replays a cumulative total. The enclosing `engine_index` is both the routing identity and the Prometheus `engine` label. The scheduler's final idle publish settles the gauges back to 0. For a single-partition model, `EngineHandle::with_load_watch` keeps the original one-engine contract. Qwen3.5 uses that contract for both its single-GPU backend and its TP backend because both execute one logical request stream through one scheduler. A partitioned scheduler uses `with_load_watches`, and the frontend launch declares the same engine count; a mismatch fails startup. GLM5.2 EP8 therefore registers engines 0–7, each bound to its own pending queue and KV pool. TP8 registers only engine 0 because its eight workers mirror one logical request stream. @@ -20,12 +20,63 @@ Measured cost is noise in both covered configurations: ## What deliberately reads zero (state at capture time) -- `prefix_cache_queries/hits` and the by-reason waiting split (`reason="deferred"` is driven by a skipped-request counter we don't report; all waiting shows as `reason="capacity"`). +- The by-reason waiting split (`reason="deferred"` is driven by a skipped-request counter we don't report; all waiting shows as `reason="capacity"`). - Spec-decode counters, per-GPU FLOPs/bytes estimates, KV-block residency histograms, cudagraph stats — the bridge sends `SchedulerStats::default()` for these fields. - Every model crate whose scheduler doesn't publish a `LoadSnapshot` watch (currently deepseek and kimi) gets path 1 only; its engine gauges are absent, not lying-zero — the bridge skips the stats task for that partition when no watch exists. ## Validated coverage and next step -Qwen3.5 single-GPU live RTX 5090 validation confirmed that running and KV gauges rise during generation, waiting rises under batch-slot pressure, and all three return to zero after drain and recovery. The commands and metric samples are recorded in [Qwen3.5 Scheduler LoadSnapshot](../../models/qwen35/load-snapshot.md#validation-boundary). TP uses the same scheduler publication path but was not part of that live run. +Wire `LoadSnapshot` publishing into the qwen35 scheduler (the other schedulers can follow the same recipe). A future partitioned model must expose its logical scheduler partitions instead of averaging them behind engine 0. -Next, wire the DeepSeek-V2-Lite and Kimi-K2 schedulers using the same recipe, and report real prefix-cache query/hit counters instead of zeros. A future partitioned model must expose its logical scheduler partitions instead of averaging them behind engine 0. +## Preparation — issue #603 + +- **Read**: + - `docs/index.md` — routed the work to the existing frontend metrics record and the Qwen3 prefix-cache record. + - `docs/subsystems/frontend/prometheus-metrics.md` — the bridge already publishes scheduler gauges from a coalescing `LoadSnapshot` watch, while prefix-cache counters are deliberately zero. + - `docs/models/qwen3/prefix-cache.md` — Qwen3 matches 16-token full blocks, skips matching for echo requests, and always leaves at least one prompt token uncached. + - `docs/subsystems/scheduler/scheduler.md` — the scheduler owns request state and publishes load at step boundaries from one GPU thread. + - `openinfer-engine/src/engine.rs`, `openinfer-qwen3/src/scheduler.rs`, `openinfer-qwen3/src/executor.rs`, and `openinfer-vllm-frontend/src/bridge.rs` — traced the snapshot, first-prefill result, and stats-only bridge paths. + - vLLM's pinned `rust/src/engine-core-client/src/{protocol/stats.rs,metrics.rs}` and `vllm/v1/{metrics/stats.py,core/kv_cache_manager.py}` — upstream records prompt-token queries and cached-token hits as interval deltas consumed with `inc_by`; skipped cache reads are not counted. + - [GitHub issue #603](https://github.com/openinfer-project/openinfer/issues/603) and parent #602 — require real Qwen3 query/hit counters, a bridge assertion, and a repeated-prompt live `/metrics` gate on one consumer GPU. +- **Relevant history**: + - `docs/subsystems/frontend/prometheus-metrics.md` — the stats watch intentionally coalesces scheduler steps, so raw per-step deltas would be lossy. + - `docs/models/qwen3/prefix-cache.md` — the existing `cached_tokens` first-prefill result is authoritative for local hits. +- **Risks / open questions**: + - A watch receiver may skip intermediate values; differencing monotonic totals is required to avoid losing per-step deltas. + - Live validation depends on a compatible CUDA GPU and locally available Qwen3-4B weights. + +## Execution Log — issue #603 + +### Steps 1–3: scheduler counters and bridge transport + +- Extended `openinfer_engine::LoadSnapshot` with explicitly cumulative prefix-cache query/hit token totals; unrelated schedulers retain zero through `Default`. +- Tagged actual Qwen3 first-prefill cache reads in the executor, excluding echo, later chunks, and cache-disabled execution. +- Accumulated upstream-compatible `queries = prompt_tokens` and `hits = cached_tokens` in the plain and LoRA-control scheduler loops. +- Differenced the cumulative values in `openinfer-vllm-frontend/src/bridge.rs` before filling `SchedulerStats.prefix_cache_stats`, preserving increments across coalesced watch updates without replaying earlier totals. +- Added focused Qwen3 scheduler and bridge assertions; validation commands are recorded after they run. + +### Step 4: release validation + +- `cargo test --release -p openinfer-vllm-frontend --lib`: 22 passed. +- `cargo test --release -p openinfer-engine --lib`: 10 passed. +- `cargo test --release -p openinfer-qwen3 --lib`: 73 passed, including `load_snapshot_accumulates_prefix_cache_query_and_hit_tokens`. +- `cargo test --release -p openinfer-sim --test frontend_e2e --no-run`: compiled successfully, covering every updated simulated `LoadSnapshot` producer. +- The host's libclang 22 makes pinned `rdma-mummy-sys` 0.2.4 generate opaque verbs structs; Qwen validation used a build-only `/tmp` source override with bindgen 0.72.1 and pthread pointer compatibility casts. No dependency or lockfile change was retained. + +### Step 5: live gate + +- Host GPU: NVIDIA GeForce RTX 2080 Ti, compute capability 7.5, 11,264 MiB total / 10,561 MiB free. +- Model search: no repository `models/` directory and no local Qwen `config.json` found under the usual `/home/zxh`, `/data`, `/mnt`, or `/models` roots. +- Result: not launched. This is not a suitable Qwen3 BF16 live-gate environment, and the required model weights are absent. + +## Debrief — issue #603 + +- **Outcome**: Qwen3 now reports real local prefix-cache prompt-token queries and cached-token hits through `/metrics`. The scheduler publishes monotonic totals, while the bridge emits interval deltas compatible with upstream vLLM Prometheus semantics. +- **Pitfalls encountered**: + - Publishing raw per-step deltas on a watch channel would lose counts when updates coalesce; cumulative producer state plus consumer differencing avoids that loss. + - The pinned RDMA binding crate is incompatible with this host's libclang 22 generation behavior, independent of the metrics change. +- **Lessons learned**: + - Prefix-cache query counters use prompt-token units, not block units; hits use cached-token units. + - Record only actual cache reads: echo requests, cache-disabled requests, and later prefill chunks do not enter the denominator. +- **Follow-ups**: + - Run the two-identical-prompt `/metrics` scrape on an Ampere-or-newer consumer GPU with local Qwen3-4B weights. diff --git a/openinfer-engine/src/engine.rs b/openinfer-engine/src/engine.rs index a21323f4..ae1d1446 100644 --- a/openinfer-engine/src/engine.rs +++ b/openinfer-engine/src/engine.rs @@ -352,6 +352,16 @@ pub struct LoadSnapshot { pub num_running_reqs: u64, /// Requests admitted but not yet running (KV pressure, prefetch wait). pub num_waiting_reqs: u64, + /// Prompt tokens queried against the local prefix cache since engine start. + /// + /// This is a monotonic total because the snapshot rides a coalescing watch + /// channel. Consumers that feed delta counters must difference consecutive + /// snapshots rather than forwarding this value directly. + pub prefix_cache_queries_total: u64, + /// Prompt tokens served from the local prefix cache since engine start. + /// Monotonic for the same coalescing-safe reason as + /// [`Self::prefix_cache_queries_total`]. + pub prefix_cache_hits_total: u64, } /// One full KV block that just became reusable from this engine's prefix cache. diff --git a/openinfer-glm52/src/scheduler/load.rs b/openinfer-glm52/src/scheduler/load.rs index 8578e0e3..2a0c214f 100644 --- a/openinfer-glm52/src/scheduler/load.rs +++ b/openinfer-glm52/src/scheduler/load.rs @@ -36,6 +36,7 @@ pub(super) fn publish_load( kv_total_blocks: kv_total_blocks as u64, num_running_reqs: slots[rank].iter().flatten().count() as u64, num_waiting_reqs: pending[rank].len() as u64, + ..LoadSnapshot::default() }); } } diff --git a/openinfer-qwen3/src/executor.rs b/openinfer-qwen3/src/executor.rs index 3ffbfb62..d5b63561 100644 --- a/openinfer-qwen3/src/executor.rs +++ b/openinfer-qwen3/src/executor.rs @@ -95,6 +95,9 @@ pub struct PrefillStepItem { /// Set by the executor after matching; the forward pass only computes /// the remaining suffix. pub(crate) cached_tokens: usize, + /// Whether this step performed the request's one local prefix-cache lookup. + /// False for later chunks, echo requests, and cache-disabled execution. + pub(crate) prefix_cache_queried: bool, /// Scheduler-set cap on prompt tokens forwarded this step (chunked /// prefill). The executor clamps it to the tokens actually remaining. pub(crate) chunk_budget: usize, @@ -125,6 +128,7 @@ impl PrefillStepItem { echo, lora_adapter: None, cached_tokens: 0, + prefix_cache_queried: false, chunk_budget: usize::MAX, chunk_start: 0, chunk_tokens, @@ -307,6 +311,7 @@ fn build_prefill_request_results( first_token_logprob: first_token_logprobs[i].take(), prompt_logprobs, cached_tokens: req.cached_tokens, + prefix_cache_queried: req.prefix_cache_queried, completed, prefill_pos: req.chunk_start + req.chunk_tokens, }); @@ -805,6 +810,10 @@ pub struct PrefillRequestResult { pub prompt_logprobs: Option>>, /// Prompt tokens served from the prefix cache (KV reused, not recomputed). pub cached_tokens: usize, + /// Whether the executor actually queried the local prefix cache for this + /// request. The scheduler uses this to exclude echo and cache-disabled + /// prefills from the aggregate query denominator. + pub prefix_cache_queried: bool, /// Whether the prompt is fully prefilled. When false this step ran a /// non-final chunk and `first_token` is meaningless. pub completed: bool, @@ -1841,6 +1850,7 @@ impl Qwen3Executor { // Echo needs logits for every prompt position; cached positions // are never forwarded, so echo requests prefill from scratch. if self.prefix_cache_enabled && !req.echo { + req.prefix_cache_queried = true; req.cached_tokens = rkv.match_and_add_prefix(self.kv_mgr.pool())?; } self.request_kvs.insert(req.request_id, rkv); diff --git a/openinfer-qwen3/src/scheduler.rs b/openinfer-qwen3/src/scheduler.rs index 56df3fac..2401f087 100644 --- a/openinfer-qwen3/src/scheduler.rs +++ b/openinfer-qwen3/src/scheduler.rs @@ -93,6 +93,23 @@ pub(super) struct PendingRequest { pub(super) cached_tokens: usize, } +/// Lifetime prefix-cache token totals published through `LoadSnapshot`. +/// Keeping totals here makes the coalescing watch lossless; the frontend bridge +/// differences observed snapshots into the interval deltas vLLM expects. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct PrefixCacheTotals { + queries: u64, + hits: u64, +} + +impl PrefixCacheTotals { + fn record(&mut self, prompt_tokens: usize, cached_tokens: usize) { + debug_assert!(cached_tokens <= prompt_tokens); + self.queries = self.queries.saturating_add(prompt_tokens as u64); + self.hits = self.hits.saturating_add(cached_tokens as u64); + } +} + impl PendingRequest { fn from_scheduler_request(request_id: RequestId, req: GenerateRequest) -> Self { Self { @@ -498,12 +515,15 @@ fn publish_load( executor: &E, num_running_reqs: u64, num_waiting_reqs: u64, + prefix_cache_totals: PrefixCacheTotals, ) { load_tx.send_replace(LoadSnapshot { kv_used_blocks: kv_total.saturating_sub(executor.available_blocks() as u64), kv_total_blocks: kv_total, num_running_reqs, num_waiting_reqs, + prefix_cache_queries_total: prefix_cache_totals.queries, + prefix_cache_hits_total: prefix_cache_totals.hits, }); } @@ -532,6 +552,7 @@ fn scheduler_loop( // Decode-overlap async prefill: pending requests whose prefill is in-flight // on the prefill overlap stream. `None` when no async prefill is running. let mut inflight_prefill_pending: Option> = None; + let mut prefix_cache_totals = PrefixCacheTotals::default(); info!("Scheduler ready"); @@ -544,6 +565,7 @@ fn scheduler_loop( + prefilling.len() + inflight_prefill_pending.as_ref().map_or(0, Vec::len)) as u64, (deferred.len() + loading.len()) as u64, + prefix_cache_totals, ); // Flush the prior step's cache changes to a router (no-op unless the // event feed is on). Top-of-loop, like `publish_load`: one pass per @@ -570,7 +592,13 @@ fn scheduler_loop( scheduled_at_unix_s, }; let effects = resolve_step(&executor, &active, artifacts); - apply_effects(&mut executor, &mut active, &mut prefilling, effects); + apply_effects( + &mut executor, + &mut active, + &mut prefilling, + &mut prefix_cache_totals, + effects, + ); } } @@ -703,7 +731,13 @@ fn scheduler_loop( // Only apply decode effects from the unified result. let effects = resolve_step(&executor, &active, artifacts); - apply_effects(&mut executor, &mut active, &mut prefilling, effects); + apply_effects( + &mut executor, + &mut active, + &mut prefilling, + &mut prefix_cache_totals, + effects, + ); // Track the pending prefill for next-iteration polling. inflight_prefill_pending = Some(pending_for_poll); @@ -721,7 +755,13 @@ fn scheduler_loop( } }; let effects = resolve_step(&executor, &active, artifacts); - apply_effects(&mut executor, &mut active, &mut prefilling, effects); + apply_effects( + &mut executor, + &mut active, + &mut prefilling, + &mut prefix_cache_totals, + effects, + ); } } @@ -743,6 +783,7 @@ fn scheduler_loop_with_lora_control( let mut prefilling: Vec = Vec::new(); let mut pending_control: VecDeque = VecDeque::new(); let mut post_control_deferred: Vec = Vec::new(); + let mut prefix_cache_totals = PrefixCacheTotals::default(); info!("Scheduler ready with LoRA control"); @@ -753,6 +794,7 @@ fn scheduler_loop_with_lora_control( &executor, (active.len() + prefilling.len()) as u64, (deferred.len() + loading.len() + post_control_deferred.len()) as u64, + prefix_cache_totals, ); // 1. Drain incoming commands. Generation submitted after a pending @@ -873,7 +915,13 @@ fn scheduler_loop_with_lora_control( } }; let effects = resolve_step(&executor, &active, artifacts); - apply_effects(&mut executor, &mut active, &mut prefilling, effects); + apply_effects( + &mut executor, + &mut active, + &mut prefilling, + &mut prefix_cache_totals, + effects, + ); } } diff --git a/openinfer-qwen3/src/scheduler/effects.rs b/openinfer-qwen3/src/scheduler/effects.rs index 6a7eab47..782605aa 100644 --- a/openinfer-qwen3/src/scheduler/effects.rs +++ b/openinfer-qwen3/src/scheduler/effects.rs @@ -7,6 +7,9 @@ use super::ActiveRequestState; use super::PendingRequest; use super::TokenEvent; use crate::executor::RequestId; +use openinfer_core::engine::{FinishReason, TokenLogprob, TokenSink}; + +use super::{ActiveRequestState, PendingRequest, PrefixCacheTotals, TokenEvent}; pub(super) struct PromptEchoEffect { pub(super) token_tx: TokenSink, @@ -24,6 +27,7 @@ pub(super) struct ScheduledEffect { pub(super) scheduled_at_unix_s: f64, pub(super) prompt_tokens: usize, pub(super) cached_tokens: usize, + pub(super) prefix_cache_queried: bool, } pub(super) enum PendingEffect { @@ -110,6 +114,7 @@ pub(super) fn apply_effects( executor: &mut impl crate::executor::ModelExecutor, active: &mut Vec, prefilling: &mut Vec, + prefix_cache_totals: &mut PrefixCacheTotals, effects: StepEffects, ) { // `Finished` events are not sent inline: they are collected here and @@ -121,6 +126,9 @@ pub(super) fn apply_effects( let mut finishes: Vec<(TokenSink, TokenEvent)> = Vec::new(); for scheduled in effects.scheduled { + if scheduled.prefix_cache_queried { + prefix_cache_totals.record(scheduled.prompt_tokens, scheduled.cached_tokens); + } let _ = scheduled.token_tx.send(TokenEvent::Scheduled { queued_at_unix_s: scheduled .queued_at_unix_s diff --git a/openinfer-qwen3/src/scheduler/plan.rs b/openinfer-qwen3/src/scheduler/plan.rs index fa5d479e..70eea125 100644 --- a/openinfer-qwen3/src/scheduler/plan.rs +++ b/openinfer-qwen3/src/scheduler/plan.rs @@ -225,6 +225,7 @@ fn build_prefill_items(pending: &[PendingRequest], indices: &[usize]) -> Vec
>>,
     prefetch_offers: Arc>>,
     stop_token: Option,
+    prefix_cache_hits: VecDeque,
 }
 
 impl FakeExecutor {
@@ -52,6 +53,7 @@ impl FakeExecutor {
             dropped,
             prefetch_offers: Arc::new(Mutex::new(Vec::new())),
             stop_token: None,
+            prefix_cache_hits: VecDeque::new(),
         }
     }
 
@@ -75,6 +77,11 @@ impl FakeExecutor {
         self
     }
 
+    fn with_prefix_cache_hits(mut self, hits: &[usize]) -> Self {
+        self.prefix_cache_hits = hits.iter().copied().collect();
+        self
+    }
+
     /// Advance a request's prompt by one chunk, mirroring the real
     /// executor: clamp the scheduler's budget to the tokens remaining
     /// and report the new authoritative position.
@@ -92,12 +99,16 @@ impl FakeExecutor {
         } else {
             self.prefill_positions.insert(req.request_id, prefill_pos);
         }
+        let cached_tokens = (start == 0)
+            .then(|| self.prefix_cache_hits.pop_front())
+            .flatten();
         PrefillRequestResult {
             request_id: req.request_id,
             first_token: 100 + req.request_id.get() as u32,
             first_token_logprob: None,
             prompt_logprobs: None,
-            cached_tokens: 0,
+            cached_tokens: cached_tokens.unwrap_or(0),
+            prefix_cache_queried: cached_tokens.is_some(),
             completed,
             prefill_pos,
         }
@@ -838,6 +849,36 @@ fn wait_until(timeout: Duration, mut predicate: impl FnMut() -> bool) -> bool {
     false
 }
 
+#[test]
+fn load_snapshot_accumulates_prefix_cache_query_and_hit_tokens() {
+    let dropped = Arc::new(Mutex::new(Vec::new()));
+    let executor = FakeExecutor::new(8, Arc::clone(&dropped)).with_prefix_cache_hits(&[0, 16]);
+    let handle = start_with_executor(executor, 42, DEFAULT_MAX_PREFILL_TOKENS);
+    let load_rx = handle.load_watch().expect("qwen3 publishes load snapshots");
+
+    for _ in 0..2 {
+        let (req, mut token_rx) = request(32, 1);
+        handle.submit(req).expect("submit cache-metrics request");
+        assert!(matches!(
+            recv_skipping_scheduled(&mut token_rx),
+            Some(TokenEvent::Token { .. })
+        ));
+        assert!(matches!(
+            recv_skipping_scheduled(&mut token_rx),
+            Some(TokenEvent::Finished { .. })
+        ));
+    }
+
+    assert!(
+        wait_until(Duration::from_secs(1), || {
+            let snapshot = *load_rx.borrow();
+            snapshot.prefix_cache_queries_total == 64 && snapshot.prefix_cache_hits_total == 16
+        }),
+        "two 32-token cache lookups should publish 64 queried and 16 hit tokens; got {:?}",
+        *load_rx.borrow()
+    );
+}
+
 #[test]
 fn unknown_lora_request_is_rejected_without_blocking_base_request() {
     let dropped = Arc::new(Mutex::new(Vec::new()));
@@ -962,6 +1003,7 @@ fn retiring_multiple_active_requests_tolerates_unsorted_indices() {
         &mut executor,
         &mut active,
         &mut Vec::new(),
+        &mut PrefixCacheTotals::default(),
         effects::StepEffects {
             scheduled: Vec::new(),
             prompt_echoes: Vec::new(),
diff --git a/openinfer-sim/tests/frontend_e2e.rs b/openinfer-sim/tests/frontend_e2e.rs
index f1ebcd24..ddf279a3 100644
--- a/openinfer-sim/tests/frontend_e2e.rs
+++ b/openinfer-sim/tests/frontend_e2e.rs
@@ -273,6 +273,7 @@ async fn one_http_endpoint_exports_per_engine_scheduler_metrics() -> Result<()>
             kv_total_blocks: 100,
             num_running_reqs: 1,
             num_waiting_reqs: 0,
+            ..LoadSnapshot::default()
         },
     )?;
     server.publish_load(
@@ -282,6 +283,7 @@ async fn one_http_endpoint_exports_per_engine_scheduler_metrics() -> Result<()>
             kv_total_blocks: 100,
             num_running_reqs: 0,
             num_waiting_reqs: 2,
+            ..LoadSnapshot::default()
         },
     )?;
     wait_for_metrics(
@@ -304,6 +306,7 @@ async fn one_http_endpoint_exports_per_engine_scheduler_metrics() -> Result<()>
             kv_total_blocks: 100,
             num_running_reqs: 3,
             num_waiting_reqs: 4,
+            ..LoadSnapshot::default()
         },
     )?;
     server.publish_load(
@@ -313,6 +316,7 @@ async fn one_http_endpoint_exports_per_engine_scheduler_metrics() -> Result<()>
             kv_total_blocks: 100,
             num_running_reqs: 5,
             num_waiting_reqs: 6,
+            ..LoadSnapshot::default()
         },
     )?;
     wait_for_metrics(
diff --git a/openinfer-vllm-frontend/src/bridge.rs b/openinfer-vllm-frontend/src/bridge.rs
index e1530a87..92cdf30b 100644
--- a/openinfer-vllm-frontend/src/bridge.rs
+++ b/openinfer-vllm-frontend/src/bridge.rs
@@ -599,9 +599,20 @@ async fn publish_scheduler_stats(
     output_tx: mpsc::UnboundedSender,
     shutdown: CancellationToken,
 ) -> Result<()> {
+    let mut previous_prefix_cache_queries = 0;
+    let mut previous_prefix_cache_hits = 0;
     loop {
         let snapshot = *load_rx.borrow_and_update();
-        let stats = SchedulerStats {
+        let prefix_cache_queries = snapshot
+            .prefix_cache_queries_total
+            .saturating_sub(previous_prefix_cache_queries);
+        let prefix_cache_hits = snapshot
+            .prefix_cache_hits_total
+            .saturating_sub(previous_prefix_cache_hits);
+        previous_prefix_cache_queries = snapshot.prefix_cache_queries_total;
+        previous_prefix_cache_hits = snapshot.prefix_cache_hits_total;
+
+        let mut stats = SchedulerStats {
             num_running_reqs: snapshot.num_running_reqs,
             num_waiting_reqs: snapshot.num_waiting_reqs,
             kv_cache_usage: if snapshot.kv_total_blocks == 0 {
@@ -611,6 +622,12 @@ async fn publish_scheduler_stats(
             },
             ..SchedulerStats::default()
         };
+        // Upstream records these with Prometheus `inc_by`, so only the delta
+        // since this bridge's previous observation belongs in SchedulerStats.
+        // The scheduler-side values stay cumulative because watch updates may
+        // coalesce; differencing a jump preserves every skipped increment.
+        stats.prefix_cache_stats.base.queries = prefix_cache_queries;
+        stats.prefix_cache_stats.base.hits = prefix_cache_hits;
         let outputs = RequestBatchOutputs {
             engine_index,
             scheduler_stats: Some(Box::new(stats)),
diff --git a/openinfer-vllm-frontend/src/bridge/tests.rs b/openinfer-vllm-frontend/src/bridge/tests.rs
index ab9bddac..e0a51c05 100644
--- a/openinfer-vllm-frontend/src/bridge/tests.rs
+++ b/openinfer-vllm-frontend/src/bridge/tests.rs
@@ -456,8 +456,9 @@ fn rejected_request_is_reported_as_error() {
 
 /// The scheduler-stats task turns each load-watch snapshot into a stats-only
 /// batch (no request outputs, no finished set) with the queue gauges and the
-/// fractional KV usage the frontend records into Prometheus, sends the current
-/// snapshot up front, and follows every watch update with exactly one message.
+/// fractional KV usage and interval prefix-cache deltas the frontend records
+/// into Prometheus, sends the current snapshot up front, and follows every
+/// watch update with exactly one message.
 #[tokio::test]
 async fn load_snapshots_become_stats_only_batches() {
     let (load_tx, load_rx) = tokio::sync::watch::channel(LoadSnapshot {
@@ -465,6 +466,8 @@ async fn load_snapshots_become_stats_only_batches() {
         kv_total_blocks: 100,
         num_running_reqs: 2,
         num_waiting_reqs: 1,
+        prefix_cache_queries_total: 32,
+        prefix_cache_hits_total: 16,
     });
     let (output_tx, mut output_rx) = mpsc::unbounded_channel();
     let shutdown = CancellationToken::new();
@@ -486,8 +489,17 @@ async fn load_snapshots_become_stats_only_batches() {
     assert_eq!(stats.num_running_reqs, 2);
     assert_eq!(stats.num_waiting_reqs, 1);
     assert!((stats.kv_cache_usage - 0.25).abs() < 1e-9);
-
-    load_tx.send_replace(LoadSnapshot::default());
+    assert_eq!(stats.prefix_cache_stats.base.queries, 32);
+    assert_eq!(stats.prefix_cache_stats.base.hits, 16);
+
+    // The cumulative source jumps over multiple possible scheduler steps; the
+    // bridge must forward only the unobserved interval, without losing it to
+    // watch-channel coalescing or replaying the earlier totals.
+    load_tx.send_replace(LoadSnapshot {
+        prefix_cache_queries_total: 96,
+        prefix_cache_hits_total: 48,
+        ..LoadSnapshot::default()
+    });
     let batch = match output_rx.recv().await.expect("drained stats batch") {
         EngineCoreOutputs::RequestBatch(batch) => batch,
         other => panic!("expected a stats batch, got {other:?}"),
@@ -495,6 +507,8 @@ async fn load_snapshots_become_stats_only_batches() {
     let stats = batch.scheduler_stats.expect("scheduler stats");
     assert_eq!(stats.num_running_reqs, 0);
     assert_eq!(stats.kv_cache_usage.to_bits(), 0.0_f64.to_bits());
+    assert_eq!(stats.prefix_cache_stats.base.queries, 64);
+    assert_eq!(stats.prefix_cache_stats.base.hits, 32);
 
     shutdown.cancel();
     task.await

From 23dd73b4de1472e92413c2cc29ba22a6deab6972 Mon Sep 17 00:00:00 2001
From: Kokoro2336 <2529677678@qq.com>
Date: Fri, 17 Jul 2026 14:33:13 +0800
Subject: [PATCH 2/6] chore: remove redundant docs.

Signed-off-by: Kokoro2336 <2529677678@qq.com>
---
 .../subsystems/frontend/prometheus-metrics.md | 83 +++----------------
 1 file changed, 12 insertions(+), 71 deletions(-)

diff --git a/docs/subsystems/frontend/prometheus-metrics.md b/docs/subsystems/frontend/prometheus-metrics.md
index 164d2ded..db422a56 100644
--- a/docs/subsystems/frontend/prometheus-metrics.md
+++ b/docs/subsystems/frontend/prometheus-metrics.md
@@ -1,82 +1,23 @@
-# Prometheus /metrics via the vLLM frontend
+# Prometheus `/metrics` via the vLLM frontend
 
-**TL;DR:** `/metrics` exposes request histograms for every model and engine stats for schedulers that publish `LoadSnapshot`: Qwen3 reports real prefix-cache query/hit counters plus gauges, while GLM5.2 reports rank-local gauges. Monotonic scheduler totals are differenced by the bridge so coalesced watch updates remain delta-safe.
+**TL;DR:** `/metrics` exposes request metrics for every model. Schedulers that publish `LoadSnapshot` also expose load and KV gauges; Qwen3 additionally reports real prefix-cache query and hit token counters.
 
 Last touched: 2026-07
 
-## How the numbers flow
+## Metric sources
 
-Two independent paths feed the upstream Prometheus registry (`vllm-metrics`, served by `vllm-server` at `/metrics` with its HTTP middleware counters):
+- Request metrics come from frontend request events and include latency histograms, prompt/generated token totals, and request outcomes.
+- Scheduler metrics come from `LoadSnapshot`. They are present only for model schedulers that publish a load watch.
+- Each logical scheduler partition has its own `engine` label.
 
-1. **Per-request path (works for every model crate).** The bridge stamps each request's first output with `Queued`/`Scheduled` timestamps and `PrefillStats` (prompt/computed/cached token split). The upstream `RequestMetricsTracker` turns those into `time_to_first_token_seconds`, `inter_token_latency_seconds`, `request_queue_time_seconds`, `prompt_tokens_total`, `generation_tokens_total`, `request_success_total`, `prompt_tokens_by_source_total`, … unconditionally — `disable_log_stats` only gates the periodic *text* logger, not Prometheus.
-2. **Engine-stats path (needs one `LoadSnapshot` watch per scheduler partition).** The scheduler publishes occupancy gauges at scheduler boundaries; Qwen3 also includes monotonic `prefix_cache_queries_total` / `prefix_cache_hits_total` token counts. One bridge identity per partition forwards a stats-only `RequestBatchOutputs`. The bridge differences the prefix totals against its previous observation before filling vLLM's interval-valued `SchedulerStats`, so a watch update that coalesces several steps loses no increments and Prometheus `inc_by` never replays a cumulative total. The enclosing `engine_index` is both the routing identity and the Prometheus `engine` label. The scheduler's final idle publish settles the gauges back to 0.
+Qwen3 publishes monotonic prefix-cache totals. The frontend converts them to interval deltas before passing them to vLLM's Prometheus collector, so coalesced load-watch updates do not lose or double-count increments.
 
-For a single-partition model, `EngineHandle::with_load_watch` keeps the original one-engine contract. Qwen3.5 uses that contract for both its single-GPU backend and its TP backend because both execute one logical request stream through one scheduler. A partitioned scheduler uses `with_load_watches`, and the frontend launch declares the same engine count; a mismatch fails startup. GLM5.2 EP8 therefore registers engines 0–7, each bound to its own pending queue and KV pool. TP8 registers only engine 0 because its eight workers mirror one logical request stream.
+`vllm:prefix_cache_queries_total` counts prompt tokens submitted to an actual first-prefix lookup. `vllm:prefix_cache_hits_total` counts tokens restored from matching full cache blocks. Echo requests, cache-disabled requests, and later chunks of the same prefill do not add queries. Qwen3 cache blocks contain 16 tokens.
 
-Measured cost is noise in both covered configurations:
+## Check prefix-cache counters
 
-- Qwen3 TPOT: 10.6387 ms (main) vs 10.6395 ms (metrics branch) over 828 tokens.
-- GLM5.2 EP8, three-run median at concurrency 64: 1268.58 vs 1264.82 output tok/s (-0.30%); TPOT p50 41.76 vs 41.35 ms.
+Send the same prompt of at least 16 tokens twice, then scrape `/metrics` and inspect `vllm:prefix_cache_queries_total` and `vllm:prefix_cache_hits_total`. Both counters are cumulative. Queries should increase for each request; hits should increase when the repeated prompt reuses cached blocks.
 
-## What deliberately reads zero (state at capture time)
+## Coverage limits
 
-- The by-reason waiting split (`reason="deferred"` is driven by a skipped-request counter we don't report; all waiting shows as `reason="capacity"`).
-- Spec-decode counters, per-GPU FLOPs/bytes estimates, KV-block residency histograms, cudagraph stats — the bridge sends `SchedulerStats::default()` for these fields.
-- Every model crate whose scheduler doesn't publish a `LoadSnapshot` watch (currently deepseek and kimi) gets path 1 only; its engine gauges are absent, not lying-zero — the bridge skips the stats task for that partition when no watch exists.
-
-## Validated coverage and next step
-
-Wire `LoadSnapshot` publishing into the qwen35 scheduler (the other schedulers can follow the same recipe). A future partitioned model must expose its logical scheduler partitions instead of averaging them behind engine 0.
-
-## Preparation — issue #603
-
-- **Read**:
-  - `docs/index.md` — routed the work to the existing frontend metrics record and the Qwen3 prefix-cache record.
-  - `docs/subsystems/frontend/prometheus-metrics.md` — the bridge already publishes scheduler gauges from a coalescing `LoadSnapshot` watch, while prefix-cache counters are deliberately zero.
-  - `docs/models/qwen3/prefix-cache.md` — Qwen3 matches 16-token full blocks, skips matching for echo requests, and always leaves at least one prompt token uncached.
-  - `docs/subsystems/scheduler/scheduler.md` — the scheduler owns request state and publishes load at step boundaries from one GPU thread.
-  - `openinfer-engine/src/engine.rs`, `openinfer-qwen3/src/scheduler.rs`, `openinfer-qwen3/src/executor.rs`, and `openinfer-vllm-frontend/src/bridge.rs` — traced the snapshot, first-prefill result, and stats-only bridge paths.
-  - vLLM's pinned `rust/src/engine-core-client/src/{protocol/stats.rs,metrics.rs}` and `vllm/v1/{metrics/stats.py,core/kv_cache_manager.py}` — upstream records prompt-token queries and cached-token hits as interval deltas consumed with `inc_by`; skipped cache reads are not counted.
-  - [GitHub issue #603](https://github.com/openinfer-project/openinfer/issues/603) and parent #602 — require real Qwen3 query/hit counters, a bridge assertion, and a repeated-prompt live `/metrics` gate on one consumer GPU.
-- **Relevant history**:
-  - `docs/subsystems/frontend/prometheus-metrics.md` — the stats watch intentionally coalesces scheduler steps, so raw per-step deltas would be lossy.
-  - `docs/models/qwen3/prefix-cache.md` — the existing `cached_tokens` first-prefill result is authoritative for local hits.
-- **Risks / open questions**:
-  - A watch receiver may skip intermediate values; differencing monotonic totals is required to avoid losing per-step deltas.
-  - Live validation depends on a compatible CUDA GPU and locally available Qwen3-4B weights.
-
-## Execution Log — issue #603
-
-### Steps 1–3: scheduler counters and bridge transport
-
-- Extended `openinfer_engine::LoadSnapshot` with explicitly cumulative prefix-cache query/hit token totals; unrelated schedulers retain zero through `Default`.
-- Tagged actual Qwen3 first-prefill cache reads in the executor, excluding echo, later chunks, and cache-disabled execution.
-- Accumulated upstream-compatible `queries = prompt_tokens` and `hits = cached_tokens` in the plain and LoRA-control scheduler loops.
-- Differenced the cumulative values in `openinfer-vllm-frontend/src/bridge.rs` before filling `SchedulerStats.prefix_cache_stats`, preserving increments across coalesced watch updates without replaying earlier totals.
-- Added focused Qwen3 scheduler and bridge assertions; validation commands are recorded after they run.
-
-### Step 4: release validation
-
-- `cargo test --release -p openinfer-vllm-frontend --lib`: 22 passed.
-- `cargo test --release -p openinfer-engine --lib`: 10 passed.
-- `cargo test --release -p openinfer-qwen3 --lib`: 73 passed, including `load_snapshot_accumulates_prefix_cache_query_and_hit_tokens`.
-- `cargo test --release -p openinfer-sim --test frontend_e2e --no-run`: compiled successfully, covering every updated simulated `LoadSnapshot` producer.
-- The host's libclang 22 makes pinned `rdma-mummy-sys` 0.2.4 generate opaque verbs structs; Qwen validation used a build-only `/tmp` source override with bindgen 0.72.1 and pthread pointer compatibility casts. No dependency or lockfile change was retained.
-
-### Step 5: live gate
-
-- Host GPU: NVIDIA GeForce RTX 2080 Ti, compute capability 7.5, 11,264 MiB total / 10,561 MiB free.
-- Model search: no repository `models/` directory and no local Qwen `config.json` found under the usual `/home/zxh`, `/data`, `/mnt`, or `/models` roots.
-- Result: not launched. This is not a suitable Qwen3 BF16 live-gate environment, and the required model weights are absent.
-
-## Debrief — issue #603
-
-- **Outcome**: Qwen3 now reports real local prefix-cache prompt-token queries and cached-token hits through `/metrics`. The scheduler publishes monotonic totals, while the bridge emits interval deltas compatible with upstream vLLM Prometheus semantics.
-- **Pitfalls encountered**:
-  - Publishing raw per-step deltas on a watch channel would lose counts when updates coalesce; cumulative producer state plus consumer differencing avoids that loss.
-  - The pinned RDMA binding crate is incompatible with this host's libclang 22 generation behavior, independent of the metrics change.
-- **Lessons learned**:
-  - Prefix-cache query counters use prompt-token units, not block units; hits use cached-token units.
-  - Record only actual cache reads: echo requests, cache-disabled requests, and later prefill chunks do not enter the denominator.
-- **Follow-ups**:
-  - Run the two-identical-prompt `/metrics` scrape on an Ampere-or-newer consumer GPU with local Qwen3-4B weights.
+Unsupported scheduler fields remain at their upstream defaults, including speculative-decoding counters, GPU FLOP/byte estimates, KV-residency histograms, and CUDA Graph statistics. Models without a `LoadSnapshot` watch expose request metrics but no scheduler gauges.

From 0be85fdf02ed456abe604ece2e5b3cccbc56aa04 Mon Sep 17 00:00:00 2001
From: Kokoro2336 <2529677678@qq.com>
Date: Fri, 17 Jul 2026 14:31:07 +0000
Subject: [PATCH 3/6] fix: some bugs

Signed-off-by: Kokoro2336 <2529677678@qq.com>
---
 openinfer-qwen3/Cargo.toml                   |   3 +
 openinfer-qwen3/src/executor.rs              | 114 +++++++++++++++----
 openinfer-qwen3/src/executor/remote_fetch.rs |   1 +
 3 files changed, 97 insertions(+), 21 deletions(-)

diff --git a/openinfer-qwen3/Cargo.toml b/openinfer-qwen3/Cargo.toml
index 66c7c6be..da7a6a94 100644
--- a/openinfer-qwen3/Cargo.toml
+++ b/openinfer-qwen3/Cargo.toml
@@ -11,6 +11,8 @@ clap = { workspace = true, optional = true }
 comfy-table = { workspace = true, optional = true }
 crossbeam-channel = { workspace = true }
 cudarc = { workspace = true }
+openinfer-kv-cache = { workspace = true }
+openinfer-kv-offload = { workspace = true, optional = true }
 fastrace = { workspace = true }
 half = { workspace = true }
 hex = { workspace = true, optional = true }
@@ -31,6 +33,7 @@ tokio = { workspace = true, features = ["sync"] }
 toml = { workspace = true, optional = true }
 
 [features]
+kv-offload = ["dep:openinfer-kv-offload"]
 kernel-call-trace = ["openinfer-core/kernel-call-trace"]
 kernel-report = [
   "dep:clap",
diff --git a/openinfer-qwen3/src/executor.rs b/openinfer-qwen3/src/executor.rs
index d5b63561..70a60811 100644
--- a/openinfer-qwen3/src/executor.rs
+++ b/openinfer-qwen3/src/executor.rs
@@ -18,21 +18,16 @@ use openinfer_core::engine::panic_message;
 use openinfer_core::kv_pool::KvLayout;
 use openinfer_core::ops;
 use openinfer_core::sampler::SamplingParams;
-use openinfer_core::tensor::DeviceContext;
-use openinfer_core::tensor::HiddenStates;
-use openinfer_core::weight_loader::WeightPrefetch;
-use openinfer_core::weight_loader::load_shard_info;
-use openinfer_kv_cache::KvBlockGuard;
-use openinfer_kv_cache::KvBuffer;
-use openinfer_kv_cache::KvCacheEvent;
-use openinfer_kv_cache::KvCacheManager;
-use openinfer_kv_cache::KvView;
-use openinfer_kv_cache::LoadReservation;
-use openinfer_kv_cache::PrefixProbe;
-use openinfer_kv_cache::RegisteredBlock;
-use openinfer_kv_offload::LoadHandle;
-use openinfer_kv_offload::OffloadConfig;
-use openinfer_kv_offload::OffloadEngine;
+use openinfer_core::tensor::{DeviceContext, DeviceVec, HiddenStates};
+use openinfer_kv_cache::{
+    KvBuffer, KvCacheEvent, KvCacheManager, KvView, RegisteredBlock,
+};
+#[cfg(feature = "kv-offload")]
+use openinfer_kv_cache::{KvBlockGuard, LoadReservation, PrefixProbe};
+#[cfg(feature = "kv-offload")]
+use openinfer_kv_offload::{LoadHandle, OffloadConfig, OffloadEngine};
+#[cfg(not(feature = "kv-offload"))]
+type OffloadEngine = ();
 use tokio::sync::broadcast;
 
 use crate::Qwen3LoraOptions;
@@ -49,10 +44,10 @@ use crate::weights::Qwen3Model;
 
 mod dflash_lane;
 mod dflash_prefill;
+#[cfg(feature = "kv-offload")]
 mod remote_fetch;
-use remote_fetch::QueryView;
-use remote_fetch::RemoteFetchAction;
-use remote_fetch::remote_fetch_action;
+#[cfg(feature = "kv-offload")]
+use remote_fetch::{QueryView, RemoteFetchAction, remote_fetch_action};
 mod spec;
 
 use dflash_lane::DFlashLaneState;
@@ -1084,6 +1079,7 @@ struct SendEvent(cudarc::driver::sys::CUevent);
 unsafe impl Send for SendEvent {}
 
 /// One request's in-flight CPU-tier KV prefetch.
+#[cfg(feature = "kv-offload")]
 ///
 /// `probe` holds the GPU-hit prefix resident for the request's whole parked
 /// life; `phase` tracks where the missing prefix currently is.
@@ -1092,6 +1088,10 @@ struct PrefetchState {
     phase: PrefetchPhase,
 }
 
+#[cfg(not(feature = "kv-offload"))]
+struct PrefetchState;
+
+#[cfg(feature = "kv-offload")]
 enum PrefetchPhase {
     /// pegaflow is pulling the missing prefix from a remote peer (P2P RDMA)
     /// into the local host tier; the executor re-queries each scheduler tick
@@ -1141,6 +1141,7 @@ const MISS_BREAKER_THRESHOLD: u32 = 3;
 
 /// vLLM-compat P/D mode, derived from [`crate::Qwen3VllmCompatOptions`] at
 /// executor build time (the hasher needs the resolved KV block size).
+#[cfg(feature = "kv-offload")]
 struct VllmCompatState {
     hasher: openinfer_kv_offload::VllmBlockHasher,
     /// Zero-hit wait window: how long a cold request re-queries before
@@ -1152,6 +1153,9 @@ struct VllmCompatState {
     consecutive_miss_windows: u32,
 }
 
+#[cfg(not(feature = "kv-offload"))]
+struct VllmCompatState;
+
 impl Qwen3Executor {
     pub(crate) fn dump_decode_graph_png(&self, png_path: &Path) -> Result {
         self.primary.dump_decode_graph_png(png_path.to_path_buf())
@@ -1210,6 +1214,7 @@ impl Qwen3Executor {
         let offload = build_offload(offload_opts, &kv_mgr, model.config(), model.device_ctx())?;
         let total_blocks = kv_mgr.pool().total_blocks();
         let padding_block_id = kv_mgr.pool().padding_block_id();
+        #[cfg(feature = "kv-offload")]
         let vllm_compat = match offload_opts.vllm_compat.as_ref() {
             None => None,
             Some(c) => {
@@ -1247,6 +1252,8 @@ impl Qwen3Executor {
                 })
             }
         };
+        #[cfg(not(feature = "kv-offload"))]
+        let vllm_compat = None;
         Ok(Self {
             metadata,
             kv_mgr,
@@ -1748,6 +1755,7 @@ impl Qwen3Executor {
     /// query can see them. A persistence barrier for handoff and tests; no-op
     /// without offload.
     pub fn flush_offload_saves(&self) {
+        #[cfg(feature = "kv-offload")]
         if let Some(offload) = &self.offload {
             offload.flush_saves();
         }
@@ -1791,6 +1799,7 @@ impl Qwen3Executor {
     /// tier (fire-and-forget). Safe to call right after `apply_prefill`/
     /// `apply_decode`: the producing step's token read-back has already
     /// synchronized the compute stream, so the sealed KV is fully written.
+    #[cfg(feature = "kv-offload")]
     fn save_sealed_blocks(&mut self, request_id: RequestId) {
         if self.offload.is_none() {
             return;
@@ -1834,6 +1843,9 @@ impl Qwen3Executor {
             .save(&block_ids, &block_hashes, pins);
     }
 
+    #[cfg(not(feature = "kv-offload"))]
+    fn save_sealed_blocks(&mut self, _request_id: RequestId) {}
+
     // ── Chunked prefill ────────────────────────────────────────────────
 
     /// Prepare one prefill step for `req`: create its `RequestKv` on the
@@ -1907,6 +1919,7 @@ impl Qwen3Executor {
     /// reserved blocks are staged + registered (held by the probe until the
     /// request prefills); on failure the state is dropped so the request
     /// prefills from scratch.
+    #[cfg(feature = "kv-offload")]
     fn settle_prefetch(
         &mut self,
         id: RequestId,
@@ -1933,6 +1946,7 @@ impl Qwen3Executor {
     /// answer either starts the H2D load (probe → Loading, still parked → and
     /// so returns `false`) or, on zero hit / reservation pressure / deadline,
     /// drops the prefetch so the request prefills from scratch.
+    #[cfg(feature = "kv-offload")]
     fn poll_remote_fetch(&mut self, id: RequestId, reserve_floor: usize) -> bool {
         let Some(st) = self.prefetch.get(&id) else {
             return true;
@@ -2046,6 +2060,7 @@ impl Qwen3Executor {
     /// Miss-breaker bookkeeping: one more request exhausted the whole
     /// zero-hit wait window. At the threshold the breaker opens and
     /// [`Self::begin_kv_prefetch`] stops parking new requests.
+    #[cfg(feature = "kv-offload")]
     fn note_miss_window_exhausted(&mut self) {
         let Some(compat) = self.vllm_compat.as_mut() else {
             return;
@@ -2062,6 +2077,7 @@ impl Qwen3Executor {
 
     /// Miss-breaker bookkeeping: remote content is visible again (leased
     /// hit), so cold requests may wait on the P/D handoff race once more.
+    #[cfg(feature = "kv-offload")]
     fn note_remote_hit(&mut self) {
         if let Some(compat) = self.vllm_compat.as_mut() {
             compat.consecutive_miss_windows = 0;
@@ -2073,6 +2089,7 @@ impl Qwen3Executor {
     /// of `probe`, which keeps the GPU-hit prefix resident meanwhile).
     /// `Err(())` means the prefetch was abandoned (lease released, no state
     /// kept) and the request should prefill from scratch.
+    #[cfg(feature = "kv-offload")]
     fn start_prefetch_load(
         &mut self,
         id: RequestId,
@@ -2182,6 +2199,7 @@ fn profile_kv_budget_on_worker(
 /// is disabled. Registers the fused KV buffer with pegaflow against the model's
 /// device/stream — must be called while that stream is still owned by the model
 /// (before it moves into the `RankWorker`).
+#[cfg(feature = "kv-offload")]
 fn build_offload(
     opts: &Qwen3OffloadOptions,
     kv_mgr: &KvCacheManager,
@@ -2241,6 +2259,21 @@ fn build_offload(
     Ok(Some(engine))
 }
 
+#[cfg(not(feature = "kv-offload"))]
+fn build_offload(
+    opts: &Qwen3OffloadOptions,
+    _kv_mgr: &KvCacheManager,
+    _config: &Config,
+    _ctx: &DeviceContext,
+) -> Result> {
+    anyhow::ensure!(
+        !opts.enabled && opts.p2p.is_none() && opts.vllm_compat.is_none(),
+        "Qwen3 KV offload is not compiled in; rebuild with `--features qwen3-kv-offload` \
+         for the server or `--features kv-offload` for openinfer-qwen3"
+    );
+    Ok(None)
+}
+
 fn ensure_lora_capacity(
     loaded_lora_adapters: &HashSet,
     lora_name: &str,
@@ -2297,12 +2330,19 @@ impl ModelExecutor for Qwen3Executor {
         self.metadata.stop_token_ids.contains(&token_id)
     }
 
+    #[cfg(feature = "kv-offload")]
     fn prefetched_blocks(&self, request_id: RequestId) -> usize {
         self.prefetch
             .get(&request_id)
             .map_or(0, |st| st.probe.held_blocks())
     }
 
+    #[cfg(not(feature = "kv-offload"))]
+    fn prefetched_blocks(&self, _request_id: RequestId) -> usize {
+        0
+    }
+
+    #[cfg(feature = "kv-offload")]
     fn release_finished_events(&self, finishes: Vec<(TokenSink, TokenEvent)>) {
         match &self.offload {
             // P/D prefill role: the peer treats our HTTP response as the
@@ -2316,6 +2356,11 @@ impl ModelExecutor for Qwen3Executor {
         }
     }
 
+    #[cfg(not(feature = "kv-offload"))]
+    fn release_finished_events(&self, finishes: Vec<(TokenSink, TokenEvent)>) {
+        send_finished_events(finishes);
+    }
+
     fn drop_request(&mut self, request_id: RequestId) -> Result<()> {
         // Remove and drop — RAII on SchedulableSequence's block guards
         // returns all allocated blocks regardless of lifecycle state. The same
@@ -2340,9 +2385,12 @@ impl ModelExecutor for Qwen3Executor {
         // phase has no local reservation yet — pegaflow owns that fetch and its
         // pinned-pool destination; dropping the state simply orphans the query,
         // which pegaflow's own req-scoped prefetch GC cleans up.)
-        if let Some(state) = self.prefetch.remove(&request_id) {
-            if let PrefetchPhase::Loading { handle, .. } = state.phase {
-                let _ = handle.wait();
+        #[cfg(feature = "kv-offload")]
+        {
+            if let Some(state) = self.prefetch.remove(&request_id) {
+                if let PrefetchPhase::Loading { handle, .. } = state.phase {
+                    let _ = handle.wait();
+                }
             }
         }
         self.saved_cursor.remove(&request_id);
@@ -2372,6 +2420,7 @@ impl ModelExecutor for Qwen3Executor {
         runs
     }
 
+    #[cfg(feature = "kv-offload")]
     fn begin_kv_prefetch(
         &mut self,
         request_id: RequestId,
@@ -2492,6 +2541,18 @@ impl ModelExecutor for Qwen3Executor {
         }
     }
 
+    #[cfg(not(feature = "kv-offload"))]
+    fn begin_kv_prefetch(
+        &mut self,
+        _request_id: RequestId,
+        _prompt_tokens: &[u32],
+        _lora_adapter: Option<&str>,
+        _reserve_floor: usize,
+    ) -> bool {
+        false
+    }
+
+    #[cfg(feature = "kv-offload")]
     fn drain_ready_prefetch(&mut self, reserve_floor: usize) -> Vec {
         let ids: Vec = self.prefetch.keys().copied().collect();
         let mut done = Vec::new();
@@ -2521,6 +2582,12 @@ impl ModelExecutor for Qwen3Executor {
         done
     }
 
+    #[cfg(not(feature = "kv-offload"))]
+    fn drain_ready_prefetch(&mut self, _reserve_floor: usize) -> Vec {
+        Vec::new()
+    }
+
+    #[cfg(feature = "kv-offload")]
     fn wait_ready_prefetch(&mut self, reserve_floor: usize) -> Vec {
         let mut done = Vec::new();
         if let Some(id) = self
@@ -2562,6 +2629,11 @@ impl ModelExecutor for Qwen3Executor {
         done
     }
 
+    #[cfg(not(feature = "kv-offload"))]
+    fn wait_ready_prefetch(&mut self, _reserve_floor: usize) -> Vec {
+        Vec::new()
+    }
+
     fn execute_prefill(&mut self, plan: PrefillPlan<'_>) -> Result {
         // 1. Create RequestKvs (first chunk only), clamp chunk budgets,
         // schedule KV for this step's tokens
diff --git a/openinfer-qwen3/src/executor/remote_fetch.rs b/openinfer-qwen3/src/executor/remote_fetch.rs
index de1de59d..4ed792b8 100644
--- a/openinfer-qwen3/src/executor/remote_fetch.rs
+++ b/openinfer-qwen3/src/executor/remote_fetch.rs
@@ -26,6 +26,7 @@ pub(super) enum QueryView {
     Ready { lease: Option, num_blocks: usize },
 }
 
+#[cfg(feature = "kv-offload")]
 impl From for QueryView {
     fn from(outcome: openinfer_kv_offload::QueryOutcome) -> Self {
         match outcome {

From 7eb7c1a08704d2121fdb19d4040fa8f9a22b6f0c Mon Sep 17 00:00:00 2001
From: Kokoro2336 <2529677678@qq.com>
Date: Fri, 17 Jul 2026 14:48:03 +0000
Subject: [PATCH 4/6] chore: fmt

Signed-off-by: Kokoro2336 <2529677678@qq.com>
---
 openinfer-qwen3/src/executor.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/openinfer-qwen3/src/executor.rs b/openinfer-qwen3/src/executor.rs
index 70a60811..146a28df 100644
--- a/openinfer-qwen3/src/executor.rs
+++ b/openinfer-qwen3/src/executor.rs
@@ -19,11 +19,9 @@ use openinfer_core::kv_pool::KvLayout;
 use openinfer_core::ops;
 use openinfer_core::sampler::SamplingParams;
 use openinfer_core::tensor::{DeviceContext, DeviceVec, HiddenStates};
-use openinfer_kv_cache::{
-    KvBuffer, KvCacheEvent, KvCacheManager, KvView, RegisteredBlock,
-};
 #[cfg(feature = "kv-offload")]
 use openinfer_kv_cache::{KvBlockGuard, LoadReservation, PrefixProbe};
+use openinfer_kv_cache::{KvBuffer, KvCacheEvent, KvCacheManager, KvView, RegisteredBlock};
 #[cfg(feature = "kv-offload")]
 use openinfer_kv_offload::{LoadHandle, OffloadConfig, OffloadEngine};
 #[cfg(not(feature = "kv-offload"))]

From b17a03ce56f77facf1b18e6e2395ca121939b1af Mon Sep 17 00:00:00 2001
From: Kokoro2336 <2529677678@qq.com>
Date: Wed, 22 Jul 2026 12:14:48 +0800
Subject: [PATCH 5/6] fix: remove cfg

Signed-off-by: Kokoro2336 <2529677678@qq.com>
---
 openinfer-qwen3/Cargo.toml                   |  3 +-
 openinfer-qwen3/src/executor.rs              | 95 ++------------------
 openinfer-qwen3/src/executor/remote_fetch.rs |  1 -
 3 files changed, 8 insertions(+), 91 deletions(-)

diff --git a/openinfer-qwen3/Cargo.toml b/openinfer-qwen3/Cargo.toml
index da7a6a94..ac01b82a 100644
--- a/openinfer-qwen3/Cargo.toml
+++ b/openinfer-qwen3/Cargo.toml
@@ -12,7 +12,7 @@ comfy-table = { workspace = true, optional = true }
 crossbeam-channel = { workspace = true }
 cudarc = { workspace = true }
 openinfer-kv-cache = { workspace = true }
-openinfer-kv-offload = { workspace = true, optional = true }
+openinfer-kv-offload = { workspace = true }
 fastrace = { workspace = true }
 half = { workspace = true }
 hex = { workspace = true, optional = true }
@@ -33,7 +33,6 @@ tokio = { workspace = true, features = ["sync"] }
 toml = { workspace = true, optional = true }
 
 [features]
-kv-offload = ["dep:openinfer-kv-offload"]
 kernel-call-trace = ["openinfer-core/kernel-call-trace"]
 kernel-report = [
   "dep:clap",
diff --git a/openinfer-qwen3/src/executor.rs b/openinfer-qwen3/src/executor.rs
index 146a28df..3937d518 100644
--- a/openinfer-qwen3/src/executor.rs
+++ b/openinfer-qwen3/src/executor.rs
@@ -19,13 +19,11 @@ use openinfer_core::kv_pool::KvLayout;
 use openinfer_core::ops;
 use openinfer_core::sampler::SamplingParams;
 use openinfer_core::tensor::{DeviceContext, DeviceVec, HiddenStates};
-#[cfg(feature = "kv-offload")]
-use openinfer_kv_cache::{KvBlockGuard, LoadReservation, PrefixProbe};
-use openinfer_kv_cache::{KvBuffer, KvCacheEvent, KvCacheManager, KvView, RegisteredBlock};
-#[cfg(feature = "kv-offload")]
+use openinfer_kv_cache::{
+    KvBlockGuard, KvBuffer, KvCacheEvent, KvCacheManager, KvView, LoadReservation, PrefixProbe,
+    RegisteredBlock,
+};
 use openinfer_kv_offload::{LoadHandle, OffloadConfig, OffloadEngine};
-#[cfg(not(feature = "kv-offload"))]
-type OffloadEngine = ();
 use tokio::sync::broadcast;
 
 use crate::Qwen3LoraOptions;
@@ -42,9 +40,7 @@ use crate::weights::Qwen3Model;
 
 mod dflash_lane;
 mod dflash_prefill;
-#[cfg(feature = "kv-offload")]
 mod remote_fetch;
-#[cfg(feature = "kv-offload")]
 use remote_fetch::{QueryView, RemoteFetchAction, remote_fetch_action};
 mod spec;
 
@@ -1077,7 +1073,6 @@ struct SendEvent(cudarc::driver::sys::CUevent);
 unsafe impl Send for SendEvent {}
 
 /// One request's in-flight CPU-tier KV prefetch.
-#[cfg(feature = "kv-offload")]
 ///
 /// `probe` holds the GPU-hit prefix resident for the request's whole parked
 /// life; `phase` tracks where the missing prefix currently is.
@@ -1086,10 +1081,6 @@ struct PrefetchState {
     phase: PrefetchPhase,
 }
 
-#[cfg(not(feature = "kv-offload"))]
-struct PrefetchState;
-
-#[cfg(feature = "kv-offload")]
 enum PrefetchPhase {
     /// pegaflow is pulling the missing prefix from a remote peer (P2P RDMA)
     /// into the local host tier; the executor re-queries each scheduler tick
@@ -1139,7 +1130,6 @@ const MISS_BREAKER_THRESHOLD: u32 = 3;
 
 /// vLLM-compat P/D mode, derived from [`crate::Qwen3VllmCompatOptions`] at
 /// executor build time (the hasher needs the resolved KV block size).
-#[cfg(feature = "kv-offload")]
 struct VllmCompatState {
     hasher: openinfer_kv_offload::VllmBlockHasher,
     /// Zero-hit wait window: how long a cold request re-queries before
@@ -1151,9 +1141,6 @@ struct VllmCompatState {
     consecutive_miss_windows: u32,
 }
 
-#[cfg(not(feature = "kv-offload"))]
-struct VllmCompatState;
-
 impl Qwen3Executor {
     pub(crate) fn dump_decode_graph_png(&self, png_path: &Path) -> Result {
         self.primary.dump_decode_graph_png(png_path.to_path_buf())
@@ -1212,7 +1199,6 @@ impl Qwen3Executor {
         let offload = build_offload(offload_opts, &kv_mgr, model.config(), model.device_ctx())?;
         let total_blocks = kv_mgr.pool().total_blocks();
         let padding_block_id = kv_mgr.pool().padding_block_id();
-        #[cfg(feature = "kv-offload")]
         let vllm_compat = match offload_opts.vllm_compat.as_ref() {
             None => None,
             Some(c) => {
@@ -1250,8 +1236,6 @@ impl Qwen3Executor {
                 })
             }
         };
-        #[cfg(not(feature = "kv-offload"))]
-        let vllm_compat = None;
         Ok(Self {
             metadata,
             kv_mgr,
@@ -1753,7 +1737,6 @@ impl Qwen3Executor {
     /// query can see them. A persistence barrier for handoff and tests; no-op
     /// without offload.
     pub fn flush_offload_saves(&self) {
-        #[cfg(feature = "kv-offload")]
         if let Some(offload) = &self.offload {
             offload.flush_saves();
         }
@@ -1797,7 +1780,6 @@ impl Qwen3Executor {
     /// tier (fire-and-forget). Safe to call right after `apply_prefill`/
     /// `apply_decode`: the producing step's token read-back has already
     /// synchronized the compute stream, so the sealed KV is fully written.
-    #[cfg(feature = "kv-offload")]
     fn save_sealed_blocks(&mut self, request_id: RequestId) {
         if self.offload.is_none() {
             return;
@@ -1841,9 +1823,6 @@ impl Qwen3Executor {
             .save(&block_ids, &block_hashes, pins);
     }
 
-    #[cfg(not(feature = "kv-offload"))]
-    fn save_sealed_blocks(&mut self, _request_id: RequestId) {}
-
     // ── Chunked prefill ────────────────────────────────────────────────
 
     /// Prepare one prefill step for `req`: create its `RequestKv` on the
@@ -1917,7 +1896,6 @@ impl Qwen3Executor {
     /// reserved blocks are staged + registered (held by the probe until the
     /// request prefills); on failure the state is dropped so the request
     /// prefills from scratch.
-    #[cfg(feature = "kv-offload")]
     fn settle_prefetch(
         &mut self,
         id: RequestId,
@@ -1944,7 +1922,6 @@ impl Qwen3Executor {
     /// answer either starts the H2D load (probe → Loading, still parked → and
     /// so returns `false`) or, on zero hit / reservation pressure / deadline,
     /// drops the prefetch so the request prefills from scratch.
-    #[cfg(feature = "kv-offload")]
     fn poll_remote_fetch(&mut self, id: RequestId, reserve_floor: usize) -> bool {
         let Some(st) = self.prefetch.get(&id) else {
             return true;
@@ -2058,7 +2035,6 @@ impl Qwen3Executor {
     /// Miss-breaker bookkeeping: one more request exhausted the whole
     /// zero-hit wait window. At the threshold the breaker opens and
     /// [`Self::begin_kv_prefetch`] stops parking new requests.
-    #[cfg(feature = "kv-offload")]
     fn note_miss_window_exhausted(&mut self) {
         let Some(compat) = self.vllm_compat.as_mut() else {
             return;
@@ -2075,7 +2051,6 @@ impl Qwen3Executor {
 
     /// Miss-breaker bookkeeping: remote content is visible again (leased
     /// hit), so cold requests may wait on the P/D handoff race once more.
-    #[cfg(feature = "kv-offload")]
     fn note_remote_hit(&mut self) {
         if let Some(compat) = self.vllm_compat.as_mut() {
             compat.consecutive_miss_windows = 0;
@@ -2087,7 +2062,6 @@ impl Qwen3Executor {
     /// of `probe`, which keeps the GPU-hit prefix resident meanwhile).
     /// `Err(())` means the prefetch was abandoned (lease released, no state
     /// kept) and the request should prefill from scratch.
-    #[cfg(feature = "kv-offload")]
     fn start_prefetch_load(
         &mut self,
         id: RequestId,
@@ -2197,7 +2171,6 @@ fn profile_kv_budget_on_worker(
 /// is disabled. Registers the fused KV buffer with pegaflow against the model's
 /// device/stream — must be called while that stream is still owned by the model
 /// (before it moves into the `RankWorker`).
-#[cfg(feature = "kv-offload")]
 fn build_offload(
     opts: &Qwen3OffloadOptions,
     kv_mgr: &KvCacheManager,
@@ -2257,21 +2230,6 @@ fn build_offload(
     Ok(Some(engine))
 }
 
-#[cfg(not(feature = "kv-offload"))]
-fn build_offload(
-    opts: &Qwen3OffloadOptions,
-    _kv_mgr: &KvCacheManager,
-    _config: &Config,
-    _ctx: &DeviceContext,
-) -> Result> {
-    anyhow::ensure!(
-        !opts.enabled && opts.p2p.is_none() && opts.vllm_compat.is_none(),
-        "Qwen3 KV offload is not compiled in; rebuild with `--features qwen3-kv-offload` \
-         for the server or `--features kv-offload` for openinfer-qwen3"
-    );
-    Ok(None)
-}
-
 fn ensure_lora_capacity(
     loaded_lora_adapters: &HashSet,
     lora_name: &str,
@@ -2328,19 +2286,12 @@ impl ModelExecutor for Qwen3Executor {
         self.metadata.stop_token_ids.contains(&token_id)
     }
 
-    #[cfg(feature = "kv-offload")]
     fn prefetched_blocks(&self, request_id: RequestId) -> usize {
         self.prefetch
             .get(&request_id)
             .map_or(0, |st| st.probe.held_blocks())
     }
 
-    #[cfg(not(feature = "kv-offload"))]
-    fn prefetched_blocks(&self, _request_id: RequestId) -> usize {
-        0
-    }
-
-    #[cfg(feature = "kv-offload")]
     fn release_finished_events(&self, finishes: Vec<(TokenSink, TokenEvent)>) {
         match &self.offload {
             // P/D prefill role: the peer treats our HTTP response as the
@@ -2354,11 +2305,6 @@ impl ModelExecutor for Qwen3Executor {
         }
     }
 
-    #[cfg(not(feature = "kv-offload"))]
-    fn release_finished_events(&self, finishes: Vec<(TokenSink, TokenEvent)>) {
-        send_finished_events(finishes);
-    }
-
     fn drop_request(&mut self, request_id: RequestId) -> Result<()> {
         // Remove and drop — RAII on SchedulableSequence's block guards
         // returns all allocated blocks regardless of lifecycle state. The same
@@ -2383,12 +2329,9 @@ impl ModelExecutor for Qwen3Executor {
         // phase has no local reservation yet — pegaflow owns that fetch and its
         // pinned-pool destination; dropping the state simply orphans the query,
         // which pegaflow's own req-scoped prefetch GC cleans up.)
-        #[cfg(feature = "kv-offload")]
-        {
-            if let Some(state) = self.prefetch.remove(&request_id) {
-                if let PrefetchPhase::Loading { handle, .. } = state.phase {
-                    let _ = handle.wait();
-                }
+        if let Some(state) = self.prefetch.remove(&request_id) {
+            if let PrefetchPhase::Loading { handle, .. } = state.phase {
+                let _ = handle.wait();
             }
         }
         self.saved_cursor.remove(&request_id);
@@ -2418,7 +2361,6 @@ impl ModelExecutor for Qwen3Executor {
         runs
     }
 
-    #[cfg(feature = "kv-offload")]
     fn begin_kv_prefetch(
         &mut self,
         request_id: RequestId,
@@ -2539,18 +2481,6 @@ impl ModelExecutor for Qwen3Executor {
         }
     }
 
-    #[cfg(not(feature = "kv-offload"))]
-    fn begin_kv_prefetch(
-        &mut self,
-        _request_id: RequestId,
-        _prompt_tokens: &[u32],
-        _lora_adapter: Option<&str>,
-        _reserve_floor: usize,
-    ) -> bool {
-        false
-    }
-
-    #[cfg(feature = "kv-offload")]
     fn drain_ready_prefetch(&mut self, reserve_floor: usize) -> Vec {
         let ids: Vec = self.prefetch.keys().copied().collect();
         let mut done = Vec::new();
@@ -2580,12 +2510,6 @@ impl ModelExecutor for Qwen3Executor {
         done
     }
 
-    #[cfg(not(feature = "kv-offload"))]
-    fn drain_ready_prefetch(&mut self, _reserve_floor: usize) -> Vec {
-        Vec::new()
-    }
-
-    #[cfg(feature = "kv-offload")]
     fn wait_ready_prefetch(&mut self, reserve_floor: usize) -> Vec {
         let mut done = Vec::new();
         if let Some(id) = self
@@ -2627,11 +2551,6 @@ impl ModelExecutor for Qwen3Executor {
         done
     }
 
-    #[cfg(not(feature = "kv-offload"))]
-    fn wait_ready_prefetch(&mut self, _reserve_floor: usize) -> Vec {
-        Vec::new()
-    }
-
     fn execute_prefill(&mut self, plan: PrefillPlan<'_>) -> Result {
         // 1. Create RequestKvs (first chunk only), clamp chunk budgets,
         // schedule KV for this step's tokens
diff --git a/openinfer-qwen3/src/executor/remote_fetch.rs b/openinfer-qwen3/src/executor/remote_fetch.rs
index 4ed792b8..de1de59d 100644
--- a/openinfer-qwen3/src/executor/remote_fetch.rs
+++ b/openinfer-qwen3/src/executor/remote_fetch.rs
@@ -26,7 +26,6 @@ pub(super) enum QueryView {
     Ready { lease: Option, num_blocks: usize },
 }
 
-#[cfg(feature = "kv-offload")]
 impl From for QueryView {
     fn from(outcome: openinfer_kv_offload::QueryOutcome) -> Self {
         match outcome {

From 13d43a48837815da03d51fadeb9b6ba305ba98b5 Mon Sep 17 00:00:00 2001
From: Kokoro2336 <2529677678@qq.com>
Date: Wed, 22 Jul 2026 09:47:44 +0000
Subject: [PATCH 6/6] fix: some bugs.

Signed-off-by: Kokoro2336 <2529677678@qq.com>
---
 openinfer-qwen3/Cargo.toml               |  2 --
 openinfer-qwen3/src/executor.rs          | 25 +++++++++++++++++-------
 openinfer-qwen3/src/scheduler/effects.rs |  4 +---
 3 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/openinfer-qwen3/Cargo.toml b/openinfer-qwen3/Cargo.toml
index ac01b82a..66c7c6be 100644
--- a/openinfer-qwen3/Cargo.toml
+++ b/openinfer-qwen3/Cargo.toml
@@ -11,8 +11,6 @@ clap = { workspace = true, optional = true }
 comfy-table = { workspace = true, optional = true }
 crossbeam-channel = { workspace = true }
 cudarc = { workspace = true }
-openinfer-kv-cache = { workspace = true }
-openinfer-kv-offload = { workspace = true }
 fastrace = { workspace = true }
 half = { workspace = true }
 hex = { workspace = true, optional = true }
diff --git a/openinfer-qwen3/src/executor.rs b/openinfer-qwen3/src/executor.rs
index 3937d518..d5b63561 100644
--- a/openinfer-qwen3/src/executor.rs
+++ b/openinfer-qwen3/src/executor.rs
@@ -18,12 +18,21 @@ use openinfer_core::engine::panic_message;
 use openinfer_core::kv_pool::KvLayout;
 use openinfer_core::ops;
 use openinfer_core::sampler::SamplingParams;
-use openinfer_core::tensor::{DeviceContext, DeviceVec, HiddenStates};
-use openinfer_kv_cache::{
-    KvBlockGuard, KvBuffer, KvCacheEvent, KvCacheManager, KvView, LoadReservation, PrefixProbe,
-    RegisteredBlock,
-};
-use openinfer_kv_offload::{LoadHandle, OffloadConfig, OffloadEngine};
+use openinfer_core::tensor::DeviceContext;
+use openinfer_core::tensor::HiddenStates;
+use openinfer_core::weight_loader::WeightPrefetch;
+use openinfer_core::weight_loader::load_shard_info;
+use openinfer_kv_cache::KvBlockGuard;
+use openinfer_kv_cache::KvBuffer;
+use openinfer_kv_cache::KvCacheEvent;
+use openinfer_kv_cache::KvCacheManager;
+use openinfer_kv_cache::KvView;
+use openinfer_kv_cache::LoadReservation;
+use openinfer_kv_cache::PrefixProbe;
+use openinfer_kv_cache::RegisteredBlock;
+use openinfer_kv_offload::LoadHandle;
+use openinfer_kv_offload::OffloadConfig;
+use openinfer_kv_offload::OffloadEngine;
 use tokio::sync::broadcast;
 
 use crate::Qwen3LoraOptions;
@@ -41,7 +50,9 @@ use crate::weights::Qwen3Model;
 mod dflash_lane;
 mod dflash_prefill;
 mod remote_fetch;
-use remote_fetch::{QueryView, RemoteFetchAction, remote_fetch_action};
+use remote_fetch::QueryView;
+use remote_fetch::RemoteFetchAction;
+use remote_fetch::remote_fetch_action;
 mod spec;
 
 use dflash_lane::DFlashLaneState;
diff --git a/openinfer-qwen3/src/scheduler/effects.rs b/openinfer-qwen3/src/scheduler/effects.rs
index 782605aa..3b7eea71 100644
--- a/openinfer-qwen3/src/scheduler/effects.rs
+++ b/openinfer-qwen3/src/scheduler/effects.rs
@@ -5,11 +5,9 @@ use openinfer_core::engine::TokenSink;
 
 use super::ActiveRequestState;
 use super::PendingRequest;
+use super::PrefixCacheTotals;
 use super::TokenEvent;
 use crate::executor::RequestId;
-use openinfer_core::engine::{FinishReason, TokenLogprob, TokenSink};
-
-use super::{ActiveRequestState, PendingRequest, PrefixCacheTotals, TokenEvent};
 
 pub(super) struct PromptEchoEffect {
     pub(super) token_tx: TokenSink,