Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion tokenspeed-scheduler/csrc/fsm/forward_states.h
Original file line number Diff line number Diff line change
Expand Up @@ -175,14 +175,34 @@ struct ForwardState : public BaseState {
std::int32_t GetReqPoolIndex() const { return req_pool_index_ ? req_pool_index_->slot_ : -1; }

// Flat: group-0 SAMPLE (ids -- and with per-group block sizes, counts -- differ per group);
// feeds one-group stats and the radix-era req_to_page fallback, never cross-group accounting.
// feeds one-group stats and the radix-era req_to_page fallback, never cross-group accounting
// (that is GetOccupiedPagesAllGroups).
std::vector<std::int32_t> GetOccupiedPages() const {
if (!block_tables_.empty()) {
return BlockTablePageIds(block_tables_[0]); // flat: first-group sample (see above)
}
return GetPageContainer().Pages(); // radix
}

// Flat: real pages of EVERY group's block table (null holes excluded — block 0 is the
// never-allocated punch placeholder, not occupancy). Block ids are pool-wide and each id
// is owned by one group, so concatenation has no cross-group duplicates. This is the
// pool-wide occupancy counterpart of the group-0 sample above.
std::vector<std::int32_t> GetOccupiedPagesAllGroups() const {
if (block_tables_.empty()) {
return GetPageContainer().Pages(); // radix
}
std::vector<std::int32_t> ids;
for (const BlockTable& table : block_tables_) {
for (CacheBlock* b : table.Blocks()) {
if (!b->IsNull()) {
ids.push_back(b->BlockId());
}
}
}
return ids;
}

std::vector<std::int32_t> GetLocalAllocatorPages() const {
if (!block_tables_.empty()) {
return BlockTablePageIds(block_tables_[0]); // flat: first-group sample (see GetOccupiedPages)
Expand Down
16 changes: 16 additions & 0 deletions tokenspeed-scheduler/csrc/scheduler/request.h
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,22 @@ class Request {
state_);
}

// Pool-wide occupancy: every group's real pages on the flat path (radix: same as
// GetOccupiedPages). Forward states only — the occupancy stats that need this never
// query queued or terminal requests.
std::vector<std::int32_t> GetOccupiedPagesAllGroups() const {
return std::visit(Overloaded{
[]<typename T>(const T& s) -> std::vector<std::int32_t>
requires std::derived_from<T, fsm::ForwardState>
{ return s.GetOccupiedPagesAllGroups(); },
[this](const auto&) -> std::vector<std::int32_t> {
throw std::logic_error("Request::GetOccupiedPagesAllGroups: expected a forward state; got state=" +
StateName());
},
},
state_);
}

// Flat KV-cache: true when the current state carries no per-group block
// tables (radix path, or a non-forward state). Forward states on the flat
// path return false once allocation has populated block_tables_.
Expand Down
6 changes: 5 additions & 1 deletion tokenspeed-scheduler/csrc/scheduler/scheduler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -260,10 +260,14 @@ std::size_t Scheduler::AvailableKvPages() const {
}

std::size_t Scheduler::ActiveKvPages() const {
// Distinct pages pinned by running requests, in the same units as AvailableKvPages():
// flat pool ids across ALL groups (a group-0 sample here understated the ratio that
// Python monitoring derives against the whole pool), radix device pages otherwise.
// The set dedups pages shared between requests via prefix hits.
std::unordered_set<std::int32_t> active_pages;
for (const auto& [_, req] : requests_) {
if (req->Is<fsm::Prefilling>() || req->Is<fsm::PrefillDone>() || req->Is<fsm::Decoding>()) {
for (std::int32_t page : req->GetOccupiedPages()) {
for (std::int32_t page : req->GetOccupiedPagesAllGroups()) {
active_pages.insert(page);
}
}
Expand Down
Loading