Skip to content

fix(web,api): truthful instances everywhere — node names, stacked in Server view, live status, fleet-wide update banner #59

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
webdevtodayjason merged 1 commit into from
Jul 7, 2026
Merged

fix(web,api): truthful instances everywhere — node names, stacked in Server view, live status, fleet-wide update banner #59

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 35 additions & 5 deletions ainode/api/server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,37 @@ async def _engine_serving(backend, loop) -> bool:
return False


async def _live_instance_records(manager, loop) -> list:
"""Probe every managed instance; return the records whose engine answers.

Also flips each live record's status to ``serving`` (F3). The record is
stamped ``starting`` at load time and was never updated once the engine
came up, so the announcement advertised a phantom ``starting`` forever and
the dashboard kept painting a "STARTING · 8%" progress bar for an instance
that was actually serving traffic. The localhost /v1/models probe (via
``_engine_serving``) is the truthful liveness signal, so a passing probe is
exactly when the status should read ``serving``.
"""
live = []
for inst in manager.instances():
if await _engine_serving(inst.backend, loop):
if inst.record.status != "serving":
inst.record.status = "serving"
live.append(inst.record)
elif inst.record.status == "serving":
# Truthful reset (the other half of F3): the `serving` stamp is a
# latch — set once when the engine first answered and, before this,
# never cleared. An instance that was serving but whose engine no
# longer answers has crashed or been killed out-of-band; leaving the
# latch at `serving` makes every consumer that reads `record.status`
# without its own probe (e.g. the Server view's LOADED MODELS list)
# paint a dead instance as READY forever. Flip it back to `failed`
# so status tracks liveness in both directions. If the engine
# recovers, the next cycle re-flips it to `serving`.
inst.record.status = "failed"
return live


async def _cluster_sync_loop(app: web.Application) -> None:
"""Periodically sync the listener registry into ClusterState."""
try:
Expand Down Expand Up @@ -433,11 +464,10 @@ async def _cluster_sync_loop(app: web.Application) -> None:
manager = app.get("instances")
if manager is not None and not manager.is_empty():
# Only advertise instances whose engine actually answers — a dead
# stacked instance drops out of the broadcast within one cycle.
live_records = []
for inst in manager.instances():
if await _engine_serving(inst.backend, loop):
live_records.append(inst.record)
# stacked instance drops out of the broadcast within one cycle —
# and flip each live record's status to `serving` so the UI stops
# showing a phantom `starting` progress bar (F3).
live_records = await _live_instance_records(manager, loop)
updates["instances"] = [r.to_dict() for r in live_records]
else:
updates["instances"] = (
Expand Down
84 changes: 81 additions & 3 deletions ainode/api/server_routes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,14 +198,18 @@ async def handle_server_status(request: web.Request) -> web.Response:
web_port = getattr(config, "web_port", 3000)
host = getattr(config, "host", "0.0.0.0")

# Collect loaded models (local + cluster members)
local_models = await _probe_loaded_models(session, getattr(config, "api_port", 8000))
# Collect loaded models (local primary + local stacked + cluster members).
primary_port = getattr(config, "api_port", 8000)
local_models = await _probe_loaded_models(session, primary_port)
loaded_models: list[dict] = []
for mid in local_models:
loaded_models.append({
"id": mid,
"node_hostname": config.node_name or "local",
"node_id": config.node_id or "local",
"port": primary_port,
"ready": True,
"ejectable": True,
"type": "llm",
"format": "SafeTensors",
"quantization": None,
Expand All @@ -215,6 +219,45 @@ async def handle_server_status(request: web.Request) -> web.Response:
"loaded_at": start_time,
})

# Local STACKED instances (2nd+ model on this node, ports 8001+) live in the
# InstanceManager, not on the primary vLLM port the probe above hits — so
# they were invisible in the Server view (F2). Add a row per stacked
# instance. These are local, so the eject endpoint can target them.
manager = request.app.get("instances")
if manager is not None:
try:
for inst in manager.instances():
rec = inst.record
if not rec.model or rec.api_port == primary_port:
continue # primary already covered by the probe above
# Truthful readiness: probe the stacked instance's own OpenAI
# port live rather than trusting rec.status. rec.status is a
# latch stamped `serving` when the engine first answered; a
# stacked engine that later crashes or is killed out-of-band
# keeps reading `serving` (so it would show a green READY badge
# and be chat-targetable indefinitely). The /v1/models probe is
# the same liveness signal the primary uses two blocks up, so
# views and routing agree. The row stays ejectable either way so
# a dead instance can still be cleaned up from the UI.
stacked_live = await _probe_loaded_models(session, rec.api_port)
loaded_models.append({
"id": rec.model,
"node_hostname": config.node_name or "local",
"node_id": config.node_id or "local",
"port": rec.api_port,
"ready": bool(stacked_live),
"ejectable": True,
"type": "llm",
"format": "SafeTensors",
"quantization": None,
"size_bytes": 0,
"parallel": 1,
"capabilities": ["chat", "completions"],
"loaded_at": start_time,
})
except Exception:
logger.exception("failed to list local stacked instances")

# Include loaded embedding models (in-process, via EmbeddingManager)
embedding_manager = request.app.get("embedding_manager")
if embedding_manager is not None:
Expand All @@ -226,6 +269,9 @@ async def handle_server_status(request: web.Request) -> web.Response:
"id": emb["id"],
"node_hostname": hostname,
"node_id": config.node_id or "local",
"port": primary_port,
"ready": True,
"ejectable": True,
"type": "embed",
"format": "SafeTensors",
"quantization": None,
Expand All @@ -247,11 +293,43 @@ async def handle_server_status(request: web.Request) -> web.Response:
for m in members:
if m.node_id == config.node_id:
continue
member_port = getattr(m, "api_port", 8000) or 8000
# Remote instances can't be ejected from here — the eject
# endpoint only targets this node's local InstanceManager.
if m.model:
loaded_models.append({
"id": m.model,
"node_hostname": m.node_name,
"node_id": m.node_id,
"port": member_port,
"ready": True, # model is only broadcast once serving
"ejectable": False,
"type": "llm",
"format": "SafeTensors",
"quantization": None,
"size_bytes": 0,
"parallel": 1,
"capabilities": ["chat", "completions"],
"loaded_at": getattr(m, "last_seen", start_time),
})
# Remote STACKED instances (ports 8001+) carried on the peer's
# announcement — same source /api/nodes exposes (F2).
for inst in (getattr(m, "instances", []) or []):
if not isinstance(inst, dict):
continue
im = inst.get("model")
if not im:
continue
inst_port = inst.get("api_port") or member_port
if im == m.model and inst_port == member_port:
continue # primary already added above
loaded_models.append({
"id": im,
"node_hostname": m.node_name,
"node_id": m.node_id,
"port": inst_port,
"ready": inst.get("status") == "serving",
"ejectable": False,
"type": "llm",
"format": "SafeTensors",
"quantization": None,
Expand All @@ -261,7 +339,7 @@ async def handle_server_status(request: web.Request) -> web.Response:
"loaded_at": getattr(m, "last_seen", start_time),
})
except Exception:
pass
logger.exception("failed to list cluster-member instances")

# Request counters
counters = request.app.get("api_log_counters") or {}
Expand Down
60 changes: 48 additions & 12 deletions ainode/web/static/js/app.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -518,16 +518,30 @@ const AINode = {
badge.innerHTML = '⬆ Update available: v' + info.latest;
badge.title = 'Click to update to v' + info.latest;
badge.addEventListener('click', function () {
if (!confirm('Update AINode to v' + info.latest + '? The service will restart.')) return;
// Fleet-wide update: POST /api/cluster/update-all (resolves one target tag
// and threads it to every peer, master last) instead of the head-only
// /api/engine/update. F4.
var nodeCount = (self.state.nodes || []).length || 1;
var nodeWord = nodeCount === 1 ? 'node' : 'nodes';
if (!confirm('Update all ' + nodeCount + ' ' + nodeWord + ' to v' + info.latest + '? Services restart one by one.')) return;
badge.textContent = 'Updating...';
badge.disabled = true;
fetch('/api/engine/update', { method: 'POST' })
fetch('/api/cluster/update-all', { method: 'POST' })
.then(function (r) { return r.json(); })
.then(function () {
self.toast('Update started — service will restart in a moment', 'success');
badge.textContent = 'Restarting...';
// Re-check version in 60 seconds
setTimeout(function () { self.checkVersion(); }, 60000);
.then(function (data) {
if (data && data.error) {
badge.textContent = '⬆ Update available: v' + info.latest;
badge.disabled = false;
self.toast('Update failed: ' + data.error, 'error');
return;
}
self.toast('Fleet update started — nodes restart one by one', 'success');
badge.textContent = 'Updating fleet...';
// Surface live per-node progress in the cluster panel if it's mounted.
if (data && data.update_id) self._pollClusterUpdate(data.update_id);
// Re-check version in 90 seconds (bumped from 60s — a rolling fleet
// restart takes longer than a single-node one).
setTimeout(function () { self.checkVersion(); }, 90000);
})
.catch(function () {
badge.textContent = '⬆ Update available: v' + info.latest;
Expand Down Expand Up @@ -710,7 +724,10 @@ const AINode = {
// the federated /v1/models the chat dropdown uses. Skip the distributed one.
var seen = {};
(this.state.nodes || []).forEach(function (n) {
var host = n.hostname || n.node_id;
// Show the friendly node NAME (Spark-2-DGX), not the raw node-id hex.
// node_name is the id→name mapping carried on every /api/nodes entry;
// fall back to hostname, then the id, when the name is unknown (F1).
var host = n.node_name || n.hostname || n.node_id;
var modelName = n.model;
if (modelName && !(distModel && modelName === distModel)) {
var key = modelName + '@' + (n.node_id || n.hostname);
Expand Down Expand Up @@ -739,7 +756,13 @@ const AINode = {
model: im,
strategy: 'stacked',
nodes: [host + (inst.api_port ? ':' + inst.api_port : '')],
status: (inst.status === 'serving' || n.engine_ready) ? 'READY' : 'STARTING',
// Use the stacked instance's OWN status, not the node's primary
// readiness. n.engine_ready reflects the PRIMARY engine (and for the
// local node is hardcoded online → always true), so OR-ing it in made
// a still-loading or dead stacked model read READY the moment the
// primary was up. inst.status is the per-instance truth (the head
// only advertises live instances, flipped to `serving`).
status: inst.status === 'serving' ? 'READY' : 'STARTING',
badge: 'STACKED' + (inst.api_port ? ' · :' + inst.api_port : ''),
});
});
Expand Down Expand Up @@ -5278,8 +5301,15 @@ const AINode = {
_renderLoadedCard(m, idx) {
var id = m.id || 'unknown';
var nodeHost = m.node_hostname || m.node_id || 'local';
// Stacked instances live on ports 8001+ — show the port so two models on
// one node are distinguishable (F2). Primary instances omit it.
var portStr = (m.port && m.port !== 8000) ? ' · :' + m.port : '';
var type = m.type || 'llm';
var isEmbed = type === 'embed';
var ready = m.ready !== false;
// Remote instances (loaded on a peer) can't be ejected from here — the eject
// endpoint only targets this node's local InstanceManager (F2).
var ejectable = m.ejectable !== false;
var sizeStr = m.size_bytes > 0 ? this.formatBytes(m.size_bytes) : '—';
var parallel = m.parallel || 1;
var selected = (this._serverState.selectedModelId === id) ? ' selected' : '';
Expand All @@ -5292,10 +5322,16 @@ const AINode = {
var primaryIconBtn = isEmbed
? ' <button class="server-icon-btn" data-action="show-info" data-model="' + this.esc(id) + '" title="Embedding info">ℹ</button>'
: ' <button class="server-icon-btn" data-action="open-chat" data-model="' + this.esc(id) + '" title="Open in Chat">🔍</button>';
var statusBadge = ready
? '<span class="server-badge ready">READY</span>'
: '<span class="server-badge">STARTING</span>';
var ejectBtn = ejectable
? ' <button class="server-eject-btn" data-action="eject" data-model="' + this.esc(id) + '">Eject</button>'
: '';
return '<div class="server-loaded-card' + selected + '" data-model-id="' + this.esc(id) + '" data-idx="' + idx + '">' +
'<div class="server-loaded-left">' +
' <span class="server-badge ready">READY</span>' +
' <span class="server-node-pill">' + this.esc(nodeHost) + '</span>' +
' ' + statusBadge +
' <span class="server-node-pill">' + this.esc(nodeHost + portStr) + '</span>' +
' <span class="server-type-tag"' + typeTagStyle + '>' + this.esc(type) + '</span>' +
' <span class="server-model-id mono" data-copy="' + this.esc(id) + '" title="Click to copy">' + this.esc(id) + '</span>' +
'</div>' +
Expand All @@ -5306,7 +5342,7 @@ const AINode = {
' <button class="server-icon-btn" data-action="preview" title="Preview">👁</button>' +
primaryIconBtn +
' <button class="server-icon-btn" data-action="copy-curl" data-model="' + this.esc(id) + '" data-type="' + this.esc(type) + '" title="Copy curl">⎘</button>' +
' <button class="server-eject-btn" data-action="eject" data-model="' + this.esc(id) + '">Eject</button>' +
ejectBtn +
'</div>' +
'</div>';
},
Expand Down
62 changes: 61 additions & 1 deletion tests/test_engine_serving.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import asyncio

from ainode.api.server import _engine_serving
from ainode.api.server import _engine_serving, _live_instance_records


class _Backend:
Expand Down Expand Up @@ -49,10 +49,70 @@ def test_no_backend():
assert _run(None) is False


# --- _live_instance_records: status must track liveness in BOTH directions ---


class _Record:
def __init__(self, status):
self.status = status


class _Instance:
def __init__(self, health, status):
self.backend = _Backend(health)
self.record = _Record(status)


class _Manager:
def __init__(self, insts):
self._insts = insts

def instances(self):
return self._insts


def _run_live(manager):
loop = asyncio.new_event_loop()
try:
return loop.run_until_complete(_live_instance_records(manager, loop))
finally:
loop.close()


def test_live_flips_starting_to_serving():
inst = _Instance({"api_responding": True}, "starting")
live = _run_live(_Manager([inst]))
assert inst.record.status == "serving"
assert live == [inst.record]


def test_live_resets_stale_serving_latch_to_failed():
# F3 both-directions: an instance stamped `serving` whose engine no longer
# answers must NOT keep reading `serving` (that painted a dead stacked
# instance READY forever in the Server view). It drops out of the live list
# AND its status latch is cleared.
inst = _Instance({"api_responding": False, "process_alive": False}, "serving")
live = _run_live(_Manager([inst]))
assert inst.record.status == "failed"
assert live == []


def test_live_leaves_never_served_starting_untouched():
# A still-loading instance (never reached serving) is not falsely failed —
# it's simply excluded from the live list until its api answers.
inst = _Instance({"api_responding": False, "process_alive": True}, "starting")
live = _run_live(_Manager([inst]))
assert inst.record.status == "starting"
assert live == []


if __name__ == "__main__":
test_serving_when_api_responds()
test_not_serving_while_loading()
test_not_serving_when_crashed()
test_not_serving_on_probe_error()
test_no_backend()
test_live_flips_starting_to_serving()
test_live_resets_stale_serving_latch_to_failed()
test_live_leaves_never_served_starting_untouched()
print("ok")
Loading
Loading