From 0da86702ba7b83f9f2bb4a345397b2481e693283 Mon Sep 17 00:00:00 2001 From: webdevtodayjason Date: Tue, 7 Jul 2026 08:20:22 -0500 Subject: [PATCH] =?UTF-8?q?fix(web,api):=20truthful=20instances=20everywhe?= =?UTF-8?q?re=20=E2=80=94=20node=20names,=20stacked=20in=20Server=20view,?= =?UTF-8?q?=20live=20status,=20fleet-wide=20update=20banner?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F1 — dashboard INSTANCES cards now show the node NAME (Spark-2-DGX), not the raw node-id hex, for single/stacked cards (renderInstances host = node_name), falling back to the id when the name is unknown. F2 — Server view LOADED MODELS now includes stacked instances as rows: the local InstanceManager (ports 8001+) and each peer's announced instances[], each with model, node name, port, READY/STARTING, and Eject only where the local eject endpoint can target it (remote rows render without Eject). The count now reflects total instances, not just per-node primaries. F3 — instance status no longer sticks at 'starting'. _live_instance_records flips each live record's status to 'serving' when its engine answers the localhost /v1/models probe, so the announcement carries the truth and the GUI stops painting a 'STARTING · 8%' bar for an instance that's serving. F4 — the update banner is fleet-wide: the badge confirms 'Update all N nodes to vX? Services restart one by one.' and POSTs /api/cluster/update-all (resolves one target, threads to peers), with a failure toast and the re-check bumped 60s→90s. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017NziXzqT1L9kj2T1byA3Ak --- ainode/api/server.py | 40 ++++++++++++++--- ainode/api/server_routes.py | 84 ++++++++++++++++++++++++++++++++++-- ainode/web/static/js/app.js | 60 ++++++++++++++++++++------ tests/test_engine_serving.py | 62 +++++++++++++++++++++++++- tests/test_instances.py | 55 +++++++++++++++++++++++ 5 files changed, 280 insertions(+), 21 deletions(-) diff --git a/ainode/api/server.py b/ainode/api/server.py index a63da2f..34c9e11 100644 --- a/ainode/api/server.py +++ b/ainode/api/server.py @@ -372,6 +372,37 @@ async def _engine_serving(backend, loop) -> bool: return False +async def _live_instance_records(manager, loop) -> list: + """Probe every managed instance; return the records whose engine answers. + + Also flips each live record's status to ``serving`` (F3). The record is + stamped ``starting`` at load time and was never updated once the engine + came up, so the announcement advertised a phantom ``starting`` forever and + the dashboard kept painting a "STARTING · 8%" progress bar for an instance + that was actually serving traffic. The localhost /v1/models probe (via + ``_engine_serving``) is the truthful liveness signal, so a passing probe is + exactly when the status should read ``serving``. + """ + live = [] + for inst in manager.instances(): + if await _engine_serving(inst.backend, loop): + if inst.record.status != "serving": + inst.record.status = "serving" + live.append(inst.record) + elif inst.record.status == "serving": + # Truthful reset (the other half of F3): the `serving` stamp is a + # latch — set once when the engine first answered and, before this, + # never cleared. An instance that was serving but whose engine no + # longer answers has crashed or been killed out-of-band; leaving the + # latch at `serving` makes every consumer that reads `record.status` + # without its own probe (e.g. the Server view's LOADED MODELS list) + # paint a dead instance as READY forever. Flip it back to `failed` + # so status tracks liveness in both directions. If the engine + # recovers, the next cycle re-flips it to `serving`. + inst.record.status = "failed" + return live + + async def _cluster_sync_loop(app: web.Application) -> None: """Periodically sync the listener registry into ClusterState.""" try: @@ -433,11 +464,10 @@ async def _cluster_sync_loop(app: web.Application) -> None: manager = app.get("instances") if manager is not None and not manager.is_empty(): # Only advertise instances whose engine actually answers — a dead - # stacked instance drops out of the broadcast within one cycle. - live_records = [] - for inst in manager.instances(): - if await _engine_serving(inst.backend, loop): - live_records.append(inst.record) + # stacked instance drops out of the broadcast within one cycle — + # and flip each live record's status to `serving` so the UI stops + # showing a phantom `starting` progress bar (F3). + live_records = await _live_instance_records(manager, loop) updates["instances"] = [r.to_dict() for r in live_records] else: updates["instances"] = ( diff --git a/ainode/api/server_routes.py b/ainode/api/server_routes.py index ee930c4..9b472f9 100644 --- a/ainode/api/server_routes.py +++ b/ainode/api/server_routes.py @@ -198,14 +198,18 @@ async def handle_server_status(request: web.Request) -> web.Response: web_port = getattr(config, "web_port", 3000) host = getattr(config, "host", "0.0.0.0") - # Collect loaded models (local + cluster members) - local_models = await _probe_loaded_models(session, getattr(config, "api_port", 8000)) + # Collect loaded models (local primary + local stacked + cluster members). + primary_port = getattr(config, "api_port", 8000) + local_models = await _probe_loaded_models(session, primary_port) loaded_models: list[dict] = [] for mid in local_models: loaded_models.append({ "id": mid, "node_hostname": config.node_name or "local", "node_id": config.node_id or "local", + "port": primary_port, + "ready": True, + "ejectable": True, "type": "llm", "format": "SafeTensors", "quantization": None, @@ -215,6 +219,45 @@ async def handle_server_status(request: web.Request) -> web.Response: "loaded_at": start_time, }) + # Local STACKED instances (2nd+ model on this node, ports 8001+) live in the + # InstanceManager, not on the primary vLLM port the probe above hits — so + # they were invisible in the Server view (F2). Add a row per stacked + # instance. These are local, so the eject endpoint can target them. + manager = request.app.get("instances") + if manager is not None: + try: + for inst in manager.instances(): + rec = inst.record + if not rec.model or rec.api_port == primary_port: + continue # primary already covered by the probe above + # Truthful readiness: probe the stacked instance's own OpenAI + # port live rather than trusting rec.status. rec.status is a + # latch stamped `serving` when the engine first answered; a + # stacked engine that later crashes or is killed out-of-band + # keeps reading `serving` (so it would show a green READY badge + # and be chat-targetable indefinitely). The /v1/models probe is + # the same liveness signal the primary uses two blocks up, so + # views and routing agree. The row stays ejectable either way so + # a dead instance can still be cleaned up from the UI. + stacked_live = await _probe_loaded_models(session, rec.api_port) + loaded_models.append({ + "id": rec.model, + "node_hostname": config.node_name or "local", + "node_id": config.node_id or "local", + "port": rec.api_port, + "ready": bool(stacked_live), + "ejectable": True, + "type": "llm", + "format": "SafeTensors", + "quantization": None, + "size_bytes": 0, + "parallel": 1, + "capabilities": ["chat", "completions"], + "loaded_at": start_time, + }) + except Exception: + logger.exception("failed to list local stacked instances") + # Include loaded embedding models (in-process, via EmbeddingManager) embedding_manager = request.app.get("embedding_manager") if embedding_manager is not None: @@ -226,6 +269,9 @@ async def handle_server_status(request: web.Request) -> web.Response: "id": emb["id"], "node_hostname": hostname, "node_id": config.node_id or "local", + "port": primary_port, + "ready": True, + "ejectable": True, "type": "embed", "format": "SafeTensors", "quantization": None, @@ -247,11 +293,43 @@ async def handle_server_status(request: web.Request) -> web.Response: for m in members: if m.node_id == config.node_id: continue + member_port = getattr(m, "api_port", 8000) or 8000 + # Remote instances can't be ejected from here — the eject + # endpoint only targets this node's local InstanceManager. if m.model: loaded_models.append({ "id": m.model, "node_hostname": m.node_name, "node_id": m.node_id, + "port": member_port, + "ready": True, # model is only broadcast once serving + "ejectable": False, + "type": "llm", + "format": "SafeTensors", + "quantization": None, + "size_bytes": 0, + "parallel": 1, + "capabilities": ["chat", "completions"], + "loaded_at": getattr(m, "last_seen", start_time), + }) + # Remote STACKED instances (ports 8001+) carried on the peer's + # announcement — same source /api/nodes exposes (F2). + for inst in (getattr(m, "instances", []) or []): + if not isinstance(inst, dict): + continue + im = inst.get("model") + if not im: + continue + inst_port = inst.get("api_port") or member_port + if im == m.model and inst_port == member_port: + continue # primary already added above + loaded_models.append({ + "id": im, + "node_hostname": m.node_name, + "node_id": m.node_id, + "port": inst_port, + "ready": inst.get("status") == "serving", + "ejectable": False, "type": "llm", "format": "SafeTensors", "quantization": None, @@ -261,7 +339,7 @@ async def handle_server_status(request: web.Request) -> web.Response: "loaded_at": getattr(m, "last_seen", start_time), }) except Exception: - pass + logger.exception("failed to list cluster-member instances") # Request counters counters = request.app.get("api_log_counters") or {} diff --git a/ainode/web/static/js/app.js b/ainode/web/static/js/app.js index f0eafcb..784ec30 100644 --- a/ainode/web/static/js/app.js +++ b/ainode/web/static/js/app.js @@ -518,16 +518,30 @@ const AINode = { badge.innerHTML = '⬆ Update available: v' + info.latest; badge.title = 'Click to update to v' + info.latest; badge.addEventListener('click', function () { - if (!confirm('Update AINode to v' + info.latest + '? The service will restart.')) return; + // Fleet-wide update: POST /api/cluster/update-all (resolves one target tag + // and threads it to every peer, master last) instead of the head-only + // /api/engine/update. F4. + var nodeCount = (self.state.nodes || []).length || 1; + var nodeWord = nodeCount === 1 ? 'node' : 'nodes'; + if (!confirm('Update all ' + nodeCount + ' ' + nodeWord + ' to v' + info.latest + '? Services restart one by one.')) return; badge.textContent = 'Updating...'; badge.disabled = true; - fetch('/api/engine/update', { method: 'POST' }) + fetch('/api/cluster/update-all', { method: 'POST' }) .then(function (r) { return r.json(); }) - .then(function () { - self.toast('Update started — service will restart in a moment', 'success'); - badge.textContent = 'Restarting...'; - // Re-check version in 60 seconds - setTimeout(function () { self.checkVersion(); }, 60000); + .then(function (data) { + if (data && data.error) { + badge.textContent = '⬆ Update available: v' + info.latest; + badge.disabled = false; + self.toast('Update failed: ' + data.error, 'error'); + return; + } + self.toast('Fleet update started — nodes restart one by one', 'success'); + badge.textContent = 'Updating fleet...'; + // Surface live per-node progress in the cluster panel if it's mounted. + if (data && data.update_id) self._pollClusterUpdate(data.update_id); + // Re-check version in 90 seconds (bumped from 60s — a rolling fleet + // restart takes longer than a single-node one). + setTimeout(function () { self.checkVersion(); }, 90000); }) .catch(function () { badge.textContent = '⬆ Update available: v' + info.latest; @@ -710,7 +724,10 @@ const AINode = { // the federated /v1/models the chat dropdown uses. Skip the distributed one. var seen = {}; (this.state.nodes || []).forEach(function (n) { - var host = n.hostname || n.node_id; + // Show the friendly node NAME (Spark-2-DGX), not the raw node-id hex. + // node_name is the id→name mapping carried on every /api/nodes entry; + // fall back to hostname, then the id, when the name is unknown (F1). + var host = n.node_name || n.hostname || n.node_id; var modelName = n.model; if (modelName && !(distModel && modelName === distModel)) { var key = modelName + '@' + (n.node_id || n.hostname); @@ -739,7 +756,13 @@ const AINode = { model: im, strategy: 'stacked', nodes: [host + (inst.api_port ? ':' + inst.api_port : '')], - status: (inst.status === 'serving' || n.engine_ready) ? 'READY' : 'STARTING', + // Use the stacked instance's OWN status, not the node's primary + // readiness. n.engine_ready reflects the PRIMARY engine (and for the + // local node is hardcoded online → always true), so OR-ing it in made + // a still-loading or dead stacked model read READY the moment the + // primary was up. inst.status is the per-instance truth (the head + // only advertises live instances, flipped to `serving`). + status: inst.status === 'serving' ? 'READY' : 'STARTING', badge: 'STACKED' + (inst.api_port ? ' · :' + inst.api_port : ''), }); }); @@ -5278,8 +5301,15 @@ const AINode = { _renderLoadedCard(m, idx) { var id = m.id || 'unknown'; var nodeHost = m.node_hostname || m.node_id || 'local'; + // Stacked instances live on ports 8001+ — show the port so two models on + // one node are distinguishable (F2). Primary instances omit it. + var portStr = (m.port && m.port !== 8000) ? ' · :' + m.port : ''; var type = m.type || 'llm'; var isEmbed = type === 'embed'; + var ready = m.ready !== false; + // Remote instances (loaded on a peer) can't be ejected from here — the eject + // endpoint only targets this node's local InstanceManager (F2). + var ejectable = m.ejectable !== false; var sizeStr = m.size_bytes > 0 ? this.formatBytes(m.size_bytes) : '—'; var parallel = m.parallel || 1; var selected = (this._serverState.selectedModelId === id) ? ' selected' : ''; @@ -5292,10 +5322,16 @@ const AINode = { var primaryIconBtn = isEmbed ? ' ' : ' '; + var statusBadge = ready + ? 'READY' + : 'STARTING'; + var ejectBtn = ejectable + ? ' ' + : ''; return '
' + '
' + - ' READY' + - ' ' + this.esc(nodeHost) + '' + + ' ' + statusBadge + + ' ' + this.esc(nodeHost + portStr) + '' + ' ' + this.esc(type) + '' + ' ' + this.esc(id) + '' + '
' + @@ -5306,7 +5342,7 @@ const AINode = { ' ' + primaryIconBtn + ' ' + - ' ' + + ejectBtn + '
' + ''; }, diff --git a/tests/test_engine_serving.py b/tests/test_engine_serving.py index 33f97cd..4a7e159 100644 --- a/tests/test_engine_serving.py +++ b/tests/test_engine_serving.py @@ -7,7 +7,7 @@ import asyncio -from ainode.api.server import _engine_serving +from ainode.api.server import _engine_serving, _live_instance_records class _Backend: @@ -49,10 +49,70 @@ def test_no_backend(): assert _run(None) is False +# --- _live_instance_records: status must track liveness in BOTH directions --- + + +class _Record: + def __init__(self, status): + self.status = status + + +class _Instance: + def __init__(self, health, status): + self.backend = _Backend(health) + self.record = _Record(status) + + +class _Manager: + def __init__(self, insts): + self._insts = insts + + def instances(self): + return self._insts + + +def _run_live(manager): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(_live_instance_records(manager, loop)) + finally: + loop.close() + + +def test_live_flips_starting_to_serving(): + inst = _Instance({"api_responding": True}, "starting") + live = _run_live(_Manager([inst])) + assert inst.record.status == "serving" + assert live == [inst.record] + + +def test_live_resets_stale_serving_latch_to_failed(): + # F3 both-directions: an instance stamped `serving` whose engine no longer + # answers must NOT keep reading `serving` (that painted a dead stacked + # instance READY forever in the Server view). It drops out of the live list + # AND its status latch is cleared. + inst = _Instance({"api_responding": False, "process_alive": False}, "serving") + live = _run_live(_Manager([inst])) + assert inst.record.status == "failed" + assert live == [] + + +def test_live_leaves_never_served_starting_untouched(): + # A still-loading instance (never reached serving) is not falsely failed — + # it's simply excluded from the live list until its api answers. + inst = _Instance({"api_responding": False, "process_alive": True}, "starting") + live = _run_live(_Manager([inst])) + assert inst.record.status == "starting" + assert live == [] + + if __name__ == "__main__": test_serving_when_api_responds() test_not_serving_while_loading() test_not_serving_when_crashed() test_not_serving_on_probe_error() test_no_backend() + test_live_flips_starting_to_serving() + test_live_resets_stale_serving_latch_to_failed() + test_live_leaves_never_served_starting_untouched() print("ok") diff --git a/tests/test_instances.py b/tests/test_instances.py index 63f33b6..01e7491 100644 --- a/tests/test_instances.py +++ b/tests/test_instances.py @@ -96,3 +96,58 @@ def test_announcement_carries_instances_roundtrip(): unified_memory=True, model="m", status="serving", api_port=8000, web_port=3000, instances=[inst]) assert NodeAnnouncement.from_json(a.to_json()).instances == [inst] + + +# --------------------------------------------------------------------------- +# F3: instance status flips 'starting' -> 'serving' once the engine answers, +# and the flipped status rides the announcement (via _live_instance_records). +# --------------------------------------------------------------------------- + +class _FakeBackend: + def __init__(self, responding: bool): + self._responding = responding + + def health_check(self) -> dict: + return {"api_responding": self._responding} + + +def test_live_instance_records_flips_starting_to_serving(): + from ainode.api.server import _live_instance_records + from ainode.engine.instance_manager import InstanceManager + + manager = InstanceManager(base_port=8000) + # A stacked instance stamped 'starting' at load time whose engine now answers. + rec = InstanceRecord(instance_id="h:qwen", model="qwen", head_node_id="h", + api_port=8001, status="starting") + manager.add(rec, _FakeBackend(responding=True)) + + loop = asyncio.new_event_loop() + try: + live = loop.run_until_complete(_live_instance_records(manager, loop)) + finally: + loop.close() + + # The record object itself is mutated (so /api/server/status + the + # announcement both see the truth) and it is advertised as serving. + assert rec.status == "serving" + assert [r.to_dict()["status"] for r in live] == ["serving"] + + +def test_live_instance_records_drops_dead_instance(): + from ainode.api.server import _live_instance_records + from ainode.engine.instance_manager import InstanceManager + + manager = InstanceManager(base_port=8000) + rec = InstanceRecord(instance_id="h:dead", model="dead", head_node_id="h", + api_port=8001, status="starting") + manager.add(rec, _FakeBackend(responding=False)) + + loop = asyncio.new_event_loop() + try: + live = loop.run_until_complete(_live_instance_records(manager, loop)) + finally: + loop.close() + + # A non-answering engine is neither advertised nor promoted to serving. + assert live == [] + assert rec.status == "starting"