diff --git a/src/auth.js b/src/auth.js index e95c79dc..b254ed1b 100644 --- a/src/auth.js +++ b/src/auth.js @@ -827,8 +827,9 @@ const MODEL_CATALOG_CONFIRM_RETRY_MS = 30_000; // the live set back down. Hence per-account rows here, unioned before every write. // // Existence and entitlement are deliberately different questions. This union -// answers "does the upstream have this selector at all" — the pool-wide view. Who -// may CALL it stays per-account (isConnectSelectorAllowedForAccount → tier bucket). +// answers "does at least one active account's live catalog contain this selector?" +// — the pool-wide discovery view. Who may CALL it stays per-account: the tier +// bucket and that account's own catalog must both allow it. const _connectCatalogRowsByAccount = new Map(); // account id → decoded catalog rows const _connectCatalogSyncedKeys = new Map(); // account id → apiKey already synced const _connectCatalogSyncedAt = new Map(); // account id → successful refresh timestamp @@ -1409,7 +1410,20 @@ export function isConnectSelectorAllowedForAccount(account, selector) { // Paid selector: require a paid bucket. 'unknown' (unprobed new account) is // allowed — it self-heals to 'free' after a probe, then gets blocked here on // the next request, matching MODEL_TIER_ACCESS.unknown's optimistic policy. - return bucket === 'pro' || bucket === 'unknown'; + if (bucket !== 'pro' && bucket !== 'unknown') return false; + + // A successful GetCliModelConfigs response is authoritative for THIS account. + // Paid tier alone is not proof that every selector in the frozen snapshot still + // exists. Mixed pools expose the union, then this check keeps routing within the + // selected account's own contribution. If this account has never produced a + // non-empty catalog, retain the existing tier-based fail-open so a cold start or + // transient catalog failure does not take the account offline. + const rows = _connectCatalogRowsByAccount.get(account.id); + if (!Array.isArray(rows) || rows.length === 0) return true; + return rows.some((row) => { + const candidate = typeof row === 'string' ? row : row?.selector; + return typeof candidate === 'string' && candidate.trim() === selector; + }); } // True if at least one active account is entitled to this connect selector. diff --git a/src/dashboard/api.js b/src/dashboard/api.js index 1b291207..236935f5 100644 --- a/src/dashboard/api.js +++ b/src/dashboard/api.js @@ -39,7 +39,7 @@ import { getLogs, subscribeToLogs, unsubscribeFromLogs } from './logger.js'; import { getProxyConfig, getProxyConfigMasked, setGlobalProxy, setAccountProxy, removeProxy, getEffectiveProxy } from './proxy-config.js'; import { MODELS, MODEL_TIER_ACCESS as _TIER_TABLE, getTierModels as _getTierModels, filterModelKeysByCloudCatalog } from '../models.js'; import { buildConnectReachability } from '../handlers/models.js'; -import { FREE_REACHABLE_SELECTORS } from '../devin-connect-models.js'; +import { FREE_REACHABLE_SELECTORS, getLiveCatalog } from '../devin-connect-models.js'; import { windsurfLogin, refreshFirebaseToken, reRegisterWithCodeium } from './windsurf-login.js'; import { getModelAccessConfig, setModelAccessMode, setModelAccessList, addModelToList, removeModelFromList, setDefaultModel } from './model-access.js'; import { checkMessageRateLimit } from '../windsurf-api.js'; @@ -1903,6 +1903,9 @@ export async function handleDashboardApi(method, subpath, body, req, res) { // usable rate table, which is what lets `currentlyFree` below distinguish "costs quota" // from "we do not know yet". const freeSet = getCurrentlyFreeConnectSelectors(); + const connectCurrentlyFree = (selector) => (!selector ? null + : isConnectSelectorCurrentlyFree(selector) ? true + : (freeSet === null ? null : false)); const models = filterModelKeysByCloudCatalog().map((id) => { const info = MODELS[id]; const { reachable, selector } = isReachable(id); @@ -1931,9 +1934,7 @@ export async function handleDashboardApi(method, subpath, body, req, res) { // Hence the explicit freeSet null-check: auth.js:304 returns null for "no table // anywhere" precisely so callers do not have to guess, and flattening it here would // throw away the one thing that function went out of its way to preserve. - currentlyFree: !selector ? null - : isConnectSelectorCurrentlyFree(selector) ? true - : (freeSet === null ? null : false), + currentlyFree: connectCurrentlyFree(selector), }; }); // Selectors that ARE serveable but have no MODELS row (`swe-1-6-slow` is in neither the @@ -1949,6 +1950,33 @@ export async function handleDashboardApi(method, subpath, body, req, res) { // test/dashboard-models-connect-parity.test.js. if (getBackendSwitch('devinConnect')) { const seen = new Set(models.map((m) => m.id)); + const representedSelectors = new Set(models + .map((m) => m.connectSelector) + .filter(Boolean)); + + // /v1/models has a second producer for selectors present in the live Connect + // catalog but absent from the shared MODELS table. Mirror it here; otherwise + // the API advertises live-only selectors that the Dashboard cannot display, + // inspect for pricing, or manage. Keep rows annotated rather than filtered: + // a free account's catalog still lists paid models, and the operator needs to + // see those as unreachable rather than have them disappear. + for (const row of getLiveCatalog()) { + const id = (typeof row === 'string' ? row : row?.selector)?.trim(); + if (!id || seen.has(id) || representedSelectors.has(id)) continue; + seen.add(id); + const { reachable, selector } = isReachable(id); + if (selector) representedSelectors.add(selector); + models.push({ + id, + name: row.label || id, + provider: row.provider || 'windsurf', + credit: null, + reachable, + connectSelector: selector, + currentlyFree: connectCurrentlyFree(selector), + }); + } + for (const selector of FREE_REACHABLE_SELECTORS) { if (seen.has(selector)) continue; seen.add(selector); diff --git a/src/devin-connect-models.js b/src/devin-connect-models.js index db8b37f1..afd34373 100644 --- a/src/devin-connect-models.js +++ b/src/devin-connect-models.js @@ -172,11 +172,12 @@ export const FREE_REACHABLE_SELECTORS = new Set(['swe-1-6-slow']); // (chat.js) as "not a valid model", despite being genuinely runnable. // // Fix: a runtime-populated live selector set, refreshed from GetCliModelConfigs -// (devin-connect-catalog.js:fetchCatalog) by auth.js on catalog sync. The -// existence checks below treat "snapshot ∪ live" as the source of truth — the -// snapshot degrades to a cold-start fallback + the catalog-drift test baseline, -// exactly the single-source-of-truth principle converged on cross-project. -// Empty until the first sync (cold start falls back to snapshot alone). +// (devin-connect-catalog.js:fetchCatalog) by auth.js on catalog sync. A NON-EMPTY +// live response is authoritative: keeping `snapshot ∪ live` after a successful +// sync advertises selectors omitted by upstream account-level restrictions. The +// snapshot is therefore only a cold-start / failed-sync fallback. Empty responses +// never replace a prior good live set, so making live authoritative does not turn +// a transient fetch failure into an empty catalog. const _liveSelectors = new Set(); // Full decoded catalog rows ({ selector, label, provider, alias, ... }) from the // last good sync. Kept alongside _liveSelectors so /v1/models can synthesize @@ -205,8 +206,15 @@ export function setLiveCatalogSelectors(catalog) { : (catalog instanceof Set ? [...catalog] : []); if (!items.length) return; const next = new Set(); + const rowsBySelector = new Map(); for (const it of items) { - if (typeof it === 'string') { if (it.trim()) next.add(it.trim()); continue; } + if (typeof it === 'string') { + const selector = it.trim(); + if (!selector) continue; + next.add(selector); + if (!rowsBySelector.has(selector)) rowsBySelector.set(selector, { selector }); + continue; + } if (it && typeof it === 'object') { // ONLY the canonical `selector` (the full, upstream-accepted form) goes into // the live existence set. The catalog's `alias` is a FAMILY shortcut @@ -221,16 +229,19 @@ export function setLiveCatalogSelectors(catalog) { // by the hand-maintained SELECTOR_MAP (which resolves them to a real selector); // an alias the map doesn't know must fail closed, not pass through raw. // (ultracode review 2026-07-12; real-account confirmed gpt-5.6-sol regression) - if (typeof it.selector === 'string' && it.selector.trim()) next.add(it.selector.trim()); + const selector = typeof it.selector === 'string' ? it.selector.trim() : ''; + if (!selector) continue; + next.add(selector); + if (!rowsBySelector.has(selector)) rowsBySelector.set(selector, { ...it, selector }); } } if (!next.size) return; // never blank out a good set on a bad fetch _liveSelectors.clear(); for (const s of next) _liveSelectors.add(s); - // Retain the full rows too (only when we were handed decoded objects, not a - // bare string/Set) so /v1/models can synthesize live-only entries. - const rows = items.filter((it) => it && typeof it === 'object' && typeof it.selector === 'string' && it.selector.trim()); - if (rows.length) _liveCatalog = rows; + // Retain normalized rows so every consumer sees the same canonical selector + // strings as the existence set. String-only test seams become minimal rows + // instead of leaving stale metadata from an earlier object catalog behind. + _liveCatalog = [...rowsBySelector.values()]; } /** @@ -247,9 +258,26 @@ export function clearLiveCatalogSelectors() { _liveCatalog = []; } -/** A selector exists if the frozen snapshot OR the live catalog knows it. */ -function selectorExists(name) { - return CATALOG_SELECTORS.has(name) || _liveSelectors.has(name); +// Synthetic selectors do not appear in GetCliModelConfigs but remain valid routing +// targets. Keep this list deliberately tiny: everything else must come from the +// authoritative live catalog once one has been fetched. +const ALWAYS_KNOWN_SELECTORS = new Set([ + ...FREE_REACHABLE_SELECTORS, + 'subagent-default', +]); + +/** + * Does the currently authoritative Connect catalog contain this selector? + * + * Before the first successful live sync, fall back to the frozen snapshot so a + * cold-start or transient catalog failure stays usable. Once live data exists, + * use it exclusively so removed snapshot selectors are not advertised or routed. + */ +export function isKnownConnectSelector(name) { + if (ALWAYS_KNOWN_SELECTORS.has(name)) return true; + return _liveSelectors.size > 0 + ? _liveSelectors.has(name) + : CATALOG_SELECTORS.has(name); } /** @@ -259,43 +287,54 @@ function selectorExists(name) { * unmapped alias — it degrades to the one selector that always works. * * @param {string} model + * @param {object} [opts] + * @param {boolean} [opts.warnOnFallback=true] set false for read-only catalog + * probes; a later real request will still emit the one-time downgrade warning * @returns {{ selector: string, mapped: boolean }} */ -export function resolveConnectSelector(model) { +export function resolveConnectSelector(model, { warnOnFallback = true } = {}) { const raw = String(model || '').trim(); if (!raw) return { selector: FREE_TIER_SELECTOR, mapped: false }; - // Direct hit (covers both dash-form and enum-form selectors passed verbatim). - if (SELECTOR_MAP.has(raw)) return { selector: SELECTOR_MAP.get(raw), mapped: true }; + // A hand-maintained alias is valid only while its TARGET exists in the + // authoritative catalog. Otherwise a stale map entry can keep a removed model + // routable forever even after the live sync proved it is gone. + const directTarget = SELECTOR_MAP.get(raw); + if (directTarget && isKnownConnectSelector(directTarget)) { + return { selector: directTarget, mapped: true }; + } // Normalize: lowercase, collapse dots to dashes, strip a leading provider // prefix some clients prepend (e.g. "anthropic/claude-..."). const norm = raw.toLowerCase().replace(/^[a-z]+\//, '').replace(/\./g, '-'); - if (SELECTOR_MAP.has(norm)) return { selector: SELECTOR_MAP.get(norm), mapped: true }; + const normalizedTarget = SELECTOR_MAP.get(norm); + if (normalizedTarget && isKnownConnectSelector(normalizedTarget)) { + return { selector: normalizedTarget, mapped: true }; + } // A normalized dash-form that IS a real catalog selector (e.g. client sent the // dotted "gpt-5.5-medium" → norm "gpt-5-5-medium" which the catalog exposes but // the alias map doesn't list). Without this, a valid selector written with dots // silently degraded to the free tier. Checked after the map so an alias still // wins, before the free-tier fallback. - if (selectorExists(norm)) return { selector: norm, mapped: true }; + if (isKnownConnectSelector(norm)) return { selector: norm, mapped: true }; // Enum-form passthrough — ONLY when the catalog actually exposes it. A blind // MODEL_* passthrough is what re-introduces UPSTREAM_INTERNAL on drift: any // bogus MODEL_DOES_NOT_EXIST would otherwise be written raw to #21. - if (/^MODEL_[A-Z0-9_]+$/.test(raw) && selectorExists(raw)) { + if (/^MODEL_[A-Z0-9_]+$/.test(raw) && isKnownConnectSelector(raw)) { return { selector: raw, mapped: true }; } - // A verbatim dash-form selector that IS in the catalog (snapshot ∪ live) but - // missing from the alias map (e.g. a lowercased/prefixed valid enum, or a - // selector the upstream added after the frozen snapshot — qwen-3/glm-5/etc.) - // should still go through rather than silently degrade a paid request to free. - if (selectorExists(raw)) return { selector: raw, mapped: true }; + // A verbatim dash-form selector that IS in the authoritative catalog but is + // missing from the alias map (e.g. a selector the upstream added after the + // frozen snapshot — qwen-3/glm-5/etc.) should still go through rather than + // silently degrade a paid request to free. + if (isKnownConnectSelector(raw)) return { selector: raw, mapped: true }; // Unmapped: degrade to the always-available free selector, but make it // OBSERVABLE (one-time per distinct model) so a caller ignoring mapped:false // still gets an operator signal that a paid model was downgraded to free. - if (!degradeWarned.has(raw)) { + if (warnOnFallback && !degradeWarned.has(raw)) { degradeWarned.add(raw); log.warn( `[devin-connect] unmapped model "${raw}" not in catalog — degrading to ` diff --git a/src/handlers/models.js b/src/handlers/models.js index 5ef42b99..d6abf019 100644 --- a/src/handlers/models.js +++ b/src/handlers/models.js @@ -1,5 +1,7 @@ import { listModels } from '../models.js'; -import { resolveConnectSelector, getLiveCatalog, FREE_REACHABLE_SELECTORS, __testing } from '../devin-connect-models.js'; +import { + resolveConnectSelector, getLiveCatalog, FREE_REACHABLE_SELECTORS, +} from '../devin-connect-models.js'; import { getBackendSwitch } from '../runtime-config.js'; import { hasConnectEntitledAccount, getAccountCount } from '../auth.js'; @@ -51,31 +53,24 @@ export function buildConnectReachability(env = process.env) { if (!getBackendSwitch('devinConnect', effectiveEnv)) { return () => ({ reachable: true, selector: null }); } - const known = (selector) => __testing.CATALOG_SELECTORS.has(selector) || __testing._liveSelectors.has(selector); const skipEntitlement = shouldSkipEntitlementFilter(effectiveEnv, getAccountCount().total); const entitled = (selector) => skipEntitlement || hasConnectEntitledAccount(selector); - // NOTE on what is deliberately NOT here: a FREE_REACHABLE_SELECTORS short-circuit. - // - // It looks like it belongs — `swe-1-6-slow` is callable by any account and is absent from - // both the snapshot and the live catalog, so `known()` answers false for it. But every - // caller passes a MODELS-derived id (`handleModels` passes `m._windsurf_id`, the Dashboard - // passes the MODELS key), and the free selector is NEITHER a MODELS key nor any entry's - // `_windsurf_id` — measured against all 163 entries. So the branch never executed. It was - // written, then mutation-verified to be unreachable: deleting it failed zero assertions. - // - // The floor is real, it just does not live here — both views SYNTHESIZE the free selector - // as a row (/v1/models' third producer, and the Dashboard route's equivalent), which is - // where it actually works. v3.9.13 shipped a defect of exactly this shape: an exported - // helper with no production caller whose mutation guard was watching unreachable code. - // - // And re-testing the RESOLVED selector instead would be worse than dead — with - // STRICT_MODEL=0 an unmapped paid name degrades to `swe-1-6-slow`, so testing after - // resolution reports `claude-4-sonnet` as reachable on a free-only pool, which is exactly - // the lie #234 is about. `connect-discovery-rebuild.test.js` caught that on the first run - // when this was first written the wrong way round. + // Do not short-circuit on the RESOLVED free selector. With STRICT_MODEL=0 an + // unmapped paid name resolves to `swe-1-6-slow` with mapped:false; treating the + // selector alone as reachable would re-advertise every unsupported paid name. + // The real free floor is synthesized by both callers after this predicate runs. return (windsurfId) => { - const { selector, mapped } = resolveConnectSelector(windsurfId); - return { reachable: !!(mapped && known(selector) && entitled(selector)), selector: mapped ? selector : null }; + // Discovery probes every MODELS row, including intentionally unsupported + // Cascade-only names. They are not paid requests and must not consume the + // resolver's one-time downgrade warning; a later real chat request still will. + const { selector, mapped } = resolveConnectSelector(windsurfId, { warnOnFallback: false }); + return { + // resolveConnectSelector already validates aliases and direct selectors + // against the authoritative catalog. Keep entitlement as the second, + // independent gate; re-checking existence here would duplicate policy. + reachable: !!(mapped && entitled(selector)), + selector: mapped ? selector : null, + }; }; } @@ -87,25 +82,38 @@ export function handleModels(env = process.env) { let data = listModels({ env: effectiveEnv }); if (getBackendSwitch('devinConnect', effectiveEnv)) { const liveCatalog = getLiveCatalog(); + const catalogSelector = (row) => ( + typeof row === 'string' ? row : row?.selector + )?.trim(); const imageCapabilityBySelector = new Map( - liveCatalog - .filter((row) => typeof row?.selector === 'string' && typeof row?.supportsImages === 'boolean') - .map((row) => [row.selector, row.supportsImages]), + liveCatalog.flatMap((row) => { + const selector = catalogSelector(row); + return selector && typeof row?.supportsImages === 'boolean' + ? [[selector, row.supportsImages]] + : []; + }), ); // Row producer #1: the MODELS table, filtered to what this deployment can serve. // - // The rule (existence = snapshot ∪ live, plus the per-account entitlement check, plus - // the FREE_REACHABLE floor) now lives in buildConnectReachability because the Dashboard - // needs the identical answer. Existence alone used to be the only test here, so a + // The rule (existence = authoritative live catalog, with snapshot as cold-start + // fallback, plus per-account entitlement and the FREE_REACHABLE floor) now lives in + // buildConnectReachability because the Dashboard needs the identical answer. Existence + // alone used to be the only test here, so a // free-only pool advertised every paid selector the upstream publishes and the client // got a 403 at chat (#234 / #231 in the connect namespace). #232 fixed that for the // Cascade namespace, but its filters early-return unfiltered when devinConnect is on // (models.js isModelAllowedByCloudCatalog / filterModelKeysByCloudCatalog), which is // correct as a namespace boundary and is why the check has to be redone here. const isReachable = buildConnectReachability(effectiveEnv); + // Discovery is a selector catalog, not an alias catalog. Several public names can + // resolve to the same upstream selector (for example `claude-opus-4.6` and + // `claude-opus-4-6`). Keep the first stable client-facing name and suppress the + // rest, otherwise one entitled upstream model appears two or three times. + const representedSelectors = new Set(); data = data.flatMap((m) => { const reachability = isReachable(m._windsurf_id); - if (!reachability.reachable) return []; + if (!reachability.reachable || representedSelectors.has(reachability.selector)) return []; + representedSelectors.add(reachability.selector); const supportsImages = imageCapabilityBySelector.get(reachability.selector); return [typeof supportsImages === 'boolean' ? { ...m, supports_images: supportsImages } : m]; }); @@ -126,18 +134,19 @@ export function handleModels(env = process.env) { // first one left a free-only pool still advertising every live-only paid // selector (measured: 86 rows survived a filter applied to producer #1 alone). for (const row of liveCatalog) { - const id = row.selector; - if (!id || seen.has(id)) continue; + const id = catalogSelector(row); + if (!id || seen.has(id) || representedSelectors.has(id)) continue; if (!entitled(id)) continue; seen.add(id); + representedSelectors.add(id); data.push({ id, object: 'model', created: ts, - owned_by: row.provider || 'windsurf', + owned_by: row?.provider || 'windsurf', _windsurf_id: id, _source: 'live_catalog', - ...(row.label ? { _label: row.label } : {}), + ...(row?.label ? { _label: row.label } : {}), ...(typeof row.supportsImages === 'boolean' ? { supports_images: row.supportsImages } : {}), }); } diff --git a/test/connect-catalog-delatch.test.js b/test/connect-catalog-delatch.test.js index e80209b0..cb15202d 100644 --- a/test/connect-catalog-delatch.test.js +++ b/test/connect-catalog-delatch.test.js @@ -13,7 +13,8 @@ import { afterEach, beforeEach, describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { __resetModelCatalogState, __setModelCatalogDeps, __waitForModelCatalogSync, - addAccountByKey, removeAccount, getAccountInternal, setAccountStatus, trySyncModelCatalog, + addAccountByKey, removeAccount, getAccountInternal, setAccountStatus, + isConnectSelectorAllowedForAccount, trySyncModelCatalog, } from '../src/auth.js'; const created = []; @@ -148,6 +149,27 @@ describe('connect catalog de-latch (#234)', () => { 'the resolver must hold the union of both accounts, not just the last one'); }); + it('keeps paid-account routing inside that account\'s own live catalog', async () => { + installDeps({ + perAccount: { + 'sk-opus-acct': [{ selector: 'claude-opus-4-8-medium' }], + 'sk-gpt-acct': [{ selector: 'gpt-5-5-low' }], + }, + }); + + const opus = mk('sk-opus-acct', 'pro'); + await __waitForModelCatalogSync(); + const gpt = mk('sk-gpt-acct', 'pro'); + await __waitForModelCatalogSync(); + + assert.equal(isConnectSelectorAllowedForAccount(opus, 'claude-opus-4-8-medium'), true); + assert.equal(isConnectSelectorAllowedForAccount(opus, 'gpt-5-5-low'), false, + 'a paid tier is not permission to route a selector absent from this account catalog'); + assert.equal(isConnectSelectorAllowedForAccount(gpt, 'gpt-5-5-low'), true); + assert.equal(isConnectSelectorAllowedForAccount(gpt, 'claude-opus-4-8-medium'), false, + 'mixed-pool union must not erase per-account routing boundaries'); + }); + it('does not let an empty response shrink the union', async () => { // Same asymmetry as the Cascade empty-catalog guard: an empty response is no // data, not "this account reaches nothing". diff --git a/test/connect-discovery-rebuild.test.js b/test/connect-discovery-rebuild.test.js index 166bad99..a08f472e 100644 --- a/test/connect-discovery-rebuild.test.js +++ b/test/connect-discovery-rebuild.test.js @@ -26,7 +26,7 @@ import { } from '../src/auth.js'; import { handleModels } from '../src/handlers/models.js'; import { - setLiveCatalogSelectors, clearLiveCatalogSelectors, + setLiveCatalogSelectors, clearLiveCatalogSelectors, __testing, } from '../src/devin-connect-models.js'; const FREE_SELECTOR = 'swe-1-6-slow'; @@ -105,6 +105,47 @@ describe('connect discovery — entitlement filter (#234)', () => { assert.ok(withPaid.includes(FREE_SELECTOR), 'the free selector stays listed'); }); + it('treats a non-empty live catalog as authoritative over the frozen snapshot', () => { + connectEnv(); + mk('pro'); + setLiveCatalogSelectors([ + { selector: 'claude-opus-4-8-medium', provider: 'anthropic' }, + ]); + liveCatalogDirty = true; + + const rows = ids(); + assert.ok(rows.includes('claude-opus-4-8-medium'), + 'the selector confirmed by the live catalog must be advertised'); + assert.ok(!rows.includes('gpt-5.5'), + 'a snapshot-only model must not be advertised after a successful live sync'); + assert.ok(rows.includes(FREE_SELECTOR), 'the universal free floor stays advertised'); + }); + + it('does not log paid-request downgrade warnings while building discovery', () => { + connectEnv(); + mk('pro'); + setLiveCatalogSelectors([ + { selector: 'claude-opus-4-8-medium', provider: 'anthropic' }, + ]); + liveCatalogDirty = true; + __testing.degradeWarned.clear(); + + const warnings = []; + const originalWarn = console.warn; + console.warn = (...args) => { warnings.push(args.join(' ')); }; + try { + ids(); + } finally { + console.warn = originalWarn; + } + + assert.deepEqual( + warnings.filter((line) => line.includes('paid request downgraded to free tier')), + [], + 'GET /v1/models is a read-only catalog probe, not a paid request', + ); + }); + it('fails open on an empty pool instead of returning nothing', () => { // hasConnectEntitledAccount is Array.some over the pool, so it returns false // for EVERY selector when the pool is empty. Filtering without this arm would diff --git a/test/dashboard-models-connect-parity.test.js b/test/dashboard-models-connect-parity.test.js index 078c992c..18ad9fd2 100644 --- a/test/dashboard-models-connect-parity.test.js +++ b/test/dashboard-models-connect-parity.test.js @@ -38,6 +38,7 @@ import { } from '../src/auth.js'; import { handleDashboardApi } from '../src/dashboard/api.js'; import { handleModels } from '../src/handlers/models.js'; +import { setLiveCatalogSelectors, clearLiveCatalogSelectors } from '../src/devin-connect-models.js'; import { _resetRuntimeConfigForTests } from '../src/runtime-config.js'; // Written literally rather than imported from FREE_REACHABLE_SELECTORS: importing the set @@ -51,6 +52,7 @@ const ORIGINAL_ALLOW_NO_AUTH = process.env.DASHBOARD_ALLOW_NO_AUTH; const ORIGINAL_DEVIN_CONNECT = process.env.DEVIN_CONNECT; const ORIGINAL_DASHBOARD_PASSWORD = config.dashboardPassword; const created = []; +let liveCatalogDirty = false; function fakeRes() { return { @@ -102,6 +104,12 @@ beforeEach(() => { process.env.DASHBOARD_ALLOW_NO_AUTH = '1'; config.dashboardPassword = ''; configureBindHost('127.0.0.1'); + // These are namespace/view tests. Keep them deterministic and offline: account + // additions must not launch real catalog RPCs with fixture tokens. + __setModelCatalogDeps({ + disableConnectSync: true, + getCascadeModelConfigs: async () => ({ configs: [] }), + }); }); afterEach(async () => { @@ -111,6 +119,10 @@ afterEach(async () => { await __waitForModelCatalogSync(); __resetModelCatalogState(); __setModelCatalogDeps(null); + if (liveCatalogDirty) { + clearLiveCatalogSelectors(); + liveCatalogDirty = false; + } _resetRuntimeConfigForTests(); if (ORIGINAL_DEVIN_CONNECT === undefined) delete process.env.DEVIN_CONNECT; else process.env.DEVIN_CONNECT = ORIGINAL_DEVIN_CONNECT; @@ -170,6 +182,27 @@ describe('dashboard /models agrees with /v1/models on the Connect namespace (#23 } }); + it('includes live-only selectors that /v1/models synthesizes', async () => { + seed('pro'); + const liveOnly = 'grok-4-5-medium-dashboard-parity'; + const aliasBacked = 'claude-opus-4-6'; + setLiveCatalogSelectors([ + { selector: ` ${liveOnly} `, provider: 'xai', label: 'Grok live only' }, + { selector: aliasBacked, provider: 'anthropic', label: 'Claude Opus 4.8 Medium' }, + ]); + liveCatalogDirty = true; + + const ids = v1Ids(); + assert.ok(ids.has(liveOnly), 'precondition: /v1/models synthesized the live-only selector'); + const rows = await dashboardModels(); + const row = rows.find((r) => r.id === liveOnly); + assert.ok(row, 'Dashboard omitted a selector that /v1/models advertises'); + assert.equal(row.reachable, true); + assert.equal(row.connectSelector, liveOnly); + assert.equal(rows.filter((r) => r.reachable && r.connectSelector === aliasBacked).length, 1, + 'Dashboard must not append a canonical row when a visible alias already represents it'); + }); + it('widens when a paid account joins — the flag is computed, not baked in', async () => { seed('free'); const before = await dashboardModels(); @@ -186,42 +219,6 @@ describe('dashboard /models agrees with /v1/models on the Connect namespace (#23 ); }); - // Pins the premise that makes a documented survivor harmless rather than pretending the - // survivor is covered (mutations spec: "the existence term (known()) is dropped"). - // - // The predicate is `mapped && known(selector) && entitled(selector)`. Dropping `known` - // changes nothing today because no MODELS entry is in the mapped-but-unknown state: every - // entry that resolves mapped also exists in snapshot ∪ live. That is a property of the - // current alias map, NOT a structural guarantee — resolveConnectSelector's first branch - // (devin-connect-models.js:257) returns mapped:true straight from SELECTOR_MAP without - // consulting the catalog, so an alias left pointing at a selector a snapshot rotation - // removed would be mapped-but-unknown and `known` would become the only thing rejecting - // it. When that day comes this assertion fails, which is the signal to write the fixture - // the mutation currently cannot have. - it('no MODELS entry is mapped-but-unknown — the premise behind a documented survivor', async () => { - seed('free'); - const { MODELS } = await import('../src/models.js'); - const dcm = await import('../src/devin-connect-models.js'); - const { CATALOG_SELECTORS, _liveSelectors } = dcm.__testing; - const mappedUnknown = []; - let mapped = 0; - for (const key of Object.keys(MODELS)) { - const r = dcm.resolveConnectSelector(MODELS[key]?._windsurf_id || key); - if (!r.mapped) continue; - mapped++; - if (!CATALOG_SELECTORS.has(r.selector) && !_liveSelectors.has(r.selector)) { - mappedUnknown.push(`${key} -> ${r.selector}`); - } - } - assert.ok(mapped > 0, `precondition: some entry must resolve mapped (got ${mapped}) — a ` - + 'zero here would make the assertion below vacuously true'); - assert.deepEqual(mappedUnknown, [], - 'a MODELS entry now resolves to a selector absent from snapshot ∪ live. The existence ' - + 'term in buildConnectReachability just became load-bearing, so its mutation should ' - + 'now be CAUGHT — write the fixture and flip expectCaught in ' - + 'test/mutations/dashboard-connect-parity.json'); - }); - // #235: the panel must say whether a model COSTS QUOTA, and must not guess when it cannot // tell. The reporter had a pro account, set GLM-5-2 believing it was free on that plan, and // burned the whole weekly allowance. The rate-table wiring that answers this shipped in diff --git a/test/devin-connect-models.test.js b/test/devin-connect-models.test.js index f2858275..ef886600 100644 --- a/test/devin-connect-models.test.js +++ b/test/devin-connect-models.test.js @@ -54,7 +54,8 @@ describe('resolveConnectSelector — live catalog (audit 2026-07-12 snapshot sta // after it was captured (proven on a live account 2026-07-12: qwen-3, glm-5, // kimi-k2.5, deepseek-v3, minimax-*) were absent from CATALOG_SELECTORS and // got mapped:false → 400'd by the strict gate despite being runnable. The live - // catalog set (populated from GetCliModelConfigs) fixes this as "snapshot ∪ live". + // catalog set (populated from GetCliModelConfigs) becomes authoritative after + // the first successful sync; the snapshot remains the cold-start fallback. const STALE = ['qwen-3', 'glm-5', 'kimi-k2.5', 'deepseek-v3']; it('cold start (no live sync): a genuinely-runnable-but-unsnapshotted selector is mapped:false', async () => { @@ -102,6 +103,19 @@ describe('resolveConnectSelector — live catalog (audit 2026-07-12 snapshot sta assert.equal(m.resolveConnectSelector('glm-5.2').selector, 'glm-5-2'); }); + it('retires a snapshot alias when its target is absent from the live catalog', async () => { + const m = await import(`../src/devin-connect-models.js?fresh=${Date.now()}-c3`); + m.setLiveCatalogSelectors([ + { selector: 'claude-opus-4-8-medium', alias: 'claude-opus-4.8' }, + ]); + + assert.equal(m.resolveConnectSelector('claude-opus-4.8').mapped, true, + 'precondition: an alias targeting a live selector remains valid'); + const retired = m.resolveConnectSelector('gpt-5.5', { warnOnFallback: false }); + assert.deepEqual(retired, { selector: m.FREE_TIER_SELECTOR, mapped: false }, + 'chat preflight must not keep a snapshot-only alias routable after live sync'); + }); + it('a bad/empty sync never blanks out a good live set', async () => { const m = await import(`../src/devin-connect-models.js?fresh=${Date.now()}-d`); m.setLiveCatalogSelectors([{ selector: 'qwen-3' }]); @@ -111,6 +125,14 @@ describe('resolveConnectSelector — live catalog (audit 2026-07-12 snapshot sta assert.equal(m.resolveConnectSelector('qwen-3').mapped, true, 'prior good set survives a bad sync'); }); + it('normalizes string-only seam rows into the readable live catalog', async () => { + const m = await import(`../src/devin-connect-models.js?fresh=${Date.now()}-strings`); + m.setLiveCatalogSelectors([' qwen-3 ', '']); + + assert.deepEqual(m.getLiveCatalog(), [{ selector: 'qwen-3' }]); + assert.deepEqual(m.resolveConnectSelector('qwen-3'), { selector: 'qwen-3', mapped: true }); + }); + it('genuine junk still degrades even with a live catalog present', async () => { const m = await import(`../src/devin-connect-models.js?fresh=${Date.now()}-e`); m.setLiveCatalogSelectors([{ selector: 'qwen-3' }]); diff --git a/test/models-live-catalog.test.js b/test/models-live-catalog.test.js index 0f5190cc..799f84d1 100644 --- a/test/models-live-catalog.test.js +++ b/test/models-live-catalog.test.js @@ -1,19 +1,20 @@ // audit 2026-07-12 (v3.2.4 regression fix): after v3.2.3 made resolveConnectSelector -// recognize live-synced selectors (snapshot ∪ live), the /v1/models handler still +// recognize live-synced selectors, the /v1/models handler still // filtered against the frozen CATALOG_SELECTORS snapshot ONLY, and the 37 upstream- // added selectors (gpt-5-6-*/grok-4-5-*/nemotron) aren't in the hardcoded MODELS // table either — so they ran fine at /v1/chat/completions but were absent from // /v1/models, leaving Codex/clients unable to discover them. handleModels now -// (a) filters on snapshot ∪ live and (b) synthesizes entries for live-only selectors. +// (a) filters on the authoritative live catalog (snapshot only before sync) and +// (b) synthesizes entries for live-only selectors. // // NOTE: handleModels imports devin-connect-models via a plain (cached) import, so // these tests use the SAME cached singleton (no ?fresh= — that would give the -// handler a different instance than the one we seed). The live catalog is additive -// and cleared per test via setLiveCatalogSelectors, so cross-test leakage is bounded. +// handler a different instance than the one we seed). Each non-empty seed replaces +// the prior live catalog, so cross-test leakage inside this file is bounded. import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { setLiveCatalogSelectors } from '../src/devin-connect-models.js'; +import { setLiveCatalogSelectors, resolveConnectSelector } from '../src/devin-connect-models.js'; import { handleModels } from '../src/handlers/models.js'; const ENV_ON = { DEVIN_CONNECT: '1' }; @@ -23,7 +24,7 @@ describe('/v1/models — live-catalog synthesis (audit v3.2.4)', () => { // grok-4-5-medium: proven live-catalog-only 2026-07-12 (runs at chat, absent // from the 130-model MODELS table and the 105-model snapshot). setLiveCatalogSelectors([ - { selector: 'grok-4-5-medium', provider: 'xai', label: 'Grok 4.5 (medium)', supportsImages: true }, + { selector: ' grok-4-5-medium ', provider: 'xai', label: 'Grok 4.5 (medium)', supportsImages: true }, { selector: 'gpt-5-6-sol-medium', provider: 'openai', label: 'GPT-5.6 Sol (medium)', supportsImages: false }, ]); const { data } = handleModels(ENV_ON); @@ -35,6 +36,8 @@ describe('/v1/models — live-catalog synthesis (audit v3.2.4)', () => { assert.equal(grok.owned_by, 'xai'); assert.equal(grok._source, 'live_catalog'); assert.equal(grok.supports_images, true); + assert.equal(data.some((m) => m.id !== m.id.trim()), false, + 'live catalog whitespace must not leak into public model ids'); assert.equal(data.find((m) => m.id === 'gpt-5-6-sol-medium').supports_images, false); }); @@ -45,6 +48,19 @@ describe('/v1/models — live-catalog synthesis (audit v3.2.4)', () => { assert.ok(count <= 1, `swe-1-6-slow must not be duplicated (got ${count})`); }); + it('deduplicates a live canonical selector already represented by a client alias', () => { + const selector = 'claude-opus-4-6'; + setLiveCatalogSelectors([{ selector, provider: 'anthropic' }]); + const { data } = handleModels(ENV_ON); + const matching = data.filter((m) => { + if (m._source === 'live_catalog') return m.id === selector; + return resolveConnectSelector(m._windsurf_id, { warnOnFallback: false }).selector === selector; + }); + + assert.equal(matching.length, 1, + `one upstream selector must produce one discovery row, got: ${matching.map((m) => m.id).join(', ')}`); + }); + it('non-DEVIN_CONNECT deployment returns the full list without live synthesis', () => { setLiveCatalogSelectors([{ selector: 'grok-4-5-medium', provider: 'xai' }]); const { data } = handleModels({}); // devinConnect off diff --git a/test/mutations/dashboard-connect-parity.json b/test/mutations/dashboard-connect-parity.json index c2043c4b..b58be02d 100644 --- a/test/mutations/dashboard-connect-parity.json +++ b/test/mutations/dashboard-connect-parity.json @@ -4,13 +4,13 @@ "test/connect-discovery-rebuild.test.js", "test/cloud-catalog-backend-boundary.test.js" ], - "expectBaselinePass": 22, + "expectBaselinePass": 24, "mutations": [ { "name": "the original defect: dashboard stops annotating and reports everything reachable", "file": "src/dashboard/api.js", - "anchor": " const { reachable, selector } = isReachable(id);", - "replacement": " const { selector } = isReachable(id); const reachable = true;" + "anchor": " const info = MODELS[id];\n const { reachable, selector } = isReachable(id);", + "replacement": " const info = MODELS[id];\n const { selector } = isReachable(id); const reachable = true;" }, { "name": "dashboard narrows instead of annotating — an allow-list entry becomes unclickable", @@ -37,41 +37,40 @@ "replacement": " // look identical in the UI otherwise." }, { - "name": "the FREE_REACHABLE floor is re-tested AFTER resolution — a degraded paid name reads as reachable", - "file": "src/handlers/models.js", - "anchor": " const { selector, mapped } = resolveConnectSelector(windsurfId);\n return { reachable: !!(mapped && known(selector) && entitled(selector)), selector: mapped ? selector : null };", - "replacement": " const { selector, mapped } = resolveConnectSelector(windsurfId);\n if (FREE_REACHABLE_SELECTORS.has(selector)) return { reachable: true, selector };\n return { reachable: !!(mapped && known(selector) && entitled(selector)), selector: mapped ? selector : null };" + "name": "Dashboard stops synthesizing live-only selectors advertised by /v1/models", + "file": "src/dashboard/api.js", + "anchor": " for (const row of getLiveCatalog()) {", + "replacement": " for (const row of []) {" }, { - "name": "entitlement term dropped from the predicate — a free-only pool advertises paid selectors again", + "name": "the FREE_REACHABLE floor is re-tested AFTER resolution — a degraded paid name reads as reachable", "file": "src/handlers/models.js", - "anchor": " return { reachable: !!(mapped && known(selector) && entitled(selector)), selector: mapped ? selector : null };", - "replacement": " return { reachable: !!(mapped && known(selector)), selector: mapped ? selector : null };" + "anchor": " reachable: !!(mapped && entitled(selector)),", + "replacement": " reachable: FREE_REACHABLE_SELECTORS.has(selector) || !!(mapped && entitled(selector))," }, { - "name": "DOCUMENTED SURVIVOR: the existence term (known()) is dropped. Measured: of 163 MODELS entries, 58 resolve mapped and ALL 58 are known (snapshot ∪ live), 105 are unmapped — so no fixture puts a row in the mapped-but-unknown state this term exists to reject, and dropping it changes nothing observable. It is NOT dead code: resolveConnectSelector's first branch (devin-connect-models.js:257) returns mapped:true straight from SELECTOR_MAP without consulting the catalog, so an alias left pointing at a selector a snapshot rotation removed would be mapped-but-unknown and this term is the only thing rejecting it. Reproducing that needs a SELECTOR_MAP entry diverging from the catalog, i.e. reaching into module internals to manufacture drift; the term is kept and the premise that makes it currently unexercised is pinned by the fixture-shape assertion in the parity test instead.", + "name": "entitlement term dropped from the predicate — a free-only pool advertises paid selectors again", "file": "src/handlers/models.js", - "anchor": " return { reachable: !!(mapped && known(selector) && entitled(selector)), selector: mapped ? selector : null };", - "replacement": " return { reachable: !!(mapped && entitled(selector)), selector: mapped ? selector : null };", - "expectCaught": false + "anchor": " reachable: !!(mapped && entitled(selector)),", + "replacement": " reachable: !!mapped," }, { "name": "#235 unknown cost is flattened to 'billable' — a guess rendered as a fact", "file": "src/dashboard/api.js", - "anchor": " currentlyFree: !selector ? null\n : isConnectSelectorCurrentlyFree(selector) ? true\n : (freeSet === null ? null : false),", - "replacement": " currentlyFree: selector ? isConnectSelectorCurrentlyFree(selector) : null," + "anchor": " const connectCurrentlyFree = (selector) => (!selector ? null\n : isConnectSelectorCurrentlyFree(selector) ? true\n : (freeSet === null ? null : false));", + "replacement": " const connectCurrentlyFree = (selector) => (selector ? isConnectSelectorCurrentlyFree(selector) : null);" }, { "name": "#235 unknown cost is flattened to 'free' — the reporter's exact loss, reintroduced", "file": "src/dashboard/api.js", - "anchor": " currentlyFree: !selector ? null\n : isConnectSelectorCurrentlyFree(selector) ? true\n : (freeSet === null ? null : false),", - "replacement": " currentlyFree: !selector ? null : (freeSet === null ? true : isConnectSelectorCurrentlyFree(selector))," + "anchor": " const connectCurrentlyFree = (selector) => (!selector ? null\n : isConnectSelectorCurrentlyFree(selector) ? true\n : (freeSet === null ? null : false));", + "replacement": " const connectCurrentlyFree = (selector) => (!selector ? null : (freeSet === null ? true : isConnectSelectorCurrentlyFree(selector)));" }, { "name": "#235 the field is dropped entirely — the panel goes back to not answering the question", "file": "src/dashboard/api.js", - "anchor": " currentlyFree: !selector ? null\n : isConnectSelectorCurrentlyFree(selector) ? true\n : (freeSet === null ? null : false),", - "replacement": "" + "anchor": " const connectCurrentlyFree = (selector) => (!selector ? null\n : isConnectSelectorCurrentlyFree(selector) ? true\n : (freeSet === null ? null : false));", + "replacement": " const connectCurrentlyFree = () => undefined;" }, { "name": "predicate ignores the transport — a Cascade deployment gets its panel filtered by connect rules", @@ -83,7 +82,7 @@ "name": "handleModels stops using the shared predicate — the two views can diverge again", "file": "src/handlers/models.js", "anchor": " const reachability = isReachable(m._windsurf_id);", - "replacement": " const { selector, mapped } = resolveConnectSelector(m._windsurf_id); const reachability = { reachable: mapped && entitled(selector), selector: mapped ? selector : null };" + "replacement": " const { selector, mapped } = resolveConnectSelector(m._windsurf_id, { warnOnFallback: false }); const reachability = { reachable: mapped, selector: mapped ? selector : null };" } ] }