From f5d3d42e66e46509f8084e2d2e95dfd280b34108 Mon Sep 17 00:00:00 2001 From: David Abutbul Date: Thu, 23 Jul 2026 02:27:40 +0300 Subject: [PATCH] fix(suite): fail closed on catalog lifecycle --- skills/clawsec-suite/CHANGELOG.md | 14 + skills/clawsec-suite/SKILL.md | 36 +- .../scripts/discover_skill_catalog.mjs | 421 +++++++-- skills/clawsec-suite/skill.json | 8 +- .../test/skill_catalog_discovery.test.mjs | 805 ++++++++++++++---- 5 files changed, 1040 insertions(+), 244 deletions(-) diff --git a/skills/clawsec-suite/CHANGELOG.md b/skills/clawsec-suite/CHANGELOG.md index 7021be70..639c3993 100644 --- a/skills/clawsec-suite/CHANGELOG.md +++ b/skills/clawsec-suite/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [0.1.17] - 2026-07-23 + +### Changed + +- Made catalog discovery distinguish lifecycle-screened remote candidates from non-authorizing historical or suite-local context. +- Added exact requested-skill discovery so callers can resolve one catalog identity before beginning the separate advisory-gated install flow. +- Classified the bundled `skill.json` catalog as descriptive fallback context rather than installation authority. +- Marked the projected catalog as a denial overlay that does not replace independent signed stable-catalog authorization. + +### Security + +- Fail closed without install commands when the remote index is unavailable, malformed, contains invalid lifecycle metadata, or omits the requested skill. +- Treat explicit `installable: false` as a denial that local defaults cannot override, while preserving missing-field compatibility for existing remote records. + ## [0.1.16] - 2026-07-14 ### Fixed diff --git a/skills/clawsec-suite/SKILL.md b/skills/clawsec-suite/SKILL.md index 788093af..4fb4f964 100644 --- a/skills/clawsec-suite/SKILL.md +++ b/skills/clawsec-suite/SKILL.md @@ -1,6 +1,6 @@ --- name: clawsec-suite -version: 0.1.16 +version: 0.1.17 description: ClawSec suite manager with embedded advisory-feed monitoring, cryptographic signature verification, approval-gated malicious-skill response, and guided setup for additional security skills. homepage: https://clawsec.prompt.security clawdis: @@ -54,9 +54,22 @@ SUITE_DIR="${INSTALL_ROOT:-$HOME/.openclaw/skills}/clawsec-suite" node "$SUITE_DIR/scripts/discover_skill_catalog.mjs" ``` -Fallback behavior: -- If the remote catalog index is reachable and valid, the suite uses it. -- If the remote index is unavailable or malformed, the script falls back to suite-local catalog metadata in `skill.json`. +Lifecycle behavior: +- A remote record with `installable: true`, or a legacy record with no `installable` field, may appear as a lifecycle-screened candidate. +- A remote record with `installable: false` may appear only as non-authorizing historical context. It never receives an install command. +- A present non-Boolean `installable` value invalidates the whole remote index and disables install recommendations. +- An unavailable or malformed remote index produces no installable results. Suite-local metadata in `skill.json` may be shown only as unverified context; it never authorizes an install. +- Local metadata cannot override a remote denial or satisfy a record that is missing from a valid remote index. +- This projected index is a denial overlay. A non-denied record is necessary but not sufficient for an official stable install; signed stable-catalog authorization remains a separate gate. +- The Suite's final SemVer does not itself assert stable status. Version `0.1.17` remains a stable-intent candidate until the release and active-catalog gates authorize that exact payload. + +Resolve one exact skill before installing it: + +```bash +node "$SUITE_DIR/scripts/discover_skill_catalog.mjs" --skill +``` + +Only continue to the next checks when that command returns a lifecycle-screened candidate. A withdrawn, missing, malformed, or unavailable record is denied without an install command. Discovery does not replace signed stable-catalog authorization or the advisory/operator gates. ## Installation @@ -183,7 +196,9 @@ Restart the OpenClaw gateway after enabling the hook. Then run `/new` once to fo ## Guarded Skill Install Flow (Double Confirmation) -When the user asks to install a skill, treat that as the first request and run a guarded install check: +Lifecycle screening and advisory gating are separate checks. First resolve the exact skill through `discover_skill_catalog.mjs --skill `. Do not pass a denied, missing, or unverified-local record to the installer. + +When catalog discovery does not deny the requested skill, independently verify stable-catalog authorization, then treat the user's request as the first confirmation and run the guarded advisory check: ```bash SUITE_DIR="${INSTALL_ROOT:-$HOME/.openclaw/skills}/clawsec-suite" @@ -390,20 +405,21 @@ Audit entries (with check identifiers like `skills.code_safety`) are only matche ## Optional Skill Installation -Discover currently available installable skills dynamically, then install the ones you want: +Discover currently available installable skills dynamically, then resolve one exact skill before installing it: ```bash SUITE_DIR="${INSTALL_ROOT:-$HOME/.openclaw/skills}/clawsec-suite" node "$SUITE_DIR/scripts/discover_skill_catalog.mjs" - -# then install any discovered skill by name -npx clawhub@latest install +node "$SUITE_DIR/scripts/discover_skill_catalog.mjs" --skill ``` -Machine-readable output is also available for automation: +The requested form emits an install command only for an exact lifecycle-screened record from a valid remote index. That command is a candidate recommendation, not proof of stable authorization. Explicitly withdrawn records, missing records, malformed lifecycle data, and remote failures emit no install command. Local fallback data is context only. + +Machine-readable output follows the same denial rules and identifies stable authorization as not evaluated: ```bash node "$SUITE_DIR/scripts/discover_skill_catalog.mjs" --json +node "$SUITE_DIR/scripts/discover_skill_catalog.mjs" --json --skill ``` ## Security Notes diff --git a/skills/clawsec-suite/scripts/discover_skill_catalog.mjs b/skills/clawsec-suite/scripts/discover_skill_catalog.mjs index 51a84489..6ff87e38 100644 --- a/skills/clawsec-suite/scripts/discover_skill_catalog.mjs +++ b/skills/clawsec-suite/scripts/discover_skill_catalog.mjs @@ -6,6 +6,10 @@ import { loadTextFile } from "./local_file_io.mjs"; const DEFAULT_INDEX_URL = "https://clawsec.prompt.security/skills/index.json"; const DEFAULT_TIMEOUT_MS = 5000; +const EXIT_RECOMMENDATION_DENIED = 2; +const SAFE_SKILL_ID = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; +const SAFE_CATALOG_VERSION = /^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)$/; +const SAFE_CATALOG_UPDATED = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/; const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)); const SUITE_DIR = path.resolve(SCRIPT_DIR, ".."); @@ -15,10 +19,53 @@ function isObject(value) { return typeof value === "object" && value !== null && !Array.isArray(value); } -function normalizeSkillId(value) { - return String(value ?? "") - .trim() - .toLowerCase(); +function canonicalSkillId(value) { + return value.normalize("NFKC").toLowerCase(); +} + +function requireExactString(value, label) { + if (typeof value !== "string" || value.length === 0 || value.trim() !== value) { + throw new Error(`${label} must be an exact non-empty string`); + } + return value; +} + +function optionalString(value, label) { + if (value === undefined || value === null) return null; + if (typeof value !== "string") { + throw new Error(`${label} must be a string when present`); + } + return value.trim() || null; +} + +function optionalStringArray(value, label) { + if (value === undefined || value === null) return []; + if (!Array.isArray(value)) { + throw new Error(`${label} must be an array when present`); + } + return value.map((entry, index) => requireExactString(entry, `${label} entry at index ${index}`)); +} + +function optionalCatalogVersion(value) { + if (value === undefined || value === null) return null; + const version = requireExactString(value, "Catalog index version"); + if (!SAFE_CATALOG_VERSION.test(version)) { + throw new Error("Catalog index version must use numeric major.minor.patch form"); + } + return version; +} + +function optionalCatalogUpdated(value) { + if (value === undefined || value === null) return null; + const updated = requireExactString(value, "Catalog index updated"); + const timestamp = Date.parse(updated); + const normalized = Number.isFinite(timestamp) + ? new Date(timestamp).toISOString().replace(".000Z", "Z") + : null; + if (!SAFE_CATALOG_UPDATED.test(updated) || normalized !== updated) { + throw new Error("Catalog index updated must be a valid UTC timestamp"); + } + return updated; } function normalizeBoolean(value) { @@ -49,16 +96,52 @@ function parseTimeoutMs() { return parsed; } +function indexUrlForOutput(value) { + try { + const parsed = new URL(value); + if (parsed.protocol !== "https:" && parsed.protocol !== "http:") return null; + parsed.username = ""; + parsed.password = ""; + parsed.search = ""; + parsed.hash = ""; + return parsed.toString(); + } catch { + return null; + } +} + +function validateRequestedSkill(value) { + const skillId = requireExactString(value, "--skill value"); + if (skillId !== canonicalSkillId(skillId) || !SAFE_SKILL_ID.test(skillId)) { + throw new Error("Invalid --skill value. Use canonical lowercase letters, digits, and single hyphens only."); + } + return skillId; +} + function parseArgs(argv) { const args = { json: false, + skill: null, }; - for (const token of argv) { + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (token === "--json") { + if (args.json) throw new Error("Duplicate argument: --json"); args.json = true; continue; } + if (token === "--skill") { + if (args.skill !== null) throw new Error("Duplicate argument: --skill"); + const value = argv[index + 1]; + if (value === undefined || value.startsWith("-")) { + throw new Error("Missing value for --skill"); + } + args.skill = validateRequestedSkill(value); + index += 1; + continue; + } if (token === "--help" || token === "-h") { printUsage(); process.exit(0); @@ -74,16 +157,24 @@ function printUsage() { process.stdout.write( [ "Usage:", - " node scripts/discover_skill_catalog.mjs [--json]", + " node scripts/discover_skill_catalog.mjs [--skill ] [--json]", "", "Behavior:", - " - Fetches dynamic catalog from CLAWSEC_SKILLS_INDEX_URL (default: https://clawsec.prompt.security/skills/index.json)", - " - Falls back to suite-local catalog metadata in skill.json when remote index is unavailable/invalid", + " - Fetches the projected catalog from CLAWSEC_SKILLS_INDEX_URL", + " - Lists only exact remote records that the lifecycle projection does not deny", + " - Treats suite-local catalog metadata as non-authorizing context only", + " - Emits no install command when the remote catalog is unavailable, malformed, missing the requested record, or explicitly non-installable", + " - Acts only as a denial overlay; positive stable-install authorization is evaluated separately", "", "Environment:", " CLAWSEC_SKILLS_INDEX_URL Override remote catalog index URL", " CLAWSEC_SKILLS_INDEX_TIMEOUT_MS HTTP timeout in milliseconds (default: 5000)", "", + "Exit codes:", + " 0 catalog report or lifecycle-screened candidate", + ` ${EXIT_RECOMMENDATION_DENIED} requested skill was not eligible for recommendation`, + " 1 invalid arguments or runtime error", + "", ].join("\n"), ); } @@ -98,56 +189,102 @@ function normalizeRemoteSkills(payload) { throw new Error("Catalog index missing skills array"); } - const dedup = new Map(); + const exactIds = new Set(); + const canonicalIds = new Map(); + const records = []; - for (const entry of rawSkills) { - if (!isObject(entry)) continue; + for (let index = 0; index < rawSkills.length; index += 1) { + const entry = rawSkills[index]; + const label = `Catalog skill at index ${index}`; + if (!isObject(entry)) { + throw new Error(`${label} must be a JSON object`); + } - const id = normalizeSkillId(entry.id ?? entry.name); - if (!id) continue; + const id = requireExactString(entry.id, `${label} id`); + const name = requireExactString(entry.name, `${label} name`); + const version = requireExactString(entry.version, `${label} version`); + const tag = requireExactString(entry.tag, `${label} tag`); + const canonicalId = canonicalSkillId(id); - dedup.set(id, { + if (exactIds.has(id)) { + throw new Error(`Catalog index contains duplicate skill id: ${id}`); + } + const canonicalOwner = canonicalIds.get(canonicalId); + if (canonicalOwner !== undefined) { + throw new Error(`Catalog index contains canonical skill-id collision: ${canonicalOwner} and ${id}`); + } + exactIds.add(id); + canonicalIds.set(canonicalId, id); + + if (id !== canonicalId || !SAFE_SKILL_ID.test(id)) { + throw new Error(`${label} id must use canonical lowercase letters, digits, and single hyphens only`); + } + if (id !== name) { + throw new Error(`${label} id and name must match exactly`); + } + if (tag !== `${name}-v${version}`) { + throw new Error(`${label} tag must equal ${name}-v${version}`); + } + + const installableExplicitlyDeclared = Object.hasOwn(entry, "installable"); + if (installableExplicitlyDeclared && typeof entry.installable !== "boolean") { + throw new Error(`${label} installable must be a boolean when present`); + } + const installable = installableExplicitlyDeclared ? entry.installable : true; + + records.push({ id, - name: String(entry.name ?? id), - version: String(entry.version ?? "").trim() || null, - description: String(entry.description ?? "").trim() || null, - emoji: String(entry.emoji ?? "").trim() || null, - category: String(entry.category ?? "").trim() || null, - tag: String(entry.tag ?? "").trim() || null, - trust: entry.trust ?? null, + name, + version, + description: optionalString(entry.description, `${label} description`), + emoji: optionalString(entry.emoji, `${label} emoji`), + category: optionalString(entry.category, `${label} category`), + platforms: optionalStringArray(entry.platforms, `${label} platforms`), + tag, + trust: optionalString(entry.trust, `${label} trust`), + installable, + installable_explicitly_declared: installableExplicitlyDeclared, + lifecycle_status: installable ? "not_denied" : "historical", source: "remote", }); } return { - version: String(payload.version ?? "").trim() || null, - updated: String(payload.updated ?? "").trim() || null, - skills: [...dedup.values()].sort((a, b) => a.id.localeCompare(b.id)), + version: optionalCatalogVersion(payload.version), + updated: optionalCatalogUpdated(payload.updated), + records: records.sort((left, right) => left.id.localeCompare(right.id)), }; } async function loadFallbackCatalog() { const raw = await loadTextFile(SUITE_SKILL_JSON); const parsed = JSON.parse(raw); + if (!isObject(parsed)) { + throw new Error("Suite skill.json must contain a JSON object"); + } - const catalogSkills = isObject(parsed?.catalog?.skills) ? parsed.catalog.skills : {}; - const dedup = new Map(); + const catalogSkills = isObject(parsed.catalog?.skills) ? parsed.catalog.skills : {}; + const context = []; for (const [rawId, meta] of Object.entries(catalogSkills)) { - const id = normalizeSkillId(rawId); - if (!id) continue; + if (rawId !== canonicalSkillId(rawId) || !SAFE_SKILL_ID.test(rawId)) { + throw new Error(`Suite-local catalog contains invalid skill id: ${rawId}`); + } const safeMeta = isObject(meta) ? meta : {}; - - dedup.set(id, { - id, - name: id, + context.push({ + id: rawId, + name: rawId, version: null, - description: String(safeMeta.description ?? "").trim() || null, + description: optionalString(safeMeta.description, `Suite-local catalog ${rawId} description`), emoji: null, category: null, + platforms: [], tag: null, trust: null, + installable: false, + installable_explicitly_declared: false, + lifecycle_status: "local_context_only", source: "fallback", integrated_in_suite: normalizeBoolean(safeMeta.integrated_in_suite), requires_explicit_consent: normalizeBoolean(safeMeta.requires_explicit_consent), @@ -155,37 +292,45 @@ async function loadFallbackCatalog() { }); } - return { - version: null, - updated: null, - skills: [...dedup.values()].sort((a, b) => a.id.localeCompare(b.id)), - }; + return context.sort((left, right) => left.id.localeCompare(right.id)); } -function mergeWithFallbackMetadata(remoteSkills, fallbackSkills) { - const fallbackById = new Map(fallbackSkills.map((skill) => [skill.id, skill])); +function enrichWithFallbackMetadata(remoteRecords, fallbackContext) { + const fallbackById = new Map(fallbackContext.map((skill) => [skill.id, skill])); - return remoteSkills.map((skill) => { + return remoteRecords.map((skill) => { const fallback = fallbackById.get(skill.id); - if (!fallback) { - return { - ...skill, - integrated_in_suite: false, - requires_explicit_consent: false, - default_install: false, - }; - } - return { ...skill, - description: skill.description || fallback.description || null, - integrated_in_suite: normalizeBoolean(fallback.integrated_in_suite), - requires_explicit_consent: normalizeBoolean(fallback.requires_explicit_consent), - default_install: normalizeBoolean(fallback.default_install), + description: skill.description || fallback?.description || null, + integrated_in_suite: normalizeBoolean(fallback?.integrated_in_suite), + requires_explicit_consent: normalizeBoolean(fallback?.requires_explicit_consent), + default_install: normalizeBoolean(fallback?.default_install), }; }); } +function toNonAuthorizingSummary(skill) { + return { + id: skill.id, + name: skill.id, + version: null, + description: null, + emoji: null, + category: null, + platforms: [], + tag: null, + trust: null, + installable: false, + installable_explicitly_declared: normalizeBoolean(skill.installable_explicitly_declared), + lifecycle_status: skill.lifecycle_status, + source: skill.source, + integrated_in_suite: normalizeBoolean(skill.integrated_in_suite), + requires_explicit_consent: normalizeBoolean(skill.requires_explicit_consent), + default_install: normalizeBoolean(skill.default_install), + }; +} + async function loadRemoteCatalog(indexUrl, timeoutMs) { if (typeof globalThis.fetch !== "function") { throw new Error("fetch is unavailable in this runtime"); @@ -228,84 +373,188 @@ function formatFlags(skill) { flags.push("explicit opt-in"); } if (skill.default_install) { - flags.push("recommended default"); + flags.push( + skill.installable + ? "suite default preference (lifecycle screened only)" + : "packaged default preference (non-authorizing)", + ); } return flags; } +function buildRecommendation(requestedSkill, remoteRecords, catalogAvailable) { + if (!requestedSkill) return null; + + if (!catalogAvailable) { + return { + requested_skill: requestedSkill, + status: "denied", + reason: "catalog_unavailable", + }; + } + + const record = remoteRecords.find((skill) => skill.id === requestedSkill); + if (!record) { + return { + requested_skill: requestedSkill, + status: "denied", + reason: "missing_remote_record", + }; + } + if (!record.installable) { + return { + requested_skill: requestedSkill, + status: "denied", + reason: "non_installable", + }; + } + + return { + requested_skill: requestedSkill, + status: "eligible_candidate", + reason: "remote_lifecycle_not_denied", + stable_authorization: "not_evaluated", + version: record.version, + tag: record.tag, + install_command: `npx clawhub@latest install ${record.id}`, + }; +} + +function printSkillDetails(skill) { + const label = skill.version ? `${skill.id} (v${skill.version})` : skill.id; + process.stdout.write(`- ${label}\n`); + if (skill.description) { + process.stdout.write(` ${skill.description}\n`); + } + + const flags = formatFlags(skill); + if (flags.length > 0) { + process.stdout.write(` notes: ${flags.join("; ")}\n`); + } +} + function printHumanSummary(result) { process.stdout.write("=== ClawSec Skill Catalog Discovery ===\n"); process.stdout.write(`Source: ${result.source}\n`); - process.stdout.write(`Index URL: ${result.index_url}\n`); + process.stdout.write(`Status: ${result.status}\n`); + if (result.index_url) { + process.stdout.write(`Index URL: ${result.index_url}\n`); + } if (result.updated) { process.stdout.write(`Catalog updated: ${result.updated}\n`); } if (result.warning) { - process.stdout.write(`Fallback reason: ${result.warning}\n`); + process.stdout.write(`Catalog warning: ${result.warning}\n`); } - process.stdout.write("\nAvailable installable skills:\n"); - - if (!Array.isArray(result.skills) || result.skills.length === 0) { - process.stdout.write("- none\n"); + if (result.recommendation) { + process.stdout.write("\nRequested skill lifecycle screen:\n"); + process.stdout.write(`- skill: ${result.recommendation.requested_skill}\n`); + process.stdout.write(`- status: ${result.recommendation.status}\n`); + process.stdout.write(`- reason: ${result.recommendation.reason}\n`); + if (result.recommendation.status === "eligible_candidate") { + process.stdout.write(`- install: ${result.recommendation.install_command}\n`); + } else { + process.stdout.write("- no install command was emitted\n"); + } return; } - for (const skill of result.skills) { - const label = skill.version ? `${skill.id} (v${skill.version})` : skill.id; - process.stdout.write(`- ${label}\n`); - if (skill.description) { - process.stdout.write(` ${skill.description}\n`); + process.stdout.write("\nLifecycle-screened candidates (not stable authorization):\n"); + if (result.skills.length === 0) { + process.stdout.write("- none\n"); + } else { + for (const skill of result.skills) { + printSkillDetails(skill); + process.stdout.write(` install: npx clawhub@latest install ${skill.id}\n`); } + } - const flags = formatFlags(skill); - if (flags.length > 0) { - process.stdout.write(` notes: ${flags.join("; ")}\n`); + if (result.historical.length > 0) { + process.stdout.write("\nHistorical, non-installable remote records:\n"); + for (const skill of result.historical) { + printSkillDetails(skill); + process.stdout.write(" lifecycle: historical; no install recommendation\n"); } + } - process.stdout.write(` install: npx clawhub@latest install ${skill.id}\n`); + if (result.context.length > 0) { + process.stdout.write("\nSuite-local context (non-authorizing):\n"); + for (const skill of result.context) { + printSkillDetails(skill); + process.stdout.write(" lifecycle: local context only; no install recommendation\n"); + } } } -async function discoverCatalog() { +async function discoverCatalog(requestedSkill) { const indexUrl = envVar("CLAWSEC_SKILLS_INDEX_URL") || DEFAULT_INDEX_URL; + const outputIndexUrl = indexUrlForOutput(indexUrl); const timeoutMs = parseTimeoutMs(); - const fallback = await loadFallbackCatalog(); + const fallbackContext = await loadFallbackCatalog(); try { const remote = await loadRemoteCatalog(indexUrl, timeoutMs); + const records = enrichWithFallbackMetadata(remote.records, fallbackContext); + const skills = records.filter((skill) => skill.installable); + const historical = records + .filter((skill) => !skill.installable) + .map(toNonAuthorizingSummary); + const context = fallbackContext.map(toNonAuthorizingSummary); + const recommendation = buildRecommendation(requestedSkill, records, true); + const requestDenied = recommendation?.status === "denied"; + const requestedOnly = (entries) => + requestedSkill ? entries.filter((entry) => entry.id === requestedSkill) : entries; return { source: "remote", - index_url: indexUrl, - version: remote.version, - updated: remote.updated, - skills: mergeWithFallbackMetadata(remote.skills, fallback.skills), + status: "available", + catalog_role: "denial_overlay", + stable_authorization: "not_evaluated", + index_url: outputIndexUrl, + version: requestDenied ? null : remote.version, + updated: requestDenied ? null : remote.updated, + skills: requestDenied ? [] : requestedOnly(skills), + historical: requestedOnly(historical), + context: requestedOnly(context), + recommendation, warning: null, }; - } catch (error) { + } catch { return { - source: "fallback", - index_url: indexUrl, - version: fallback.version, - updated: fallback.updated, - skills: fallback.skills, - warning: String(error), + source: "unavailable", + status: "catalog_unavailable", + catalog_role: "denial_overlay", + stable_authorization: "not_evaluated", + failure_reason: "remote_catalog_unavailable_or_invalid", + index_url: outputIndexUrl, + version: null, + updated: null, + skills: [], + historical: [], + context: fallbackContext + .map(toNonAuthorizingSummary) + .filter((entry) => !requestedSkill || entry.id === requestedSkill), + recommendation: buildRecommendation(requestedSkill, [], false), + warning: "Remote catalog unavailable or invalid; lifecycle-screened install recommendations are disabled.", }; } } async function main() { const args = parseArgs(process.argv.slice(2)); - const result = await discoverCatalog(); + const result = await discoverCatalog(args.skill); if (args.json) { process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); - return; + } else { + printHumanSummary(result); } - printHumanSummary(result); + if (result.recommendation?.status === "denied") { + process.exitCode = EXIT_RECOMMENDATION_DENIED; + } } main().catch((error) => { diff --git a/skills/clawsec-suite/skill.json b/skills/clawsec-suite/skill.json index 356566ed..44a6603c 100644 --- a/skills/clawsec-suite/skill.json +++ b/skills/clawsec-suite/skill.json @@ -1,6 +1,6 @@ { "name": "clawsec-suite", - "version": "0.1.16", + "version": "0.1.17", "description": "ClawSec suite manager with embedded advisory-feed monitoring, cryptographic signature verification, approval-gated malicious-skill response, and guided setup for additional security skills.", "author": "prompt-security", "license": "AGPL-3.0-or-later", @@ -138,7 +138,7 @@ { "path": "scripts/discover_skill_catalog.mjs", "required": true, - "description": "Dynamic skill-catalog discovery with remote index fetch and suite-local fallback metadata" + "description": "Strict lifecycle-aware catalog discovery with fail-closed remote validation and non-authorizing suite-local context" }, { "path": "scripts/local_file_io.mjs", @@ -187,7 +187,7 @@ } }, "catalog": { - "description": "Available protections in the ClawSec suite", + "description": "Descriptive suite-local context only; a valid non-denied remote record is necessary but not sufficient, and signed stable-catalog authorization remains separate", "base_url": "https://clawsec.prompt.security/releases/download", "skills": { "clawsec-feed": { @@ -259,6 +259,8 @@ "CLAWSEC_FEED_PUBLIC_KEY", "CLAWSEC_ALLOW_UNSIGNED_FEED", "CLAWSEC_VERIFY_CHECKSUM_MANIFEST", + "CLAWSEC_SKILLS_INDEX_URL", + "CLAWSEC_SKILLS_INDEX_TIMEOUT_MS", "CLAWSEC_HOOK_INTERVAL_SECONDS", "CLAWSEC_ADVISORY_CRON_NAME", "CLAWSEC_ADVISORY_CRON_EVERY" diff --git a/skills/clawsec-suite/test/skill_catalog_discovery.test.mjs b/skills/clawsec-suite/test/skill_catalog_discovery.test.mjs index b9fbaf5b..ee1a4537 100644 --- a/skills/clawsec-suite/test/skill_catalog_discovery.test.mjs +++ b/skills/clawsec-suite/test/skill_catalog_discovery.test.mjs @@ -1,28 +1,101 @@ #!/usr/bin/env node /** - * Dynamic skill catalog discovery tests for clawsec-suite. + * Fail-closed catalog lifecycle tests for clawsec-suite. * - * Tests cover: - * - Remote index fetch and normalization - * - Enrichment with suite-local metadata (non-breaking compatibility) - * - Fallback behavior when remote index is invalid/unavailable - * - * Run: node skills/clawsec-suite/test/skill_catalog_discovery.test.mjs + * Run only in an isolated test environment: + * node skills/clawsec-suite/test/skill_catalog_discovery.test.mjs */ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { cp, mkdtemp, rm, writeFile } from "node:fs/promises"; import http from "node:http"; +import { tmpdir } from "node:os"; import path from "node:path"; -import { spawn } from "node:child_process"; import { fileURLToPath } from "node:url"; import { pass, fail, report, exitWithResults } from "./lib/test_harness.mjs"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const SCRIPT_PATH = path.resolve(__dirname, "..", "scripts", "discover_skill_catalog.mjs"); +const SUITE_DIR = path.resolve(__dirname, ".."); +const SCRIPT_PATH = path.join(SUITE_DIR, "scripts", "discover_skill_catalog.mjs"); +const SCRIPTS_DIR = path.join(SUITE_DIR, "scripts"); +const INSTALL_COMMAND = "npx clawhub@latest install"; + +function catalogRecord(name, overrides = {}) { + const version = overrides.version ?? "1.2.3"; + return { + id: name, + name, + version, + description: `${name} catalog fixture`, + emoji: "🛡️", + category: "security", + platforms: ["openclaw"], + tag: `${name}-v${version}`, + ...overrides, + }; +} + +function catalogPayload(skills) { + return { + version: "1.0.0", + updated: "2026-07-23T00:00:00Z", + skills, + }; +} + +function countOccurrences(text, needle) { + return text.split(needle).length - 1; +} + +function combinedOutput(result) { + return `${result.stdout}\n${result.stderr}`; +} -function runCatalogScript(args, env = {}) { - return new Promise((resolve) => { - const proc = spawn("node", [SCRIPT_PATH, ...args], { +function assertNoInstallCommand(result) { + assert.equal( + result.stdout.includes(INSTALL_COMMAND), + false, + `stdout unexpectedly exposed an install command:\n${result.stdout}`, + ); + assert.equal( + result.stderr.includes(INSTALL_COMMAND), + false, + `stderr unexpectedly exposed an install command:\n${result.stderr}`, + ); +} + +function parseJsonStdout(result) { + try { + return JSON.parse(result.stdout); + } catch (error) { + throw new Error( + `Expected machine-readable JSON output, got exit ${result.code}:\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}\n${error}`, + ); + } +} + +function assertUnavailableList(result) { + const payload = parseJsonStdout(result); + assert.equal(payload.source, "unavailable", `Unexpected unavailable payload: ${result.stdout}`); + assert.deepEqual(payload.skills, [], `Unavailable catalog must have no installable skills: ${result.stdout}`); + assert.ok( + typeof payload.warning === "string" && payload.warning.length > 0, + `Unavailable catalog must explain why it is unavailable: ${result.stdout}`, + ); + assertNoInstallCommand(result); + return payload; +} + +function assertDeniedRequest(result) { + assert.notEqual(result.code, 0, `Denied request must exit non-zero: ${result.stdout}`); + assertNoInstallCommand(result); +} + +function runCatalogScript(args, env = {}, scriptPath = SCRIPT_PATH) { + return new Promise((resolve, reject) => { + const proc = spawn(process.execPath, [scriptPath, ...args], { env: { ...process.env, ...env }, stdio: ["ignore", "pipe", "pipe"], }); @@ -33,11 +106,10 @@ function runCatalogScript(args, env = {}) { proc.stdout.on("data", (chunk) => { stdout += chunk.toString(); }); - proc.stderr.on("data", (chunk) => { stderr += chunk.toString(); }); - + proc.on("error", reject); proc.on("close", (code) => { resolve({ code, stdout, stderr }); }); @@ -48,181 +120,624 @@ function withServer(handler) { return new Promise((resolve, reject) => { const server = http.createServer(handler); server.listen(0, "127.0.0.1", () => { - const addr = server.address(); - if (!addr || typeof addr === "string") { - reject(new Error("Failed to bind test server")); + const address = server.address(); + if (!address || typeof address === "string") { + reject(new Error("Failed to bind catalog fixture server")); return; } resolve({ - url: `http://127.0.0.1:${addr.port}`, + url: `http://127.0.0.1:${address.port}`, close: () => new Promise((done) => { + server.closeAllConnections?.(); server.close(() => done()); }), }); }); - server.on("error", reject); }); } -// ----------------------------------------------------------------------------- -// Test: remote index is used when valid -// ----------------------------------------------------------------------------- -async function testRemoteCatalogSuccess() { - const testName = "discover_skill_catalog: uses remote index when valid"; - let fixture = null; +async function withCatalogResponse( + { status = 200, body = "", contentType = "application/json", hang = false }, + callback, +) { + const fixture = await withServer((_request, response) => { + if (hang) return; + response.writeHead(status, { "Content-Type": contentType }); + response.end(typeof body === "string" ? body : JSON.stringify(body)); + }); try { - fixture = await withServer((req, res) => { - if (req.url !== "/index.json") { - res.writeHead(404, { "Content-Type": "application/json" }); - res.end(JSON.stringify({ error: "not found" })); - return; + return await callback(`${fixture.url}/index.json`); + } finally { + await fixture.close(); + } +} + +function catalogEnvironment(indexUrl, overrides = {}) { + return { + CLAWSEC_SKILLS_INDEX_URL: indexUrl, + CLAWSEC_SKILLS_INDEX_TIMEOUT_MS: "1000", + ...overrides, + }; +} + +async function runCase(name, callback) { + try { + await callback(); + pass(name); + } catch (error) { + fail(name, error); + } +} + +async function testValidListLifecycleScreening() { + const withdrawnCommand = `${INSTALL_COMMAND} withdrawn-skill`; + const payload = catalogPayload([ + catalogRecord("explicit-true", { installable: true }), + catalogRecord("legacy-absent"), + catalogRecord("withdrawn-skill", { + installable: false, + description: `Untrusted historical text: ${withdrawnCommand}`, + version: `1.2.3 ${withdrawnCommand}`, + }), + ]); + + await withCatalogResponse({ body: payload }, async (indexUrl) => { + const env = catalogEnvironment(indexUrl); + + const jsonResult = await runCatalogScript(["--json"], env); + assert.equal(jsonResult.code, 0, jsonResult.stderr); + const json = parseJsonStdout(jsonResult); + assert.equal(json.source, "remote"); + assert.equal(json.catalog_role, "denial_overlay"); + assert.equal(json.stable_authorization, "not_evaluated"); + assert.deepEqual( + json.skills.map((skill) => skill.id).sort(), + ["explicit-true", "legacy-absent"], + `Only explicit true and legacy-absent records may pass lifecycle screening: ${jsonResult.stdout}`, + ); + assert.equal(json.skills.some((skill) => skill.id === "withdrawn-skill"), false); + assert.ok(Array.isArray(json.historical), `Missing historical records: ${jsonResult.stdout}`); + const historical = json.historical.find((skill) => skill.id === "withdrawn-skill"); + assert.ok(historical, `Explicit false must remain visible as historical: ${jsonResult.stdout}`); + assert.equal(historical.id, "withdrawn-skill"); + assert.equal(historical.installable, false); + assert.equal(historical.lifecycle_status, "historical"); + assert.equal(Object.hasOwn(historical, "install_command"), false); + assert.equal(Object.hasOwn(historical, "command"), false); + assert.equal(JSON.stringify(historical).includes(INSTALL_COMMAND), false); + assert.equal(jsonResult.stdout.includes(withdrawnCommand), false, jsonResult.stdout); + assert.equal(jsonResult.stderr.includes(withdrawnCommand), false, jsonResult.stderr); + + const humanResult = await runCatalogScript([], env); + assert.equal(humanResult.code, 0, humanResult.stderr); + assert.equal( + countOccurrences(humanResult.stdout, `${INSTALL_COMMAND} explicit-true`), + 1, + humanResult.stdout, + ); + assert.equal( + countOccurrences(humanResult.stdout, `${INSTALL_COMMAND} legacy-absent`), + 1, + humanResult.stdout, + ); + assert.equal( + humanResult.stdout.includes(withdrawnCommand), + false, + humanResult.stdout, + ); + assert.equal(humanResult.stderr.includes(withdrawnCommand), false, humanResult.stderr); + assert.equal(countOccurrences(combinedOutput(humanResult), INSTALL_COMMAND), 2); + }); +} + +async function testExactRequestedEligibility() { + const payload = catalogPayload([ + catalogRecord("explicit-true", { installable: true }), + catalogRecord("legacy-absent"), + catalogRecord("historical-false", { installable: false }), + ]); + + await withCatalogResponse({ body: payload }, async (indexUrl) => { + const env = catalogEnvironment(indexUrl); + + for (const skillId of ["explicit-true", "legacy-absent"]) { + const result = await runCatalogScript(["--skill", skillId], env); + assert.equal(result.code, 0, `${skillId}: ${result.stderr}`); + assert.equal( + countOccurrences(combinedOutput(result), INSTALL_COMMAND), + 1, + `Requested mode must emit exactly one command: ${combinedOutput(result)}`, + ); + assert.equal( + result.stdout.includes(`${INSTALL_COMMAND} ${skillId}`), + true, + `Requested mode emitted the wrong command: ${result.stdout}`, + ); + + const jsonResult = await runCatalogScript(["--json", "--skill", skillId], env); + assert.equal(jsonResult.code, 0, `${skillId}: ${jsonResult.stderr}`); + const json = parseJsonStdout(jsonResult); + assert.equal(json.catalog_role, "denial_overlay"); + assert.equal(json.stable_authorization, "not_evaluated"); + const screenedRecord = json.skills.find((skill) => skill.id === skillId); + assert.ok(screenedRecord, `Requested record missing from screened candidates: ${jsonResult.stdout}`); + assert.equal(screenedRecord.id, skillId); + assert.equal(screenedRecord.name, skillId); + assert.equal(screenedRecord.version, "1.2.3"); + assert.equal(screenedRecord.tag, `${skillId}-v1.2.3`); + assert.equal(json.recommendation?.status, "eligible_candidate"); + assert.equal(json.recommendation?.requested_skill, skillId); + assert.equal(json.recommendation?.version, "1.2.3"); + assert.equal(json.recommendation?.tag, `${skillId}-v1.2.3`); + assert.equal(json.recommendation?.install_command, `${INSTALL_COMMAND} ${skillId}`); + assert.equal(json.recommendation?.stable_authorization, "not_evaluated"); + assert.equal(countOccurrences(JSON.stringify(json.recommendation), INSTALL_COMMAND), 1); + } + + const historical = await runCatalogScript(["--skill", "historical-false"], env); + assertDeniedRequest(historical); + + // This ID exists in suite-local metadata and is marked default_install=true. + // A missing remote record must not resurrect it. + const missingLocalDefault = await runCatalogScript( + ["--skill", "openclaw-audit-watchdog"], + env, + ); + assertDeniedRequest(missingLocalDefault); + }); +} + +async function testDeniedRequestedOutputDoesNotEchoUnrelatedMetadata() { + const unrelatedCommand = `${INSTALL_COMMAND} unrelated-eligible`; + const payload = catalogPayload([ + catalogRecord("unrelated-eligible", { + installable: true, + description: `Untrusted unrelated text: ${unrelatedCommand}`, + version: `1.2.3 ${unrelatedCommand}`, + }), + catalogRecord("withdrawn-target", { installable: false }), + ]); + + const cases = [ + { + label: "missing remote record with matching suite-local context", + requestedSkill: "openclaw-audit-watchdog", + expectedReason: "missing_remote_record", + expectedHistoricalIds: [], + expectedContextIds: ["openclaw-audit-watchdog"], + }, + { + label: "explicit remote false", + requestedSkill: "withdrawn-target", + expectedReason: "non_installable", + expectedHistoricalIds: ["withdrawn-target"], + expectedContextIds: [], + }, + ]; + + await withCatalogResponse({ body: payload }, async (indexUrl) => { + const env = catalogEnvironment(indexUrl); + + for (const testCase of cases) { + const result = await runCatalogScript( + ["--json", "--skill", testCase.requestedSkill], + env, + ); + assertDeniedRequest(result); + const json = parseJsonStdout(result); + + assert.equal(json.source, "remote", `${testCase.label}: ${result.stdout}`); + assert.equal(json.catalog_role, "denial_overlay"); + assert.equal(json.stable_authorization, "not_evaluated"); + assert.deepEqual(json.skills, [], `${testCase.label}: ${result.stdout}`); + assert.equal(json.version, null, `${testCase.label}: ${result.stdout}`); + assert.equal(json.updated, null, `${testCase.label}: ${result.stdout}`); + assert.equal(json.recommendation?.status, "denied"); + assert.equal(json.recommendation?.requested_skill, testCase.requestedSkill); + assert.equal(json.recommendation?.reason, testCase.expectedReason); + assert.equal(Object.hasOwn(json.recommendation ?? {}, "install_command"), false); + + assert.deepEqual( + (json.historical ?? []).map((record) => record.id), + testCase.expectedHistoricalIds, + `${testCase.label}: historical output was not request-scoped: ${result.stdout}`, + ); + for (const historical of json.historical ?? []) { + assert.equal(historical.id, testCase.requestedSkill); + assert.equal(historical.installable, false); + assert.equal(historical.lifecycle_status, "historical"); + assert.equal(Object.hasOwn(historical, "install_command"), false); + assert.equal(Object.hasOwn(historical, "command"), false); } - res.writeHead(200, { "Content-Type": "application/json" }); - res.end( - JSON.stringify({ - version: "1.0.0", - updated: "2026-02-16T08:20:00Z", - skills: [ - { - id: "soul-guardian", - name: "soul-guardian", - version: "9.9.9", - description: "Remote skill metadata", - emoji: "👻", - category: "security", - tag: "soul-guardian-v9.9.9", - }, - { - id: "clawtributor", - name: "clawtributor", - version: "1.2.3", - description: "Remote clawtributor metadata", - emoji: "🤝", - category: "security", - tag: "clawtributor-v1.2.3", - }, - ], - }), + assert.deepEqual( + (json.context ?? []).map((record) => record.id), + testCase.expectedContextIds, + `${testCase.label}: local context was not request-scoped: ${result.stdout}`, ); - }); + for (const context of json.context ?? []) { + assert.equal(context.id, testCase.requestedSkill); + assert.equal(context.installable, false); + assert.equal(context.lifecycle_status, "local_context_only"); + assert.equal(Object.hasOwn(context, "install_command"), false); + assert.equal(Object.hasOwn(context, "command"), false); + } + + assert.equal(result.stdout.includes(unrelatedCommand), false, result.stdout); + assert.equal(result.stderr.includes(unrelatedCommand), false, result.stderr); + assert.equal(result.stdout.includes("unrelated-eligible"), false, result.stdout); + } + }); +} - const result = await runCatalogScript(["--json"], { - CLAWSEC_SKILLS_INDEX_URL: `${fixture.url}/index.json`, - CLAWSEC_SKILLS_INDEX_TIMEOUT_MS: "2000", +async function testMalformedRootMetadataDoesNotLeakIntoWarning() { + const injectedCommand = `${INSTALL_COMMAND} root-metadata`; + const validSkill = catalogRecord("otherwise-valid", { installable: true }); + const cases = [ + { + label: "malformed root version", + payload: { + ...catalogPayload([validSkill]), + version: `1.0.0 ${injectedCommand}`, + }, + }, + { + label: "malformed root updated timestamp", + payload: { + ...catalogPayload([validSkill]), + updated: `2026-07-23T00:00:00Z ${injectedCommand}`, + }, + }, + ]; + + for (const testCase of cases) { + await runCase(`catalog diagnostics: ${testCase.label} is not reflected`, async () => { + await withCatalogResponse({ body: testCase.payload }, async (indexUrl) => { + const result = await runCatalogScript(["--json"], catalogEnvironment(indexUrl)); + const json = assertUnavailableList(result); + assert.equal(json.version, null); + assert.equal(json.updated, null); + assert.match(json.warning, /catalog.*(?:unavailable|invalid)/i); + assert.equal(json.warning.includes(injectedCommand), false, json.warning); + assert.equal(result.stdout.includes(injectedCommand), false, result.stdout); + assert.equal(result.stderr.includes(injectedCommand), false, result.stderr); + }); }); + } +} - if (result.code !== 0) { - fail(testName, `Expected exit 0, got ${result.code}: ${result.stderr}`); - return; - } +async function testInvalidLifecycleRejectsWholeIndex() { + const invalidValues = [ + ["null", null], + ["string", "false"], + ["number", 0], + ["object", { value: false }], + ["array", []], + ]; + + for (const [label, invalidValue] of invalidValues) { + await runCase(`catalog lifecycle: ${label} rejects the whole index`, async () => { + const payload = catalogPayload([ + catalogRecord("otherwise-valid", { installable: true }), + catalogRecord("invalid-lifecycle", { installable: invalidValue }), + ]); + + await withCatalogResponse({ body: payload }, async (indexUrl) => { + const env = catalogEnvironment(indexUrl); + const listResult = await runCatalogScript(["--json"], env); + assertUnavailableList(listResult); + assert.equal(listResult.stdout.includes("otherwise-valid"), false); + + const requestedResult = await runCatalogScript(["--skill", "otherwise-valid"], env); + assertDeniedRequest(requestedResult); + }); + }); + } +} - const payload = JSON.parse(result.stdout); - const clawtributor = payload.skills.find((entry) => entry.id === "clawtributor"); - const soulGuardian = payload.skills.find((entry) => entry.id === "soul-guardian"); - - if ( - payload.source === "remote" && - payload.updated === "2026-02-16T08:20:00Z" && - soulGuardian?.version === "9.9.9" && - clawtributor?.requires_explicit_consent === true - ) { - pass(testName); - } else { - fail(testName, `Unexpected payload: ${result.stdout}`); - } - } catch (error) { - fail(testName, error); - } finally { - if (fixture) { - await fixture.close(); - } +async function testMalformedCatalogRejectsWholeIndex() { + const duplicate = catalogRecord("duplicate"); + const canonicalCollision = catalogRecord("case-collision"); + const caseVariant = catalogRecord("Case-Collision"); + const otherwiseValid = catalogRecord("otherwise-valid", { installable: true }); + const malformedCases = [ + ["invalid JSON", "{not-json", "application/json"], + ["null root", "null", "application/json"], + ["array root", "[]", "application/json"], + ["missing skills", JSON.stringify({ version: "1.0.0" }), "application/json"], + ["non-array skills", JSON.stringify({ skills: {} }), "application/json"], + [ + "non-object record", + JSON.stringify(catalogPayload([otherwiseValid, "invalid-record"])), + "application/json", + ], + [ + "blank id", + JSON.stringify(catalogPayload([otherwiseValid, catalogRecord("blank-id", { id: " " })])), + "application/json", + ], + [ + "missing name", + JSON.stringify(catalogPayload([ + otherwiseValid, + (() => { + const record = catalogRecord("missing-name"); + delete record.name; + return record; + })(), + ])), + "application/json", + ], + [ + "id-name mismatch", + JSON.stringify(catalogPayload([ + otherwiseValid, + catalogRecord("identity", { name: "different-identity" }), + ])), + "application/json", + ], + [ + "blank version", + JSON.stringify(catalogPayload([otherwiseValid, catalogRecord("blank-version", { version: " " })])), + "application/json", + ], + [ + "missing tag", + JSON.stringify(catalogPayload([ + otherwiseValid, + (() => { + const record = catalogRecord("missing-tag"); + delete record.tag; + return record; + })(), + ])), + "application/json", + ], + [ + "tag-version mismatch", + JSON.stringify(catalogPayload([ + otherwiseValid, + catalogRecord("wrong-tag", { tag: "wrong-tag-v9.9.9" }), + ])), + "application/json", + ], + [ + "duplicate identity", + JSON.stringify(catalogPayload([otherwiseValid, duplicate, { ...duplicate }])), + "application/json", + ], + [ + "canonical case collision", + JSON.stringify(catalogPayload([otherwiseValid, canonicalCollision, caseVariant])), + "application/json", + ], + ]; + + for (const [label, body, contentType] of malformedCases) { + await runCase(`catalog structure: ${label} rejects the whole index`, async () => { + await withCatalogResponse({ body, contentType }, async (indexUrl) => { + const env = catalogEnvironment(indexUrl); + const listResult = await runCatalogScript(["--json"], env); + assertUnavailableList(listResult); + assert.equal(listResult.stdout.includes("otherwise-valid"), false); + + const requestedResult = await runCatalogScript(["--skill", "otherwise-valid"], env); + assertDeniedRequest(requestedResult); + }); + }); } } -// ----------------------------------------------------------------------------- -// Test: invalid remote payload falls back to suite-local catalog -// ----------------------------------------------------------------------------- -async function testInvalidRemotePayloadFallsBack() { - const testName = "discover_skill_catalog: invalid remote payload falls back"; - let fixture = null; +async function testUnsafeAndMalformedCliArguments() { + const env = catalogEnvironment("http://127.0.0.1:9/index.json", { + CLAWSEC_SKILLS_INDEX_TIMEOUT_MS: "50", + }); + const unsafeRequestedIds = [ + "UPPER", + "../escape", + "skill/name", + "double--dash", + " leading", + "trailing ", + "skill;command", + "ｃlawsec-suite", + ]; + + for (const skillId of unsafeRequestedIds) { + await runCase(`catalog CLI: unsafe requested id ${JSON.stringify(skillId)} fails closed`, async () => { + const result = await runCatalogScript(["--skill", skillId], env); + assertDeniedRequest(result); + }); + } - try { - fixture = await withServer((_req, res) => { - res.writeHead(200, { "Content-Type": "application/json" }); - res.end(JSON.stringify({ version: "1.0.0", note: "missing skills" })); + const malformedArgumentCases = [ + ["missing --skill value", ["--skill"]], + ["option used as --skill value", ["--skill", "--json"]], + ["duplicate --skill", ["--skill", "safe-id", "--skill", "other-id"]], + ["duplicate --json", ["--json", "--json"]], + ["unknown option", ["--unknown"]], + ["unexpected positional argument", ["safe-id"]], + ]; + + for (const [label, args] of malformedArgumentCases) { + await runCase(`catalog CLI: ${label} fails without a command`, async () => { + const result = await runCatalogScript(args, env); + assertDeniedRequest(result); }); + } +} - const result = await runCatalogScript(["--json"], { - CLAWSEC_SKILLS_INDEX_URL: `${fixture.url}/index.json`, - CLAWSEC_SKILLS_INDEX_TIMEOUT_MS: "2000", +async function testRemoteFailureNeverAuthorizesFallback() { + const httpCases = [ + ["HTTP 404", { status: 404, body: { error: "not found" } }], + ["HTTP 500", { status: 500, body: { error: "server error" } }], + ["HTML fallback", { body: "fallback", contentType: "text/html" }], + ]; + + for (const [label, fixtureOptions] of httpCases) { + await runCase(`catalog availability: ${label} yields no authorization`, async () => { + await withCatalogResponse(fixtureOptions, async (indexUrl) => { + const env = catalogEnvironment(indexUrl); + const listResult = await runCatalogScript(["--json"], env); + assertUnavailableList(listResult); + + const requestedResult = await runCatalogScript( + ["--skill", "openclaw-audit-watchdog"], + env, + ); + assertDeniedRequest(requestedResult); + }); }); + } - if (result.code !== 0) { - fail(testName, `Expected exit 0, got ${result.code}: ${result.stderr}`); - return; - } + await runCase("catalog availability: unreachable remote yields no authorization", async () => { + const env = catalogEnvironment("http://127.0.0.1:9/index.json", { + CLAWSEC_SKILLS_INDEX_TIMEOUT_MS: "100", + }); + const listResult = await runCatalogScript(["--json"], env); + assertUnavailableList(listResult); + + const requestedResult = await runCatalogScript( + ["--skill", "openclaw-audit-watchdog"], + env, + ); + assertDeniedRequest(requestedResult); + }); - const payload = JSON.parse(result.stdout); - const hasSoulGuardian = Array.isArray(payload.skills) - ? payload.skills.some((entry) => entry.id === "soul-guardian") - : false; + await runCase("catalog availability: timeout yields no authorization", async () => { + await withCatalogResponse({ hang: true }, async (indexUrl) => { + const env = catalogEnvironment(indexUrl, { + CLAWSEC_SKILLS_INDEX_TIMEOUT_MS: "50", + }); + const listResult = await runCatalogScript(["--json"], env); + assertUnavailableList(listResult); - if (payload.source === "fallback" && hasSoulGuardian && String(payload.warning).includes("skills array")) { - pass(testName); - } else { - fail(testName, `Unexpected payload: ${result.stdout}`); - } - } catch (error) { - fail(testName, error); - } finally { - if (fixture) { - await fixture.close(); - } - } + const requestedResult = await runCatalogScript( + ["--skill", "openclaw-audit-watchdog"], + env, + ); + assertDeniedRequest(requestedResult); + }); + }); } -// ----------------------------------------------------------------------------- -// Test: unreachable remote index falls back to suite-local catalog -// ----------------------------------------------------------------------------- -async function testUnreachableRemoteFallsBack() { - const testName = "discover_skill_catalog: unreachable remote index falls back"; +async function testLocalFlagsCannotOverrideRemoteLifecycle() { + const fixtureRoot = await mkdtemp(path.join(tmpdir(), "clawsec-suite-catalog-test-")); + const fixtureScripts = path.join(fixtureRoot, "scripts"); + const fixtureScript = path.join(fixtureScripts, "discover_skill_catalog.mjs"); try { - const result = await runCatalogScript(["--json"], { - CLAWSEC_SKILLS_INDEX_URL: "http://127.0.0.1:9/index.json", - CLAWSEC_SKILLS_INDEX_TIMEOUT_MS: "250", + await cp(SCRIPTS_DIR, fixtureScripts, { recursive: true }); + await writeFile( + path.join(fixtureRoot, "skill.json"), + `${JSON.stringify({ + name: "clawsec-suite", + version: "0.0.0-test", + catalog: { + skills: { + "local-override": { + description: "Must remain non-authorizing local context", + default_install: true, + installable: true, + }, + }, + }, + }, null, 2)}\n`, + ); + + const remotePayload = catalogPayload([ + catalogRecord("remote-eligible", { installable: true }), + ]); + await withCatalogResponse({ body: remotePayload }, async (indexUrl) => { + const env = catalogEnvironment(indexUrl); + + const listResult = await runCatalogScript(["--json"], env, fixtureScript); + assert.equal(listResult.code, 0, listResult.stderr); + const listPayload = parseJsonStdout(listResult); + assert.deepEqual( + listPayload.skills.map((skill) => skill.id), + ["remote-eligible"], + `Local default_install/installable flags must not join remote installable output: ${listResult.stdout}`, + ); + + const requestedResult = await runCatalogScript( + ["--skill", "local-override"], + env, + fixtureScript, + ); + assertDeniedRequest(requestedResult); }); - if (result.code !== 0) { - fail(testName, `Expected exit 0, got ${result.code}: ${result.stderr}`); - return; - } + const remoteDenialPayload = catalogPayload([ + catalogRecord("local-override", { installable: false }), + ]); + await withCatalogResponse({ body: remoteDenialPayload }, async (indexUrl) => { + const env = catalogEnvironment(indexUrl); + + const listResult = await runCatalogScript(["--json"], env, fixtureScript); + assert.equal(listResult.code, 0, listResult.stderr); + const listPayload = parseJsonStdout(listResult); + assert.deepEqual(listPayload.skills, []); + const historical = listPayload.historical?.find((skill) => skill.id === "local-override"); + assert.ok(historical, `Remote false must remain historical: ${listResult.stdout}`); + assert.equal(historical.installable, false); + assert.equal(historical.default_install, true); + assert.equal(JSON.stringify(historical).includes(INSTALL_COMMAND), false); + assertNoInstallCommand(listResult); + + const requestedResult = await runCatalogScript( + ["--skill", "local-override"], + env, + fixtureScript, + ); + assertDeniedRequest(requestedResult); + }); - const payload = JSON.parse(result.stdout); - if (payload.source === "fallback" && Array.isArray(payload.skills) && payload.skills.length > 0) { - pass(testName); - } else { - fail(testName, `Unexpected payload: ${result.stdout}`); - } - } catch (error) { - fail(testName, error); + const unavailableEnv = catalogEnvironment("http://127.0.0.1:9/index.json", { + CLAWSEC_SKILLS_INDEX_TIMEOUT_MS: "100", + }); + const unavailableList = await runCatalogScript(["--json"], unavailableEnv, fixtureScript); + assertUnavailableList(unavailableList); + + const unavailableRequested = await runCatalogScript( + ["--skill", "local-override"], + unavailableEnv, + fixtureScript, + ); + assertDeniedRequest(unavailableRequested); + } finally { + await rm(fixtureRoot, { recursive: true, force: true }); } } -// ----------------------------------------------------------------------------- -// Main test runner -// ----------------------------------------------------------------------------- async function runTests() { - console.log("=== ClawSec Skill Catalog Discovery Tests ===\n"); - - await testRemoteCatalogSuccess(); - await testInvalidRemotePayloadFallsBack(); - await testUnreachableRemoteFallsBack(); + console.log("=== ClawSec Skill Catalog Lifecycle Tests ===\n"); + + await runCase( + "catalog lifecycle: list screens true and legacy absent as eligible and records false as historical", + testValidListLifecycleScreening, + ); + await runCase( + "catalog lifecycle: requested mode emits lifecycle-screened eligible candidates only", + testExactRequestedEligibility, + ); + await runCase( + "catalog lifecycle: denied requested JSON cannot echo unrelated eligible metadata", + testDeniedRequestedOutputDoesNotEchoUnrelatedMetadata, + ); + await testMalformedRootMetadataDoesNotLeakIntoWarning(); + await testInvalidLifecycleRejectsWholeIndex(); + await testMalformedCatalogRejectsWholeIndex(); + await testUnsafeAndMalformedCliArguments(); + await testRemoteFailureNeverAuthorizesFallback(); + await runCase( + "catalog fallback: local default_install/installable flags cannot override remote lifecycle", + testLocalFlagsCannotOverrideRemoteLifecycle, + ); report(); exitWithResults();