From e32a021f49d81122a5904f72c9775e06718abf14 Mon Sep 17 00:00:00 2001 From: Lucenx9 Date: Sat, 15 Aug 2026 03:17:53 +0200 Subject: [PATCH 1/3] chore: prepare self-hosted v0.2.0 certification --- .github/workflows/self-hosted-release.yml | 35 +- distribution/self-hosted/README.md | 16 +- .../moderated-usability.schema.json | 434 +++++++++++++ .../self-hosted/supported-matrix.json | 7 + docs/self-hosted-moderated-usability.md | 117 ++++ docs/self-hosted-release-evidence.md | 38 +- package.json | 2 +- scripts/build-self-hosted-release.ts | 13 +- scripts/verify-self-hosted-release-tag.ts | 190 ++++++ scripts/verify-self-hosted-release.ts | 11 + .../contracts/release-and-recovery.md | 12 +- .../004-self-hosted-onboarding/data-model.md | 4 +- .../004-self-hosted-onboarding/quickstart.md | 2 +- specs/004-self-hosted-onboarding/research.md | 6 +- specs/004-self-hosted-onboarding/spec.md | 4 +- src/onboarding/domain/moderated-usability.ts | 581 ++++++++++++++++++ src/onboarding/domain/release-manifest.ts | 7 +- .../release/moderated-usability.test.ts | 327 ++++++++++ .../release/release-manifest-schema.test.ts | 57 ++ tests/contract/release/release-tag.test.ts | 230 +++++++ .../release/self-hosted-matrix.test.ts | 34 + .../contract/release/signing-workflow.test.ts | 31 + .../release/trust-policy-schema.test.ts | 20 +- 23 files changed, 2125 insertions(+), 53 deletions(-) create mode 100644 distribution/self-hosted/moderated-usability.schema.json create mode 100644 docs/self-hosted-moderated-usability.md create mode 100644 scripts/verify-self-hosted-release-tag.ts create mode 100644 src/onboarding/domain/moderated-usability.ts create mode 100644 tests/contract/release/moderated-usability.test.ts create mode 100644 tests/contract/release/release-tag.test.ts diff --git a/.github/workflows/self-hosted-release.yml b/.github/workflows/self-hosted-release.yml index bc27775..109bbde 100644 --- a/.github/workflows/self-hosted-release.yml +++ b/.github/workflows/self-hosted-release.yml @@ -48,7 +48,14 @@ jobs: - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 with: node-version: 24.18.0 - cache: pnpm + - name: Verify annotated release identity before repository code + shell: bash + run: | + set -euo pipefail + node scripts/verify-self-hosted-release-tag.ts \ + --repository "${GITHUB_WORKSPACE}" \ + --ref "${GITHUB_REF}" \ + --sha "${GITHUB_SHA}" - run: corepack enable && corepack prepare "pnpm@${PNPM_VERSION}" --activate - run: pnpm install --frozen-lockfile @@ -196,6 +203,7 @@ jobs: steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: + fetch-depth: 0 persist-credentials: false - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 with: @@ -203,20 +211,21 @@ jobs: - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 with: node-version: 24.18.0 - cache: pnpm - - run: pnpm install --frozen-lockfile - - name: Complete all source gates before signing - run: - pnpm format:check && pnpm lint && pnpm typecheck && pnpm build && pnpm - test - id: release shell: bash run: | set -euo pipefail + node scripts/verify-self-hosted-release-tag.ts \ + --repository "${GITHUB_WORKSPACE}" \ + --ref "${GITHUB_REF}" \ + --sha "${GITHUB_SHA}" version="$(node -p "require('./package.json').version")" - test "${GITHUB_REF}" = "refs/tags/self-hosted-v${version}" - test "$(git rev-parse HEAD)" = "${GITHUB_SHA}" printf 'version=%s\n' "${version}" >> "${GITHUB_OUTPUT}" + - run: pnpm install --frozen-lockfile + - name: Complete all source gates before signing + run: + pnpm format:check && pnpm lint && pnpm typecheck && pnpm build && pnpm + test - name: Acquire and verify pinned Cosign shell: bash env: @@ -323,6 +332,12 @@ jobs: "${ARCH}" cp distribution/self-hosted/trust-policy.v1.json \ "${RUNNER_TEMP}/release-${ARCH}/skillwire-trust-policy-v1.json" + manifest="${RUNNER_TEMP}/release-${ARCH}/skillwire-${VERSION}-linux-${ARCH}.release.json" + node scripts/verify-self-hosted-release-tag.ts \ + --repository "${GITHUB_WORKSPACE}" \ + --ref "${GITHUB_REF}" \ + --sha "${GITHUB_SHA}" \ + --manifest "${manifest}" - name: Sign external canonical manifest shell: bash env: @@ -498,4 +513,4 @@ jobs: assets[index]="release-assets/${assets[index]}" done gh release create "${GITHUB_REF_NAME}" "${assets[@]}" \ - --verify-tag --title "SkillWire self-hosted ${VERSION}" + --verify-tag --title "SkillWire Self-Hosted v${VERSION}" diff --git a/distribution/self-hosted/README.md b/distribution/self-hosted/README.md index 8fe8a13..d1a3783 100644 --- a/distribution/self-hosted/README.md +++ b/distribution/self-hosted/README.md @@ -20,7 +20,10 @@ Do not use `latest`, `curl | sh`, an unverified package manager, or a Cosign binary shipped only inside the candidate archive. Obtain the current Sigstore production `trusted_root.json` through its signed TUF repository and verify its SHA-256 against the trust policy. The first source-pinned root in this release -is `distribution/self-hosted/trusted-root.v1.json`. +is `distribution/self-hosted/trusted-root.v1.json`. Its media type must be +exactly `application/vnd.dev.sigstore.trustedroot+json;version=0.1`, the only +value accepted by the Cosign 3.1.3 / sigstore-go 1.2.2 loader. Do not substitute +the unsupported `application/vnd.dev.sigstore.trustedroot.v0.2+json` value. ## 2. Verify the signed external manifest offline @@ -42,6 +45,11 @@ overlap quorum of two, a second sibling bundle named manifest names both signer IDs and both exact bundle paths; an extra bundle is never accepted by convention alone. +The manifest names bundle files and signer IDs but never contains bundle bytes, +media types, or digests. Each bundle signs the exact canonical manifest bytes; +including its own digest in that manifest would be circular. Cosign, the exact +certificate claims, and the manifest message digest bind the external bundle. + Disconnect outbound networking (or run inside an already network-isolated namespace) and invoke the independently verified Cosign directly: @@ -61,8 +69,10 @@ cosign verify-blob \ skillwire-VERSION-linux-ARCH.release.json ``` -The version and source commit are fields of the canonical manifest; compare them -to the protected tag independently before using them in the command. Never add +The version and source commit are fields of the canonical manifest. The only +accepted tag is the annotated `self-hosted-vVERSION` tag; recursively peel it, +require the result to equal the manifest source commit, and require that commit +to be reachable from protected `main` before using it in the command. Never add an insecure SCT/transparency bypass or a regular-expression identity. Verification must finish without a network lookup. diff --git a/distribution/self-hosted/moderated-usability.schema.json b/distribution/self-hosted/moderated-usability.schema.json new file mode 100644 index 0000000..b6b7f83 --- /dev/null +++ b/distribution/self-hosted/moderated-usability.schema.json @@ -0,0 +1,434 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://skillwire.dev/schemas/skillwire.moderated-usability.v1.json", + "title": "SkillWire self-hosted moderated usability cohort evidence", + "description": "Privacy-safe evidence only. Prompt content, response content, credentials, and participant personal data are forbidden.", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "evidenceKind", + "cohortId", + "targetMilliseconds", + "release", + "documentation", + "participants", + "aggregation" + ], + "properties": { + "schemaVersion": { "const": "skillwire.moderated-usability/v1" }, + "evidenceKind": { + "enum": ["synthetic-fixture", "certified-observation"] + }, + "cohortId": { "pattern": "^cohort-[0-9a-f]{16}$", "type": "string" }, + "targetMilliseconds": { "const": 900000 }, + "release": { "$ref": "#/$defs/cohortRelease" }, + "documentation": { "$ref": "#/$defs/documentation" }, + "participants": { + "type": "array", + "minItems": 10, + "maxItems": 10, + "items": { "$ref": "#/$defs/participant" } + }, + "aggregation": { + "type": "object", + "additionalProperties": false, + "required": [ + "cohortSize", + "completedWithinTarget", + "sc001Required", + "sc001Passed", + "unassistedCompletions", + "sc014Required", + "sc014Passed" + ], + "properties": { + "cohortSize": { "const": 10 }, + "completedWithinTarget": { + "type": "integer", + "minimum": 0, + "maximum": 10 + }, + "sc001Required": { "const": 10 }, + "sc001Passed": { "type": "boolean" }, + "unassistedCompletions": { + "type": "integer", + "minimum": 0, + "maximum": 10 + }, + "sc014Required": { "const": 9 }, + "sc014Passed": { "type": "boolean" } + } + } + }, + "$defs": { + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "publicCode": { + "type": "string", + "pattern": "^[A-Z][A-Z0-9_]{2,63}$" + }, + "asset": { + "type": "object", + "additionalProperties": false, + "required": ["path", "size", "sha256"], + "properties": { + "path": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,159}$" + }, + "size": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 17179869184 + }, + "sha256": { "$ref": "#/$defs/sha256" } + } + }, + "cohortRelease": { + "type": "object", + "additionalProperties": false, + "required": ["version", "tag", "sourceCommit", "assets"], + "properties": { + "version": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+$" }, + "tag": { + "type": "string", + "pattern": "^self-hosted-v\\d+\\.\\d+\\.\\d+$" + }, + "sourceCommit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "assets": { + "type": "array", + "minItems": 7, + "maxItems": 7, + "items": { "$ref": "#/$defs/asset" } + } + } + }, + "participantRelease": { + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "tag", + "sourceCommit", + "assets", + "manifestSha256" + ], + "properties": { + "version": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+$" }, + "tag": { + "type": "string", + "pattern": "^self-hosted-v\\d+\\.\\d+\\.\\d+$" + }, + "sourceCommit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "assets": { + "type": "array", + "minItems": 7, + "maxItems": 7, + "items": { "$ref": "#/$defs/asset" } + }, + "manifestSha256": { "$ref": "#/$defs/sha256" } + } + }, + "documentation": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha256"], + "properties": { + "path": { "const": "distribution/self-hosted/README.md" }, + "sha256": { "$ref": "#/$defs/sha256" } + } + }, + "operatingSystem": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["id", "version"], + "properties": { + "id": { "const": "ubuntu" }, + "version": { "const": "24.04" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["id", "version"], + "properties": { + "id": { "const": "debian" }, + "version": { "const": "12" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["id", "version"], + "properties": { + "id": { "const": "debian" }, + "version": { "const": "13" } + } + } + ] + }, + "milestoneName": { + "enum": [ + "releaseVerified", + "serviceReady", + "clientIntegrated", + "installationStateIdentified", + "nextSafeRecoveryActionIdentified", + "sixToolDiscovery", + "mcpSearch", + "exactSkillLoad", + "optionalResourceJourney", + "cleanup" + ] + }, + "milestone": { + "type": "object", + "additionalProperties": false, + "required": ["status", "publicErrorCodes"], + "properties": { + "status": { + "enum": ["passed", "failed", "timeout", "abandoned", "not-applicable"] + }, + "publicErrorCodes": { + "type": "array", + "maxItems": 16, + "items": { "$ref": "#/$defs/publicCode" } + } + } + }, + "participant": { + "type": "object", + "additionalProperties": false, + "required": [ + "participantId", + "independent", + "previouslyInstalledSkillWire", + "attemptNumber", + "replacementForParticipantId", + "exclusionReason", + "environment", + "startingState", + "startedAt", + "endedAt", + "elapsedMilliseconds", + "release", + "documentation", + "usedOnlyPublishedQuickstart", + "manualConfigurationEdited", + "milestones", + "publicErrors", + "moderatorInterventions", + "documentationClarifications", + "serviceReady", + "discoveredTools", + "journey", + "cleanup", + "completed", + "unassisted" + ], + "properties": { + "participantId": { + "type": "string", + "pattern": "^participant-[0-9a-f]{16}$" + }, + "independent": { "const": true }, + "previouslyInstalledSkillWire": { "const": false }, + "attemptNumber": { "const": 1 }, + "replacementForParticipantId": { "type": "null" }, + "exclusionReason": { + "type": ["string", "null"], + "enum": [ + "unsupported-environment", + "prior-install", + "not-independent", + "invalid-starting-state", + null + ] + }, + "environment": { + "type": "object", + "additionalProperties": false, + "required": [ + "environmentId", + "clean", + "operatingSystem", + "architecture", + "dockerMode" + ], + "properties": { + "environmentId": { + "type": "string", + "pattern": "^environment-[0-9a-f]{16}$" + }, + "clean": { "const": true }, + "operatingSystem": { "$ref": "#/$defs/operatingSystem" }, + "architecture": { "enum": ["amd64", "arm64"] }, + "dockerMode": { "enum": ["rootful", "rootless"] } + } + }, + "startingState": { + "type": "object", + "additionalProperties": false, + "required": [ + "skillWireAbsent", + "serviceAbsent", + "selectedClientHasNoSkillWireIntegration", + "noRetainedSkillWireData" + ], + "properties": { + "skillWireAbsent": { "const": true }, + "serviceAbsent": { "const": true }, + "selectedClientHasNoSkillWireIntegration": { "const": true }, + "noRetainedSkillWireData": { "const": true } + } + }, + "startedAt": { "type": "string", "format": "date-time" }, + "endedAt": { "type": "string", "format": "date-time" }, + "elapsedMilliseconds": { + "type": "integer", + "minimum": 0, + "maximum": 86400000 + }, + "release": { "$ref": "#/$defs/participantRelease" }, + "documentation": { "$ref": "#/$defs/documentation" }, + "usedOnlyPublishedQuickstart": { "const": true }, + "manualConfigurationEdited": { "const": false }, + "milestones": { + "type": "object", + "additionalProperties": false, + "required": [ + "releaseVerified", + "serviceReady", + "clientIntegrated", + "installationStateIdentified", + "nextSafeRecoveryActionIdentified", + "sixToolDiscovery", + "mcpSearch", + "exactSkillLoad", + "optionalResourceJourney", + "cleanup" + ], + "properties": { + "releaseVerified": { "$ref": "#/$defs/milestone" }, + "serviceReady": { "$ref": "#/$defs/milestone" }, + "clientIntegrated": { "$ref": "#/$defs/milestone" }, + "installationStateIdentified": { "$ref": "#/$defs/milestone" }, + "nextSafeRecoveryActionIdentified": { + "$ref": "#/$defs/milestone" + }, + "sixToolDiscovery": { "$ref": "#/$defs/milestone" }, + "mcpSearch": { "$ref": "#/$defs/milestone" }, + "exactSkillLoad": { "$ref": "#/$defs/milestone" }, + "optionalResourceJourney": { "$ref": "#/$defs/milestone" }, + "cleanup": { "$ref": "#/$defs/milestone" } + } + }, + "publicErrors": { + "type": "array", + "maxItems": 32, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["code", "milestone", "recovered"], + "properties": { + "code": { "$ref": "#/$defs/publicCode" }, + "milestone": { "$ref": "#/$defs/milestoneName" }, + "recovered": { "type": "boolean" } + } + } + }, + "moderatorInterventions": { + "type": "array", + "maxItems": 32, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["occurredAt", "category", "milestone", "publicCode"], + "properties": { + "occurredAt": { "type": "string", "format": "date-time" }, + "category": { + "enum": [ + "undocumented-command", + "correction", + "procedural-instruction" + ] + }, + "milestone": { "$ref": "#/$defs/milestoneName" }, + "publicCode": { "$ref": "#/$defs/publicCode" } + } + } + }, + "documentationClarifications": { + "type": "array", + "maxItems": 32, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "occurredAt", + "criterion", + "documentPath", + "sectionId" + ], + "properties": { + "occurredAt": { "type": "string", "format": "date-time" }, + "criterion": { "const": "visible-document-location-only" }, + "documentPath": { + "const": "distribution/self-hosted/README.md" + }, + "sectionId": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" + } + } + } + }, + "serviceReady": { "type": "boolean" }, + "discoveredTools": { + "type": "array", + "maxItems": 6, + "uniqueItems": true, + "items": { + "enum": [ + "search_skills", + "load_skill", + "get_skill_resource", + "import_repository", + "list_repositories", + "remove_repository" + ] + } + }, + "journey": { + "type": "object", + "additionalProperties": false, + "required": [ + "searchCompleted", + "exactSkillLoaded", + "optionalResourceOutcome" + ], + "properties": { + "searchCompleted": { "type": "boolean" }, + "exactSkillLoaded": { "type": "boolean" }, + "optionalResourceOutcome": { + "enum": ["completed", "not-applicable"] + } + } + }, + "cleanup": { + "type": "object", + "additionalProperties": false, + "required": ["verified", "completedAt"], + "properties": { + "verified": { "type": "boolean" }, + "completedAt": { "type": "string", "format": "date-time" } + } + }, + "completed": { "type": "boolean" }, + "unassisted": { "type": "boolean" } + } + } + } +} diff --git a/distribution/self-hosted/supported-matrix.json b/distribution/self-hosted/supported-matrix.json index c11a643..f9893a2 100644 --- a/distribution/self-hosted/supported-matrix.json +++ b/distribution/self-hosted/supported-matrix.json @@ -6,6 +6,13 @@ { "id": "debian", "version": "13" } ], "architectures": ["amd64", "arm64"], + "dockerModes": ["rootful", "rootless"], + "certification": { + "cellCount": 12, + "observationsPerCell": "exactly-one", + "releaseIdentity": "same-final-tag-and-seven-assets", + "failedOrIncomplete": "not-certified-no-replacement-or-exclusion" + }, "docker": { "minimum": "29.7.2", "tested": "29.7.2" }, "compose": { "minimum": "5.4.0", "tested": "5.4.0" }, "postgresql": "17.10-alpine", diff --git a/docs/self-hosted-moderated-usability.md b/docs/self-hosted-moderated-usability.md new file mode 100644 index 0000000..81dfde6 --- /dev/null +++ b/docs/self-hosted-moderated-usability.md @@ -0,0 +1,117 @@ +# Self-hosted moderated usability protocol + +This is the normative moderator protocol for the external T161 release +certification. Running repository tests does not create participant evidence and +does not satisfy this protocol. Test fixtures use +`evidenceKind: synthetic-fixture`; only a completed external session may use +`certified-observation`. + +## Freeze the cohort and candidate + +Before recruiting, freeze one immutable release identity for the whole cohort: + +- release version and exact tag `self-hosted-v`; +- the annotated tag's 40-hex target commit; +- the exact seven published asset filenames, byte sizes, and SHA-256 values; +- the SHA-256 of the published `distribution/self-hosted/README.md` used by + every participant. + +Recruit exactly ten independent participants who have never installed SkillWire. +Pre-screening happens before cohort assignment. Every assigned participant must +use a clean supported Ubuntu 24.04, Debian 12, or Debian 13 environment on +`amd64` or `arm64` with rootful or rootless Docker. Assign only a privacy-safe +opaque participant ID and a distinct opaque environment ID. Do not record a +name, email address, account identity, repository identity, prompt, response, +credential, client login state, or unrelated profile content. + +After the ten IDs are frozen, a participant cannot be replaced, rerun, or +excluded. A timeout, abandonment, unrecovered error, dirty starting state, or +eligibility discrepancy is a failed first attempt. The evidence schema includes +`exclusionReason` so an invalid collection attempt cannot be hidden, but the +semantic validator rejects an exclusion inside the frozen cohort. + +## Fixed starting state + +Immediately before the UTC start timestamp, the moderator records only the +following booleans and supported environment identity: + +- SkillWire has never been installed by the participant; +- no SkillWire service or retained SkillWire data exists; +- the selected ordinary Codex or Claude profile has no SkillWire integration; +- the environment is clean, supported, and unique to this participant; +- this is attempt number one and is not a replacement attempt. + +The moderator then gives exactly this instruction: + +> Open the published SkillWire self-hosted quickstart and use only that document +> to verify the release, install SkillWire with one ordinary Codex or Claude +> client, determine the final installation state and next safe recovery action, +> complete the documented SkillWire tool journey, and clean up as documented. +> Tell me when you are finished or cannot continue. + +Start the UTC timer when the participant opens the frozen quickstart. The target +is 900,000 milliseconds. The moderator must not provide a command, correction, +procedural hint, alternative document, or service-internal explanation. + +## Observation milestones + +Record only status and public error codes for each milestone. Do not transcribe +terminal output or participant content. + +1. The exact archive, architecture manifest, signature bundle, trust policy, + annotated tag, and source commit are verified. +2. First-party setup reaches service readiness and integrates one normal Codex + or Claude profile without a wrapper or alternate production profile. +3. From the final setup or doctor output, the participant identifies the + installation state and the next safe recovery action without inspecting + service internals. +4. The participant discovers exactly `search_skills`, `load_skill`, + `get_skill_resource`, `import_repository`, `list_repositories`, and + `remove_repository`. +5. The participant completes MCP search, loads the exact selected skill, and + uses the optional resource step when the selected skill exposes a resource; + otherwise that one step is recorded `not-applicable`. +6. The participant completes the documented cleanup and the moderator verifies + that its disposable resources are absent. + +For each milestone use `passed`, `failed`, `timeout`, or `abandoned`; +`not-applicable` is permitted only for the optional resource milestone. Record +the UTC end timestamp and the exact derived duration. Stop at 900,000 +milliseconds if the journey is not complete. An abandonment, timeout, any +required milestone that is not passed, unverified cleanup, or an unrecovered +public error makes `completed` false. + +## Assistance rule + +An intervention is any undocumented command, correction, or procedural +instruction required from the moderator. Record its UTC time, category, +milestone, and privacy-safe public code. A participant with any intervention is +not unassisted even if the journey later completes. + +A clarification is non-intervening only when it identifies the location of +information already visible in the frozen quickstart and supplies no command, +correction, interpretation, ordering, or procedural guidance. Record it with +criterion `visible-document-location-only`, the frozen document path, and its +public section ID. Anything more is an intervention. Clarifications never erase +an error, timeout, or intervention. + +## Evidence and calculation + +Serialize the complete cohort with +`distribution/self-hosted/moderated-usability.schema.json`, then run the +semantic validator exported by `src/onboarding/domain/moderated-usability.ts`. +All ten records must carry the same candidate, seven-asset set, and +documentation identity. Participant and environment IDs must be unique. The +validator recomputes completion, unassisted status, durations, release identity, +and aggregate values; supplied summary booleans are not trusted. + +- **SC-001 (95%)**: exactly 10 of 10 assigned participants must complete the + required onboarding journey within 900,000 milliseconds. Nine of ten is 90% + and fails this criterion. +- **SC-014 (90%)**: at least 9 of 10 must complete without a moderator + intervention. A permitted location-only clarification is recorded separately + and is not an intervention. + +No synthetic fixture, partial cohort, replacement attempt, excluded failure, or +recalculated denominator is release evidence. Preserve only the privacy-safe +schema fields and redacted public codes required for external review. diff --git a/docs/self-hosted-release-evidence.md b/docs/self-hosted-release-evidence.md index 2c4f1cd..c7ef341 100644 --- a/docs/self-hosted-release-evidence.md +++ b/docs/self-hosted-release-evidence.md @@ -23,6 +23,15 @@ the 28-scenario gate. CI fixtures may prove deterministic logic but do not replace a real Docker, PostgreSQL, client-manager, D-Bus/keyring, or architecture boundary named by an acceptance contract. +Certification requires exactly one observation for each of the 12 Cartesian +cells. Every observation is bound to the same final +`self-hosted-v` annotated tag, source commit, and exact seven +published assets: an archive, canonical manifest, and corresponding signature +bundle for each architecture, plus the trust policy. A failed or incomplete cell +remains failed or incomplete; it cannot be replaced, rerun as a substitute, or +silently excluded from the matrix. No cell is claimed as passed by this +preparation patch. + ## Required deterministic gates - formatting, ESLint, strict TypeScript, build, migrations and idempotent rerun; @@ -43,26 +52,15 @@ not turn a deterministic setup pass into an autonomous-activation claim. ## Duration and moderated usability -The participant target of 15 minutes is informational and never a CI timeout or -pass threshold. A duration record contains only environment ID, start/end -monotonic duration, result category, source commit and manifest hash. A -moderated result uses this redacted format: - -```json -{ - "schemaVersion": "skillwire.moderated-usability/v1", - "participantId": "opaque-session-id", - "matrixCell": "ubuntu-24.04-amd64-rootless", - "sourceCommit": "40-lowercase-hex", - "manifestSha256": "64-lowercase-hex", - "completed": true, - "elapsedMilliseconds": 0, - "assistanceCategories": [], - "failureCode": null, - "credentialOrContentCaptured": false, - "cleanupVerified": true -} -``` +The 15-minute target is not a deterministic CI wall-clock timeout, but it is a +normative moderated-release threshold. Follow +`docs/self-hosted-moderated-usability.md` with exactly ten independent, +first-attempt participants and validate the privacy-safe cohort against +`distribution/self-hosted/moderated-usability.schema.json` plus the semantic +validator. SC-001 requires 10/10 participants to complete within 900,000 +milliseconds. SC-014 requires at least 9/10 completed journeys without a +moderator intervention. Timeout, abandonment, unrecovered error, replacement, +rerun, or post-assignment exclusion never improves the denominator. No participant run is recorded for the current uncommitted candidate. Release readiness therefore remains blocked until immutable commit-bound artifacts and diff --git a/package.json b/package.json index 4737db6..5f1c733 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "skillwire", - "version": "0.1.0", + "version": "0.2.0", "private": true, "type": "module", "description": "Remote MCP skill discovery service", diff --git a/scripts/build-self-hosted-release.ts b/scripts/build-self-hosted-release.ts index 3961694..58ef0ca 100644 --- a/scripts/build-self-hosted-release.ts +++ b/scripts/build-self-hosted-release.ts @@ -315,11 +315,22 @@ async function main(): Promise { const images = ReleaseManifestSchema.shape.images.parse( JSON.parse(imagesJson) as unknown, ); + const packageVersion = ReleaseManifestSchema.shape.releaseVersion.parse( + ( + JSON.parse(await readFile(resolve("package.json"), "utf8")) as { + version?: unknown; + } + ).version, + ); + const releaseVersion = + process.env["SKILLWIRE_RELEASE_VERSION"] ?? packageVersion; + if (releaseVersion !== packageVersion) + throw new Error("Release version does not match package.json"); await buildSelfHostedRelease({ payloadRoot, outputDirectory, architecture, - releaseVersion: process.env["SKILLWIRE_RELEASE_VERSION"] ?? "0.1.0", + releaseVersion, releaseSequence: Number(process.env["SKILLWIRE_RELEASE_SEQUENCE"] ?? "1"), publishedAt: process.env["SKILLWIRE_PUBLISHED_AT"] ?? "1970-01-01T00:00:00.000Z", diff --git a/scripts/verify-self-hosted-release-tag.ts b/scripts/verify-self-hosted-release-tag.ts new file mode 100644 index 0000000..7cce63d --- /dev/null +++ b/scripts/verify-self-hosted-release-tag.ts @@ -0,0 +1,190 @@ +import { execFile } from "node:child_process"; +import { constants } from "node:fs"; +import { open, readFile } from "node:fs/promises"; +import { isAbsolute, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const SEMVER = /^\d+\.\d+\.\d+$/; +const COMMIT_SHA = /^[0-9a-f]{40}$/; +const MAXIMUM_JSON_BYTES = 16 * 1024 * 1024; + +export interface SelfHostedReleaseTagOptions { + readonly repositoryRoot: string; + readonly githubRef: string; + readonly githubSha: string; + readonly manifestPath?: string | undefined; +} + +export interface VerifiedSelfHostedReleaseTag { + readonly packageVersion: string; + readonly tagRef: string; + readonly targetCommit: string; +} + +function requireRecord(value: unknown, label: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) + throw new Error(`${label} must be a JSON object`); + return value as Record; +} + +function requireCommitSha(value: string, label: string): string { + if (!COMMIT_SHA.test(value)) throw new Error(`${label} must be a commit SHA`); + return value; +} + +async function readBoundedJson(path: string): Promise> { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const metadata = await handle.stat(); + if (!metadata.isFile() || metadata.size > MAXIMUM_JSON_BYTES) + throw new Error("Release identity JSON must be a bounded regular file"); + return requireRecord( + JSON.parse(await handle.readFile("utf8")) as unknown, + "Release identity JSON", + ); + } finally { + await handle.close(); + } +} + +async function git( + repositoryRoot: string, + args: readonly string[], + acceptExitCodes: readonly number[] = [0], +): Promise<{ readonly code: number; readonly stdout: string }> { + try { + const result = await execFileAsync( + "/usr/bin/git", + ["-c", `safe.directory=${repositoryRoot}`, "-C", repositoryRoot, ...args], + { + encoding: "utf8", + env: { PATH: "/usr/bin:/bin", LANG: "C.UTF-8" }, + timeout: 10_000, + maxBuffer: 16 * 1024, + }, + ); + return { code: 0, stdout: result.stdout.trim() }; + } catch (error: unknown) { + const code = + typeof error === "object" && error !== null && "code" in error + ? Number(error.code) + : -1; + if (acceptExitCodes.includes(code)) return { code, stdout: "" }; + throw new Error("Release tag Git verification failed", { cause: error }); + } +} + +export async function verifySelfHostedReleaseTag( + options: SelfHostedReleaseTagOptions, +): Promise { + if (!isAbsolute(options.repositoryRoot)) + throw new Error("Release repository root must be absolute"); + const repositoryRoot = resolve(options.repositoryRoot); + const githubSha = requireCommitSha(options.githubSha, "Workflow SHA"); + const packageDocument = requireRecord( + JSON.parse( + await readFile(resolve(repositoryRoot, "package.json"), "utf8"), + ) as unknown, + "Package document", + ); + if ( + typeof packageDocument["version"] !== "string" || + !SEMVER.test(packageDocument["version"]) + ) + throw new Error("Package version must be an exact semantic version"); + + const expectedRef = `refs/tags/self-hosted-v${packageDocument["version"]}`; + if (options.githubRef !== expectedRef) + throw new Error( + "Self-hosted release ref does not match the package version", + ); + + const objectType = await git(repositoryRoot, [ + "cat-file", + "-t", + options.githubRef, + ]); + if (objectType.stdout !== "tag") + throw new Error("Self-hosted releases require an annotated tag object"); + + const peeled = requireCommitSha( + ( + await git(repositoryRoot, [ + "rev-parse", + "--verify", + `${options.githubRef}^{commit}`, + ]) + ).stdout, + "Annotated tag target", + ); + if (peeled !== githubSha) + throw new Error("Annotated tag target does not match the workflow SHA"); + + const reachable = await git( + repositoryRoot, + ["merge-base", "--is-ancestor", peeled, "refs/remotes/origin/main"], + [0, 1], + ); + if (reachable.code !== 0) + throw new Error("Release tag target is not reachable from protected main"); + + if (options.manifestPath !== undefined) { + const manifest = await readBoundedJson(options.manifestPath); + if ( + manifest["releaseVersion"] !== packageDocument["version"] || + manifest["sourceCommit"] !== githubSha + ) { + throw new Error( + "Release manifest version or source commit does not match the annotated tag", + ); + } + } + + return { + packageVersion: packageDocument["version"], + tagRef: expectedRef, + targetCommit: peeled, + }; +} + +function argument(name: string): string | undefined { + const index = process.argv.indexOf(name); + return index < 0 ? undefined : process.argv[index + 1]; +} + +async function main(): Promise { + const repositoryRoot = argument("--repository"); + const githubRef = argument("--ref"); + const githubSha = argument("--sha"); + if ( + repositoryRoot === undefined || + githubRef === undefined || + githubSha === undefined + ) { + throw new Error( + "Usage: verify-self-hosted-release-tag --repository --ref --sha [--manifest ]", + ); + } + await verifySelfHostedReleaseTag({ + repositoryRoot, + githubRef, + githubSha, + manifestPath: argument("--manifest"), + }); +} + +if ( + process.argv[1] !== undefined && + resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)) +) { + main().catch((error: unknown) => { + const message = + error instanceof Error + ? error.message + : "Release tag verification failed"; + process.stderr.write(`${message.replaceAll(/[\r\n]/g, " ")}\n`); + process.exitCode = 1; + }); +} diff --git a/scripts/verify-self-hosted-release.ts b/scripts/verify-self-hosted-release.ts index 3d9e8a0..83548b5 100644 --- a/scripts/verify-self-hosted-release.ts +++ b/scripts/verify-self-hosted-release.ts @@ -140,6 +140,17 @@ const CertifiedMatrixSchema = z z.object({ id: z.literal("debian"), version: z.literal("13") }).strict(), ]), architectures: z.tuple([z.literal("amd64"), z.literal("arm64")]), + dockerModes: z.tuple([z.literal("rootful"), z.literal("rootless")]), + certification: z + .object({ + cellCount: z.literal(12), + observationsPerCell: z.literal("exactly-one"), + releaseIdentity: z.literal("same-final-tag-and-seven-assets"), + failedOrIncomplete: z.literal( + "not-certified-no-replacement-or-exclusion", + ), + }) + .strict(), docker: z .object({ minimum: z.literal("29.7.2"), diff --git a/specs/004-self-hosted-onboarding/contracts/release-and-recovery.md b/specs/004-self-hosted-onboarding/contracts/release-and-recovery.md index 827175b..5bf2f81 100644 --- a/specs/004-self-hosted-onboarding/contracts/release-and-recovery.md +++ b/specs/004-self-hosted-onboarding/contracts/release-and-recovery.md @@ -17,7 +17,7 @@ Bootstrap runs `cosign verify-blob` with outbound network blocked, the local tru ## Signing and trust-policy contract -`.github/workflows/self-hosted-release.yml` is the only release signer. The protected-tag workflow checks out the exact `v` commit, builds and completes all acceptance gates before signing, grants only `contents: read` and `id-token: write`, and invokes: +`.github/workflows/self-hosted-release.yml` is the only release signer. The protected-tag workflow accepts only `self-hosted-v`, requires an annotated tag object, recursively peels it to the exact workflow SHA, proves the target is reachable from protected `main`, and checks manifest version/source identity before signing. It uses only command-scoped `safe.directory`, builds and completes all acceptance gates before signing, and grants only `contents: read` and `id-token: write`. ```text cosign sign-blob --yes \ @@ -30,7 +30,7 @@ cosign sign-blob --yes \ Cosign is pinned to 3.1.3. The bundle media type is exactly `application/vnd.dev.sigstore.bundle.v0.3+json` and contains the message signature, Fulcio certificate, signed transparency/timestamp evidence, and inclusion proof needed for offline verification. Publication fails when signing or required transparency material is absent. -The exact issuer is `https://token.actions.githubusercontent.com`. For release ``, the exact certificate identity is `https://github.com/Lucenx9/skillwire/.github/workflows/self-hosted-release.yml@refs/tags/v`. Verification additionally pins repository `Lucenx9/skillwire`, workflow ref, tag ref, and exact workflow commit-SHA claims. +The exact issuer is `https://token.actions.githubusercontent.com`. For release ``, the exact certificate identity is `https://github.com/Lucenx9/skillwire/.github/workflows/self-hosted-release.yml@refs/tags/self-hosted-v`. Verification additionally pins repository `Lucenx9/skillwire`, workflow ref, tag ref, and exact workflow commit-SHA claims. With outbound network denied, installer/bootstrap verification invokes the policy-pinned Cosign binary directly with no shell and this exact interface: @@ -40,10 +40,10 @@ cosign verify-blob \ --bundle skillwire--linux-.release.sigstore.json \ --trusted-root \ --certificate-identity \ - https://github.com/Lucenx9/skillwire/.github/workflows/self-hosted-release.yml@refs/tags/v \ + https://github.com/Lucenx9/skillwire/.github/workflows/self-hosted-release.yml@refs/tags/self-hosted-v \ --certificate-oidc-issuer https://token.actions.githubusercontent.com \ --certificate-github-workflow-repository Lucenx9/skillwire \ - --certificate-github-workflow-ref refs/tags/v \ + --certificate-github-workflow-ref refs/tags/self-hosted-v \ --certificate-github-workflow-sha \ skillwire--linux-.release.json ``` @@ -52,6 +52,8 @@ The verifier does not pass `--insecure-ignore-sct`, `--insecure-ignore-tlog`, a `skillwire.trust-policy/v1` is RFC 8785 canonical JSON and contains a monotonic policy sequence, validity window, exact accepted signer claims, TrustedRoot media type/path/SHA-256, allowed Cosign version/platform/SHA-256 entries, minimum release sequence, deny set, and signer-rotation evidence. The first policy is pinned in source and the bootstrap README. Installation state records the highest accepted policy and release sequences; lower sequences, changed hashes, unknown policies, invalid/legacy bundles, denied material, wrong claims, missing overlap proof, or invalid transparency/timestamp evidence fail before extraction or mutation. +For the pinned Cosign 3.1.3 path, the TrustedRoot media type is exactly `application/vnd.dev.sigstore.trustedroot+json;version=0.1`. Cosign 3.1.3 pins `sigstore-go` 1.2.2, whose TrustedRoot loader accepts only that value. The conflicting `application/vnd.dev.sigstore.trustedroot.v0.2+json` value from the newer protobuf contract is rejected by this release rather than treated as an alias. + For an update, the active installed policy first verifies the release manifest; the manifest's signed next-policy hash then authenticates that policy before it can become active. Signer rotation requires an old-policy-authorized policy update and an overlap release carrying the normal bundle plus an additional `skillwire--linux-.release..sigstore.json`; the manifest enumerates both ordered filenames and signer identities, each Bundle v0.3 independently binds the canonical manifest digest, and the verifier hashes the bundle bytes to reject duplicate evidence before verifying each exact signer policy. Bundle digests cannot be fields of the manifest the bundles themselves sign because that would create a circular hash dependency. A later old-policy-authorized update may remove the old signer. Emergency revocation is an old-policy-authorized deny-set update. If no trusted signer survives, automatic update stops and recovery requires a separately authenticated out-of-band bootstrap. Offline verification cannot discover a later revocation, so every new install/upgrade must use the latest explicitly refreshed policy; cached evidence is scoped to its recorded policy sequence. ## Release bundle @@ -93,7 +95,7 @@ Paths are normalized UTF-8 relative paths with no empty, dot, traversal, absolut - exact path, byte size, and SHA-256 of Feature 003's `distribution/codex-marketplace/release-integrity.json`; - minimum/maximum compatible database schema and rollback rules; - required trust-policy schema/sequence/sibling filename/byte size/SHA-256; -- one normal, or two signer-overlap, external Sigstore Bundle v0.3 filenames/media types/digests and the canonical manifest SHA-256. +- one normal, or two signer-overlap, exact external Sigstore Bundle v0.3 filenames and signer IDs. Bundle bytes, media types, and digests are not manifest fields because the bundle signs the exact canonical manifest bytes; those properties are validated directly during Cosign verification. The manifest contains no registry credential, client key, GitHub token, database password, pepper, account ID, repository identity, or host path. A changed artifact requires a new manifest/release; a signature never authorizes bytes omitted from the artifact list. diff --git a/specs/004-self-hosted-onboarding/data-model.md b/specs/004-self-hosted-onboarding/data-model.md index cb5c765..7be37bf 100644 --- a/specs/004-self-hosted-onboarding/data-model.md +++ b/specs/004-self-hosted-onboarding/data-model.md @@ -55,7 +55,7 @@ Immutable release input and its verified local materialization. | `adapters` | map | Codex and Claude plugin/marketplace identities, versions, inventories, and hashes. | | `feature003Integrity` | record | Exact path, byte size, and SHA-256 for `distribution/codex-marketplace/release-integrity.json`. | | `trustPolicy` | record | Required schema, policy sequence, sibling filename, byte size, and SHA-256. | -| `signing` | record | One normal, or two signer-overlap, external Sigstore Bundle v0.3 filename/media-type/digest identities plus exact canonical-manifest digest; no private key. | +| `signing` | record | One normal, or two signer-overlap, exact external Sigstore Bundle v0.3 filenames and signer IDs. Bundle media type/evidence and its binding to the canonical manifest bytes are verified outside the manifest; no bundle digest is embedded in the manifest it signs and no private key is stored. | | `rollbackCompatibility` | record | Minimum/maximum schema and pre/post-010 application compatibility. | The release manifest is an external UTF-8 RFC 8785 canonical JSON file with no BOM or trailing newline; it is never embedded in the archive it hashes. An Installed Release adds `installedPath`, verified manifest/archive/bundle/policy identities, verified file identities, image inspection results, verifier/trusted-root identities, and `installedAt`. It is immutable after verification. Changing any byte produces a different release or a failed integrity finding. @@ -70,7 +70,7 @@ Versioned release-verification authority, independent of mutable network state. | `policySequence` | positive integer | Monotonic; a lower sequence is never accepted after a higher one. | | `validity` | record | Explicit not-before/not-after timestamps; staleness is a blocking finding for a new install/upgrade. | | `acceptedSigners` | non-empty ordered array | Exact Fulcio identity, issuer, repository, workflow ref, tag-ref form, and required workflow commit-SHA claims. | -| `trustedRoot` | record | Sigstore TrustedRoot media type `application/vnd.dev.sigstore.trustedroot.v0.2+json`, local filename, and SHA-256. | +| `trustedRoot` | record | Exact Cosign 3.1.3 / sigstore-go 1.2.2 TrustedRoot media type `application/vnd.dev.sigstore.trustedroot+json;version=0.1`, local filename, and SHA-256; the unsupported conflicting v0.2 value is rejected. | | `cosignVerifiers` | ordered array | Allowed exact Cosign version/platform/filename/SHA-256 identities. | | `minimumReleaseSequence` | non-negative integer | Releases below this boundary are denied. | | `denySet` | ordered array | Revoked manifest, archive, bundle, certificate, or signer identities/digests. | diff --git a/specs/004-self-hosted-onboarding/quickstart.md b/specs/004-self-hosted-onboarding/quickstart.md index d4b2fe4..7757bdc 100644 --- a/specs/004-self-hosted-onboarding/quickstart.md +++ b/specs/004-self-hosted-onboarding/quickstart.md @@ -116,7 +116,7 @@ skillwire-trust-policy-v.json A signer-rotation overlap release is the documented exception: it also emits `skillwire--linux-.release..sigstore.json`, and the signed manifest binds both bundle identities. Verification must accept the release under the active old policy, verify both old/new signer bundles, authenticate the referenced next-policy hash, and only then activate that policy. -Before extracting or running the archive, follow `distribution/self-hosted/README.md`: verify the exact Cosign 3.1.3 binary with Sigstore's official TUF `artifact.pub` and release bundle, match its platform SHA-256 to the policy, then run network-blocked `cosign verify-blob` with the local TrustedRoot, Bundle v0.3 file, issuer `https://token.actions.githubusercontent.com`, and certificate identity `https://github.com/Lucenx9/skillwire/.github/workflows/self-hosted-release.yml@refs/tags/v`. The policy also checks repository, workflow ref, tag ref, and exact workflow commit SHA. Never use `latest`, an unverified action download, or `curl | sh`. +Before extracting or running the archive, follow `distribution/self-hosted/README.md`: verify the exact Cosign 3.1.3 binary with Sigstore's official TUF `artifact.pub` and release bundle, match its platform SHA-256 to the policy, then run network-blocked `cosign verify-blob` with the local TrustedRoot, Bundle v0.3 file, issuer `https://token.actions.githubusercontent.com`, and certificate identity `https://github.com/Lucenx9/skillwire/.github/workflows/self-hosted-release.yml@refs/tags/self-hosted-v`. The policy also checks repository, workflow ref, tag ref, and exact workflow commit SHA. Never use `latest`, an unverified action download, or `curl | sh`. Expected release validation: diff --git a/specs/004-self-hosted-onboarding/research.md b/specs/004-self-hosted-onboarding/research.md index dff8293..e12d894 100644 --- a/specs/004-self-hosted-onboarding/research.md +++ b/specs/004-self-hosted-onboarding/research.md @@ -198,9 +198,9 @@ The dispatcher installs bounded `SIGINT` and `SIGTERM` handlers, aborts the shar Each platform normally publishes four sibling assets: `skillwire--linux-.tar.zst`, `skillwire--linux-.release.json`, `skillwire--linux-.release.sigstore.json`, and `skillwire-trust-policy-v.json`. A signer-overlap release additionally publishes `skillwire--linux-.release..sigstore.json`, and the manifest enumerates both exact ordered bundle filenames and signer identities; each bundle independently binds the canonical manifest digest and the verifier hashes bundle bytes to reject duplicate evidence. Bundle digests cannot be manifest fields because the bundles sign that manifest, which would create a circular hash dependency. The overlap bundle is the sole exception to the normal four-asset set. The release JSON is UTF-8 RFC 8785 JSON Canonicalization Scheme output with no BOM and no trailing newline. It uses `skillwire.release/v1`, includes a monotonic `releaseSequence`, the archive byte size/SHA-256, the complete extracted payload inventory and identities, exact image/platform digests, catalog/advisory/migration/adapter hashes, the Feature 003 `distribution/codex-marketplace/release-integrity.json` identity, compatibility, and the required trust-policy version/hash. Because the manifest is outside the archive, it can bind the archive without a circular digest. -The protected tag workflow `.github/workflows/self-hosted-release.yml` checks out the exact `v` tag commit, builds and verifies before signing, grants only `contents: read` and `id-token: write`, and invokes `cosign sign-blob --yes --timeout 2m --oidc-provider github-actions --signing-algorithm ecdsa-sha2-256-nistp256 --bundle .sigstore.json `. Cosign 3 emits canonical proto3 JSON Sigstore Bundle media type `application/vnd.dev.sigstore.bundle.v0.3+json`, containing the message signature, Fulcio certificate, signed timestamp, and transparency-log proof. The expected OIDC issuer is exactly `https://token.actions.githubusercontent.com`; the exact certificate identity is `https://github.com/Lucenx9/skillwire/.github/workflows/self-hosted-release.yml@refs/tags/v`. Verification also pins repository `Lucenx9/skillwire`, workflow ref, tag ref, and exact workflow commit SHA claims. Signing failure or missing transparency material fails publication. +The protected tag workflow `.github/workflows/self-hosted-release.yml` accepts only the annotated `self-hosted-v` tag, recursively peels it to the exact workflow SHA, proves the target is reachable from protected `main`, and checks package/manifest/source identity before signing. It builds and verifies before signing, grants only `contents: read` and `id-token: write`, and invokes `cosign sign-blob --yes --timeout 2m --oidc-provider github-actions --signing-algorithm ecdsa-sha2-256-nistp256 --bundle .sigstore.json `. Cosign 3 emits canonical proto3 JSON Sigstore Bundle media type `application/vnd.dev.sigstore.bundle.v0.3+json`, containing the message signature, Fulcio certificate, signed timestamp, and transparency-log proof. The expected OIDC issuer is exactly `https://token.actions.githubusercontent.com`; the exact certificate identity is `https://github.com/Lucenx9/skillwire/.github/workflows/self-hosted-release.yml@refs/tags/self-hosted-v`. Verification also pins repository `Lucenx9/skillwire`, workflow ref, tag ref, and exact workflow commit SHA claims. Signing failure or missing transparency material fails publication. -Bootstrap uses an independently verified Cosign 3.1.3 and a locally supplied Sigstore TrustedRoot JSON (`application/vnd.dev.sigstore.trustedroot.v0.2+json`) acquired through Sigstore TUF. With outbound network blocked, it directly runs `cosign verify-blob --timeout 30s --bundle --trusted-root --certificate-identity --certificate-oidc-issuer https://token.actions.githubusercontent.com --certificate-github-workflow-repository Lucenx9/skillwire --certificate-github-workflow-ref refs/tags/v --certificate-github-workflow-sha `. It never uses the insecure SCT/tlog bypasses or identity/issuer regular expressions, then independently validates bundle media type/evidence, policy sequences/deny set, canonical manifest, and archive digest before extraction or execution. After safe extraction, `skillwire` repeats verification using the release-pinned verifier and installed policy before any mutation. No verification path performs an unbounded transparency lookup or silently refreshes trust; a missing/stale trusted root yields exact instructions for an explicit bounded TUF refresh before retry. +Bootstrap uses an independently verified Cosign 3.1.3 and a locally supplied Sigstore TrustedRoot JSON with exact media type `application/vnd.dev.sigstore.trustedroot+json;version=0.1` acquired through Sigstore TUF. Cosign 3.1.3 pins `sigstore-go` 1.2.2, whose loader accepts only that value; the conflicting `application/vnd.dev.sigstore.trustedroot.v0.2+json` value described by the newer protobuf specification is unsupported by this pinned verifier and fails closed. With outbound network blocked, bootstrap directly runs `cosign verify-blob --timeout 30s --bundle --trusted-root --certificate-identity --certificate-oidc-issuer https://token.actions.githubusercontent.com --certificate-github-workflow-repository Lucenx9/skillwire --certificate-github-workflow-ref refs/tags/self-hosted-v --certificate-github-workflow-sha `. It never uses the insecure SCT/tlog bypasses or identity/issuer regular expressions, then independently validates bundle media type/evidence, policy sequences/deny set, canonical manifest, and archive digest before extraction or execution. After safe extraction, `skillwire` repeats verification using the release-pinned verifier and installed policy before any mutation. No verification path performs an unbounded transparency lookup or silently refreshes trust; a missing/stale trusted root yields exact instructions for an explicit bounded TUF refresh before retry. `skillwire.trust-policy/v1` is a canonical, versioned policy containing a monotonic policy sequence, accepted exact signer identities/claims, issuer, workflow, tag form, trusted-root media type/hash, Cosign version/hash set, minimum release sequence, denied manifest/certificate identities or digests, and validity window. The first policy is pinned in source/distribution documentation and embedded in the verified installer. For an update, the active installed policy first verifies the release manifest; the manifest's signed next-policy hash then authenticates that policy before it can become active. A policy/identity rotation release must be accepted under the old policy and carry the separately named valid bundles from both the old and proposed signer during at least one published overlap release; only a later old-policy-authorized update may remove the old signer. Emergency revocation is an old-policy-authorized policy update adding the compromised identity/certificate/manifest digest to the deny set. If no trusted signer survives, automatic update stops and recovery requires a separately authenticated out-of-band trust bootstrap. Offline verification cannot discover later revocation, so every new install/upgrade must use the latest explicitly refreshed policy; cached verification remains scoped to its recorded policy sequence. @@ -215,7 +215,7 @@ Install state records the highest accepted policy and release sequences. Setup/u - Trust any GitHub Actions certificate from the repository: rejected because workflow, tag, SHA, issuer, and certificate identity must all match. - Put the archive digest inside a manifest embedded in that archive: rejected because it creates a circular digest. -**Evidence**: [Sigstore CI quickstart](https://docs.sigstore.dev/quickstart/quickstart-ci/), [Sigstore bundle format](https://docs.sigstore.dev/about/bundle/), [Cosign 3.1.3 signing reference](https://github.com/sigstore/cosign/blob/v3.1.3/doc/cosign_sign-blob.md), [Cosign 3.1.3 verification reference](https://github.com/sigstore/cosign/blob/v3.1.3/doc/cosign_verify-blob.md), [Cosign installation and release verification](https://docs.sigstore.dev/cosign/system_config/installation/), [Cosign 3.1.3 release](https://github.com/sigstore/cosign/releases/tag/v3.1.3), [GitHub Actions OIDC claims](https://docs.github.com/en/actions/reference/security/oidc). +**Evidence**: [Sigstore CI quickstart](https://docs.sigstore.dev/quickstart/quickstart-ci/), [Sigstore bundle format](https://docs.sigstore.dev/about/bundle/), [Cosign 3.1.3 signing reference](https://github.com/sigstore/cosign/blob/v3.1.3/doc/cosign_sign-blob.md), [Cosign 3.1.3 verification reference](https://github.com/sigstore/cosign/blob/v3.1.3/doc/cosign_verify-blob.md), [Cosign 3.1.3 dependency pin](https://github.com/sigstore/cosign/blob/v3.1.3/go.mod), [sigstore-go 1.2.2 TrustedRoot loader](https://github.com/sigstore/sigstore-go/blob/v1.2.2/pkg/root/trusted_root.go), [Sigstore protobuf TrustedRoot media-type contract](https://github.com/sigstore/protobuf-specs/blob/v0.5.1/protos/sigstore_trustroot.proto), [Cosign installation and release verification](https://docs.sigstore.dev/cosign/system_config/installation/), [Cosign 3.1.3 release](https://github.com/sigstore/cosign/releases/tag/v3.1.3), [GitHub Actions OIDC claims](https://docs.github.com/en/actions/reference/security/oidc). ## 16. Separate deterministic timing and real-session evidence from usability claims diff --git a/specs/004-self-hosted-onboarding/spec.md b/specs/004-self-hosted-onboarding/spec.md index b203b18..1809457 100644 --- a/specs/004-self-hosted-onboarding/spec.md +++ b/specs/004-self-hosted-onboarding/spec.md @@ -356,7 +356,7 @@ Repository evidence on the same date pins the existing adapter and current relea ### Measurable Outcomes -- **SC-001**: On every supported OS/architecture fixture, at least 95% of first-attempt participants who meet prerequisites complete release verification, service setup, and one selected-client integration in 15 minutes or less without manually editing configuration files. +- **SC-001**: In the fixed cohort of exactly ten independent first-attempt participants who meet prerequisites, 10/10 complete release verification, service setup, one selected-client integration, and the defined onboarding journey in 15 minutes or less without manually editing configuration files. A timeout, abandonment, or unrecovered error fails the participant; replacement, rerun, or post-assignment exclusion is forbidden. - **SC-002**: 100% of successful selected-client installations expose exactly six named SkillWire tools and complete one authenticated search followed by an exact immutable load from a fresh ordinary client process. - **SC-003**: Across Codex-only, Claude-only, and dual-client acceptance suites, 100% of ordinary `codex` and `claude` startup attempts succeed when SkillWire is healthy, stopped, unreachable, unauthenticated, or misconfigured. - **SC-004**: Install, repeated setup, repair, upgrade, and uninstall preserve 100% of seeded unrelated client configuration and authentication state in disposable-profile tests, with zero repository client-directory writes. @@ -369,7 +369,7 @@ Repository evidence on the same date pins the existing adapter and current relea - **SC-011**: Permanent removal deletes only the exact separately confirmed SkillWire-owned paths and volumes in 100% of tests and cannot proceed from the default-uninstall confirmation alone. - **SC-012**: Both no-schema-change and forward-only upgrade suites preserve repository memory, client credentials, and unrelated configuration; 100% of unsafe image-only rollback attempts across migration `010` are refused. - **SC-013**: Every release candidate passes all Feature 001 through 003 regression suites, all 28 numbered Feature 004 acceptance scenarios, the six-tool contract tests, profile-preservation tests, and secret-disclosure tests before release. -- **SC-014**: In a moderated usability check, at least 90% of supported developers can identify installation state and the next safe recovery action from final setup or doctor output without consulting service internals. +- **SC-014**: In the same fixed ten-participant moderated cohort, at least 9/10 complete the journey and identify installation state and the next safe recovery action from final setup or doctor output without moderator intervention or consulting service internals. A location-only clarification of already visible frozen documentation is recorded separately and supplies no procedural guidance. - **SC-015**: 100% of installations that pass deterministic client verification retain that integration when the separate automatic diagnostic observes no invocation, and 100% of autonomous-activation release claims are withheld until the applicable fresh-client evidence meets Feature 003's attributable acceptance target. - **SC-016**: In every dual-client partial-failure fixture, the verified client and healthy service remain unchanged, the failed client's original profile is restored and its newly created key is revoked, and the command returns the documented incomplete non-success class. - **SC-017**: In every equivalent pre-existing component fixture, install creates zero duplicates and repair, upgrade, and uninstall produce zero mutations to the reused user-owned component. diff --git a/src/onboarding/domain/moderated-usability.ts b/src/onboarding/domain/moderated-usability.ts new file mode 100644 index 0000000..7caa4a0 --- /dev/null +++ b/src/onboarding/domain/moderated-usability.ts @@ -0,0 +1,581 @@ +import { isDeepStrictEqual } from "node:util"; + +import { z } from "zod"; + +const Sha256Schema = z.string().regex(/^[0-9a-f]{64}$/); +const PublicCodeSchema = z.string().regex(/^[A-Z][A-Z0-9_]{2,63}$/); +const OpaqueIdSchema = z + .string() + .regex(/^(?:participant|environment|cohort)-[0-9a-f]{16}$/); + +const AssetIdentitySchema = z + .object({ + path: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,159}$/), + size: z + .number() + .int() + .positive() + .max(16 * 1024 ** 3), + sha256: Sha256Schema, + }) + .strict(); + +const CohortReleaseSchema = z + .object({ + version: z.string().regex(/^\d+\.\d+\.\d+$/), + tag: z.string().regex(/^self-hosted-v\d+\.\d+\.\d+$/), + sourceCommit: z.string().regex(/^[0-9a-f]{40}$/), + assets: z.array(AssetIdentitySchema).length(7), + }) + .strict() + .superRefine((release, context) => { + if (release.tag !== `self-hosted-v${release.version}`) { + context.addIssue({ + code: "custom", + path: ["tag"], + message: "release tag does not match release version", + }); + } + const expected = [ + `skillwire-${release.version}-linux-amd64.release.json`, + `skillwire-${release.version}-linux-amd64.release.sigstore.json`, + `skillwire-${release.version}-linux-amd64.tar.zst`, + `skillwire-${release.version}-linux-arm64.release.json`, + `skillwire-${release.version}-linux-arm64.release.sigstore.json`, + `skillwire-${release.version}-linux-arm64.tar.zst`, + "skillwire-trust-policy-v1.json", + ]; + if ( + !isDeepStrictEqual( + release.assets.map(({ path }) => path), + expected, + ) + ) { + context.addIssue({ + code: "custom", + path: ["assets"], + message: "release evidence requires the exact seven ordered assets", + }); + } + if ( + new Set(release.assets.map(({ path }) => path)).size !== + release.assets.length + ) { + context.addIssue({ + code: "custom", + path: ["assets"], + message: "release asset identities must be unique", + }); + } + }); + +const DocumentationSchema = z + .object({ + path: z.literal("distribution/self-hosted/README.md"), + sha256: Sha256Schema, + }) + .strict(); + +const MilestoneNameSchema = z.enum([ + "releaseVerified", + "serviceReady", + "clientIntegrated", + "installationStateIdentified", + "nextSafeRecoveryActionIdentified", + "sixToolDiscovery", + "mcpSearch", + "exactSkillLoad", + "optionalResourceJourney", + "cleanup", +]); + +const MilestoneSchema = z + .object({ + status: z.enum([ + "passed", + "failed", + "timeout", + "abandoned", + "not-applicable", + ]), + publicErrorCodes: z.array(PublicCodeSchema).max(16), + }) + .strict(); + +const ParticipantReleaseSchema = CohortReleaseSchema.extend({ + manifestSha256: Sha256Schema, +}).strict(); + +const REQUIRED_TOOLS = [ + "search_skills", + "load_skill", + "get_skill_resource", + "import_repository", + "list_repositories", + "remove_repository", +] as const; +const ToolNameSchema = z.enum(REQUIRED_TOOLS); + +const ModeratedParticipantSchema = z + .object({ + participantId: OpaqueIdSchema.refine((id) => id.startsWith("participant-")), + independent: z.literal(true), + previouslyInstalledSkillWire: z.literal(false), + attemptNumber: z.literal(1), + replacementForParticipantId: z.null(), + exclusionReason: z + .enum([ + "unsupported-environment", + "prior-install", + "not-independent", + "invalid-starting-state", + ]) + .nullable(), + environment: z + .object({ + environmentId: OpaqueIdSchema.refine((id) => + id.startsWith("environment-"), + ), + clean: z.literal(true), + operatingSystem: z.union([ + z.object({ id: z.literal("ubuntu"), version: z.literal("24.04") }), + z.object({ id: z.literal("debian"), version: z.literal("12") }), + z.object({ id: z.literal("debian"), version: z.literal("13") }), + ]), + architecture: z.enum(["amd64", "arm64"]), + dockerMode: z.enum(["rootful", "rootless"]), + }) + .strict(), + startingState: z + .object({ + skillWireAbsent: z.literal(true), + serviceAbsent: z.literal(true), + selectedClientHasNoSkillWireIntegration: z.literal(true), + noRetainedSkillWireData: z.literal(true), + }) + .strict(), + startedAt: z.iso.datetime({ offset: true }), + endedAt: z.iso.datetime({ offset: true }), + elapsedMilliseconds: z + .number() + .int() + .nonnegative() + .max(24 * 60 * 60_000), + release: ParticipantReleaseSchema, + documentation: DocumentationSchema, + usedOnlyPublishedQuickstart: z.literal(true), + manualConfigurationEdited: z.literal(false), + milestones: z + .object({ + releaseVerified: MilestoneSchema, + serviceReady: MilestoneSchema, + clientIntegrated: MilestoneSchema, + installationStateIdentified: MilestoneSchema, + nextSafeRecoveryActionIdentified: MilestoneSchema, + sixToolDiscovery: MilestoneSchema, + mcpSearch: MilestoneSchema, + exactSkillLoad: MilestoneSchema, + optionalResourceJourney: MilestoneSchema, + cleanup: MilestoneSchema, + }) + .strict(), + publicErrors: z + .array( + z + .object({ + code: PublicCodeSchema, + milestone: MilestoneNameSchema, + recovered: z.boolean(), + }) + .strict(), + ) + .max(32), + moderatorInterventions: z + .array( + z + .object({ + occurredAt: z.iso.datetime({ offset: true }), + category: z.enum([ + "undocumented-command", + "correction", + "procedural-instruction", + ]), + milestone: MilestoneNameSchema, + publicCode: PublicCodeSchema, + }) + .strict(), + ) + .max(32), + documentationClarifications: z + .array( + z + .object({ + occurredAt: z.iso.datetime({ offset: true }), + criterion: z.literal("visible-document-location-only"), + documentPath: z.literal("distribution/self-hosted/README.md"), + sectionId: z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/), + }) + .strict(), + ) + .max(32), + serviceReady: z.boolean(), + discoveredTools: z + .array(ToolNameSchema) + .max(REQUIRED_TOOLS.length) + .superRefine((tools, context) => { + if (new Set(tools).size !== tools.length) { + context.addIssue({ + code: "custom", + message: "discovered tool identities must be unique", + }); + } + }), + journey: z + .object({ + searchCompleted: z.boolean(), + exactSkillLoaded: z.boolean(), + optionalResourceOutcome: z.enum(["completed", "not-applicable"]), + }) + .strict(), + cleanup: z + .object({ + verified: z.boolean(), + completedAt: z.iso.datetime({ offset: true }), + }) + .strict(), + completed: z.boolean(), + unassisted: z.boolean(), + }) + .strict(); + +export type ModeratedUsabilityParticipant = z.infer< + typeof ModeratedParticipantSchema +>; + +export interface ModeratedUsabilityAggregation { + readonly cohortSize: 10; + readonly completedWithinTarget: number; + readonly sc001Required: 10; + readonly sc001Passed: boolean; + readonly unassistedCompletions: number; + readonly sc014Required: 9; + readonly sc014Passed: boolean; +} + +const TARGET_MILLISECONDS = 15 * 60_000; + +function participantCompletion( + participant: ModeratedUsabilityParticipant, +): boolean { + const requiredMilestones = Object.entries(participant.milestones).filter( + ([name]) => name !== "optionalResourceJourney", + ); + const optionalResource = participant.milestones.optionalResourceJourney; + return ( + participant.exclusionReason === null && + participant.elapsedMilliseconds <= TARGET_MILLISECONDS && + requiredMilestones.every( + ([, milestone]) => milestone.status === "passed", + ) && + (optionalResource.status === "passed" || + optionalResource.status === "not-applicable") && + participant.publicErrors.every(({ recovered }) => recovered) && + participant.serviceReady && + isDeepStrictEqual(participant.discoveredTools, REQUIRED_TOOLS) && + participant.journey.searchCompleted && + participant.journey.exactSkillLoaded && + participant.cleanup.verified + ); +} + +export function calculateModeratedUsabilityCohort( + participants: readonly ModeratedUsabilityParticipant[], +): ModeratedUsabilityAggregation { + if (participants.length !== 10) + throw new Error( + "Moderated usability cohort must contain exactly ten participants", + ); + const completedWithinTarget = participants.filter( + participantCompletion, + ).length; + const unassistedCompletions = participants.filter( + (participant) => + participantCompletion(participant) && + participant.moderatorInterventions.length === 0, + ).length; + return { + cohortSize: 10, + completedWithinTarget, + sc001Required: 10, + sc001Passed: completedWithinTarget === 10, + unassistedCompletions, + sc014Required: 9, + sc014Passed: unassistedCompletions >= 9, + }; +} + +const AggregationSchema = z + .object({ + cohortSize: z.literal(10), + completedWithinTarget: z.number().int().min(0).max(10), + sc001Required: z.literal(10), + sc001Passed: z.boolean(), + unassistedCompletions: z.number().int().min(0).max(10), + sc014Required: z.literal(9), + sc014Passed: z.boolean(), + }) + .strict(); + +export const ModeratedUsabilityCohortSchema = z + .object({ + schemaVersion: z.literal("skillwire.moderated-usability/v1"), + evidenceKind: z.enum(["synthetic-fixture", "certified-observation"]), + cohortId: OpaqueIdSchema.refine((id) => id.startsWith("cohort-")), + targetMilliseconds: z.literal(TARGET_MILLISECONDS), + release: CohortReleaseSchema, + documentation: DocumentationSchema, + participants: z.array(ModeratedParticipantSchema).length(10), + aggregation: AggregationSchema, + }) + .strict() + .superRefine((cohort, context) => { + const participantIds = new Set(); + const environmentIds = new Set(); + cohort.participants.forEach((participant, index) => { + if (participantIds.has(participant.participantId)) { + context.addIssue({ + code: "custom", + path: ["participants", index, "participantId"], + message: "participant identities must be unique", + }); + } + participantIds.add(participant.participantId); + if (environmentIds.has(participant.environment.environmentId)) { + context.addIssue({ + code: "custom", + path: ["participants", index, "environment", "environmentId"], + message: "independent participant environments must be unique", + }); + } + environmentIds.add(participant.environment.environmentId); + if (participant.exclusionReason !== null) { + context.addIssue({ + code: "custom", + path: ["participants", index, "exclusionReason"], + message: "an assigned cohort participant cannot be excluded", + }); + } + const elapsed = + Date.parse(participant.endedAt) - Date.parse(participant.startedAt); + if (elapsed < 0 || elapsed !== participant.elapsedMilliseconds) { + context.addIssue({ + code: "custom", + path: ["participants", index, "elapsedMilliseconds"], + message: "participant duration does not match UTC timestamps", + }); + } + const startedAt = Date.parse(participant.startedAt); + const endedAt = Date.parse(participant.endedAt); + const requireSessionTimestamp = ( + occurredAt: string, + path: readonly (string | number)[], + ): void => { + const timestamp = Date.parse(occurredAt); + if (timestamp < startedAt || timestamp > endedAt) { + context.addIssue({ + code: "custom", + path: ["participants", index, ...path], + message: "observation timestamp is outside the participant session", + }); + } + }; + participant.moderatorInterventions.forEach( + ({ occurredAt }, eventIndex) => { + requireSessionTimestamp(occurredAt, [ + "moderatorInterventions", + eventIndex, + "occurredAt", + ]); + }, + ); + participant.documentationClarifications.forEach( + ({ occurredAt }, eventIndex) => { + requireSessionTimestamp(occurredAt, [ + "documentationClarifications", + eventIndex, + "occurredAt", + ]); + }, + ); + requireSessionTimestamp(participant.cleanup.completedAt, [ + "cleanup", + "completedAt", + ]); + Object.entries(participant.milestones).forEach( + ([milestoneName, milestone]) => { + const isOptional = milestoneName === "optionalResourceJourney"; + if (!isOptional && milestone.status === "not-applicable") { + context.addIssue({ + code: "custom", + path: ["participants", index, "milestones", milestoneName], + message: + "only the optional resource milestone may be not applicable", + }); + } + const requiresErrorCode = ["failed", "timeout", "abandoned"].includes( + milestone.status, + ); + if ( + (requiresErrorCode && milestone.publicErrorCodes.length === 0) || + (!requiresErrorCode && milestone.publicErrorCodes.length !== 0) + ) { + context.addIssue({ + code: "custom", + path: [ + "participants", + index, + "milestones", + milestoneName, + "publicErrorCodes", + ], + message: "milestone status and public error codes disagree", + }); + } + milestone.publicErrorCodes.forEach((code) => { + if ( + !participant.publicErrors.some( + (error) => + error.milestone === milestoneName && error.code === code, + ) + ) { + context.addIssue({ + code: "custom", + path: ["participants", index, "publicErrors"], + message: + "milestone public error code has no matching error record", + }); + } + }); + }, + ); + participant.publicErrors.forEach((error, errorIndex) => { + const milestone = participant.milestones[error.milestone]; + if (!milestone.publicErrorCodes.includes(error.code)) { + context.addIssue({ + code: "custom", + path: ["participants", index, "publicErrors", errorIndex], + message: "public error record has no matching milestone code", + }); + } + }); + const exactToolsDiscovered = isDeepStrictEqual( + participant.discoveredTools, + REQUIRED_TOOLS, + ); + const expectedStatuses = { + serviceReady: participant.serviceReady, + sixToolDiscovery: exactToolsDiscovered, + mcpSearch: participant.journey.searchCompleted, + exactSkillLoad: participant.journey.exactSkillLoaded, + cleanup: participant.cleanup.verified, + } as const; + Object.entries(expectedStatuses).forEach(([milestoneName, expected]) => { + const milestone = + participant.milestones[ + milestoneName as keyof typeof expectedStatuses + ]; + if ((milestone.status === "passed") !== expected) { + context.addIssue({ + code: "custom", + path: ["participants", index, "milestones", milestoneName], + message: "milestone status disagrees with structured observation", + }); + } + }); + const expectedResourceStatus = + participant.journey.optionalResourceOutcome === "completed" + ? "passed" + : "not-applicable"; + if ( + participant.milestones.optionalResourceJourney.status !== + expectedResourceStatus + ) { + context.addIssue({ + code: "custom", + path: [ + "participants", + index, + "milestones", + "optionalResourceJourney", + ], + message: "optional resource milestone disagrees with journey outcome", + }); + } + const participantRelease = { + version: participant.release.version, + tag: participant.release.tag, + sourceCommit: participant.release.sourceCommit, + assets: participant.release.assets, + }; + if (!isDeepStrictEqual(participantRelease, cohort.release)) { + context.addIssue({ + code: "custom", + path: ["participants", index, "release"], + message: + "participant release identity differs from the fixed cohort release identity", + }); + } + const manifestPath = `skillwire-${cohort.release.version}-linux-${participant.environment.architecture}.release.json`; + const manifest = cohort.release.assets.find( + ({ path }) => path === manifestPath, + ); + if (manifest?.sha256 !== participant.release.manifestSha256) { + context.addIssue({ + code: "custom", + path: ["participants", index, "release", "manifestSha256"], + message: + "participant manifest identity does not match the architecture asset", + }); + } + if (!isDeepStrictEqual(participant.documentation, cohort.documentation)) { + context.addIssue({ + code: "custom", + path: ["participants", index, "documentation"], + message: + "participant documentation identity differs from the fixed cohort documentation", + }); + } + const completed = participantCompletion(participant); + const unassisted = + completed && participant.moderatorInterventions.length === 0; + if ( + participant.completed !== completed || + participant.unassisted !== unassisted + ) { + context.addIssue({ + code: "custom", + path: ["participants", index, "completed"], + message: + "participant completion or unassisted result is inconsistent with recorded evidence", + }); + } + }); + const calculated = calculateModeratedUsabilityCohort(cohort.participants); + if (!isDeepStrictEqual(calculated, cohort.aggregation)) { + context.addIssue({ + code: "custom", + path: ["aggregation"], + message: "cohort aggregation does not match participant evidence", + }); + } + }); + +export type ModeratedUsabilityCohort = z.infer< + typeof ModeratedUsabilityCohortSchema +>; + +export function validateModeratedUsabilityCohort( + input: unknown, +): ModeratedUsabilityCohort { + return ModeratedUsabilityCohortSchema.parse(input); +} diff --git a/src/onboarding/domain/release-manifest.ts b/src/onboarding/domain/release-manifest.ts index 9298bcb..3767edd 100644 --- a/src/onboarding/domain/release-manifest.ts +++ b/src/onboarding/domain/release-manifest.ts @@ -238,6 +238,9 @@ export const ReleaseManifestSchema = z }, ); +export const COSIGN_3_1_3_TRUSTED_ROOT_MEDIA_TYPE = + "application/vnd.dev.sigstore.trustedroot+json;version=0.1" as const; + export const TrustPolicySchema = z .object({ schemaVersion: z.literal("skillwire.trust-policy/v1"), @@ -247,9 +250,7 @@ export const TrustPolicySchema = z minimumReleaseSequence: z.number().int().positive(), trustedRoot: FileIdentitySchema.omit({ size: true }) .extend({ - mediaType: z.literal( - "application/vnd.dev.sigstore.trustedroot+json;version=0.1", - ), + mediaType: z.literal(COSIGN_3_1_3_TRUSTED_ROOT_MEDIA_TYPE), }) .strict(), cosign: z diff --git a/tests/contract/release/moderated-usability.test.ts b/tests/contract/release/moderated-usability.test.ts new file mode 100644 index 0000000..527151d --- /dev/null +++ b/tests/contract/release/moderated-usability.test.ts @@ -0,0 +1,327 @@ +import { readFile } from "node:fs/promises"; + +import { describe, expect, it } from "vitest"; + +import { + calculateModeratedUsabilityCohort, + type ModeratedUsabilityCohort, + ModeratedUsabilityCohortSchema, + type ModeratedUsabilityParticipant, + validateModeratedUsabilityCohort, +} from "../../../src/onboarding/domain/moderated-usability.js"; + +const SOURCE_COMMIT = "1".repeat(40); +const DOCUMENT_SHA256 = "2".repeat(64); +const MANIFEST_SHA256 = "3".repeat(64); + +const assets = [ + "skillwire-0.2.0-linux-amd64.release.json", + "skillwire-0.2.0-linux-amd64.release.sigstore.json", + "skillwire-0.2.0-linux-amd64.tar.zst", + "skillwire-0.2.0-linux-arm64.release.json", + "skillwire-0.2.0-linux-arm64.release.sigstore.json", + "skillwire-0.2.0-linux-arm64.tar.zst", + "skillwire-trust-policy-v1.json", +].map((path, index) => ({ + path, + size: 1024 + index, + sha256: + index === 0 ? MANIFEST_SHA256 : ((index + 4) % 16).toString(16).repeat(64), +})); + +function syntheticParticipant(index: number): ModeratedUsabilityParticipant { + const startedAt = new Date(Date.UTC(2026, 7, 15, 10, index, 0)); + const endedAt = new Date(startedAt.getTime() + 10 * 60_000); + const passed = { status: "passed" as const, publicErrorCodes: [] }; + return { + participantId: `participant-${index.toString(16).padStart(16, "0")}`, + independent: true, + previouslyInstalledSkillWire: false, + attemptNumber: 1, + replacementForParticipantId: null, + exclusionReason: null, + environment: { + environmentId: `environment-${index.toString(16).padStart(16, "0")}`, + clean: true, + operatingSystem: { id: "ubuntu" as const, version: "24.04" as const }, + architecture: "amd64" as const, + dockerMode: "rootless" as const, + }, + startingState: { + skillWireAbsent: true, + serviceAbsent: true, + selectedClientHasNoSkillWireIntegration: true, + noRetainedSkillWireData: true, + }, + startedAt: startedAt.toISOString(), + endedAt: endedAt.toISOString(), + elapsedMilliseconds: endedAt.getTime() - startedAt.getTime(), + release: { + version: "0.2.0", + tag: "self-hosted-v0.2.0", + sourceCommit: SOURCE_COMMIT, + manifestSha256: MANIFEST_SHA256, + assets, + }, + documentation: { + path: "distribution/self-hosted/README.md", + sha256: DOCUMENT_SHA256, + }, + usedOnlyPublishedQuickstart: true, + manualConfigurationEdited: false, + milestones: { + releaseVerified: passed, + serviceReady: passed, + clientIntegrated: passed, + installationStateIdentified: passed, + nextSafeRecoveryActionIdentified: passed, + sixToolDiscovery: passed, + mcpSearch: passed, + exactSkillLoad: passed, + optionalResourceJourney: passed, + cleanup: passed, + }, + publicErrors: [], + moderatorInterventions: [], + documentationClarifications: [], + serviceReady: true, + discoveredTools: [ + "search_skills", + "load_skill", + "get_skill_resource", + "import_repository", + "list_repositories", + "remove_repository", + ] as const, + journey: { + searchCompleted: true, + exactSkillLoaded: true, + optionalResourceOutcome: "completed" as const, + }, + cleanup: { verified: true, completedAt: endedAt.toISOString() }, + completed: true, + unassisted: true, + }; +} + +function syntheticCohort(): ModeratedUsabilityCohort { + const participants = Array.from({ length: 10 }, (_, index) => + syntheticParticipant(index + 1), + ); + return { + schemaVersion: "skillwire.moderated-usability/v1" as const, + evidenceKind: "synthetic-fixture" as const, + cohortId: "cohort-0000000000000001", + targetMilliseconds: 15 * 60_000, + release: { + version: "0.2.0", + tag: "self-hosted-v0.2.0", + sourceCommit: SOURCE_COMMIT, + assets, + }, + documentation: { + path: "distribution/self-hosted/README.md", + sha256: DOCUMENT_SHA256, + }, + participants, + aggregation: { + cohortSize: 10, + completedWithinTarget: 10, + sc001Required: 10, + sc001Passed: true, + unassistedCompletions: 10, + sc014Required: 9, + sc014Passed: true, + }, + }; +} + +function setInvalid(target: unknown, key: string, value: unknown): void { + (target as Record)[key] = value; +} + +describe("moderated self-hosted usability certification", () => { + it("calculates SC-001 as 10/10 and SC-014 as at least 9/10", () => { + const cohort = validateModeratedUsabilityCohort(syntheticCohort()); + expect(calculateModeratedUsabilityCohort(cohort.participants)).toEqual({ + cohortSize: 10, + completedWithinTarget: 10, + sc001Required: 10, + sc001Passed: true, + unassistedCompletions: 10, + sc014Required: 9, + sc014Passed: true, + }); + + const oneIntervention = structuredClone(syntheticCohort()); + const participant = oneIntervention.participants[0]; + if (participant === undefined) + throw new Error("Synthetic participant missing"); + participant.moderatorInterventions.push({ + occurredAt: participant.startedAt, + category: "procedural-instruction", + milestone: "serviceReady", + publicCode: "MODERATOR_PROCEDURE_REQUIRED", + }); + participant.unassisted = false; + oneIntervention.aggregation.unassistedCompletions = 9; + expect( + validateModeratedUsabilityCohort(oneIntervention).aggregation.sc014Passed, + ).toBe(true); + }); + + it("counts timeout, abandonment, and unrecovered errors as failures", () => { + const cohort = structuredClone(syntheticCohort()); + const participant = cohort.participants[0]; + if (participant === undefined) + throw new Error("Synthetic participant missing"); + participant.milestones.serviceReady = { + status: "timeout", + publicErrorCodes: ["SERVICE_READINESS_TIMEOUT"], + }; + participant.publicErrors.push({ + code: "SERVICE_READINESS_TIMEOUT", + milestone: "serviceReady", + recovered: false, + }); + participant.milestones.clientIntegrated = { + status: "abandoned", + publicErrorCodes: ["JOURNEY_ABANDONED"], + }; + participant.milestones.sixToolDiscovery = { + status: "abandoned", + publicErrorCodes: ["JOURNEY_ABANDONED"], + }; + participant.milestones.mcpSearch = { + status: "abandoned", + publicErrorCodes: ["JOURNEY_ABANDONED"], + }; + participant.milestones.exactSkillLoad = { + status: "abandoned", + publicErrorCodes: ["JOURNEY_ABANDONED"], + }; + participant.milestones.optionalResourceJourney = { + status: "not-applicable", + publicErrorCodes: [], + }; + for (const milestone of [ + "clientIntegrated", + "sixToolDiscovery", + "mcpSearch", + "exactSkillLoad", + ] as const) { + participant.publicErrors.push({ + code: "JOURNEY_ABANDONED", + milestone, + recovered: false, + }); + } + participant.serviceReady = false; + participant.discoveredTools = []; + participant.journey.searchCompleted = false; + participant.journey.exactSkillLoaded = false; + participant.journey.optionalResourceOutcome = "not-applicable"; + participant.completed = false; + participant.unassisted = false; + cohort.aggregation.completedWithinTarget = 9; + cohort.aggregation.sc001Passed = false; + cohort.aggregation.unassistedCompletions = 9; + + expect(validateModeratedUsabilityCohort(cohort).aggregation).toMatchObject({ + cohortSize: 10, + completedWithinTarget: 9, + sc001Required: 10, + sc001Passed: false, + unassistedCompletions: 9, + sc014Required: 9, + sc014Passed: true, + }); + }); + + it("forbids replacement, rerun, post-start exclusion, and duplicate identity", () => { + const replacement = structuredClone(syntheticCohort()); + const participant = replacement.participants[0]; + if (participant === undefined) + throw new Error("Synthetic participant missing"); + setInvalid( + participant, + "replacementForParticipantId", + replacement.participants[1]?.participantId ?? null, + ); + expect(() => validateModeratedUsabilityCohort(replacement)).toThrow( + /replacement/i, + ); + + const rerun = structuredClone(syntheticCohort()); + setInvalid(rerun.participants[0], "attemptNumber", 2); + expect(() => validateModeratedUsabilityCohort(rerun)).toThrow(); + + const excluded = structuredClone(syntheticCohort()); + setInvalid( + excluded.participants[0], + "exclusionReason", + "unsupported-environment", + ); + expect(() => validateModeratedUsabilityCohort(excluded)).toThrow( + /exclude/i, + ); + + const duplicate = structuredClone(syntheticCohort()); + const first = duplicate.participants[0]; + const second = duplicate.participants[1]; + if (first === undefined || second === undefined) + throw new Error("Synthetic participants missing"); + second.participantId = first.participantId; + expect(() => validateModeratedUsabilityCohort(duplicate)).toThrow( + /unique/i, + ); + }); + + it("requires one fixed release, tag, asset set, documentation, and clean supported environment", () => { + const cohort = structuredClone(syntheticCohort()); + const participant = cohort.participants[0]; + if (participant === undefined) + throw new Error("Synthetic participant missing"); + participant.release.tag = "self-hosted-v0.1.0"; + expect(() => validateModeratedUsabilityCohort(cohort)).toThrow( + /release identity/i, + ); + }); + + it("records visible-document clarification separately without hiding procedural intervention", () => { + const cohort = structuredClone(syntheticCohort()); + const participant = cohort.participants[0]; + if (participant === undefined) + throw new Error("Synthetic participant missing"); + participant.documentationClarifications.push({ + occurredAt: participant.startedAt, + criterion: "visible-document-location-only", + documentPath: "distribution/self-hosted/README.md", + sectionId: "verify-the-release", + }); + expect( + validateModeratedUsabilityCohort(cohort).aggregation.sc014Passed, + ).toBe(true); + }); + + it("rejects privacy-sensitive or unknown evidence fields", () => { + const cohort = syntheticCohort() as Record; + cohort["promptContent"] = "must never be collected"; + expect(() => ModeratedUsabilityCohortSchema.parse(cohort)).toThrow(); + }); + + it("ships a strict privacy-safe JSON evidence schema", async () => { + const schema = JSON.parse( + await readFile( + "distribution/self-hosted/moderated-usability.schema.json", + "utf8", + ), + ) as { + additionalProperties?: boolean; + properties?: Record; + }; + expect(schema.additionalProperties).toBe(false); + expect(schema.properties).not.toHaveProperty("promptContent"); + expect(schema.properties).not.toHaveProperty("credentialValue"); + }); +}); diff --git a/tests/contract/release/release-manifest-schema.test.ts b/tests/contract/release/release-manifest-schema.test.ts index 4d4de45..4cc957a 100644 --- a/tests/contract/release/release-manifest-schema.test.ts +++ b/tests/contract/release/release-manifest-schema.test.ts @@ -44,6 +44,63 @@ describe("external self-hosted release manifest", () => { expect(schema.additionalProperties).toBe(false); }); + it("keeps bundle bytes and digests outside the canonical manifest", () => { + const fixture = releaseManifestFixture(); + expect(fixture.signatureBundles).toEqual([ + { + signerId: "github-release-primary", + path: "skillwire-0.1.0-test.1-linux-amd64.release.sigstore.json", + }, + ]); + expect(fixture.signatureBundles[0]).not.toHaveProperty("sha256"); + expect(fixture.signatureBundles[0]).not.toHaveProperty("mediaType"); + expect(() => + ReleaseManifestSchema.parse({ + ...fixture, + signatureBundles: [ + { ...fixture.signatureBundles[0], sha256: "1".repeat(64) }, + ], + }), + ).toThrow(); + }); + + it("rejects duplicated, swapped, and cross-architecture bundle declarations", () => { + const fixture = releaseManifestFixture(); + const declaration = fixture.signatureBundles[0]; + if (declaration === undefined) throw new Error("Fixture bundle is missing"); + expect(() => + ReleaseManifestSchema.parse({ + ...fixture, + signatureBundles: [declaration, declaration], + }), + ).toThrow(); + expect(() => + ReleaseManifestSchema.parse({ + ...fixture, + signatureBundles: [ + { + ...declaration, + path: declaration.path.replace("amd64", "arm64"), + }, + ], + }), + ).toThrow(/sibling/i); + expect(() => + ReleaseManifestSchema.parse({ + ...fixture, + signatureBundles: [ + { + ...declaration, + path: declaration.path.replace( + ".release.sigstore.json", + ".release.unexpected.sigstore.json", + ), + }, + ], + }), + ).toThrow(/sibling/i); + }); + it("rejects non-sibling archives, unbounded archives, duplicate image roles, and platform drift", () => { const fixture = releaseManifestFixture(); expect(() => diff --git a/tests/contract/release/release-tag.test.ts b/tests/contract/release/release-tag.test.ts new file mode 100644 index 0000000..f00cf1f --- /dev/null +++ b/tests/contract/release/release-tag.test.ts @@ -0,0 +1,230 @@ +import { execFileSync } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { verifySelfHostedReleaseTag } from "../../../scripts/verify-self-hosted-release-tag.js"; +import { releaseManifestFixture } from "../../helpers/self-hosted-release-fixtures.js"; + +const roots = new Set(); + +function git(root: string, ...args: readonly string[]): string { + return execFileSync( + "git", + [ + "-c", + `safe.directory=${root}`, + "-c", + "user.name=SkillWire Release Test", + "-c", + "user.email=release-test@invalid.example", + ...args, + ], + { cwd: root, encoding: "utf8" }, + ).trim(); +} + +async function releaseRepository(): Promise<{ + readonly root: string; + readonly commit: string; +}> { + const root = await mkdtemp(join(tmpdir(), "skillwire-release-tag-")); + roots.add(root); + git(root, "init", "--initial-branch=main"); + await writeFile( + join(root, "package.json"), + `${JSON.stringify({ name: "skillwire", version: "0.2.0" })}\n`, + ); + git(root, "add", "package.json"); + git(root, "commit", "-m", "release source"); + const commit = git(root, "rev-parse", "HEAD"); + git(root, "update-ref", "refs/remotes/origin/main", commit); + return { root, commit }; +} + +afterEach(async () => { + await Promise.all([...roots].map((root) => rm(root, { recursive: true }))); + roots.clear(); +}); + +describe("self-hosted release tag", () => { + it("pins the authoritative project version to 0.2.0", async () => { + const packageDocument = JSON.parse( + await readFile("package.json", "utf8"), + ) as { version?: string }; + expect(packageDocument.version).toBe("0.2.0"); + }); + + it("accepts the exact annotated tag recursively peeled to the workflow SHA", async () => { + const fixture = await releaseRepository(); + git( + fixture.root, + "tag", + "--annotate", + "self-hosted-v0.2.0", + "--message", + "SkillWire Self-Hosted v0.2.0", + fixture.commit, + ); + + await expect( + verifySelfHostedReleaseTag({ + repositoryRoot: fixture.root, + githubRef: "refs/tags/self-hosted-v0.2.0", + githubSha: fixture.commit, + }), + ).resolves.toEqual({ + packageVersion: "0.2.0", + tagRef: "refs/tags/self-hosted-v0.2.0", + targetCommit: fixture.commit, + }); + + expect(() => + execFileSync( + process.execPath, + [ + resolve("scripts/verify-self-hosted-release-tag.ts"), + "--repository", + fixture.root, + "--ref", + "refs/tags/self-hosted-v0.2.0", + "--sha", + fixture.commit, + ], + { encoding: "utf8", stdio: "pipe" }, + ), + ).not.toThrow(); + }); + + it("rejects a lightweight tag", async () => { + const fixture = await releaseRepository(); + git(fixture.root, "tag", "self-hosted-v0.2.0", fixture.commit); + + await expect( + verifySelfHostedReleaseTag({ + repositoryRoot: fixture.root, + githubRef: "refs/tags/self-hosted-v0.2.0", + githubSha: fixture.commit, + }), + ).rejects.toThrow(/annotated/i); + }); + + it.each([ + "refs/tags/self-hosted-v0.1.0", + "refs/tags/v0.2.0", + "refs/tags/self-hosted-v0.2", + "refs/tags/self-hosted-v0.2.0-extra", + ])("rejects stale, generic, or malformed ref %s", async (githubRef) => { + const fixture = await releaseRepository(); + const tagName = githubRef.slice("refs/tags/".length); + git( + fixture.root, + "tag", + "--annotate", + tagName, + "--message", + "invalid release tag", + fixture.commit, + ); + + await expect( + verifySelfHostedReleaseTag({ + repositoryRoot: fixture.root, + githubRef, + githubSha: fixture.commit, + }), + ).rejects.toThrow(/package version|release ref/i); + }); + + it("rejects a tag that does not peel to GITHUB_SHA", async () => { + const fixture = await releaseRepository(); + git( + fixture.root, + "tag", + "--annotate", + "self-hosted-v0.2.0", + "--message", + "SkillWire Self-Hosted v0.2.0", + fixture.commit, + ); + const differentSha = "f".repeat(40); + + await expect( + verifySelfHostedReleaseTag({ + repositoryRoot: fixture.root, + githubRef: "refs/tags/self-hosted-v0.2.0", + githubSha: differentSha, + }), + ).rejects.toThrow(/workflow SHA/i); + }); + + it("rejects a manifest whose version or source commit disagrees with the tag", async () => { + const fixture = await releaseRepository(); + git( + fixture.root, + "tag", + "--annotate", + "self-hosted-v0.2.0", + "--message", + "SkillWire Self-Hosted v0.2.0", + fixture.commit, + ); + const manifestPath = join(fixture.root, "release.json"); + const manifest = releaseManifestFixture({ + releaseVersion: "0.2.0", + sourceCommit: "e".repeat(40), + signatureBundles: [ + { + signerId: "github-release-primary", + path: "skillwire-0.2.0-linux-amd64.release.sigstore.json", + }, + ], + archive: { + ...releaseManifestFixture().archive, + path: "skillwire-0.2.0-linux-amd64.tar.zst", + }, + }); + await writeFile(manifestPath, JSON.stringify(manifest)); + + await expect( + verifySelfHostedReleaseTag({ + repositoryRoot: fixture.root, + githubRef: "refs/tags/self-hosted-v0.2.0", + githubSha: fixture.commit, + manifestPath, + }), + ).rejects.toThrow(/manifest version or source commit/i); + }); + + it("rejects an annotated tag whose target is not reachable from protected main", async () => { + const fixture = await releaseRepository(); + git(fixture.root, "switch", "--orphan", "unmerged-release"); + await writeFile( + join(fixture.root, "package.json"), + `${JSON.stringify({ name: "skillwire", version: "0.2.0" })}\n`, + ); + await writeFile(join(fixture.root, "unmerged.txt"), "unmerged\n"); + git(fixture.root, "add", "package.json", "unmerged.txt"); + git(fixture.root, "commit", "-m", "unmerged release"); + const unmergedCommit = git(fixture.root, "rev-parse", "HEAD"); + git( + fixture.root, + "tag", + "--annotate", + "self-hosted-v0.2.0", + "--message", + "SkillWire Self-Hosted v0.2.0", + unmergedCommit, + ); + + await expect( + verifySelfHostedReleaseTag({ + repositoryRoot: fixture.root, + githubRef: "refs/tags/self-hosted-v0.2.0", + githubSha: unmergedCommit, + }), + ).rejects.toThrow(/protected main/i); + }); +}); diff --git a/tests/contract/release/self-hosted-matrix.test.ts b/tests/contract/release/self-hosted-matrix.test.ts index 23b0349..70ca177 100644 --- a/tests/contract/release/self-hosted-matrix.test.ts +++ b/tests/contract/release/self-hosted-matrix.test.ts @@ -59,6 +59,13 @@ describe("Feature 004 certified release matrix", () => { ) as { operatingSystems: unknown[]; architectures: unknown[]; + dockerModes: unknown[]; + certification: { + cellCount: number; + observationsPerCell: string; + releaseIdentity: string; + failedOrIncomplete: string; + }; docker: { minimum: string; tested: string }; compose: { minimum: string; tested: string }; node: string; @@ -73,6 +80,13 @@ describe("Feature 004 certified release matrix", () => { { id: "debian", version: "13" }, ], architectures: ["amd64", "arm64"], + dockerModes: ["rootful", "rootless"], + certification: { + cellCount: 12, + observationsPerCell: "exactly-one", + releaseIdentity: "same-final-tag-and-seven-assets", + failedOrIncomplete: "not-certified-no-replacement-or-exclusion", + }, docker: { minimum: "29.7.2", tested: "29.7.2" }, compose: { minimum: "5.4.0", tested: "5.4.0" }, node: "24.18.0", @@ -155,4 +169,24 @@ describe("Feature 004 certified release matrix", () => { expect(workflowSource).not.toMatch(/^\s+pull_request(?:_target)?:/mu); expect(workflowSource).not.toContain("pull-requests: write"); }); + + it("requires one non-replaceable observation per cell against one final seven-asset release", async () => { + const evidenceContract = await readFile( + "docs/self-hosted-release-evidence.md", + "utf8", + ); + expect(evidenceContract).toMatch( + /exactly one observation for each of the 12 Cartesian\s+cells/, + ); + expect(evidenceContract).toMatch( + /same final\s+`self-hosted-v` annotated tag/, + ); + expect(evidenceContract).toMatch(/exact seven\s+published assets/); + expect(evidenceContract).toContain( + "cannot be replaced, rerun as a substitute", + ); + expect(evidenceContract).toMatch( + /No cell is claimed as passed by this\s+preparation patch/, + ); + }); }); diff --git a/tests/contract/release/signing-workflow.test.ts b/tests/contract/release/signing-workflow.test.ts index 3841092..69cc41f 100644 --- a/tests/contract/release/signing-workflow.test.ts +++ b/tests/contract/release/signing-workflow.test.ts @@ -39,6 +39,37 @@ describe("protected self-hosted signing", () => { expect(workflow).toContain("sign-blob"); expect(workflow).toContain("--bundle"); expect(workflow).toContain("refs/tags/self-hosted-v"); + expect(workflow).toContain("verify-self-hosted-release-tag.ts"); + expect(workflow).toContain('--repository "${GITHUB_WORKSPACE}"'); + expect(workflow).toContain('--ref "${GITHUB_REF}"'); + expect(workflow).toContain('--sha "${GITHUB_SHA}"'); + expect(workflow).toContain('--manifest "${manifest}"'); + expect(workflow).toContain("fetch-depth: 0"); + expect(workflow).not.toContain("git config --global"); + const buildJob = workflow.slice(workflow.indexOf(" build-test-sign:")); + expect(buildJob.indexOf("verify-self-hosted-release-tag.ts")).toBeLessThan( + buildJob.indexOf("pnpm install --frozen-lockfile"), + ); + expect(buildJob.indexOf("verify-self-hosted-release-tag.ts")).toBeLessThan( + buildJob.indexOf("pnpm format:check && pnpm lint"), + ); + expect(workflow).toContain( + "node scripts/verify-self-hosted-release-tag.ts", + ); + const certifiedJob = workflow.slice( + workflow.indexOf(" certified-matrix:"), + workflow.indexOf(" build-test-sign:"), + ); + expect( + certifiedJob.indexOf("verify-self-hosted-release-tag.ts"), + ).toBeLessThan(certifiedJob.indexOf("pnpm install --frozen-lockfile")); + expect(workflow).toContain('--title "SkillWire Self-Hosted v${VERSION}"'); + const tagVerifier = readFileSync( + resolve("scripts/verify-self-hosted-release-tag.ts"), + "utf8", + ); + expect(tagVerifier).toContain("`safe.directory=${repositoryRoot}`"); + expect(tagVerifier).not.toContain("config --global"); expect(workflow).toContain("needs: build-test-sign"); expect(workflow).toContain( "github.ref == format('refs/tags/self-hosted-v{0}'", diff --git a/tests/contract/release/trust-policy-schema.test.ts b/tests/contract/release/trust-policy-schema.test.ts index 3dbe2ce..f68de27 100644 --- a/tests/contract/release/trust-policy-schema.test.ts +++ b/tests/contract/release/trust-policy-schema.test.ts @@ -4,7 +4,10 @@ import { resolve } from "node:path"; import { describe, expect, it } from "vitest"; -import { TrustPolicySchema } from "../../../src/onboarding/domain/release-manifest.js"; +import { + COSIGN_3_1_3_TRUSTED_ROOT_MEDIA_TYPE, + TrustPolicySchema, +} from "../../../src/onboarding/domain/release-manifest.js"; import { canonicalJson, trustPolicyFixture, @@ -19,7 +22,9 @@ describe("self-hosted trust policy", () => { ); expect(policy.signers[0]?.signerId).toBe("github-release-primary"); expect(policy.minimumReleaseSequence).toBe(1); - expect(policy.trustedRoot.mediaType).toContain("trustedroot"); + expect(policy.trustedRoot.mediaType).toBe( + COSIGN_3_1_3_TRUSTED_ROOT_MEDIA_TYPE, + ); expect(policy.overlap.requiredSignerCount).toBe(1); }); @@ -53,6 +58,15 @@ describe("self-hosted trust policy", () => { ], }), ).toThrow(/duplicate/i); + expect(() => + TrustPolicySchema.parse({ + ...trustPolicyFixture(), + trustedRoot: { + ...trustPolicyFixture().trustedRoot, + mediaType: "application/vnd.dev.sigstore.trustedroot.v0.2+json", + }, + }), + ).toThrow(); }); it("pins one canonical first policy and a complete local TrustedRoot identity", () => { @@ -71,9 +85,11 @@ describe("self-hosted trust policy", () => { createHash("sha256").update(trustedRoot).digest("hex"), ); const root = JSON.parse(trustedRoot.toString("utf8")) as { + mediaType?: string; tlogs?: unknown[]; certificateAuthorities?: unknown[]; }; + expect(root.mediaType).toBe(COSIGN_3_1_3_TRUSTED_ROOT_MEDIA_TYPE); expect(root.tlogs?.length).toBeGreaterThan(0); expect(root.certificateAuthorities?.length).toBeGreaterThan(0); }); From 77122c764c8afbf13d6745218dd3c8673c8bdff0 Mon Sep 17 00:00:00 2001 From: Lucenx9 Date: Sat, 15 Aug 2026 03:24:22 +0200 Subject: [PATCH 2/3] test: include release evidence in container suite --- .dockerignore | 1 + tests/contract/release/self-hosted-matrix.test.ts | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.dockerignore b/.dockerignore index d7c78ac..44dc5ba 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,6 +12,7 @@ benchmarks/results/* !benchmarks/results/.gitkeep docs/* !docs/api-keys.md +!docs/self-hosted-release-evidence.md *.log* *.tmp *.swp diff --git a/tests/contract/release/self-hosted-matrix.test.ts b/tests/contract/release/self-hosted-matrix.test.ts index 70ca177..39f2a80 100644 --- a/tests/contract/release/self-hosted-matrix.test.ts +++ b/tests/contract/release/self-hosted-matrix.test.ts @@ -171,10 +171,12 @@ describe("Feature 004 certified release matrix", () => { }); it("requires one non-replaceable observation per cell against one final seven-asset release", async () => { + const dockerIgnore = await readFile(".dockerignore", "utf8"); const evidenceContract = await readFile( "docs/self-hosted-release-evidence.md", "utf8", ); + expect(dockerIgnore).toContain("!docs/self-hosted-release-evidence.md"); expect(evidenceContract).toMatch( /exactly one observation for each of the 12 Cartesian\s+cells/, ); From 95a30f73e8c1d38cff010b943c7e25cd3039147a Mon Sep 17 00:00:00 2001 From: Lucenx9 Date: Sat, 15 Aug 2026 03:28:36 +0200 Subject: [PATCH 3/3] test: keep container evidence check self-contained --- tests/contract/release/self-hosted-matrix.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/contract/release/self-hosted-matrix.test.ts b/tests/contract/release/self-hosted-matrix.test.ts index 39f2a80..70ca177 100644 --- a/tests/contract/release/self-hosted-matrix.test.ts +++ b/tests/contract/release/self-hosted-matrix.test.ts @@ -171,12 +171,10 @@ describe("Feature 004 certified release matrix", () => { }); it("requires one non-replaceable observation per cell against one final seven-asset release", async () => { - const dockerIgnore = await readFile(".dockerignore", "utf8"); const evidenceContract = await readFile( "docs/self-hosted-release-evidence.md", "utf8", ); - expect(dockerIgnore).toContain("!docs/self-hosted-release-evidence.md"); expect(evidenceContract).toMatch( /exactly one observation for each of the 12 Cartesian\s+cells/, );