From 25c3ff97c5003e6371ee419f9ca650e01ead6547 Mon Sep 17 00:00:00 2001 From: Lucenx9 Date: Fri, 14 Aug 2026 15:08:30 +0200 Subject: [PATCH 1/7] feat: complete self-hosted operations lifecycle --- specs/004-self-hosted-onboarding/tasks.md | 84 +- src/credential-bridge/credential-resolver.ts | 5 +- .../adapters/credentials/restrictive-file.ts | 82 +- src/onboarding/adapters/docker/deployment.ts | 49 +- src/onboarding/adapters/docker/environment.ts | 65 + .../adapters/docker/writer-drain.ts | 30 + .../adapters/filesystem/release-installer.ts | 76 +- src/onboarding/adapters/postgres/backup.ts | 312 ++ .../adapters/postgres/schema-compatibility.ts | 55 + src/onboarding/application/backup.ts | 162 + .../application/client-credentials.ts | 84 +- .../application/client-lifecycle.ts | 106 + .../application/diagnostic-probes.ts | 284 ++ src/onboarding/application/doctor.ts | 18 + .../application/production-continuation.ts | 843 ++++ .../application/production-lifecycle.ts | 3815 +++++++++++++++++ .../application/production-setup.ts | 443 +- src/onboarding/application/purge.ts | 163 + src/onboarding/application/recovery.ts | 179 + src/onboarding/application/repair.ts | 121 + .../application/service-secret-rotation.ts | 263 ++ src/onboarding/application/setup.ts | 65 + src/onboarding/application/status.ts | 64 + src/onboarding/application/uninstall.ts | 141 + .../application/upgrade-recovery.ts | 34 + src/onboarding/application/upgrade.ts | 226 + src/onboarding/cli/command-router.ts | 198 +- src/onboarding/cli/main.ts | 9 +- src/onboarding/cli/output.ts | 18 +- src/onboarding/domain/diagnostics.ts | 5 + src/onboarding/domain/operation-journal.ts | 29 +- src/onboarding/domain/ownership.ts | 118 + .../contract/cli/lifecycle-operations.test.ts | 319 ++ .../default-uninstall.test.ts | 174 + .../permanent-removal.test.ts | 104 + .../reinstall-retained-data.test.ts | 71 + .../repeated-setup.test.ts | 108 + .../upgrade-preservation.test.ts | 64 + .../backup-restore-validation.test.ts | 445 ++ .../onboarding/concurrent-mutator.test.ts | 64 + .../onboarding/doctor-classification.test.ts | 59 + .../onboarding/interruption-recovery.test.ts | 278 ++ tests/integration/onboarding/repair.test.ts | 132 + .../service-secret-rotation.test.ts | 190 + .../onboarding/service-setup.test.ts | 33 + .../onboarding/upgrade-compatible.test.ts | 72 + .../upgrade-forward-only-010.test.ts | 84 + .../onboarding/upgrade-interruption.test.ts | 176 + .../onboarding/docker-environment.test.ts | 40 + .../security/onboarding/key-rotation.test.ts | 124 + .../onboarding/removal-boundaries.test.ts | 245 ++ .../upgrade-trust-downgrade.test.ts | 98 + .../unit/onboarding/release-installer.test.ts | 12 +- 53 files changed, 10932 insertions(+), 106 deletions(-) create mode 100644 src/onboarding/adapters/docker/environment.ts create mode 100644 src/onboarding/adapters/docker/writer-drain.ts create mode 100644 src/onboarding/adapters/postgres/backup.ts create mode 100644 src/onboarding/adapters/postgres/schema-compatibility.ts create mode 100644 src/onboarding/application/backup.ts create mode 100644 src/onboarding/application/diagnostic-probes.ts create mode 100644 src/onboarding/application/doctor.ts create mode 100644 src/onboarding/application/production-continuation.ts create mode 100644 src/onboarding/application/production-lifecycle.ts create mode 100644 src/onboarding/application/purge.ts create mode 100644 src/onboarding/application/recovery.ts create mode 100644 src/onboarding/application/repair.ts create mode 100644 src/onboarding/application/service-secret-rotation.ts create mode 100644 src/onboarding/application/status.ts create mode 100644 src/onboarding/application/uninstall.ts create mode 100644 src/onboarding/application/upgrade-recovery.ts create mode 100644 src/onboarding/application/upgrade.ts create mode 100644 tests/contract/cli/lifecycle-operations.test.ts create mode 100644 tests/e2e/self-hosted-onboarding/default-uninstall.test.ts create mode 100644 tests/e2e/self-hosted-onboarding/permanent-removal.test.ts create mode 100644 tests/e2e/self-hosted-onboarding/reinstall-retained-data.test.ts create mode 100644 tests/e2e/self-hosted-onboarding/repeated-setup.test.ts create mode 100644 tests/e2e/self-hosted-onboarding/upgrade-preservation.test.ts create mode 100644 tests/integration/onboarding/backup-restore-validation.test.ts create mode 100644 tests/integration/onboarding/concurrent-mutator.test.ts create mode 100644 tests/integration/onboarding/doctor-classification.test.ts create mode 100644 tests/integration/onboarding/interruption-recovery.test.ts create mode 100644 tests/integration/onboarding/repair.test.ts create mode 100644 tests/integration/onboarding/service-secret-rotation.test.ts create mode 100644 tests/integration/onboarding/upgrade-compatible.test.ts create mode 100644 tests/integration/onboarding/upgrade-forward-only-010.test.ts create mode 100644 tests/integration/onboarding/upgrade-interruption.test.ts create mode 100644 tests/security/onboarding/docker-environment.test.ts create mode 100644 tests/security/onboarding/key-rotation.test.ts create mode 100644 tests/security/onboarding/removal-boundaries.test.ts create mode 100644 tests/security/onboarding/upgrade-trust-downgrade.test.ts diff --git a/specs/004-self-hosted-onboarding/tasks.md b/specs/004-self-hosted-onboarding/tasks.md index 98d9766..048b181 100644 --- a/specs/004-self-hosted-onboarding/tasks.md +++ b/specs/004-self-hosted-onboarding/tasks.md @@ -193,26 +193,26 @@ description: "Dependency-ordered implementation tasks for Feature 004" ### Tests for User Story 4 — write and observe failure first -- [ ] T097 [P] [US4] Add failing `status`, `doctor`, `repair`, `clients rotate-key`, and `maintenance rotate-service-secret` JSON/exit/output tests in `tests/contract/cli/lifecycle-operations.test.ts` -- [ ] T098 [P] [US4] Add failing FR-061 fixture classification plus release/trust/service-secret/dispatcher/ownership/concurrency/backup/recovery findings in `tests/integration/onboarding/doctor-classification.test.ts` -- [ ] T099 [P] [US4] Add failing ten-run account/key/volume/service-secret/source/plugin/MCP identity and zero unchanged-write tests in `tests/e2e/self-hosted-onboarding/repeated-setup.test.ts` -- [ ] T100 [P] [US4] Add failing drifted-owned, ambiguous, external, data-preserving, and no-implicit-secret-rotation repair tests in `tests/integration/onboarding/repair.test.ts` -- [ ] T101 [P] [US4] Add failing process termination after every journal intent/effect/verify/compensate/commit boundary in `tests/integration/onboarding/interruption-recovery.test.ts` -- [ ] T102 [P] [US4] Add failing live-lock, proven-stale-lock, and exactly-one-mutator tests in `tests/integration/onboarding/concurrent-mutator.test.ts` -- [ ] T103 [P] [US4] Add failing replacement client-key verification, old-key retention on failure, and sibling-key isolation tests in `tests/security/onboarding/key-rotation.test.ts` -- [ ] T104 [P] [US4] Add failing database/application rotation tests for explicit preview, independent new value, retained old file, readiness commit, rollback at every boundary, and zero disclosure in `tests/integration/onboarding/service-secret-rotation.test.ts` +- [x] T097 [P] [US4] Add failing `status`, `doctor`, `repair`, `clients rotate-key`, and `maintenance rotate-service-secret` JSON/exit/output tests in `tests/contract/cli/lifecycle-operations.test.ts` +- [x] T098 [P] [US4] Add failing FR-061 fixture classification plus release/trust/service-secret/dispatcher/ownership/concurrency/backup/recovery findings in `tests/integration/onboarding/doctor-classification.test.ts` +- [x] T099 [P] [US4] Add failing ten-run account/key/volume/service-secret/source/plugin/MCP identity and zero unchanged-write tests in `tests/e2e/self-hosted-onboarding/repeated-setup.test.ts` +- [x] T100 [P] [US4] Add failing drifted-owned, ambiguous, external, data-preserving, and no-implicit-secret-rotation repair tests in `tests/integration/onboarding/repair.test.ts` +- [x] T101 [P] [US4] Add failing process termination after every journal intent/effect/verify/compensate/commit boundary in `tests/integration/onboarding/interruption-recovery.test.ts` +- [x] T102 [P] [US4] Add failing live-lock, proven-stale-lock, and exactly-one-mutator tests in `tests/integration/onboarding/concurrent-mutator.test.ts` +- [x] T103 [P] [US4] Add failing replacement client-key verification, old-key retention on failure, and sibling-key isolation tests in `tests/security/onboarding/key-rotation.test.ts` +- [x] T104 [P] [US4] Add failing database/application rotation tests for explicit preview, independent new value, retained old file, readiness commit, rollback at every boundary, and zero disclosure in `tests/integration/onboarding/service-secret-rotation.test.ts` ### Implementation for User Story 4 -- [ ] T105 [P] [US4] Implement bounded installed/live state inspection without credential retrieval or mutation in `src/onboarding/application/status.ts` -- [ ] T106 [P] [US4] Implement layered release/trust/filesystem/Docker/PostgreSQL/migration/catalog/service-secret/credential/bridge/client/source/backup/journal probes in `src/onboarding/application/diagnostic-probes.ts` -- [ ] T107 [US4] Aggregate stable redacted findings and exact safe next actions for `doctor` in `src/onboarding/application/doctor.ts` -- [ ] T108 [US4] Implement observation-based journal recovery and narrow compensation at the last validated boundary in `src/onboarding/application/recovery.ts` -- [ ] T109 [US4] Implement preview-first, ownership-proven, data-preserving repair with no implicit key/service-secret rotation in `src/onboarding/application/repair.ts` -- [ ] T110 [US4] Implement persist-and-verify-before-revoke client-key rotation in `src/onboarding/application/client-credentials.ts` -- [ ] T111 [US4] Implement explicit database/application secret rotation with old-value retention, readiness, commit, and application/config rollback in `src/onboarding/application/service-secret-rotation.ts` -- [ ] T112 [US4] Make unchanged setup a byte-for-byte no-op across secrets, state, clients, sources, catalog, volume, and account in `src/onboarding/application/setup.ts` -- [ ] T113 [US4] Wire status/doctor/repair/client-key/service-secret maintenance routes and final summaries in `src/onboarding/cli/command-router.ts` +- [x] T105 [P] [US4] Implement bounded installed/live state inspection without credential retrieval or mutation in `src/onboarding/application/status.ts` +- [x] T106 [P] [US4] Implement layered release/trust/filesystem/Docker/PostgreSQL/migration/catalog/service-secret/credential/bridge/client/source/backup/journal probes in `src/onboarding/application/diagnostic-probes.ts` +- [x] T107 [US4] Aggregate stable redacted findings and exact safe next actions for `doctor` in `src/onboarding/application/doctor.ts` +- [x] T108 [US4] Implement observation-based journal recovery and narrow compensation at the last validated boundary in `src/onboarding/application/recovery.ts` +- [x] T109 [US4] Implement preview-first, ownership-proven, data-preserving repair with no implicit key/service-secret rotation in `src/onboarding/application/repair.ts` +- [x] T110 [US4] Implement persist-and-verify-before-revoke client-key rotation in `src/onboarding/application/client-credentials.ts` +- [x] T111 [US4] Implement explicit database/application secret rotation with old-value retention, readiness, commit, and application/config rollback in `src/onboarding/application/service-secret-rotation.ts` +- [x] T112 [US4] Make unchanged setup a byte-for-byte no-op across secrets, state, clients, sources, catalog, volume, and account in `src/onboarding/application/setup.ts` +- [x] T113 [US4] Wire status/doctor/repair/client-key/service-secret maintenance routes and final summaries in `src/onboarding/cli/command-router.ts` **Checkpoint**: Operations are idempotent and recoverable; rotation is always explicit and narrowly reversible. @@ -226,23 +226,23 @@ description: "Dependency-ordered implementation tasks for Feature 004" ### Tests for User Story 5 — write and observe failure first -- [ ] T114 [P] [US5] Add failing custom-format dump, checksum, isolated restore/readiness, invalid archive, service-secret-reference-only, and no-raw-secret tests in `tests/integration/onboarding/backup-restore-validation.test.ts` -- [ ] T115 [P] [US5] Add failing no-schema-change upgrade and automatic application/config rollback tests in `tests/integration/onboarding/upgrade-compatible.test.ts` -- [ ] T116 [P] [US5] Add failing migration-010 drain, validated-backup-before-migration, live-schema readback, and unsafe pre-010 rollback refusal tests in `tests/integration/onboarding/upgrade-forward-only-010.test.ts` -- [ ] T117 [P] [US5] Add failing interruption injection for release verification, backup, drain, migration, readiness, clients, and release commit in `tests/integration/onboarding/upgrade-interruption.test.ts` -- [ ] T118 [P] [US5] Add failing repository-memory, client/service-secret, ownership, volume, backup, source, and unrelated-profile preservation tests in `tests/e2e/self-hosted-onboarding/upgrade-preservation.test.ts` -- [ ] T119 [P] [US5] Add failing upgrade rejection tests for lower release/policy sequence, stale policy, bad overlap, denied signer/material, and incompatible restored executable trust in `tests/security/onboarding/upgrade-trust-downgrade.test.ts` +- [x] T114 [P] [US5] Add failing custom-format dump, checksum, isolated restore/readiness, invalid archive, service-secret-reference-only, and no-raw-secret tests in `tests/integration/onboarding/backup-restore-validation.test.ts` +- [x] T115 [P] [US5] Add failing no-schema-change upgrade and automatic application/config rollback tests in `tests/integration/onboarding/upgrade-compatible.test.ts` +- [x] T116 [P] [US5] Add failing migration-010 drain, validated-backup-before-migration, live-schema readback, and unsafe pre-010 rollback refusal tests in `tests/integration/onboarding/upgrade-forward-only-010.test.ts` +- [x] T117 [P] [US5] Add failing interruption injection for release verification, backup, drain, migration, readiness, clients, and release commit in `tests/integration/onboarding/upgrade-interruption.test.ts` +- [x] T118 [P] [US5] Add failing repository-memory, client/service-secret, ownership, volume, backup, source, and unrelated-profile preservation tests in `tests/e2e/self-hosted-onboarding/upgrade-preservation.test.ts` +- [x] T119 [P] [US5] Add failing upgrade rejection tests for lower release/policy sequence, stale policy, bad overlap, denied signer/material, and incompatible restored executable trust in `tests/security/onboarding/upgrade-trust-downgrade.test.ts` ### Implementation for User Story 5 -- [ ] T120 [P] [US5] Implement `pg_dump -Fc`, isolated PostgreSQL 17.10 restore, safe `pg_restore`, checksums, invariants, readiness, and validation cleanup in `src/onboarding/adapters/postgres/backup.ts` -- [ ] T121 [US5] Implement protected backup-set state and recovery manifests containing only non-secret service/client credential references in `src/onboarding/application/backup.ts` -- [ ] T122 [P] [US5] Implement manifest/live-schema compatibility and forward-only rollback decisions in `src/onboarding/adapters/postgres/schema-compatibility.ts` -- [ ] T123 [P] [US5] Implement application, ingestion, and administration writer draining/restart controls in `src/onboarding/adapters/docker/writer-drain.ts` -- [ ] T124 [US5] Implement target trust/release verification, anti-downgrade, backup, confirmation, migration, readiness, client verification, and active-policy/release commit in `src/onboarding/application/upgrade.ts` -- [ ] T125 [US5] Implement compatible application/config rollback and restore-required guidance with backup identity, release, data-loss boundary, and erased-memory warning in `src/onboarding/application/upgrade-recovery.ts` -- [ ] T126 [US5] Integrate upgrade journal boundaries and atomic active release/trust-policy selection in `src/onboarding/adapters/filesystem/release-installer.ts` -- [ ] T127 [US5] Wire `backup` and `upgrade --release` previews, exit classes, backup IDs, rollback boundaries, and recovery summaries in `src/onboarding/cli/command-router.ts` +- [x] T120 [P] [US5] Implement `pg_dump -Fc`, isolated PostgreSQL 17.10 restore, safe `pg_restore`, checksums, invariants, readiness, and validation cleanup in `src/onboarding/adapters/postgres/backup.ts` +- [x] T121 [US5] Implement protected backup-set state and recovery manifests containing only non-secret service/client credential references in `src/onboarding/application/backup.ts` +- [x] T122 [P] [US5] Implement manifest/live-schema compatibility and forward-only rollback decisions in `src/onboarding/adapters/postgres/schema-compatibility.ts` +- [x] T123 [P] [US5] Implement application, ingestion, and administration writer draining/restart controls in `src/onboarding/adapters/docker/writer-drain.ts` +- [x] T124 [US5] Implement target trust/release verification, anti-downgrade, backup, confirmation, migration, readiness, client verification, and active-policy/release commit in `src/onboarding/application/upgrade.ts` +- [x] T125 [US5] Implement compatible application/config rollback and restore-required guidance with backup identity, release, data-loss boundary, and erased-memory warning in `src/onboarding/application/upgrade-recovery.ts` +- [x] T126 [US5] Integrate upgrade journal boundaries and atomic active release/trust-policy selection in `src/onboarding/adapters/filesystem/release-installer.ts` +- [x] T127 [US5] Wire `backup` and `upgrade --release` previews, exit classes, backup IDs, rollback boundaries, and recovery summaries in `src/onboarding/cli/command-router.ts` **Checkpoint**: Upgrades are signed, restore-backed, schema-aware, anti-downgrade, and interruption-safe. @@ -256,20 +256,20 @@ description: "Dependency-ordered implementation tasks for Feature 004" ### Tests for User Story 6 — write and observe failure first -- [ ] T128 [P] [US6] Add failing owned-only client removal, container stop, retained volume/backups/client and service secrets/releases/trust/ownership, and unrelated-state tests in `tests/e2e/self-hosted-onboarding/default-uninstall.test.ts` -- [ ] T129 [P] [US6] Add failing retained installation/data/secret reuse and duplicate-free account/key/MCP/plugin reinstall tests in `tests/e2e/self-hosted-onboarding/reinstall-retained-data.test.ts` -- [ ] T130 [P] [US6] Add failing separate purge preview/hash/installation-ID confirmation and exact named deletion tests in `tests/e2e/self-hosted-onboarding/permanent-removal.test.ts` -- [ ] T131 [P] [US6] Add failing external/ambiguous/drifted/concurrent/symlinked/interrupted removal tests proving zero mutation outside current ownership in `tests/security/onboarding/removal-boundaries.test.ts` +- [x] T128 [P] [US6] Add failing owned-only client removal, container stop, retained volume/backups/client and service secrets/releases/trust/ownership, and unrelated-state tests in `tests/e2e/self-hosted-onboarding/default-uninstall.test.ts` +- [x] T129 [P] [US6] Add failing retained installation/data/secret reuse and duplicate-free account/key/MCP/plugin reinstall tests in `tests/e2e/self-hosted-onboarding/reinstall-retained-data.test.ts` +- [x] T130 [P] [US6] Add failing separate purge preview/hash/installation-ID confirmation and exact named deletion tests in `tests/e2e/self-hosted-onboarding/permanent-removal.test.ts` +- [x] T131 [P] [US6] Add failing external/ambiguous/drifted/concurrent/symlinked/interrupted removal tests proving zero mutation outside current ownership in `tests/security/onboarding/removal-boundaries.test.ts` ### Implementation for User Story 6 -- [ ] T132 [US6] Implement matching-owned-only Codex/Claude MCP/plugin/marketplace/credential/key inverse operations in `src/onboarding/application/client-lifecycle.ts` -- [ ] T133 [US6] Implement data/service-secret/trust/release-preserving default uninstall and retained-state transitions in `src/onboarding/application/uninstall.ts` -- [ ] T134 [P] [US6] Implement exact owned-asset purge planning, separate confirmation scope, safe deletion, and unrecoverable inventory in `src/onboarding/application/purge.ts` -- [ ] T135 [US6] Enforce retain-by-default/remove-only-on-purge dispositions and current identity proof in `src/onboarding/domain/ownership.ts` -- [ ] T136 [US6] Implement interrupted uninstall convergence and ambiguity-safe recovery in `src/onboarding/application/recovery.ts` -- [ ] T137 [US6] Wire `clients uninstall`, `uninstall`, and `purge` previews, confirmations, and stable results in `src/onboarding/cli/command-router.ts` -- [ ] T138 [US6] Add retained installation discovery and duplicate-free reactivation to `src/onboarding/application/setup.ts` +- [x] T132 [US6] Implement matching-owned-only Codex/Claude MCP/plugin/marketplace/credential/key inverse operations in `src/onboarding/application/client-lifecycle.ts` +- [x] T133 [US6] Implement data/service-secret/trust/release-preserving default uninstall and retained-state transitions in `src/onboarding/application/uninstall.ts` +- [x] T134 [P] [US6] Implement exact owned-asset purge planning, separate confirmation scope, safe deletion, and unrecoverable inventory in `src/onboarding/application/purge.ts` +- [x] T135 [US6] Enforce retain-by-default/remove-only-on-purge dispositions and current identity proof in `src/onboarding/domain/ownership.ts` +- [x] T136 [US6] Implement interrupted uninstall convergence and ambiguity-safe recovery in `src/onboarding/application/recovery.ts` +- [x] T137 [US6] Wire `clients uninstall`, `uninstall`, and `purge` previews, confirmations, and stable results in `src/onboarding/cli/command-router.ts` +- [x] T138 [US6] Add retained installation discovery and duplicate-free reactivation to `src/onboarding/application/setup.ts` **Checkpoint**: Default removal is reversible; purge cannot reuse uninstall confirmation or delete external/ambiguous state. diff --git a/src/credential-bridge/credential-resolver.ts b/src/credential-bridge/credential-resolver.ts index c779b23..5f76eb2 100644 --- a/src/credential-bridge/credential-resolver.ts +++ b/src/credential-bridge/credential-resolver.ts @@ -38,8 +38,9 @@ const BridgeStateSchema = z credentialReference: z .string() .regex( - /^(?:restrictive-file:(?:codex|claude)|secret-service:(?:codex|claude):[0-9a-f-]{36})$/, + /^(?:restrictive-file:(?:codex|claude)(?::[0-9a-f-]{36})?|secret-service:(?:codex|claude):[0-9a-f-]{36})$/, ), + keyId: z.uuid().optional(), }) .strict(), ) @@ -106,7 +107,7 @@ export class CredentialResolver { ); if ( entry === undefined || - (entry.credentialReference !== `restrictive-file:${client}` && + (!entry.credentialReference.startsWith(`restrictive-file:${client}`) && !entry.credentialReference.startsWith(`secret-service:${client}:`)) ) { throw new BridgeFailure("BRIDGE_CREDENTIAL_UNAVAILABLE"); diff --git a/src/onboarding/adapters/credentials/restrictive-file.ts b/src/onboarding/adapters/credentials/restrictive-file.ts index f0c187e..5eb0c2f 100644 --- a/src/onboarding/adapters/credentials/restrictive-file.ts +++ b/src/onboarding/adapters/credentials/restrictive-file.ts @@ -9,7 +9,8 @@ import { validateOwnedPath, } from "../filesystem/safe-paths.js"; -export type RestrictiveFileReference = `restrictive-file:${ClientName}`; +export type RestrictiveFileReference = + `restrictive-file:${ClientName}` | `restrictive-file:${ClientName}:${string}`; export class RestrictiveFileCredentialStore { public constructor( @@ -52,6 +53,18 @@ export class RestrictiveFileCredentialStore { return root; } + private async syncDirectory(path: string): Promise { + const handle = await open( + path, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ); + try { + await handle.sync(); + } finally { + await handle.close(); + } + } + async store( client: ClientName, token: string, @@ -79,16 +92,67 @@ export class RestrictiveFileCredentialStore { } finally { await handle.close(); } + await this.syncDirectory(root); return `restrictive-file:${client}`; } + async storeReplacement( + client: ClientName, + token: string, + referenceId: string, + ): Promise { + if ( + !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test( + referenceId, + ) + ) + throw new Error("Replacement credential identity is invalid"); + if (parseApiKeyToken(token) === undefined) + throw new Error("Client credential has an invalid shape"); + const root = await this.credentialRoot(); + const path = resolve(root, `${client}.${referenceId}.key`); + const handle = await open( + path, + constants.O_WRONLY | + constants.O_CREAT | + constants.O_EXCL | + constants.O_NOFOLLOW, + 0o600, + ); + try { + await handle.writeFile(token, "ascii"); + await handle.sync(); + } finally { + await handle.close(); + } + await this.syncDirectory(root); + return `restrictive-file:${client}:${referenceId}`; + } + + private referencePath(reference: RestrictiveFileReference): { + readonly client: ClientName; + readonly name: string; + } { + const match = /^restrictive-file:(codex|claude)(?::([0-9a-f-]{36}))?$/.exec( + reference, + ); + if (match === null) throw new Error("Credential reference is invalid"); + const client = match[1] as ClientName; + const generation = match[2]; + return { + client, + name: + generation === undefined + ? `${client}.key` + : `${client}.${generation}.key`, + }; + } + async lookup(reference: RestrictiveFileReference): Promise { - const client = reference.slice("restrictive-file:".length); - if (client !== "codex" && client !== "claude") - throw new Error("Credential reference is invalid"); + const { name } = this.referencePath(reference); const root = await this.credentialRoot(); const handle = await open( - resolve(root, `${client}.key`), + resolve(root, name), constants.O_RDONLY | constants.O_NOFOLLOW, ); try { @@ -116,17 +180,16 @@ export class RestrictiveFileCredentialStore { } async remove(reference: RestrictiveFileReference): Promise { - const client = reference.slice("restrictive-file:".length); - if (client !== "codex" && client !== "claude") - throw new Error("Credential reference is invalid"); + const { name } = this.referencePath(reference); const root = await this.credentialRoot(); - const path = await validateOwnedPath(resolve(root, `${client}.key`), root); + const path = await validateOwnedPath(resolve(root, name), root); const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); try { const stats = await handle.stat(); if ( !stats.isFile() || stats.nlink !== 1 || + stats.uid !== process.getuid?.() || (stats.mode & 0o777) !== 0o600 ) throw new Error("Credential ownership is ambiguous"); @@ -134,5 +197,6 @@ export class RestrictiveFileCredentialStore { await handle.close(); } await unlink(path); + await this.syncDirectory(root); } } diff --git a/src/onboarding/adapters/docker/deployment.ts b/src/onboarding/adapters/docker/deployment.ts index acd9dd0..663c9e7 100644 --- a/src/onboarding/adapters/docker/deployment.ts +++ b/src/onboarding/adapters/docker/deployment.ts @@ -8,6 +8,7 @@ import { type CommandOptions, type CommandResult, } from "../process/command-runner.js"; +import { dockerProcessEnvironment } from "./environment.js"; type CommandExecutor = (options: CommandOptions) => Promise; @@ -22,6 +23,7 @@ export interface DeploymentOptions { readonly applicationPepperFile: string; readonly runtimeSocketDirectory: string; readonly socketPath: string; + readonly hostEnvironment?: NodeJS.ProcessEnv | undefined; readonly run?: CommandExecutor | undefined; readonly readinessProbe?: ((socketPath: string, signal: AbortSignal) => Promise) | undefined; @@ -118,8 +120,7 @@ export class DeploymentAdapter { executable: resolve(this.options.dockerExecutable), args, environment: { - PATH: "/usr/bin:/bin", - LANG: "C.UTF-8", + ...dockerProcessEnvironment(this.options.hostEnvironment ?? {}), SKILLWIRE_COMPOSE_PROJECT: this.options.projectName, SKILLWIRE_POSTGRES_VOLUME: this.options.volumeName, SKILLWIRE_IMAGE: this.options.skillwireImage, @@ -281,4 +282,48 @@ export class DeploymentAdapter { `SkillWire readiness failed${lastError instanceof Error ? `: ${lastError.message}` : ""}`, ); } + + async observeOwnedService( + service: "skillwire" | "postgres", + signal: AbortSignal, + ): Promise { + const listed = await this.command( + [ + "compose", + "--project-name", + this.options.projectName, + "--file", + this.options.composePath, + "ps", + "--all", + "--quiet", + service, + ], + signal, + ); + const identities = listed.stdout.trim().split("\n").filter(Boolean); + if (identities.length === 0) return false; + if (identities.length !== 1) + throw new Error("Owned Compose service identity is ambiguous"); + const inspected = await this.command( + [ + "container", + "inspect", + identities[0] ?? "", + "--format", + '{{index .Config.Labels "com.docker.compose.project"}}|{{index .Config.Labels "com.docker.compose.service"}}|{{.Config.Image}}', + ], + signal, + ); + const expectedImage = + service === "skillwire" + ? this.options.skillwireImage + : this.options.postgresImage; + if ( + inspected.stdout.trim() !== + `${this.options.projectName}|${service}|${expectedImage}` + ) + throw new Error("Owned Compose service labels or image identity drifted"); + return true; + } } diff --git a/src/onboarding/adapters/docker/environment.ts b/src/onboarding/adapters/docker/environment.ts new file mode 100644 index 0000000..3b4a082 --- /dev/null +++ b/src/onboarding/adapters/docker/environment.ts @@ -0,0 +1,65 @@ +import { isAbsolute } from "node:path"; + +const ROUTING_KEYS = [ + "HOME", + "XDG_CONFIG_HOME", + "XDG_RUNTIME_DIR", + "DOCKER_HOST", + "DOCKER_CONTEXT", + "DOCKER_CONFIG", + "DOCKER_CERT_PATH", + "DOCKER_TLS_VERIFY", +] as const; + +function safeRoutingValue(key: (typeof ROUTING_KEYS)[number], value: string) { + if (value.length === 0 || value.length > 4096 || /[\0\r\n]/.test(value)) + throw new Error(`Docker ${key} routing value is invalid`); + if ( + (key === "HOME" || + key === "XDG_CONFIG_HOME" || + key === "XDG_RUNTIME_DIR" || + key === "DOCKER_CONFIG" || + key === "DOCKER_CERT_PATH") && + !isAbsolute(value) + ) + throw new Error(`Docker ${key} path must be absolute`); + if ( + key === "DOCKER_HOST" && + !value.startsWith("unix://") && + !value.startsWith("npipe://") + ) + throw new Error("Only a local Docker endpoint is supported"); + if ( + key === "DOCKER_CONTEXT" && + !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/.test(value) + ) + throw new Error("Docker context name is invalid"); + if (key === "DOCKER_TLS_VERIFY" && value !== "0" && value !== "1") + throw new Error("Docker TLS routing value is invalid"); + return value; +} + +export function dockerProcessEnvironment( + ambient: NodeJS.ProcessEnv, + explicit: Readonly> = {}, +): NodeJS.ProcessEnv { + const result: NodeJS.ProcessEnv = { + PATH: "/usr/bin:/bin", + LANG: "C.UTF-8", + }; + for (const key of ROUTING_KEYS) { + const value = ambient[key]; + if (value !== undefined) result[key] = safeRoutingValue(key, value); + } + for (const [key, value] of Object.entries(explicit)) { + if ( + !/^SKILLWIRE_[A-Z0-9_]{1,96}$/.test(key) || + value.length === 0 || + value.length > 4096 || + /[\0\r\n]/.test(value) + ) + throw new Error("Explicit Docker Compose environment is invalid"); + result[key] = value; + } + return result; +} diff --git a/src/onboarding/adapters/docker/writer-drain.ts b/src/onboarding/adapters/docker/writer-drain.ts new file mode 100644 index 0000000..df4b661 --- /dev/null +++ b/src/onboarding/adapters/docker/writer-drain.ts @@ -0,0 +1,30 @@ +export interface WriterControls { + stopAdministration(signal: AbortSignal): Promise; + stopIngestion(signal: AbortSignal): Promise; + stopApplication(signal: AbortSignal): Promise; + verifyNoWriters(signal: AbortSignal): Promise; + startApplication(signal: AbortSignal): Promise; + startIngestion(signal: AbortSignal): Promise; + startAdministration(signal: AbortSignal): Promise; +} + +export async function drainWriters( + controls: WriterControls, + signal: AbortSignal, +): Promise { + if (signal.aborted) throw new Error("Writer drain cancelled"); + await controls.stopAdministration(signal); + await controls.stopIngestion(signal); + await controls.stopApplication(signal); + if (!(await controls.verifyNoWriters(signal))) + throw new Error("Writer drain could not prove a quiescent database"); +} + +export async function restartWriters( + controls: WriterControls, + signal: AbortSignal, +): Promise { + await controls.startApplication(signal); + await controls.startIngestion(signal); + await controls.startAdministration(signal); +} diff --git a/src/onboarding/adapters/filesystem/release-installer.ts b/src/onboarding/adapters/filesystem/release-installer.ts index 454bc1b..1919aaa 100644 --- a/src/onboarding/adapters/filesystem/release-installer.ts +++ b/src/onboarding/adapters/filesystem/release-installer.ts @@ -13,6 +13,7 @@ import { import { join, relative, resolve, sep } from "node:path"; import type { ReleaseManifest } from "../../domain/release-manifest.js"; +import type { OperationJournal } from "../../domain/operation-journal.js"; import { OwnershipProofSchema } from "../../domain/ownership.js"; import { atomicWriteJson } from "./atomic-state.js"; import { validateOwnedDirectory, validateOwnedPath } from "./safe-paths.js"; @@ -29,6 +30,7 @@ export interface InstallReleaseOptions { readonly manifestSha256: string; readonly trustPolicyPath: string; readonly tarExecutable?: string | undefined; + readonly activate?: boolean | undefined; } export interface InstalledReleasePaths { @@ -122,6 +124,13 @@ async function extractedInventory( return result; } +export async function releaseDirectoryIdentity(root: string): Promise { + return createHash("sha256") + .update("skillwire-release-directory-v1\0") + .update(JSON.stringify(await extractedInventory(resolve(root)))) + .digest("hex"); +} + async function persistTrustPolicy( sourcePath: string, manifest: ReleaseManifest, @@ -463,20 +472,21 @@ export async function installVerifiedRelease( await rename(launcherStage, launcherPath); await chmod(launcherPath, 0o700); } - await atomicWriteJson( - resolve(stateRoot, "active-release.json"), - { - schemaVersion: "skillwire.active-release/v1", - releaseVersion: options.manifest.releaseVersion, - releaseSequence: options.manifest.releaseSequence, - trustPolicySequence: options.manifest.trustPolicySequence, - architecture: options.manifest.architecture, - manifestSha256: options.manifestSha256, - archiveSha256: options.manifest.archive.sha256, - trustPolicyPath, - }, - stateRoot, - ); + if (options.activate !== false) + await atomicWriteJson( + resolve(stateRoot, "active-release.json"), + { + schemaVersion: "skillwire.active-release/v1", + releaseVersion: options.manifest.releaseVersion, + releaseSequence: options.manifest.releaseSequence, + trustPolicySequence: options.manifest.trustPolicySequence, + architecture: options.manifest.architecture, + manifestSha256: options.manifestSha256, + archiveSha256: options.manifest.archive.sha256, + trustPolicyPath, + }, + stateRoot, + ); await atomicWriteJson( ownershipPath, OwnershipProofSchema.parse({ @@ -496,3 +506,41 @@ export async function installVerifiedRelease( ownershipPath, }; } + +export interface ActiveReleaseSelection { + readonly schemaVersion: "skillwire.active-release/v1"; + readonly releaseVersion: string; + readonly releaseSequence: number; + readonly trustPolicySequence: number; + readonly architecture: "amd64" | "arm64"; + readonly manifestSha256: string; + readonly archiveSha256: string; + readonly trustPolicyPath: string; +} + +export async function commitActiveReleaseSelection(options: { + readonly stateRoot: string; + readonly selection: ActiveReleaseSelection; + readonly journal: OperationJournal; + readonly signal: AbortSignal; +}): Promise { + await options.journal.runEffect({ + step: "active-release-selection", + intent: { + releaseSequence: options.selection.releaseSequence, + trustPolicySequence: options.selection.trustPolicySequence, + manifestSha256: options.selection.manifestSha256, + }, + signal: options.signal, + action: () => + atomicWriteJson( + resolve(options.stateRoot, "active-release.json"), + options.selection, + options.stateRoot, + ), + verification: () => ({ + releaseSequence: options.selection.releaseSequence, + selected: true, + }), + }); +} diff --git a/src/onboarding/adapters/postgres/backup.ts b/src/onboarding/adapters/postgres/backup.ts new file mode 100644 index 0000000..0e8ef57 --- /dev/null +++ b/src/onboarding/adapters/postgres/backup.ts @@ -0,0 +1,312 @@ +import { createHash, randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { chmod, mkdir, open, rm } from "node:fs/promises"; +import { basename, isAbsolute, relative, resolve } from "node:path"; + +import { + runCommand, + type CommandOptions, + type CommandResult, +} from "../process/command-runner.js"; +import { + validateOwnedDirectory, + validateOwnedPath, +} from "../filesystem/safe-paths.js"; +import { dockerProcessEnvironment } from "../docker/environment.js"; + +const COMPOSE_KEYS = [ + "SKILLWIRE_COMPOSE_PROJECT", + "SKILLWIRE_POSTGRES_VOLUME", + "SKILLWIRE_IMAGE", + "SKILLWIRE_POSTGRES_IMAGE", + "SKILLWIRE_DATABASE_PASSWORD_SECRET_FILE", + "SKILLWIRE_APPLICATION_PEPPER_SECRET_FILE", + "SKILLWIRE_RUNTIME_SOCKET_DIRECTORY", + "SKILLWIRE_RUNTIME_UID", + "SKILLWIRE_RUNTIME_GID", +] as const; + +export interface RestoredDatabaseValidation { + readonly latestMigration: string; + readonly invariantsValid: boolean; + readonly catalogValid: boolean; + readonly ready: boolean; +} + +export interface PostgresBackupOptions { + readonly dockerExecutable: string; + readonly composePath: string; + readonly projectName: string; + readonly installationId: string; + readonly protectedRoot: string; + readonly backupsRoot: string; + readonly postgresImage: string; + readonly environment?: NodeJS.ProcessEnv | undefined; + readonly expectedLatestMigration?: string | undefined; + readonly run?: + ((options: CommandOptions) => Promise) | undefined; + readonly validateRestoredDatabase: ( + containerName: string, + signal: AbortSignal, + ) => Promise; +} + +export interface ValidatedPostgresBackup { + readonly backupId: string; + readonly archivePath: string; + readonly archiveSha256: string; + readonly validation: RestoredDatabaseValidation; +} + +export class PostgresBackupAdapter { + private readonly run: (options: CommandOptions) => Promise; + + public constructor(private readonly options: PostgresBackupOptions) { + if (!/^skillwire-[a-z0-9-]+$/.test(options.projectName)) + throw new Error("Backup project identity is invalid"); + if ( + !/^docker\.io\/library\/postgres@sha256:[0-9a-f]{64}$/.test( + options.postgresImage, + ) + ) + throw new Error("Backup restore image must be digest-pinned PostgreSQL"); + const backupRelative = relative( + resolve(options.protectedRoot), + resolve(options.backupsRoot), + ); + if ( + backupRelative === "" || + backupRelative.startsWith("..") || + isAbsolute(backupRelative) + ) + throw new Error("Backup directory must be below its protected root"); + this.run = options.run ?? runCommand; + } + + private command( + args: readonly string[], + signal: AbortSignal, + deadlineMilliseconds = 120_000, + ): Promise { + const ambient = this.options.environment ?? {}; + const explicit: Record = {}; + for (const key of COMPOSE_KEYS) { + const value = ambient[key]; + if (value !== undefined) explicit[key] = value; + } + return this.run({ + executable: resolve(this.options.dockerExecutable), + args, + environment: dockerProcessEnvironment(ambient, explicit), + deadlineMilliseconds, + maximumOutputBytes: 128 * 1024, + signal, + }); + } + + private async waitForValidationDatabase( + containerName: string, + signal: AbortSignal, + ): Promise { + let lastError: unknown; + let consecutiveReadyChecks = 0; + for (let attempt = 0; attempt < 60; attempt += 1) { + if (signal.aborted) throw new Error("Backup validation cancelled"); + try { + await this.command( + [ + "exec", + containerName, + "pg_isready", + "--username=postgres", + "--dbname=postgres", + ], + signal, + 2_000, + ); + consecutiveReadyChecks += 1; + if (consecutiveReadyChecks >= 2) return; + } catch (error) { + lastError = error; + consecutiveReadyChecks = 0; + } + await new Promise((done) => { + setTimeout(done, 250); + }); + } + throw new Error("Isolated PostgreSQL restore target did not become ready", { + cause: lastError, + }); + } + + async createAndValidate( + signal: AbortSignal, + ): Promise { + if (signal.aborted) throw new Error("Backup cancelled"); + const backupId = randomUUID(); + const protectedRoot = resolve(this.options.protectedRoot); + await validateOwnedDirectory(protectedRoot, protectedRoot); + const root = await validateOwnedPath( + resolve(this.options.backupsRoot), + protectedRoot, + ); + await mkdir(root, { recursive: true, mode: 0o700 }); + await validateOwnedDirectory(root, protectedRoot); + const backupRoot = await validateOwnedPath(resolve(root, backupId), root); + await mkdir(backupRoot, { mode: 0o700 }); + await validateOwnedDirectory(backupRoot, root); + const archivePath = await validateOwnedPath( + resolve(backupRoot, "database.dump"), + backupRoot, + ); + const containerArchive = `/tmp/skillwire-${backupId}.dump`; + const compose = [ + "compose", + "--project-name", + this.options.projectName, + "--file", + resolve(this.options.composePath), + ]; + try { + await this.command( + [ + ...compose, + "exec", + "-T", + "postgres", + "pg_dump", + "--username=skillwire", + "--dbname=skillwire", + "--format=custom", + "--no-owner", + "--no-acl", + `--file=${containerArchive}`, + ], + signal, + ); + try { + await this.command( + [...compose, "cp", `postgres:${containerArchive}`, archivePath], + signal, + ); + } finally { + await this.command( + [...compose, "exec", "-T", "postgres", "rm", "-f", containerArchive], + AbortSignal.timeout(30_000), + ).catch(() => undefined); + } + await chmod(archivePath, 0o600); + const handle = await open( + archivePath, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + let archiveSha256: string; + try { + const stats = await handle.stat(); + if ( + !stats.isFile() || + stats.nlink !== 1 || + stats.uid !== process.getuid?.() || + (stats.mode & 0o777) !== 0o600 || + stats.size < 8 || + stats.size > 16 * 1024 * 1024 * 1024 + ) + throw new Error("Backup archive is unsafe or invalid"); + const digest = createHash("sha256"); + for await (const chunk of handle.createReadStream({ + autoClose: false, + }) as AsyncIterable) + digest.update(chunk); + archiveSha256 = digest.digest("hex"); + } finally { + await handle.close(); + } + + const suffix = backupId.replaceAll("-", "").slice(0, 16); + const validationContainer = `skillwire-backup-validate-${suffix}`; + const validationVolume = `${validationContainer}_data`; + let validation: RestoredDatabaseValidation | undefined; + try { + await this.command(["volume", "create", validationVolume], signal); + await this.command( + [ + "run", + "--detach", + "--name", + validationContainer, + "--network", + "none", + "--mount", + `type=volume,source=${validationVolume},target=/var/lib/postgresql/data`, + "--env", + "POSTGRES_HOST_AUTH_METHOD=trust", + this.options.postgresImage, + ], + signal, + ); + await this.waitForValidationDatabase(validationContainer, signal); + await this.command( + [ + "cp", + archivePath, + `${validationContainer}:/tmp/${basename(archivePath)}`, + ], + signal, + ); + await this.command( + [ + "exec", + validationContainer, + "pg_restore", + "--exit-on-error", + "--single-transaction", + "--no-owner", + "--no-acl", + "--username=postgres", + "--dbname=postgres", + `/tmp/${basename(archivePath)}`, + ], + signal, + ); + validation = await this.options.validateRestoredDatabase( + validationContainer, + signal, + ); + if ( + validation.latestMigration !== + (this.options.expectedLatestMigration ?? "010") || + !validation.invariantsValid || + !validation.catalogValid || + !validation.ready + ) + throw new Error("Restored backup did not pass readiness invariants"); + } catch (error) { + throw new Error("Backup archive restore validation failed", { + cause: error, + }); + } finally { + const cleanupSignal = AbortSignal.timeout(30_000); + await this.command( + ["container", "rm", "--force", validationContainer], + cleanupSignal, + ).catch(() => undefined); + await this.command( + ["volume", "rm", validationVolume], + cleanupSignal, + ).catch(() => undefined); + } + return { backupId, archivePath, archiveSha256, validation }; + } catch (error) { + try { + await validateOwnedDirectory(backupRoot, root); + await rm(backupRoot, { recursive: true }); + } catch (cleanupError) { + throw new Error( + "Incomplete backup cleanup requires exact filesystem recovery", + { cause: cleanupError }, + ); + } + throw error; + } + } +} diff --git a/src/onboarding/adapters/postgres/schema-compatibility.ts b/src/onboarding/adapters/postgres/schema-compatibility.ts new file mode 100644 index 0000000..fdd8e07 --- /dev/null +++ b/src/onboarding/adapters/postgres/schema-compatibility.ts @@ -0,0 +1,55 @@ +export interface SchemaUpgradeDecision { + readonly kind: "same-schema" | "forward-only"; + readonly liveSchema: number; + readonly targetSchema: number; + readonly rollbackBoundary: "application-config" | "database-restore-required"; + readonly requiresBackup: true; + readonly requiresWriterDrain: boolean; +} + +export function classifySchemaUpgrade(input: { + readonly liveSchema: number; + readonly schemaMinimum: number; + readonly schemaMaximum: number; + readonly latestMigration: number; + readonly forwardOnlyMigrations?: readonly number[] | undefined; +}): SchemaUpgradeDecision { + if ( + !Number.isInteger(input.liveSchema) || + !Number.isInteger(input.schemaMinimum) || + !Number.isInteger(input.schemaMaximum) || + !Number.isInteger(input.latestMigration) + ) + throw new Error("Schema compatibility values are invalid"); + if ( + input.liveSchema < input.schemaMinimum || + input.liveSchema > input.schemaMaximum + ) + throw new Error("Live schema is incompatible with the target release"); + if (input.latestMigration < input.liveSchema) + throw new Error("Schema downgrade is forbidden"); + if (input.latestMigration === input.liveSchema) + return { + kind: "same-schema", + liveSchema: input.liveSchema, + targetSchema: input.latestMigration, + rollbackBoundary: "application-config", + requiresBackup: true, + requiresWriterDrain: false, + }; + const migrations = Array.from( + { length: input.latestMigration - input.liveSchema }, + (_, index) => input.liveSchema + index + 1, + ); + const forwardOnly = new Set(input.forwardOnlyMigrations ?? [10]); + if (!migrations.some((migration) => forwardOnly.has(migration))) + throw new Error("Unclassified forward migration is forbidden"); + return { + kind: "forward-only", + liveSchema: input.liveSchema, + targetSchema: input.latestMigration, + rollbackBoundary: "database-restore-required", + requiresBackup: true, + requiresWriterDrain: true, + }; +} diff --git a/src/onboarding/application/backup.ts b/src/onboarding/application/backup.ts new file mode 100644 index 0000000..11b8b20 --- /dev/null +++ b/src/onboarding/application/backup.ts @@ -0,0 +1,162 @@ +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { open, readdir } from "node:fs/promises"; +import { atomicWriteJson } from "../adapters/filesystem/atomic-state.js"; +import type { PostgresBackupAdapter } from "../adapters/postgres/backup.js"; +import { + BackupRecordSchema, + type ServiceSecretReferenceSchema, +} from "../domain/installation.js"; +import type { z } from "zod"; +import { dirname, resolve } from "node:path"; +import type { OperationJournal } from "../domain/operation-journal.js"; + +type ServiceSecretReference = z.infer; + +export async function createValidatedBackup(options: { + readonly installationId: string; + readonly sourceReleaseId: string; + readonly serviceSecretReferences: readonly ServiceSecretReference[]; + readonly clientCredentialReferences: readonly string[]; + readonly adapter: PostgresBackupAdapter; + readonly signal: AbortSignal; + readonly journal?: OperationJournal | undefined; +}): Promise< + z.infer & { + readonly archivePath: string; + readonly backupRoot: string; + readonly backupIdentitySha256: string; + } +> { + const backup = + options.journal === undefined + ? await options.adapter.createAndValidate(options.signal) + : await options.journal.runEffect({ + step: "backup-create-and-restore-validate", + intent: { installationId: options.installationId }, + signal: options.signal, + action: () => options.adapter.createAndValidate(options.signal), + verification: (value) => ({ + backupId: value.backupId, + archiveSha256: value.archiveSha256, + restored: value.validation.ready, + }), + }); + const record = BackupRecordSchema.parse({ + schemaVersion: "skillwire.backup/v1", + backupId: backup.backupId, + installationId: options.installationId, + status: "validated", + createdAt: new Date().toISOString(), + archiveSha256: backup.archiveSha256, + sourceReleaseId: options.sourceReleaseId, + serviceSecretReferences: options.serviceSecretReferences, + clientCredentialReferences: options.clientCredentialReferences, + }); + const root = dirname(backup.archivePath); + const protectedFileSha256 = async (path: string): Promise => { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stats = await handle.stat(); + if ( + !stats.isFile() || + stats.nlink !== 1 || + stats.uid !== process.getuid?.() || + (stats.mode & 0o777) !== 0o600 + ) + throw new Error("Backup metadata is not a protected regular file"); + return createHash("sha256") + .update(await handle.readFile()) + .digest("hex"); + } finally { + await handle.close(); + } + }; + const publish = async () => { + const recoveryManifestPath = resolve(root, "recovery-manifest.json"); + const validationPath = resolve(root, "validation.json"); + await atomicWriteJson( + recoveryManifestPath, + { + ...record, + archiveLocator: "database.dump", + }, + root, + ); + await atomicWriteJson( + validationPath, + { + schemaVersion: "skillwire.backup-validation/v1", + backupId: backup.backupId, + status: "validated", + validation: backup.validation, + }, + root, + ); + await atomicWriteJson( + resolve(root, "checksums.json"), + { + schemaVersion: "skillwire.backup-checksums/v1", + backupId: backup.backupId, + files: { + "database.dump": backup.archiveSha256, + "recovery-manifest.json": + await protectedFileSha256(recoveryManifestPath), + "validation.json": await protectedFileSha256(validationPath), + }, + }, + root, + ); + }; + if (options.journal === undefined) await publish(); + else + await options.journal.runEffect({ + step: "backup-state-publication", + intent: { backupId: backup.backupId }, + signal: options.signal, + action: publish, + verification: () => ({ backupId: backup.backupId, published: true }), + }); + return { + ...record, + archivePath: backup.archivePath, + backupRoot: root, + backupIdentitySha256: await backupDirectoryIdentity(root), + }; +} + +export async function backupDirectoryIdentity(root: string): Promise { + const expected = [ + "checksums.json", + "database.dump", + "recovery-manifest.json", + "validation.json", + ]; + const names = (await readdir(root)).sort(); + if (JSON.stringify(names) !== JSON.stringify(expected)) + throw new Error("Backup file set is incomplete or contains unknown files"); + const digest = createHash("sha256").update( + "skillwire-backup-directory-identity-v1\0", + ); + for (const name of names) { + digest.update(name).update("\0"); + const handle = await open( + resolve(root, name), + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const stats = await handle.stat(); + if ( + !stats.isFile() || + stats.nlink !== 1 || + stats.uid !== process.getuid?.() || + (stats.mode & 0o777) !== 0o600 + ) + throw new Error("Backup file set contains an unsafe object"); + digest.update(await handle.readFile()).update("\0"); + } finally { + await handle.close(); + } + } + return digest.digest("hex"); +} diff --git a/src/onboarding/application/client-credentials.ts b/src/onboarding/application/client-credentials.ts index 9f4ea34..16e1d8f 100644 --- a/src/onboarding/application/client-credentials.ts +++ b/src/onboarding/application/client-credentials.ts @@ -38,6 +38,13 @@ export class ClientCredentialService { private readonly backend: ClientCredentialBackend, ) {} + private async revokeThenRemove( + credential: PersistedClientCredential, + ): Promise { + await this.issuer.revoke(credential.keyId); + await this.backend.remove(credential.client, credential.reference); + } + async provision(client: ClientName): Promise { const created = await this.issuer.create(client); if (parseApiKeyToken(created.token) === undefined) @@ -55,13 +62,11 @@ export class ClientCredentialService { throw new Error("Persisted client credential did not verify"); return { client, keyId: created.keyId, reference }; } catch (error) { - const cleanup = await Promise.allSettled([ - ...(reference === undefined - ? [] - : [this.backend.remove(client, reference)]), - this.issuer.revoke(created.keyId), - ]); - if (cleanup.some(({ status }) => status === "rejected")) { + try { + await this.issuer.revoke(created.keyId); + if (reference !== undefined) + await this.backend.remove(client, reference); + } catch { throw new ClientCredentialRecoveryError( "Client credential provisioning failed and narrow cleanup requires recovery", ); @@ -70,12 +75,67 @@ export class ClientCredentialService { } } + async rotate( + current: PersistedClientCredential, + transition?: { + readonly activate: ( + replacement: PersistedClientCredential, + ) => Promise; + readonly verify: ( + replacement: PersistedClientCredential, + ) => Promise; + readonly rollback: (current: PersistedClientCredential) => Promise; + }, + ): Promise { + const replacement = await this.provision(current.client); + try { + await transition?.activate(replacement); + await transition?.verify(replacement); + } catch (error) { + try { + await transition?.rollback(current); + await this.revokeThenRemove(replacement); + } catch { + throw new ClientCredentialRecoveryError( + "Client key rotation failed and replacement cleanup requires recovery", + ); + } + throw new Error( + "Client key rotation could not activate its replacement", + { + cause: error, + }, + ); + } + try { + await this.issuer.revoke(current.keyId); + } catch (error) { + try { + await transition?.rollback(current); + await this.revokeThenRemove(replacement); + } catch { + throw new ClientCredentialRecoveryError( + "Client key rotation failed and replacement cleanup requires recovery", + ); + } + throw new Error("Client key rotation could not revoke the old key", { + cause: error, + }); + } + try { + await this.backend.remove(current.client, current.reference); + } catch { + throw new ClientCredentialRecoveryError( + "Client key rotation succeeded but old credential cleanup requires recovery", + ); + } + return replacement; + } + async revoke(credential: PersistedClientCredential): Promise { - const cleanup = await Promise.allSettled([ - this.backend.remove(credential.client, credential.reference), - this.issuer.revoke(credential.keyId), - ]); - if (cleanup.some(({ status }) => status === "rejected")) { + try { + await this.revokeThenRemove(credential); + } catch { throw new ClientCredentialRecoveryError( "Client credential removal is incomplete and requires recovery", ); diff --git a/src/onboarding/application/client-lifecycle.ts b/src/onboarding/application/client-lifecycle.ts index 4eb574d..b07e69a 100644 --- a/src/onboarding/application/client-lifecycle.ts +++ b/src/onboarding/application/client-lifecycle.ts @@ -315,3 +315,109 @@ export async function installClientLifecycle( }; } } + +export type RemovableClientComponent = + "mcp-entry" | "plugin" | "marketplace" | "credential"; + +export interface ClientRemovalObservation { + readonly component: RemovableClientComponent; + readonly classification: ClientComponentClassification; + readonly expectedIdentitySha256: string | null; + readonly currentIdentitySha256: string | null; +} + +export interface ClientRemovalDependencies { + inspect(): Promise; + removeMcp(): Promise; + removePlugin(): Promise; + removeMarketplace(): Promise; + revokeCredential(): Promise; + verifyAbsent(component: RemovableClientComponent): Promise; +} + +export interface ClientRemovalResult { + readonly client: ClientName; + readonly status: "removed" | "unchanged" | "recovery-required"; + readonly removed: readonly RemovableClientComponent[]; + readonly retainedExternal: readonly RemovableClientComponent[]; +} + +export async function uninstallClientLifecycle( + client: ClientName, + dependencies: ClientRemovalDependencies, + signal: AbortSignal, +): Promise { + const observations = await dependencies.inspect(); + const retainedExternal = observations + .filter(({ classification }) => classification === "external-equivalent") + .map(({ component }) => component); + const blocked = observations.find( + ({ classification }) => + classification !== "absent" && + classification !== "external-equivalent" && + classification !== "owned-equivalent", + ); + if (blocked !== undefined) + return { + client, + status: "recovery-required", + removed: [], + retainedExternal, + }; + const removable = observations.filter( + (observation) => observation.classification === "owned-equivalent", + ); + if ( + removable.some( + ({ expectedIdentitySha256, currentIdentitySha256 }) => + expectedIdentitySha256 === null || + currentIdentitySha256 === null || + expectedIdentitySha256 !== currentIdentitySha256, + ) + ) + return { + client, + status: "recovery-required", + removed: [], + retainedExternal, + }; + const inverse: Readonly< + Record Promise> + > = { + "mcp-entry": () => dependencies.removeMcp(), + plugin: () => dependencies.removePlugin(), + marketplace: () => dependencies.removeMarketplace(), + credential: () => dependencies.revokeCredential(), + }; + const order: readonly RemovableClientComponent[] = [ + "plugin", + "marketplace", + "mcp-entry", + "credential", + ]; + const removed: RemovableClientComponent[] = []; + for (const component of order) { + if (!removable.some((entry) => entry.component === component)) continue; + if (signal.aborted) + return { client, status: "recovery-required", removed, retainedExternal }; + try { + await inverse[component](); + if (!(await dependencies.verifyAbsent(component))) + return { + client, + status: "recovery-required", + removed, + retainedExternal, + }; + removed.push(component); + } catch { + return { client, status: "recovery-required", removed, retainedExternal }; + } + } + return { + client, + status: removed.length === 0 ? "unchanged" : "removed", + removed, + retainedExternal, + }; +} diff --git a/src/onboarding/application/diagnostic-probes.ts b/src/onboarding/application/diagnostic-probes.ts new file mode 100644 index 0000000..ddc3c87 --- /dev/null +++ b/src/onboarding/application/diagnostic-probes.ts @@ -0,0 +1,284 @@ +import { + DiagnosticFindingSchema, + type DiagnosticFinding, +} from "../domain/diagnostics.js"; +import { redactOutput } from "../cli/output.js"; + +export type DiagnosticCondition = + | "service-stopped" + | "postgres-unavailable" + | "migration-pending" + | "schema-incompatible" + | "schema-drifted" + | "catalog-invalid" + | "advisory-invalid" + | "client-missing" + | "client-version-unsupported" + | "plugin-missing" + | "plugin-outdated" + | "mcp-absent" + | "mcp-conflicting" + | "mcp-duplicate" + | "credential-unavailable" + | "authentication-rejected" + | "endpoint-unreachable" + | "tool-contract-mismatch" + | "activation-adapter-unavailable" + | "source-degraded" + | "release-invalid" + | "trust-policy-invalid" + | "service-secret-unsafe" + | "ownership-drifted" + | "operation-locked" + | "backup-invalid" + | "journal-recovery-required"; + +export interface DiagnosticProbe { + readonly id: string; + run(signal: AbortSignal): Promise; +} + +interface Classification { + readonly code: string; + readonly component: DiagnosticFinding["component"]; + readonly severity: DiagnosticFinding["severity"]; + readonly summary: string; + readonly nextAction: string; +} + +const CLASSIFICATIONS: Readonly> = { + "service-stopped": { + code: "SERVICE_STOPPED", + component: "docker", + severity: "error", + summary: "The owned service is stopped", + nextAction: "Run repair after inspecting the owned Compose project", + }, + "postgres-unavailable": { + code: "POSTGRES_UNAVAILABLE", + component: "postgres", + severity: "error", + summary: "PostgreSQL is unavailable", + nextAction: "Restore PostgreSQL readiness before retrying", + }, + "migration-pending": { + code: "MIGRATION_PENDING", + component: "migration", + severity: "warning", + summary: "A supported migration is pending", + nextAction: "Run a verified upgrade with a restore-validated backup", + }, + "schema-incompatible": { + code: "SCHEMA_INCOMPATIBLE", + component: "migration", + severity: "error", + summary: "The live schema is incompatible", + nextAction: "Select a release compatible with the live schema", + }, + "schema-drifted": { + code: "SCHEMA_DRIFTED", + component: "migration", + severity: "recovery-required", + summary: "The live schema identity has drifted", + nextAction: "Stop writers and follow restore recovery guidance", + }, + "catalog-invalid": { + code: "CATALOG_INTEGRITY_INVALID", + component: "catalog", + severity: "error", + summary: "Catalog integrity verification failed", + nextAction: "Restore a release-bound catalog before serving requests", + }, + "advisory-invalid": { + code: "ADVISORY_INTEGRITY_INVALID", + component: "advisory", + severity: "error", + summary: "Advisory integrity verification failed", + nextAction: "Restore the release-bound advisory chain", + }, + "client-missing": { + code: "CLIENT_MISSING", + component: "codex", + severity: "warning", + summary: "A selected normal client is unavailable", + nextAction: "Install a supported normal client and run repair", + }, + "client-version-unsupported": { + code: "CLIENT_VERSION_UNSUPPORTED", + component: "codex", + severity: "error", + summary: "The selected normal client version is unsupported", + nextAction: "Use a certified client version before repair", + }, + "plugin-missing": { + code: "PLUGIN_MISSING", + component: "activation", + severity: "warning", + summary: "The owned activation plugin is absent", + nextAction: "Preview an ownership-proven client repair", + }, + "plugin-outdated": { + code: "PLUGIN_OUTDATED", + component: "activation", + severity: "warning", + summary: "The owned activation plugin is outdated", + nextAction: "Preview an ownership-proven client repair", + }, + "mcp-absent": { + code: "MCP_CONFIGURATION_ABSENT", + component: "mcp-contract", + severity: "warning", + summary: "The expected MCP registration is absent", + nextAction: "Preview an ownership-proven client repair", + }, + "mcp-conflicting": { + code: "MCP_CONFIGURATION_CONFLICTING", + component: "mcp-contract", + severity: "error", + summary: "An MCP registration conflicts with the expected identity", + nextAction: "Resolve the external conflict outside SkillWire", + }, + "mcp-duplicate": { + code: "MCP_CONFIGURATION_DUPLICATE", + component: "mcp-contract", + severity: "error", + summary: "The effective MCP registration is ambiguous", + nextAction: "Remove the ambiguity outside SkillWire before repair", + }, + "credential-unavailable": { + code: "CREDENTIAL_UNAVAILABLE", + component: "credential", + severity: "warning", + summary: "A client credential reference is unavailable", + nextAction: "Run explicit client key rotation", + }, + "authentication-rejected": { + code: "AUTHENTICATION_REJECTED", + component: "credential", + severity: "error", + summary: "The service rejected client authentication", + nextAction: "Run explicit client key rotation", + }, + "endpoint-unreachable": { + code: "ENDPOINT_UNREACHABLE", + component: "bridge", + severity: "warning", + summary: "The local MCP endpoint is unreachable", + nextAction: "Restore service readiness or keep using the ordinary client", + }, + "tool-contract-mismatch": { + code: "TOOL_CONTRACT_MISMATCH", + component: "mcp-contract", + severity: "error", + summary: "The six-tool contract does not match", + nextAction: "Repair or upgrade the verified service and integration", + }, + "activation-adapter-unavailable": { + code: "ACTIVATION_ADAPTER_UNAVAILABLE", + component: "activation", + severity: "warning", + summary: "The activation adapter is unavailable", + nextAction: "Repair the owned plugin without changing normal startup", + }, + "source-degraded": { + code: "SOURCE_SYNCHRONIZATION_DEGRADED", + component: "source", + severity: "warning", + summary: "An opted-in source is degraded", + nextAction: "Keep verified cached content and retry source sync later", + }, + "release-invalid": { + code: "RELEASE_INTEGRITY_INVALID", + component: "release", + severity: "error", + summary: "The installed release identity is invalid", + nextAction: "Install a signed non-downgrade release", + }, + "trust-policy-invalid": { + code: "TRUST_POLICY_INVALID", + component: "trust-policy", + severity: "error", + summary: "The active trust policy is invalid", + nextAction: "Recover from a policy accepted by the current trust quorum", + }, + "service-secret-unsafe": { + code: "SERVICE_SECRET_UNSAFE", + component: "service-secret", + severity: "recovery-required", + summary: "An owned service-secret file is unsafe", + nextAction: "Stop the service and inspect the exact owned file", + }, + "ownership-drifted": { + code: "OWNERSHIP_DRIFTED", + component: "ownership", + severity: "recovery-required", + summary: "An owned asset no longer matches its recorded identity", + nextAction: "Resolve the drift without adopting external state", + }, + "operation-locked": { + code: "OPERATION_LOCKED", + component: "concurrency", + severity: "warning", + summary: "A live administrative operation holds the installation lock", + nextAction: "Wait for the live operation to finish", + }, + "backup-invalid": { + code: "BACKUP_INVALID", + component: "backup", + severity: "error", + summary: "The retained backup failed validation", + nextAction: "Create and restore-validate a new backup", + }, + "journal-recovery-required": { + code: "JOURNAL_RECOVERY_REQUIRED", + component: "journal", + severity: "recovery-required", + summary: "An operation journal has an unproven effect", + nextAction: "Run repair to observe and reconcile the last effect boundary", + }, +}; + +function safeEvidence( + evidence: Readonly>, +): Record { + const redacted = redactOutput(evidence); + if ( + redacted === null || + typeof redacted !== "object" || + Array.isArray(redacted) + ) + throw new Error("Diagnostic evidence is invalid"); + return Object.fromEntries(Object.entries(redacted).slice(0, 16)); +} + +export function diagnosticProbe( + condition: DiagnosticCondition, + evidence: Readonly> = {}, +): DiagnosticProbe { + const classification = CLASSIFICATIONS[condition]; + return { + id: condition, + run: (signal) => { + if (signal.aborted) throw new Error("Diagnostic inspection cancelled"); + return Promise.resolve( + DiagnosticFindingSchema.parse({ + ...classification, + evidence: safeEvidence(evidence), + }), + ); + }, + }; +} + +export async function runDiagnosticProbes( + probes: readonly DiagnosticProbe[], + signal: AbortSignal, +): Promise { + const findings: DiagnosticFinding[] = []; + for (const probe of probes) { + if (signal.aborted) throw new Error("Diagnostic inspection cancelled"); + const finding = await probe.run(signal); + if (finding !== null) findings.push(DiagnosticFindingSchema.parse(finding)); + } + return findings; +} diff --git a/src/onboarding/application/doctor.ts b/src/onboarding/application/doctor.ts new file mode 100644 index 0000000..c2d8508 --- /dev/null +++ b/src/onboarding/application/doctor.ts @@ -0,0 +1,18 @@ +import { + DiagnosticFindingSchema, + type DiagnosticFinding, +} from "../domain/diagnostics.js"; + +export function runDoctor( + findings: readonly DiagnosticFinding[], +): Promise { + return Promise.resolve( + findings + .map((finding) => DiagnosticFindingSchema.parse(finding)) + .sort((left, right) => + left.code === right.code + ? left.component.localeCompare(right.component) + : left.code.localeCompare(right.code), + ), + ); +} diff --git a/src/onboarding/application/production-continuation.ts b/src/onboarding/application/production-continuation.ts new file mode 100644 index 0000000..2173c2f --- /dev/null +++ b/src/onboarding/application/production-continuation.ts @@ -0,0 +1,843 @@ +import { createHash, randomUUID, timingSafeEqual } from "node:crypto"; +import { constants } from "node:fs"; +import { access, open, realpath } from "node:fs/promises"; +import { dirname, isAbsolute, resolve } from "node:path"; +import { z } from "zod"; + +import { atomicWriteJson } from "../adapters/filesystem/atomic-state.js"; +import { CodexClientAdapter } from "../adapters/clients/codex.js"; +import { ClaudeClientAdapter } from "../adapters/clients/claude.js"; +import { clientComponentIdentity } from "../adapters/clients/client-state.js"; +import { DeploymentAdapter } from "../adapters/docker/deployment.js"; +import { dockerProcessEnvironment } from "../adapters/docker/environment.js"; +import { SecretToolCredentialStore } from "../adapters/credentials/secret-tool.js"; +import { + RestrictiveFileCredentialStore, + type RestrictiveFileReference, +} from "../adapters/credentials/restrictive-file.js"; +import { + ClientKeyHandoffRecoveryError, + createClientKeyInAdminContainer, + revokeClientKeyInAdminContainer, +} from "../adapters/postgres/bootstrap-admin.js"; +import { ServiceDatabase } from "../adapters/postgres/service-database.js"; +import type { ClientName } from "../cli/main.js"; +import { ClientMutationNotStartedError } from "../domain/client-mutation.js"; +import { + ClientIntegrationSchema, + CredentialReferenceSchema, + InstallationSchema, + transitionInstallation, + type Installation, +} from "../domain/installation.js"; +import type { OperationJournal } from "../domain/operation-journal.js"; +import { + ExternalIntegrationDependencySchema, + reactivateOwnedAsset, + recordExternalIntegration, + recordOwnedAsset, + verifyOwnershipRecord, + type OwnershipLedger, +} from "../domain/ownership.js"; +import { + clientConflictFinding, + ClientProvisioningRecoveryError, + installClientLifecycle, +} from "./client-lifecycle.js"; +import { verifyClientIntegration } from "./client-verification.js"; +import type { + GuidedSetupOptions, + GuidedSetupResult, + SetupClientResult, +} from "./setup.js"; + +const DeploymentStateSchema = z + .object({ + schemaVersion: z.literal("skillwire.deployment/v1"), + installationId: z.uuid(), + releaseRoot: z.string().refine(isAbsolute), + composePath: z.string().refine(isAbsolute), + skillwireImage: z.string().min(1), + postgresImage: z.string().min(1), + databasePasswordFile: z.string().refine(isAbsolute), + applicationPepperFile: z.string().refine(isAbsolute), + runtimeSocketDirectory: z.string().refine(isAbsolute), + socketPath: z.string().refine(isAbsolute), + projectName: z.string().min(1), + volumeName: z.string().min(1), + }) + .strict(); + +const BridgeStateSchema = z + .object({ + schemaVersion: z.literal("skillwire.bridge-state/v1"), + installationId: z.uuid(), + transport: z.literal("unix-domain-socket"), + endpoint: z.literal("http://localhost/mcp"), + socketPath: z.string().refine(isAbsolute), + clients: z.array( + z + .object({ + client: z.enum(["codex", "claude"]), + credentialReference: z.string().min(1), + keyId: z.uuid(), + }) + .strict(), + ), + }) + .strict(); + +const CredentialStateSchema = z + .object({ + schemaVersion: z.literal("skillwire.credential-references/v1"), + installationId: z.uuid(), + credentials: z.array(CredentialReferenceSchema), + }) + .strict(); + +const IntegrationStateSchema = z + .object({ + schemaVersion: z.literal("skillwire.client-integrations/v1"), + installationId: z.uuid(), + integrations: z.array(ClientIntegrationSchema), + }) + .strict(); + +const ExternalStateSchema = z + .object({ + schemaVersion: z.literal("skillwire.external-integrations/v1"), + installationId: z.uuid(), + dependencies: z.array(ExternalIntegrationDependencySchema), + }) + .strict(); + +type CredentialBackend = "secret-service" | "restrictive-file" | "not-selected"; + +async function readProtectedJson(path: string): Promise { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stats = await handle.stat(); + if ( + !stats.isFile() || + stats.nlink !== 1 || + stats.uid !== process.getuid?.() || + (stats.mode & 0o777) !== 0o600 || + stats.size > 1024 * 1024 + ) + throw new Error("Retained setup state is unsafe"); + return JSON.parse(await handle.readFile("utf8")) as unknown; + } finally { + await handle.close(); + } +} + +function selectedClients( + selection: GuidedSetupOptions["clients"], +): readonly ClientName[] { + if (selection === "none") return []; + return selection === "codex,claude" ? ["codex", "claude"] : [selection]; +} + +async function executable( + client: ClientName, + environment: NodeJS.ProcessEnv, +): Promise { + for (const directory of ( + environment["PATH"] ?? "/usr/local/bin:/usr/bin:/bin" + ) + .split(":") + .filter(isAbsolute)) { + const candidate = resolve(directory, client); + try { + await access(candidate, constants.X_OK); + return await realpath(candidate); + } catch { + // Continue through the bounded normal-profile PATH. + } + } + throw new Error(`Supported ${client} client executable is unavailable`); +} + +function composeEnvironment( + deployment: z.infer, + environment: NodeJS.ProcessEnv, +): NodeJS.ProcessEnv { + return dockerProcessEnvironment(environment, { + SKILLWIRE_COMPOSE_PROJECT: deployment.projectName, + SKILLWIRE_POSTGRES_VOLUME: deployment.volumeName, + SKILLWIRE_IMAGE: deployment.skillwireImage, + SKILLWIRE_POSTGRES_IMAGE: deployment.postgresImage, + SKILLWIRE_DATABASE_PASSWORD_SECRET_FILE: deployment.databasePasswordFile, + SKILLWIRE_APPLICATION_PEPPER_SECRET_FILE: deployment.applicationPepperFile, + SKILLWIRE_RUNTIME_SOCKET_DIRECTORY: deployment.runtimeSocketDirectory, + SKILLWIRE_RUNTIME_UID: String(process.getuid?.() ?? 10001), + SKILLWIRE_RUNTIME_GID: String(process.getgid?.() ?? 10001), + }); +} + +export async function continueProductionSetup(options: { + readonly setup: GuidedSetupOptions; + readonly credentialBackend: CredentialBackend; + readonly installation: Installation; + readonly home: string; + readonly dataRoot: string; + readonly stateRoot: string; + readonly runtimeRoot: string; + readonly launcherPath: string; + readonly environment: NodeJS.ProcessEnv; + readonly signal: AbortSignal; + readonly journal: OperationJournal; +}): Promise { + const { installation, stateRoot, dataRoot, environment, signal, journal } = + options; + const deployment = DeploymentStateSchema.parse( + await readProtectedJson(resolve(stateRoot, "deployment.json")), + ); + const bridgePath = resolve( + stateRoot, + "installations", + installation.installationId, + "bridge-state.json", + ); + const bridge = BridgeStateSchema.parse(await readProtectedJson(bridgePath)); + const credentials = CredentialStateSchema.parse( + await readProtectedJson(resolve(stateRoot, "credential-references.json")), + ); + const integrations = IntegrationStateSchema.parse( + await readProtectedJson(resolve(stateRoot, "client-integrations.json")), + ); + const external = ExternalStateSchema.parse( + await readProtectedJson(resolve(stateRoot, "external-integrations.json")), + ); + let ownership = verifyOwnershipRecord( + await readProtectedJson(resolve(stateRoot, "ownership.json")), + ); + if ( + [ + deployment.installationId, + bridge.installationId, + credentials.installationId, + integrations.installationId, + external.installationId, + ownership.installationId, + ].some((value) => value !== installation.installationId) + ) + throw new Error("Retained setup installation identities differ"); + + const dockerEnvironment = composeEnvironment(deployment, environment); + if (installation.status === "data-retained") { + const adapter = new DeploymentAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: deployment.composePath, + projectName: deployment.projectName, + volumeName: deployment.volumeName, + skillwireImage: deployment.skillwireImage, + postgresImage: deployment.postgresImage, + databasePasswordFile: deployment.databasePasswordFile, + applicationPepperFile: deployment.applicationPepperFile, + runtimeSocketDirectory: deployment.runtimeSocketDirectory, + socketPath: deployment.socketPath, + hostEnvironment: environment, + }); + await journal.runEffect({ + step: "retained-service-reactivation", + intent: { installationId: installation.installationId }, + signal, + action: async () => { + await adapter.probe(signal); + await adapter.deploy(signal); + const database = new ServiceDatabase({ + dockerExecutable: "/usr/bin/docker", + projectName: deployment.projectName, + volumeName: deployment.volumeName, + composePath: deployment.composePath, + environment: dockerEnvironment, + }); + await database.verifyVolume(signal); + await database.verifySchemaAndReadiness(signal); + }, + verification: () => ({ retainedServiceReady: true }), + }); + for (const asset of ownership.assets.filter( + ({ kind, disposition }) => + (kind === "compose-project" || kind === "container") && + disposition === "removed", + )) + ownership = reactivateOwnedAsset( + ownership, + asset.assetId, + asset.expectedIdentitySha256, + ); + } + + const bridgeClients = [...bridge.clients]; + const nextCredentials = [...credentials.credentials]; + const nextIntegrations = [...integrations.integrations]; + let ledger: OwnershipLedger = { + record: ownership, + externalIntegrations: [...external.dependencies], + }; + const requested = selectedClients(options.setup.clients); + const clientsToEnsure = + installation.status === "data-retained" + ? [...requested] + : Array.from(new Set([...installation.selectedClients, ...requested])); + const clientResults: SetupClientResult[] = []; + for (const client of clientsToEnsure) { + const priorIntegration = nextIntegrations.find( + (entry) => entry.client === client, + ); + const mustReconcile = + installation.status === "data-retained" || + priorIntegration === undefined || + priorIntegration.state === "failed" || + priorIntegration.state === "removed" || + priorIntegration.state === "retained-external"; + if (!mustReconcile) { + clientResults.push({ + client, + status: + priorIntegration.state === "external-verified" + ? "external-verified" + : "verified", + compensated: false, + owned: priorIntegration.state !== "external-verified", + }); + continue; + } + const secretService = new SecretToolCredentialStore( + "/usr/bin/secret-tool", + environment, + ); + const fallback = new RestrictiveFileCredentialStore( + dataRoot, + dataRoot, + installation.installationId, + ); + const existingBridge = bridgeClients.find( + (entry) => entry.client === client, + ); + let currentReference = existingBridge?.credentialReference; + let currentKeyId = existingBridge?.keyId; + let createdCredential = false; + const vendorExecutable = await executable(client, environment); + const adapter = + client === "codex" + ? new CodexClientAdapter(vendorExecutable, environment, signal) + : new ClaudeClientAdapter( + vendorExecutable, + environment, + undefined, + undefined, + signal, + ); + const marketplacePath = resolve( + deployment.releaseRoot, + client === "codex" + ? "distribution/codex-release-marketplace" + : "distribution/claude-marketplace", + ); + const persistBridge = () => + atomicWriteJson( + bridgePath, + { ...bridge, clients: bridgeClients }, + stateRoot, + ); + const lifecycle = await installClientLifecycle(client, { + preflight: async () => { + const ownedMcpIdentity = ledger.record.assets.find( + (asset) => + asset.kind === "mcp-entry" && + asset.client === client && + asset.disposition === "present", + )?.expectedIdentitySha256; + const [mcpState, pluginState] = await Promise.all([ + adapter.reconcileMcp( + options.launcherPath, + installation.installationId, + ownedMcpIdentity, + ), + adapter.reconcilePlugin(marketplacePath), + ]); + const blocked = [mcpState, pluginState].find( + ({ classification }) => + classification !== "absent" && + classification !== "owned-equivalent" && + classification !== "external-equivalent", + ); + if (blocked !== undefined) + return { + action: "block" as const, + classification: blocked.classification as Exclude< + typeof blocked.classification, + "absent" | "owned-equivalent" | "external-equivalent" + >, + ...(blocked.observations[0] === undefined + ? {} + : { + finding: clientConflictFinding( + client, + blocked === mcpState ? "mcp-entry" : "plugin", + blocked.classification, + blocked.observations[0], + ), + }), + }; + return { + action: + mcpState.classification === "external-equivalent" && + pluginState.classification === "external-equivalent" + ? ("reuse-external" as const) + : ("proceed" as const), + mcp: + mcpState.classification === "absent" + ? ("create" as const) + : mcpState.classification === "owned-equivalent" + ? ("reuse-owned" as const) + : ("reuse-external" as const), + plugin: + pluginState.classification === "absent" + ? ("create" as const) + : pluginState.classification === "owned-equivalent" + ? ("reuse-owned" as const) + : ("reuse-external" as const), + }; + }, + provisionCredential: async () => { + if (currentReference !== undefined && currentKeyId !== undefined) + return { keyId: currentKeyId, reference: currentReference }; + if (options.credentialBackend === "not-selected") + throw new Error("A selected client requires a credential backend"); + const key = await journal + .runEffect({ + step: `continued-client-${client}-key`, + intent: { client, accountId: installation.accountId }, + signal, + action: () => + createClientKeyInAdminContainer({ + client, + dockerExecutable: "/usr/bin/docker", + composePath: deployment.composePath, + projectName: deployment.projectName, + accountId: installation.accountId, + runtimeRoot: options.runtimeRoot, + environment: dockerEnvironment, + signal, + }), + verification: (value) => ({ client, keyId: value.keyId }), + }) + .catch((error: unknown) => { + if (error instanceof ClientKeyHandoffRecoveryError) + throw new ClientProvisioningRecoveryError(error.message); + throw error; + }); + currentKeyId = key.keyId; + try { + const stored = await journal.runEffect({ + step: `continued-client-${client}-credential`, + intent: { client, backend: options.credentialBackend }, + signal, + action: async () => { + currentReference = + options.credentialBackend === "secret-service" + ? ( + await secretService.store( + installation.installationId, + client, + key.token, + signal, + ) + ).reference + : await fallback.store(client, key.token, true); + const readback = currentReference.startsWith("secret-service:") + ? await secretService.lookup( + installation.installationId, + client, + currentReference, + signal, + ) + : await fallback.lookup( + currentReference as RestrictiveFileReference, + ); + if ( + readback.length !== key.token.length || + !timingSafeEqual(Buffer.from(readback), Buffer.from(key.token)) + ) + throw new Error("Continued setup credential readback failed"); + bridgeClients.push({ + client, + credentialReference: currentReference, + keyId: key.keyId, + }); + await persistBridge(); + return { keyId: key.keyId, reference: currentReference }; + }, + verification: (value) => ({ client, reference: value.reference }), + }); + createdCredential = true; + return stored; + } catch (error) { + await revokeClientKeyInAdminContainer({ + dockerExecutable: "/usr/bin/docker", + composePath: deployment.composePath, + projectName: deployment.projectName, + keyId: key.keyId, + environment: dockerEnvironment, + signal: AbortSignal.timeout(30_000), + }).catch(() => undefined); + throw error; + } + }, + addMcp: () => + journal.runEffect({ + step: `continued-client-${client}-mcp`, + intent: { client }, + signal, + action: () => + adapter.addMcp(options.launcherPath, installation.installationId), + effectNotStarted: (error) => + error instanceof ClientMutationNotStartedError, + verification: () => ({ client, installed: true }), + }), + addPlugin: () => + adapter.addPlugin(marketplacePath, (component, action) => + journal.runEffect({ + step: `continued-client-${client}-${component}`, + intent: { client, component }, + signal, + action, + verification: () => ({ client, component, installed: true }), + }), + ), + verify: async () => { + await verifyClientIntegration({ + client, + vendorExecutable, + installationId: installation.installationId, + registration: await adapter.readMcp(), + expectedLauncher: options.launcherPath, + environment, + inventory: () => adapter.readInventory(marketplacePath), + signal, + }); + }, + removePlugin: async () => { + const recoveryAdapter = + client === "codex" + ? new CodexClientAdapter( + vendorExecutable, + environment, + AbortSignal.timeout(30_000), + ) + : new ClaudeClientAdapter( + vendorExecutable, + environment, + undefined, + undefined, + AbortSignal.timeout(30_000), + ); + await recoveryAdapter.removePlugin(marketplacePath); + }, + removeMcp: async () => { + const recoveryAdapter = + client === "codex" + ? new CodexClientAdapter( + vendorExecutable, + environment, + AbortSignal.timeout(30_000), + ) + : new ClaudeClientAdapter( + vendorExecutable, + environment, + undefined, + undefined, + AbortSignal.timeout(30_000), + ); + await recoveryAdapter.removeMcp(); + }, + revokeCredential: async (keyId, reference) => { + if (!createdCredential) return; + await revokeClientKeyInAdminContainer({ + dockerExecutable: "/usr/bin/docker", + composePath: deployment.composePath, + projectName: deployment.projectName, + keyId, + environment: dockerEnvironment, + signal: AbortSignal.timeout(30_000), + }); + if (reference.startsWith("secret-service:")) + await secretService.clear( + installation.installationId, + client, + reference, + AbortSignal.timeout(30_000), + ); + else await fallback.remove(reference as RestrictiveFileReference); + }, + profileSnapshot: { + client, + profileRoot: options.home, + stateRoot: dirname(stateRoot), + relativePaths: + client === "codex" + ? [".codex/config.toml"] + : [".claude.json", ".claude/settings.json"], + }, + mcpProfilePaths: + client === "codex" ? [".codex/config.toml"] : [".claude.json"], + pluginProfilePaths: + client === "codex" + ? [".codex/config.toml"] + : [".claude.json", ".claude/settings.json"], + }); + clientResults.push(lifecycle); + if ( + lifecycle.status !== "verified" && + lifecycle.status !== "external-verified" + ) + continue; + const integrationId = priorIntegration?.clientIntegrationId ?? randomUUID(); + const credential = nextCredentials.find((entry) => entry.client === client); + const credentialReferenceId = + currentReference === undefined || currentKeyId === undefined + ? null + : (credential?.credentialReferenceId ?? randomUUID()); + const mcpIdentity = clientComponentIdentity({ + command: options.launcherPath, + args: [ + "bridge", + "--installation", + installation.installationId, + "--client", + client, + ], + scope: "user", + }); + const pluginIdentity = clientComponentIdentity({ + plugin: "skillwire-autonomous-activation@skillwire", + marketplacePath, + }); + const integration = ClientIntegrationSchema.parse({ + schemaVersion: "skillwire.client-integration/v1", + clientIntegrationId: integrationId, + installationId: installation.installationId, + client, + clientVersion: client === "codex" ? "0.147.0" : "2.1.229", + profileScope: "normal-user", + state: lifecycle.status, + credentialReferenceId, + keyPublicIdHash: + currentKeyId === undefined + ? null + : createHash("sha256").update(currentKeyId).digest("hex"), + mcpIdentitySha256: mcpIdentity, + adapterIdentitySha256: pluginIdentity, + }); + const integrationIndex = nextIntegrations.findIndex( + (entry) => entry.client === client, + ); + if (integrationIndex < 0) nextIntegrations.push(integration); + else nextIntegrations[integrationIndex] = integration; + if ( + credentialReferenceId !== null && + currentReference !== undefined && + currentKeyId !== undefined + ) { + const nextCredential = CredentialReferenceSchema.parse({ + schemaVersion: "skillwire.credential-reference/v1", + credentialReferenceId, + installationId: installation.installationId, + client, + backend: currentReference.startsWith("secret-service:") + ? "secret-service" + : "restrictive-file", + locator: currentReference, + keyPublicIdHash: createHash("sha256") + .update(currentKeyId) + .digest("hex"), + createdByOperation: + credential?.createdByOperation ?? journal.operationId, + state: "available", + fallbackRiskConfirmed: !currentReference.startsWith("secret-service:"), + }); + const credentialIndex = nextCredentials.findIndex( + (entry) => entry.client === client, + ); + if (credentialIndex < 0) nextCredentials.push(nextCredential); + else nextCredentials[credentialIndex] = nextCredential; + const retainedAsset = ledger.record.assets.find( + (asset) => + asset.kind === "credential" && + asset.client === client && + asset.locator === currentReference && + asset.disposition === "retained", + ); + if (retainedAsset !== undefined) + ledger = { + ...ledger, + record: reactivateOwnedAsset( + ledger.record, + retainedAsset.assetId, + retainedAsset.expectedIdentitySha256, + ), + }; + else if (existingBridge === undefined) + ledger = recordOwnedAsset(ledger, { + kind: "credential", + client, + locator: currentReference, + expectedIdentitySha256: clientComponentIdentity({ + reference: currentReference, + }), + createdByOperation: journal.operationId, + retention: "retain-by-default", + disposition: "present", + }); + } + for (const [component, state, identity, locator] of [ + ["mcp-entry", lifecycle.components.mcp, mcpIdentity, "skillwire:user"], + [ + "marketplace", + lifecycle.components.plugin, + pluginIdentity, + `skillwire:${marketplacePath}`, + ], + [ + "plugin", + lifecycle.components.plugin, + pluginIdentity, + "skillwire-autonomous-activation@skillwire", + ], + ] as const) { + if (state === "created") + ledger = recordOwnedAsset(ledger, { + kind: component, + client, + locator, + expectedIdentitySha256: identity, + createdByOperation: journal.operationId, + retention: "remove-on-uninstall", + disposition: "present", + }); + else if ( + state === "external" && + !ledger.externalIntegrations.some( + (entry) => entry.client === client && entry.kind === component, + ) + ) + ledger = recordExternalIntegration(ledger, { + schemaVersion: "skillwire.external-integration/v1", + externalDependencyId: randomUUID(), + client, + kind: component, + scope: "user", + observedIdentitySha256: identity, + verification: "equivalent", + lastObservedAt: new Date().toISOString(), + }); + } + } + + const status = clientResults.some( + ({ status: state }) => state === "recovery-required", + ) + ? "recovery-required" + : clientResults.some( + ({ status: state }) => + state !== "verified" && state !== "external-verified", + ) + ? "incomplete" + : "success"; + const timestamp = new Date().toISOString(); + const selected = + installation.status === "data-retained" + ? [...requested] + : Array.from(new Set([...installation.selectedClients, ...requested])); + let nextInstallation: Installation = { + ...installation, + selectedClients: selected, + clientIntegrationIds: { + codex: selected.includes("codex") + ? (nextIntegrations.find(({ client }) => client === "codex") + ?.clientIntegrationId ?? null) + : null, + claude: selected.includes("claude") + ? (nextIntegrations.find(({ client }) => client === "claude") + ?.clientIntegrationId ?? null) + : null, + }, + status: installation.status, + updatedAt: timestamp, + }; + if (installation.status === "data-retained") + nextInstallation = transitionInstallation( + nextInstallation, + "service-ready", + ); + const targetStatus = + status === "recovery-required" + ? ("recovery-required" as const) + : status === "incomplete" + ? ("incomplete" as const) + : selected.length === 0 + ? ("service-ready" as const) + : ("complete" as const); + nextInstallation = + nextInstallation.status === targetStatus + ? { + ...nextInstallation, + updatedAt: timestamp, + lastValidatedAt: + targetStatus === "complete" || targetStatus === "service-ready" + ? timestamp + : nextInstallation.lastValidatedAt, + } + : transitionInstallation(nextInstallation, targetStatus); + await journal.runEffect({ + step: "continued-setup-state-publication", + intent: { installationId: installation.installationId, status }, + signal, + action: async () => { + await atomicWriteJson( + resolve(stateRoot, "ownership.json"), + ledger.record, + stateRoot, + ); + await atomicWriteJson( + resolve(stateRoot, "external-integrations.json"), + { ...external, dependencies: ledger.externalIntegrations }, + stateRoot, + ); + await atomicWriteJson( + resolve(stateRoot, "credential-references.json"), + { ...credentials, credentials: nextCredentials }, + stateRoot, + ); + await atomicWriteJson( + resolve(stateRoot, "client-integrations.json"), + { ...integrations, integrations: nextIntegrations }, + stateRoot, + ); + await atomicWriteJson( + bridgePath, + { ...bridge, clients: bridgeClients }, + stateRoot, + ); + await atomicWriteJson( + resolve(stateRoot, "installation.json"), + InstallationSchema.parse(nextInstallation), + stateRoot, + ); + }, + verification: () => ({ + installationId: installation.installationId, + published: true, + }), + }); + return { + status, + installationId: installation.installationId, + serviceReady: true, + clients: clientResults, + }; +} diff --git a/src/onboarding/application/production-lifecycle.ts b/src/onboarding/application/production-lifecycle.ts new file mode 100644 index 0000000..d35ac66 --- /dev/null +++ b/src/onboarding/application/production-lifecycle.ts @@ -0,0 +1,3815 @@ +import { createHash, randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { + access, + chmod, + open, + readdir, + realpath, + rename, +} from "node:fs/promises"; +import { request as httpRequest } from "node:http"; +import { basename, dirname, isAbsolute, relative, resolve } from "node:path"; +import { z } from "zod"; + +import type { AdministrativeOperations } from "../cli/command-router.js"; +import type { ParsedCommand } from "../cli/main.js"; +import { AdminResultSchema, type AdminResult } from "../cli/output.js"; +import { canonicalPreview, confirmPreview } from "../cli/confirmation.js"; +import { atomicWriteJson } from "../adapters/filesystem/atomic-state.js"; +import { clientComponentIdentity } from "../adapters/clients/client-state.js"; +import { DeploymentAdapter } from "../adapters/docker/deployment.js"; +import { dockerProcessEnvironment } from "../adapters/docker/environment.js"; +import { CodexClientAdapter } from "../adapters/clients/codex.js"; +import { ClaudeClientAdapter } from "../adapters/clients/claude.js"; +import { SecretToolCredentialStore } from "../adapters/credentials/secret-tool.js"; +import { + RestrictiveFileCredentialStore, + type RestrictiveFileReference, +} from "../adapters/credentials/restrictive-file.js"; +import { + createClientKeyInAdminContainer, + revokeClientKeyInAdminContainer, +} from "../adapters/postgres/bootstrap-admin.js"; +import { ServiceDatabase } from "../adapters/postgres/service-database.js"; +import { runCommand } from "../adapters/process/command-runner.js"; +import { + ClientIntegrationSchema, + CredentialReferenceSchema, + InstallationSchema, + ServiceSecretSetSchema, + transitionClientIntegration, + transitionInstallation, +} from "../domain/installation.js"; +import { + recordAssetDisposition, + recordOwnedAsset, + planOwnedAssetDispositions, + reactivateOwnedAsset, + replaceOwnedAssetIdentity, + verifyOwnershipRecord, + type OwnershipRecordSchema, +} from "../domain/ownership.js"; +import { + currentProcessIdentity, + InstallationLock, + OperationJournal, +} from "../domain/operation-journal.js"; +import { PostgresBackupAdapter } from "../adapters/postgres/backup.js"; +import { + installVerifiedRelease, + releaseDirectoryIdentity, +} from "../adapters/filesystem/release-installer.js"; +import { ReleaseManifestSchema } from "../domain/release-manifest.js"; +import { + ownedLauncherIdentity, + previewProductionSetup, +} from "./production-setup.js"; +import { + diagnosticProbe, + runDiagnosticProbes, + type DiagnosticProbe, +} from "./diagnostic-probes.js"; +import { runDoctor } from "./doctor.js"; +import { inspectInstalledStatus } from "./status.js"; +import { planRepair, runRepair, type RepairAsset } from "./repair.js"; +import { + ClientCredentialService, + type ClientCredentialBackend, +} from "./client-credentials.js"; +import { + previewServiceSecretRotation, + rotateServiceSecret, +} from "./service-secret-rotation.js"; +import { backupDirectoryIdentity, createValidatedBackup } from "./backup.js"; +import { + drainWriters, + restartWriters, +} from "../adapters/docker/writer-drain.js"; +import { + previewUpgrade, + runUpgrade, + upgradeFailureRequiresRecovery, + UpgradeRecoveryError, +} from "./upgrade.js"; +import { upgradeRecoveryGuidance } from "./upgrade-recovery.js"; +import { uninstallClientLifecycle } from "./client-lifecycle.js"; +import { previewDefaultUninstall, runDefaultUninstall } from "./uninstall.js"; +import { journalNeedsRecovery, recoverOperation } from "./recovery.js"; +import { + previewPurge, + removeOwnedFilesystemTree, + runPurge, + validateOwnedFilesystemTree, +} from "./purge.js"; + +type OwnershipRecord = z.infer; +type OwnedAsset = OwnershipRecord["assets"][number]; + +const DeploymentStateSchema = z + .object({ + schemaVersion: z.literal("skillwire.deployment/v1"), + installationId: z.uuid(), + releaseRoot: z.string().refine(isAbsolute), + composePath: z.string().refine(isAbsolute), + skillwireImage: z.string().min(1), + postgresImage: z.string().min(1), + databasePasswordFile: z.string().refine(isAbsolute), + applicationPepperFile: z.string().refine(isAbsolute), + runtimeSocketDirectory: z.string().refine(isAbsolute), + socketPath: z.string().refine(isAbsolute), + projectName: z.string().min(1), + volumeName: z.string().min(1), + }) + .strict(); + +const BridgeStateSchema = z + .object({ + schemaVersion: z.literal("skillwire.bridge-state/v1"), + installationId: z.uuid(), + transport: z.literal("unix-domain-socket"), + endpoint: z.literal("http://localhost/mcp"), + socketPath: z.string().refine(isAbsolute), + clients: z.array( + z + .object({ + client: z.enum(["codex", "claude"]), + credentialReference: z.string().min(1), + keyId: z.uuid().optional(), + }) + .strict(), + ), + }) + .strict(); + +const CredentialReferencesStateSchema = z + .object({ + schemaVersion: z.literal("skillwire.credential-references/v1"), + installationId: z.uuid(), + credentials: z.array(CredentialReferenceSchema), + }) + .strict(); + +const ClientIntegrationsStateSchema = z + .object({ + schemaVersion: z.literal("skillwire.client-integrations/v1"), + installationId: z.uuid(), + integrations: z.array(ClientIntegrationSchema), + }) + .strict(); + +interface LifecycleRoots { + readonly home: string; + readonly stateRoot: string; + readonly dataRoot: string; + readonly runtimeRoot: string; +} + +function rootsFor( + command: ParsedCommand, + environment: NodeJS.ProcessEnv, +): LifecycleRoots { + const home = environment["HOME"]; + const dataHome = + environment["XDG_DATA_HOME"] ?? + (home === undefined ? undefined : resolve(home, ".local/share")); + const stateHome = + environment["XDG_STATE_HOME"] ?? + (home === undefined ? undefined : resolve(home, ".local/state")); + const runtimeHome = environment["XDG_RUNTIME_DIR"]; + if ( + home === undefined || + dataHome === undefined || + stateHome === undefined || + runtimeHome === undefined || + !isAbsolute(home) || + !isAbsolute(dataHome) || + !isAbsolute(stateHome) || + !isAbsolute(runtimeHome) + ) + throw new Error( + "Absolute HOME and XDG data/state/runtime roots are required for lifecycle operations", + ); + const stateRoot = command.stateRoot ?? resolve(stateHome, "skillwire"); + const dataRoot = resolve(dataHome, "skillwire"); + const runtimeRoot = resolve(runtimeHome, "skillwire"); + if (!isAbsolute(stateRoot)) + throw new Error("Lifecycle state root must be absolute"); + return { home, stateRoot, dataRoot, runtimeRoot }; +} + +async function readProtectedJson( + path: string, + maximum = 1024 * 1024, +): Promise { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stats = await handle.stat(); + if ( + !stats.isFile() || + stats.nlink !== 1 || + stats.uid !== process.getuid?.() || + (stats.mode & 0o777) !== 0o600 || + stats.size > maximum + ) + throw new Error("Protected lifecycle state is unsafe"); + return JSON.parse(await handle.readFile("utf8")) as unknown; + } finally { + await handle.close(); + } +} + +async function ownershipAt(stateRoot: string): Promise { + return verifyOwnershipRecord( + await readProtectedJson(resolve(stateRoot, "ownership.json")), + ); +} + +async function deploymentAt(stateRoot: string) { + return DeploymentStateSchema.parse( + await readProtectedJson(resolve(stateRoot, "deployment.json")), + ); +} + +async function bridgeAt(stateRoot: string, installationId: string) { + return BridgeStateSchema.parse( + await readProtectedJson( + resolve(stateRoot, "installations", installationId, "bridge-state.json"), + ), + ); +} + +function deploymentEnvironment( + deployment: z.infer, + environment: NodeJS.ProcessEnv, +): NodeJS.ProcessEnv { + return dockerProcessEnvironment(environment, { + SKILLWIRE_COMPOSE_PROJECT: deployment.projectName, + SKILLWIRE_POSTGRES_VOLUME: deployment.volumeName, + SKILLWIRE_IMAGE: deployment.skillwireImage, + SKILLWIRE_POSTGRES_IMAGE: deployment.postgresImage, + SKILLWIRE_DATABASE_PASSWORD_SECRET_FILE: deployment.databasePasswordFile, + SKILLWIRE_APPLICATION_PEPPER_SECRET_FILE: deployment.applicationPepperFile, + SKILLWIRE_RUNTIME_SOCKET_DIRECTORY: deployment.runtimeSocketDirectory, + SKILLWIRE_RUNTIME_UID: String(process.getuid?.() ?? 10001), + SKILLWIRE_RUNTIME_GID: String(process.getgid?.() ?? 10001), + }); +} + +async function observeOwnedComposeService( + deployment: z.infer, + service: "skillwire" | "postgres", + environment: NodeJS.ProcessEnv, + signal: AbortSignal, +): Promise { + return new DeploymentAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: deployment.composePath, + projectName: deployment.projectName, + volumeName: deployment.volumeName, + skillwireImage: deployment.skillwireImage, + postgresImage: deployment.postgresImage, + databasePasswordFile: deployment.databasePasswordFile, + applicationPepperFile: deployment.applicationPepperFile, + runtimeSocketDirectory: deployment.runtimeSocketDirectory, + socketPath: deployment.socketPath, + hostEnvironment: environment, + }).observeOwnedService(service, signal); +} + +async function verifyCredentialAuthentication( + socketPath: string, + token: string, + signal: AbortSignal, +): Promise { + const body = JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "skillwire-credential-rotation", version: "1" }, + }, + }); + await new Promise((done, reject) => { + const request = httpRequest( + { + socketPath, + path: "/mcp", + method: "POST", + headers: { + host: "localhost", + authorization: `Bearer ${token}`, + "content-type": "application/json", + accept: "application/json, text/event-stream", + "content-length": Buffer.byteLength(body), + }, + signal, + timeout: 5_000, + }, + (response) => { + let size = 0; + response.on("data", (chunk: Buffer) => { + size += chunk.byteLength; + if (size > 64 * 1024) request.destroy(); + }); + response.once("end", () => { + if (response.statusCode !== undefined && response.statusCode < 400) + done(); + else + reject(new Error("Replacement client authentication was rejected")); + }); + }, + ); + request.once("timeout", () => + request.destroy(new Error("Credential verification timed out")), + ); + request.once("error", () => { + reject( + new Error("Replacement client authentication could not be verified"), + ); + }); + request.end(body); + }); +} + +async function acquireOperation(options: { + readonly roots: LifecycleRoots; + readonly installationId: string; + readonly command: string; +}): Promise<{ + readonly lock: InstallationLock; + readonly journal: OperationJournal; +}> { + const identity = await currentProcessIdentity(); + const lock = await InstallationLock.acquire( + resolve(options.roots.runtimeRoot, "locks"), + "installation", + identity, + ); + try { + const journal = await OperationJournal.create( + resolve(options.roots.stateRoot, "operations"), + randomUUID(), + options.command, + ); + return { lock, journal }; + } catch (error) { + await lock.release(); + throw error; + } +} + +async function journalsRequiringRecovery( + stateRoot: string, +): Promise { + const root = resolve(stateRoot, "operations"); + let names: string[]; + try { + names = await readdir(root); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") + return []; + throw error; + } + const journals: OperationJournal[] = []; + for (const name of names.toSorted()) { + const match = + /^([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\.jsonl$/.exec( + name, + ); + if (match?.[1] === undefined) + throw new Error("Unexpected operation journal entry"); + const journal = await OperationJournal.open(root, match[1], "recovery"); + if (journalNeedsRecovery(journal.entries)) journals.push(journal); + } + return journals; +} + +async function executable( + client: "codex" | "claude", + environment: NodeJS.ProcessEnv, +): Promise { + for (const directory of ( + environment["PATH"] ?? "/usr/local/bin:/usr/bin:/bin" + ) + .split(":") + .filter(isAbsolute)) { + const candidate = resolve(directory, client); + try { + await access(candidate, constants.X_OK); + return await realpath(candidate); + } catch { + // Continue through the bounded normal-profile PATH. + } + } + throw new Error(`Supported ${client} client executable is unavailable`); +} + +function result( + input: Omit, +): AdminResult { + return AdminResultSchema.parse({ + schemaVersion: "skillwire.admin-result/v1", + operationId: randomUUID(), + ...input, + }); +} + +function previewResult( + command: ParsedCommand["route"], + scope: Record>>, + summary: string, +): AdminResult { + const preview = canonicalPreview(command, scope); + return result({ + command, + status: "preview", + exitClass: "success", + previewHash: preview.hash, + previewScope: scope, + changed: false, + summary, + components: [], + findings: [], + recovery: { rollbackBoundary: "none", backupId: null, instructions: [] }, + }); +} + +async function statusOperation( + command: ParsedCommand, + signal: AbortSignal, + environment: NodeJS.ProcessEnv, +): Promise { + const roots = rootsFor(command, environment); + const { installation } = await inspectInstalledStatus({ + stateRoot: roots.stateRoot, + signal, + }); + const ownership = await ownershipAt(roots.stateRoot); + const deployment = await deploymentAt(roots.stateRoot); + const integrations = ClientIntegrationsStateSchema.parse( + await readProtectedJson( + resolve(roots.stateRoot, "client-integrations.json"), + ), + ); + const components: AdminResult["components"][number][] = [ + { + component: "installation", + state: installation.status, + changed: false, + owned: true, + identity: { + installationId: installation.installationId, + release: installation.activeReleaseId, + releaseSequence: installation.highestAcceptedReleaseSequence, + trustPolicySequence: installation.activeTrustPolicySequence, + }, + }, + { + component: "ownership", + state: "verified", + changed: false, + owned: true, + identity: { + revision: ownership.recordRevision, + assetCount: ownership.assets.length, + }, + }, + ]; + if (installation.status === "data-retained") { + components.push({ + component: "service", + state: "data-retained", + changed: false, + owned: true, + identity: { + postgresVolume: installation.postgresVolume, + retainedBackups: ownership.assets.filter( + ({ kind, disposition }) => + kind === "backup" && disposition !== "removed", + ).length, + }, + }); + } else { + try { + const services = await runCommand({ + executable: "/usr/bin/docker", + args: [ + "compose", + "--project-name", + deployment.projectName, + "--file", + deployment.composePath, + "ps", + "--services", + "--filter", + "status=running", + ], + environment: deploymentEnvironment(deployment, environment), + deadlineMilliseconds: 15_000, + maximumOutputBytes: 16 * 1024, + signal, + }); + const running = new Set( + services.stdout.trim().split("\n").filter(Boolean), + ); + components.push({ + component: "service", + state: + running.has("postgres") && running.has("skillwire") + ? "running" + : "degraded", + changed: false, + owned: true, + identity: { composeProject: deployment.projectName }, + }); + const database = new ServiceDatabase({ + dockerExecutable: "/usr/bin/docker", + projectName: deployment.projectName, + volumeName: deployment.volumeName, + composePath: deployment.composePath, + environment: deploymentEnvironment(deployment, environment), + }); + const ready = await database.verifySchemaAndReadiness(signal); + components.push({ + component: "postgres", + state: "ready", + changed: false, + owned: true, + identity: { + version: ready.version, + latestMigration: ready.latestMigration, + }, + }); + } catch { + components.push({ + component: "service", + state: "unavailable", + changed: false, + owned: true, + identity: { composeProject: deployment.projectName }, + }); + } + } + for (const client of installation.selectedClients) { + const integration = integrations.integrations.find( + (entry) => entry.client === client, + ); + components.push({ + component: client, + state: integration?.state ?? "missing-state", + changed: false, + owned: integration?.state !== "external-verified", + identity: { + profileScope: integration?.profileScope ?? "normal-user", + credentialReferencePresent: + integration?.credentialReferenceId !== null && + integration?.credentialReferenceId !== undefined, + }, + }); + } + const degraded = components.some(({ state }) => + ["degraded", "unavailable", "missing-state"].includes(state), + ); + return result({ + command: "status", + status: degraded ? "incomplete" : "success", + exitClass: degraded ? "degraded-or-incomplete" : "success", + previewHash: null, + changed: false, + summary: degraded + ? "Installed state is intact but a bounded live component is unavailable" + : `Installation state is ${installation.status}`, + components, + findings: [], + recovery: { rollbackBoundary: "none", backupId: null, instructions: [] }, + }); +} + +async function doctorOperation( + command: ParsedCommand, + signal: AbortSignal, + environment: NodeJS.ProcessEnv, +): Promise { + const roots = rootsFor(command, environment); + const probes: DiagnosticProbe[] = []; + let installation: z.infer | undefined; + let ownership: OwnershipRecord | undefined; + try { + installation = ( + await inspectInstalledStatus({ stateRoot: roots.stateRoot, signal }) + ).installation; + } catch { + probes.push( + diagnosticProbe("service-stopped", { installedState: "unavailable" }), + ); + } + if (installation !== undefined) { + try { + ownership = await ownershipAt(roots.stateRoot); + } catch { + probes.push(diagnosticProbe("ownership-drifted", { state: "invalid" })); + } + try { + const set = ServiceSecretSetSchema.parse( + await readProtectedJson( + resolve(roots.stateRoot, "service-secret-set.json"), + ), + ); + if (set.installationId !== installation.installationId) + throw new Error("Service-secret installation identity differs"); + for (const reference of set.secrets) { + const path = resolve( + roots.dataRoot, + "installations", + installation.installationId, + reference.relativePath, + ); + const handle = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const stats = await handle.stat(); + if ( + !stats.isFile() || + stats.nlink !== 1 || + stats.uid !== process.getuid?.() || + (stats.mode & 0o777) !== 0o600 || + stats.size !== 43 + ) + throw new Error("Unsafe service-secret file"); + const bytes = await handle.readFile(); + const identity = createHash("sha256") + .update("skillwire-service-secret-identity-v1\0") + .update(bytes) + .digest("hex"); + if (identity !== reference.identitySha256) + throw new Error("Service-secret identity drifted"); + } finally { + await handle.close(); + } + } + } catch { + probes.push( + diagnosticProbe("service-secret-unsafe", { state: "invalid" }), + ); + } + let deployment: z.infer | undefined; + try { + deployment = await deploymentAt(roots.stateRoot); + } catch { + probes.push( + diagnosticProbe("service-stopped", { deploymentState: "invalid" }), + ); + } + if (deployment !== undefined) { + let postgresRunning = false; + try { + postgresRunning = await observeOwnedComposeService( + deployment, + "postgres", + environment, + signal, + ); + } catch { + probes.push( + diagnosticProbe("postgres-unavailable", { + observedState: "unavailable", + }), + ); + } + try { + if ( + !(await observeOwnedComposeService( + deployment, + "skillwire", + environment, + signal, + )) + ) + throw new Error("application service is stopped"); + } catch { + probes.push( + diagnosticProbe("service-stopped", { + postgresRunning, + applicationRunning: false, + }), + ); + } + if (postgresRunning) { + try { + const liveMigration = await readLiveMigration( + deployment, + environment, + signal, + ); + if (liveMigration < 10) + probes.push( + diagnosticProbe("migration-pending", { liveMigration }), + ); + else if (liveMigration > 10) + probes.push(diagnosticProbe("schema-drifted", { liveMigration })); + const integrity = await runCommand({ + executable: "/usr/bin/docker", + args: [ + "compose", + "--project-name", + deployment.projectName, + "--file", + deployment.composePath, + "exec", + "-T", + "postgres", + "psql", + "--username=skillwire", + "--dbname=skillwire", + "--tuples-only", + "--no-align", + "--set=ON_ERROR_STOP=1", + "--command", + "SELECT concat((to_regclass('public.external_skill_revisions') IS NOT NULL)::text,'|',(to_regclass('public.external_advisory_chain_head') IS NOT NULL)::text)", + ], + environment: deploymentEnvironment(deployment, environment), + deadlineMilliseconds: 15_000, + maximumOutputBytes: 16 * 1024, + signal, + }); + const [catalog, advisory] = integrity.stdout.trim().split("|"); + if (catalog !== "true") + probes.push(diagnosticProbe("catalog-invalid")); + if (advisory !== "true") + probes.push(diagnosticProbe("advisory-invalid")); + } catch { + probes.push( + diagnosticProbe("postgres-unavailable", { + observedState: "query-failed", + }), + ); + } + } + } + if (ownership !== undefined) { + for (const asset of ownership.assets.filter( + ({ disposition }) => disposition !== "removed", + )) { + try { + if (asset.kind === "release") { + if ( + (await releaseDirectoryIdentity(asset.locator)) !== + asset.expectedIdentitySha256 + ) + throw new Error("release identity differs"); + } else if (asset.kind === "trust-policy") { + const handle = await open( + resolve(roots.dataRoot, asset.locator), + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const stats = await handle.stat(); + if (!stats.isFile() || stats.nlink !== 1) + throw new Error("trust policy is unsafe"); + if ( + createHash("sha256") + .update(await handle.readFile()) + .digest("hex") !== asset.expectedIdentitySha256 + ) + throw new Error("trust policy identity differs"); + } finally { + await handle.close(); + } + } else if ( + asset.kind === "backup" && + (await backupDirectoryIdentity(asset.locator)) !== + asset.expectedIdentitySha256 + ) + throw new Error("backup identity differs"); + } catch { + probes.push( + diagnosticProbe( + asset.kind === "release" + ? "release-invalid" + : asset.kind === "trust-policy" + ? "trust-policy-invalid" + : "backup-invalid", + { assetId: asset.assetId }, + ), + ); + } + } + } + try { + if (installation.selectedClients.length === 0) + throw new Error("no selected clients"); + const integrations = ClientIntegrationsStateSchema.parse( + await readProtectedJson( + resolve(roots.stateRoot, "client-integrations.json"), + ), + ); + const credentials = CredentialReferencesStateSchema.parse( + await readProtectedJson( + resolve(roots.stateRoot, "credential-references.json"), + ), + ); + for (const client of installation.selectedClients) { + const integration = integrations.integrations.find( + (entry) => entry.client === client, + ); + if (integration === undefined) { + probes.push(diagnosticProbe("client-missing", { client })); + continue; + } + if ( + integration.state !== "external-verified" && + !credentials.credentials.some( + (entry) => + entry.client === client && + entry.credentialReferenceId === + integration.credentialReferenceId && + (entry.state === "available" || entry.state === "retained"), + ) + ) + probes.push(diagnosticProbe("credential-unavailable", { client })); + if (deployment === undefined) continue; + try { + const vendor = await executable(client, environment); + const adapter = + client === "codex" + ? new CodexClientAdapter(vendor, environment, signal) + : new ClaudeClientAdapter( + vendor, + environment, + undefined, + undefined, + signal, + ); + const mcp = await adapter.reconcileMcp( + resolve(roots.home, ".local/bin/skillwire"), + installation.installationId, + integration.mcpIdentitySha256, + ); + if (mcp.classification === "absent") + probes.push(diagnosticProbe("mcp-absent", { client })); + else if (mcp.classification === "ambiguous") + probes.push(diagnosticProbe("mcp-duplicate", { client })); + else if ( + mcp.classification !== "owned-equivalent" && + mcp.classification !== "external-equivalent" + ) + probes.push(diagnosticProbe("mcp-conflicting", { client })); + const plugin = await adapter.reconcilePlugin( + resolve( + deployment.releaseRoot, + client === "codex" + ? "distribution/codex-release-marketplace" + : "distribution/claude-marketplace", + ), + ); + if (plugin.classification === "absent") + probes.push(diagnosticProbe("plugin-missing", { client })); + else if ( + plugin.classification !== "owned-equivalent" && + plugin.classification !== "external-equivalent" + ) + probes.push(diagnosticProbe("plugin-outdated", { client })); + } catch { + probes.push(diagnosticProbe("client-missing", { client })); + } + } + } catch { + if (installation.selectedClients.length > 0) + probes.push(diagnosticProbe("client-missing", { state: "invalid" })); + } + try { + for (const journal of await journalsRequiringRecovery(roots.stateRoot)) + probes.push( + diagnosticProbe("journal-recovery-required", { + operation: journal.operationId, + }), + ); + } catch { + probes.push( + diagnosticProbe("journal-recovery-required", { state: "invalid" }), + ); + } + } + const findings = await runDoctor(await runDiagnosticProbes(probes, signal)); + return result({ + command: "doctor", + status: findings.some(({ severity }) => severity === "recovery-required") + ? "recovery-required" + : findings.length === 0 + ? "success" + : "incomplete", + exitClass: findings.some(({ severity }) => severity === "recovery-required") + ? "rollback-required" + : findings.length === 0 + ? "success" + : "degraded-or-incomplete", + previewHash: null, + changed: false, + summary: + findings.length === 0 + ? "All bounded installed-state diagnostics passed" + : "Doctor found lifecycle conditions requiring attention", + components: [], + findings: findings.map(({ evidence: _evidence, ...finding }) => finding), + recovery: { + rollbackBoundary: findings.some( + ({ severity }) => severity === "recovery-required", + ) + ? "application-config" + : "none", + backupId: null, + instructions: [], + }, + }); +} + +async function repairOperation( + command: ParsedCommand, + signal: AbortSignal, + environment: NodeJS.ProcessEnv, +): Promise { + const roots = rootsFor(command, environment); + const installation = InstallationSchema.parse( + await readProtectedJson(resolve(roots.stateRoot, "installation.json")), + ); + if (installation.status === "data-retained") + throw new Error( + "Repair cannot reactivate a default-uninstalled service; run setup", + ); + const deployment = await deploymentAt(roots.stateRoot); + const ownership = await ownershipAt(roots.stateRoot); + const selectedAssets = ownership.assets.filter( + (asset) => + command.component === undefined || asset.assetId === command.component, + ); + if (command.component !== undefined && selectedAssets.length !== 1) + throw new Error("Repair component is not an exact owned asset"); + const observeAsset = async (asset: OwnedAsset): Promise => { + const base = { + assetId: asset.assetId, + kind: asset.kind, + client: asset.client, + locator: asset.locator, + expectedIdentitySha256: asset.expectedIdentitySha256, + }; + if (asset.disposition === "drifted") + return { ...base, observation: "drifted", ownershipProven: false }; + if (asset.disposition === "ambiguous") + return { ...base, observation: "ambiguous" }; + if (asset.kind === "credential" || asset.kind === "service-secret") + return { + ...base, + observation: asset.disposition === "removed" ? "missing" : "matching", + }; + if (asset.kind === "compose-project" || asset.kind === "container") { + try { + const service = + asset.kind === "container" + ? asset.locator.split(":").at(-1) + : undefined; + if ( + service !== undefined && + service !== "skillwire" && + service !== "postgres" + ) + throw new Error("Owned Compose service locator is invalid"); + const matching = + asset.kind === "compose-project" + ? (await observeOwnedComposeService( + deployment, + "postgres", + environment, + signal, + )) && + (await observeOwnedComposeService( + deployment, + "skillwire", + environment, + signal, + )) + : service !== undefined && + (await observeOwnedComposeService( + deployment, + service, + environment, + signal, + )); + return { + ...base, + observation: matching ? "matching" : "missing", + ownershipProven: true, + }; + } catch { + return { ...base, observation: "ambiguous", ownershipProven: false }; + } + } + if ( + asset.client !== null && + (asset.kind === "mcp-entry" || + asset.kind === "plugin" || + asset.kind === "marketplace") + ) { + try { + const vendor = await executable(asset.client, environment); + const adapter = + asset.client === "codex" + ? new CodexClientAdapter(vendor, environment, signal) + : new ClaudeClientAdapter( + vendor, + environment, + undefined, + undefined, + signal, + ); + const state = + asset.kind === "mcp-entry" + ? await adapter.reconcileMcp( + resolve(roots.home, ".local/bin/skillwire"), + installation.installationId, + asset.expectedIdentitySha256, + ) + : await adapter.reconcilePlugin( + resolve( + deployment.releaseRoot, + asset.client === "codex" + ? "distribution/codex-release-marketplace" + : "distribution/claude-marketplace", + ), + ); + const matching = + state.classification === "owned-equivalent" || + ((asset.kind === "plugin" || asset.kind === "marketplace") && + state.classification === "external-equivalent"); + return { + ...base, + observation: matching + ? "matching" + : state.classification === "absent" + ? "missing" + : "ambiguous", + ownershipProven: matching || state.classification === "absent", + }; + } catch { + return { ...base, observation: "missing", ownershipProven: true }; + } + } + return { + ...base, + observation: asset.disposition === "removed" ? "ambiguous" : "matching", + }; + }; + const assets: RepairAsset[] = []; + for (const asset of selectedAssets) assets.push(await observeAsset(asset)); + const plan = planRepair({ installationId: ownership.installationId, assets }); + const interrupted = await journalsRequiringRecovery(roots.stateRoot); + const scope = { + installationId: plan.installationId, + ownershipRevision: ownership.recordRevision, + actions: plan.actions.map( + ({ assetId, kind, client, locator, expectedIdentitySha256 }) => ({ + assetId, + kind, + client, + locator, + expectedIdentitySha256, + }), + ), + blocked: plan.blocked.map(({ code, assetId }) => ({ code, assetId })), + interruptedOperations: interrupted.map(({ operationId }) => operationId), + }; + const preview = canonicalPreview("repair", scope); + if (command.previewOnly) + return previewResult("repair", scope, "Ownership-bound repair preview"); + confirmPreview(preview, command.confirmPreview); + const { lock, journal } = await acquireOperation({ + roots, + installationId: installation.installationId, + command: "repair", + }); + await journal.intent("repair", { previewHash: preview.hash }); + let nextOwnership = ownership; + let serviceRepaired = false; + try { + for (const pending of interrupted) { + const current = await OperationJournal.open( + resolve(roots.stateRoot, "operations"), + pending.operationId, + "recovery", + ); + if (!journalNeedsRecovery(current.entries)) continue; + const recovered = await recoverOperation({ + journal: current, + signal, + observe: () => Promise.resolve("ambiguous"), + compensate: () => + Promise.reject( + new Error( + "Recovery refused compensation without an exact current identity", + ), + ), + }); + if (recovered.disposition === "recovery-required") { + await journal.cancel({ status: "failed" }); + return result({ + command: "repair", + status: "recovery-required", + exitClass: "rollback-required", + previewHash: preview.hash, + previewScope: scope, + changed: false, + summary: + "Repair stopped at an ambiguous interrupted-operation boundary", + components: [], + findings: [ + { + code: "JOURNAL_RECOVERY_REQUIRED", + severity: "recovery-required", + component: "journal", + summary: `Operation ${pending.operationId} has an unproven effect`, + nextAction: + "Inspect the named owned effect and resolve its current identity before retrying repair", + }, + ], + recovery: { + rollbackBoundary: "application-config", + backupId: null, + instructions: [], + }, + }); + } + } + const repaired = await runRepair({ + plan, + confirmation: plan.previewHash, + signal, + observe: async (candidate) => { + const currentOwnership = await ownershipAt(roots.stateRoot); + if (currentOwnership.recordSha256 !== ownership.recordSha256) + throw new Error("Repair ownership changed after preview"); + const currentAsset = currentOwnership.assets.find( + ({ assetId }) => assetId === candidate.assetId, + ); + if (currentAsset === undefined) + throw new Error("Repair asset disappeared after preview"); + const observed = await observeAsset(currentAsset); + return { + observation: observed.observation, + identitySha256: observed.expectedIdentitySha256, + ownershipProven: observed.ownershipProven, + }; + }, + repair: (asset) => + journal.runEffect({ + step: `repair-${asset.kind}-${asset.assetId}`, + intent: { assetId: asset.assetId, kind: asset.kind }, + signal, + action: async () => { + if ( + asset.kind === "compose-project" || + asset.kind === "container" + ) { + if (!serviceRepaired) { + const adapter = new DeploymentAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: deployment.composePath, + projectName: deployment.projectName, + volumeName: deployment.volumeName, + skillwireImage: deployment.skillwireImage, + postgresImage: deployment.postgresImage, + databasePasswordFile: deployment.databasePasswordFile, + applicationPepperFile: deployment.applicationPepperFile, + runtimeSocketDirectory: deployment.runtimeSocketDirectory, + socketPath: deployment.socketPath, + hostEnvironment: environment, + }); + await adapter.probe(signal); + await adapter.deploy(signal); + serviceRepaired = true; + } + } else if ( + asset.client !== null && + (asset.kind === "mcp-entry" || + asset.kind === "plugin" || + asset.kind === "marketplace") + ) { + const vendor = await executable(asset.client, environment); + const adapter = + asset.client === "codex" + ? new CodexClientAdapter(vendor, environment, signal) + : new ClaudeClientAdapter( + vendor, + environment, + undefined, + undefined, + signal, + ); + const marketplacePath = resolve( + deployment.releaseRoot, + asset.client === "codex" + ? "distribution/codex-release-marketplace" + : "distribution/claude-marketplace", + ); + if (asset.kind === "mcp-entry") + await adapter.addMcp( + resolve(roots.home, ".local/bin/skillwire"), + installation.installationId, + ); + else { + const pluginState = + await adapter.reconcilePlugin(marketplacePath); + if (pluginState.classification === "absent") + await adapter.addPlugin(marketplacePath); + } + } else throw new Error("Repair target has no safe exact adapter"); + const original = ownership.assets.find( + ({ assetId }) => assetId === asset.assetId, + ); + if (original?.disposition === "removed") + nextOwnership = reactivateOwnedAsset( + nextOwnership, + original.assetId, + original.expectedIdentitySha256, + ); + }, + verification: () => ({ assetId: asset.assetId, repaired: true }), + }), + rotate: () => + Promise.reject( + new Error("Repair never rotates credentials or service secrets"), + ), + }); + if (nextOwnership.recordSha256 !== ownership.recordSha256) + await journal.runEffect({ + step: "repair-ownership-publication", + intent: { ownershipRevision: nextOwnership.recordRevision }, + signal, + action: () => + atomicWriteJson( + resolve(roots.stateRoot, "ownership.json"), + nextOwnership, + roots.stateRoot, + ), + verification: () => ({ + ownershipRevision: nextOwnership.recordRevision, + }), + }); + await journal.commit({ status: "success" }); + const changed = repaired.changedAssets.length > 0; + return result({ + command: "repair", + status: plan.blocked.length === 0 ? "success" : "incomplete", + exitClass: + plan.blocked.length === 0 ? "success" : "policy-or-ownership-conflict", + previewHash: preview.hash, + previewScope: scope, + changed, + summary: changed + ? "Exact ownership-proven components were repaired and reverified" + : plan.blocked.length === 0 + ? "Owned installation already matches its repairable state" + : "Unsafe or external repair targets were left unchanged", + components: repaired.changedAssets.map((assetId) => ({ + component: assetId, + state: "repaired", + changed: true, + owned: true, + identity: {}, + })), + findings: plan.blocked.map((block) => ({ + code: block.code, + severity: "error" as const, + component: "ownership", + summary: `Repair skipped owned asset ${block.assetId}`, + nextAction: + "Resolve ownership ambiguity without adopting external state", + })), + recovery: { rollbackBoundary: "none", backupId: null, instructions: [] }, + }); + } catch (error) { + await journal + .cancel({ + status: journal.hasUnprovenEffect() ? "recovery-required" : "failed", + }) + .catch(() => undefined); + throw error; + } finally { + await lock.release(); + } +} + +async function rotateClientKeyOperation( + command: ParsedCommand, + signal: AbortSignal, + environment: NodeJS.ProcessEnv, +): Promise { + if (command.client === undefined) + throw new Error("Client key rotation requires an exact client"); + const roots = rootsFor(command, environment); + const installation = InstallationSchema.parse( + await readProtectedJson(resolve(roots.stateRoot, "installation.json")), + ); + if (installation.status === "data-retained") + throw new Error("Client key rotation requires a running installation"); + const deployment = await deploymentAt(roots.stateRoot); + const bridge = await bridgeAt(roots.stateRoot, installation.installationId); + const current = bridge.clients.find( + ({ client }) => client === command.client, + ); + if (current?.keyId === undefined) + throw new Error( + "Persisted client key identity is unavailable for safe rotation", + ); + const ownership = await ownershipAt(roots.stateRoot); + const credentialAsset = ownership.assets.find( + (asset) => + asset.kind === "credential" && + asset.client === command.client && + asset.locator === current.credentialReference && + (asset.disposition === "present" || asset.disposition === "retained"), + ); + if (credentialAsset === undefined) + throw new Error("Matching client credential ownership is unavailable"); + const scope = { + installationId: installation.installationId, + client: command.client, + ownershipRevision: ownership.recordRevision, + currentCredentialIdentity: createHash("sha256") + .update(current.credentialReference) + .digest("hex"), + effects: [ + "create replacement key", + "persist and authenticate replacement", + "switch bridge reference", + "revoke old key", + "remove old credential", + ], + }; + const preview = canonicalPreview("clients:rotate-key", scope); + if (command.previewOnly) + return previewResult( + "clients:rotate-key", + scope, + `Independent ${command.client} client-key rotation preview`, + ); + confirmPreview(preview, command.confirmPreview); + const { lock, journal } = await acquireOperation({ + roots, + installationId: installation.installationId, + command: "clients-rotate-key", + }); + await journal.intent("clients-rotate-key", { + client: command.client, + previewHash: preview.hash, + }); + try { + const currentOwnership = await ownershipAt(roots.stateRoot); + const currentBridge = await bridgeAt( + roots.stateRoot, + installation.installationId, + ); + const currentEntry = currentBridge.clients.find( + ({ client }) => client === command.client, + ); + if ( + currentOwnership.recordSha256 !== ownership.recordSha256 || + currentEntry?.keyId !== current.keyId || + currentEntry.credentialReference !== current.credentialReference + ) + throw new Error("Client credential state changed after preview"); + const dockerEnvironment = deploymentEnvironment(deployment, environment); + const secretService = new SecretToolCredentialStore( + "/usr/bin/secret-tool", + environment, + ); + const fallback = new RestrictiveFileCredentialStore( + roots.dataRoot, + roots.dataRoot, + installation.installationId, + ); + const usesSecretService = + current.credentialReference.startsWith("secret-service:"); + const backend: ClientCredentialBackend = { + store: async (client, token) => + usesSecretService + ? ( + await secretService.store( + installation.installationId, + client, + token, + signal, + ) + ).reference + : fallback.storeReplacement(client, token, randomUUID()), + lookup: async (client, reference) => + reference.startsWith("secret-service:") + ? secretService.lookup( + installation.installationId, + client, + reference, + signal, + ) + : fallback.lookup(reference as RestrictiveFileReference), + remove: async (client, reference) => + reference.startsWith("secret-service:") + ? secretService.clear( + installation.installationId, + client, + reference, + AbortSignal.timeout(30_000), + ) + : fallback.remove(reference as RestrictiveFileReference), + }; + const service = new ClientCredentialService( + { + create: async (client) => + createClientKeyInAdminContainer({ + client, + dockerExecutable: "/usr/bin/docker", + composePath: deployment.composePath, + projectName: deployment.projectName, + accountId: installation.accountId, + runtimeRoot: roots.runtimeRoot, + environment: dockerEnvironment, + signal, + }), + revoke: (keyId) => + revokeClientKeyInAdminContainer({ + dockerExecutable: "/usr/bin/docker", + composePath: deployment.composePath, + projectName: deployment.projectName, + keyId, + environment: dockerEnvironment, + signal: AbortSignal.timeout(30_000), + }), + }, + backend, + ); + const bridgePath = resolve( + roots.stateRoot, + "installations", + installation.installationId, + "bridge-state.json", + ); + const replacement = await journal.runEffect({ + step: `client-${command.client}-key-rotation`, + intent: { client: command.client }, + signal, + action: () => + service.rotate( + { + client: command.client ?? "codex", + keyId: current.keyId ?? "", + reference: current.credentialReference, + }, + { + activate: async (next) => { + await atomicWriteJson( + bridgePath, + { + ...currentBridge, + clients: currentBridge.clients.map((entry) => + entry.client === command.client + ? { + ...entry, + keyId: next.keyId, + credentialReference: next.reference, + } + : entry, + ), + }, + roots.stateRoot, + ); + }, + verify: async (next) => { + const token = await backend.lookup(next.client, next.reference); + await verifyCredentialAuthentication( + deployment.socketPath, + token, + signal, + ); + }, + rollback: async () => { + await atomicWriteJson(bridgePath, currentBridge, roots.stateRoot); + }, + }, + ), + verification: (value) => ({ + client: value.client, + replacementReferenceIdentity: createHash("sha256") + .update(value.reference) + .digest("hex"), + }), + }); + await journal.runEffect({ + step: `client-${command.client}-rotation-state`, + intent: { client: command.client }, + signal, + action: async () => { + const references = CredentialReferencesStateSchema.parse( + await readProtectedJson( + resolve(roots.stateRoot, "credential-references.json"), + ), + ); + const integrations = ClientIntegrationsStateSchema.parse( + await readProtectedJson( + resolve(roots.stateRoot, "client-integrations.json"), + ), + ); + const newReferenceId = randomUUID(); + const keyPublicIdHash = createHash("sha256") + .update(replacement.keyId) + .digest("hex"); + const nextReferences = { + ...references, + credentials: [ + ...references.credentials.map((entry) => + entry.client === command.client && entry.state === "available" + ? { ...entry, state: "removed" as const } + : entry, + ), + CredentialReferenceSchema.parse({ + schemaVersion: "skillwire.credential-reference/v1", + credentialReferenceId: newReferenceId, + installationId: installation.installationId, + client: command.client, + backend: replacement.reference.startsWith("secret-service:") + ? "secret-service" + : "restrictive-file", + locator: replacement.reference, + keyPublicIdHash, + createdByOperation: journal.operationId, + state: "available", + fallbackRiskConfirmed: + !replacement.reference.startsWith("secret-service:"), + }), + ], + }; + const nextIntegrations = { + ...integrations, + integrations: integrations.integrations.map((entry) => + entry.client === command.client + ? ClientIntegrationSchema.parse({ + ...entry, + credentialReferenceId: newReferenceId, + keyPublicIdHash, + }) + : entry, + ), + }; + const nextOwnership = replaceOwnedAssetIdentity( + currentOwnership, + credentialAsset.assetId, + { + locator: replacement.reference, + expectedIdentitySha256: clientComponentIdentity({ + reference: replacement.reference, + }), + }, + ); + await atomicWriteJson( + resolve(roots.stateRoot, "credential-references.json"), + nextReferences, + roots.stateRoot, + ); + await atomicWriteJson( + resolve(roots.stateRoot, "client-integrations.json"), + nextIntegrations, + roots.stateRoot, + ); + await atomicWriteJson( + resolve(roots.stateRoot, "ownership.json"), + nextOwnership, + roots.stateRoot, + ); + }, + verification: () => ({ + client: command.client ?? "codex", + published: true, + }), + }); + await journal.commit({ status: "success" }); + return result({ + command: "clients:rotate-key", + status: "success", + exitClass: "success", + previewHash: preview.hash, + previewScope: scope, + changed: true, + summary: `${command.client} client key rotated independently`, + components: [ + { + component: command.client, + state: "credential-rotated", + changed: true, + owned: true, + identity: { + credentialIdentity: createHash("sha256") + .update(replacement.reference) + .digest("hex"), + }, + }, + ], + findings: [], + recovery: { rollbackBoundary: "none", backupId: null, instructions: [] }, + }); + } catch (error) { + await journal.cancel({ + status: journal.hasUnprovenEffect() ? "recovery-required" : "failed", + }); + throw error; + } finally { + await lock.release(); + } +} + +async function rotateServiceSecretOperation( + command: ParsedCommand, + signal: AbortSignal, + environment: NodeJS.ProcessEnv, +): Promise { + if (command.serviceSecret === undefined) + throw new Error("Service-secret rotation requires an exact secret kind"); + const kind = command.serviceSecret; + requireProductionSecretRotationSupport(kind); + const roots = rootsFor(command, environment); + + function requireProductionSecretRotationSupport( + requestedKind: typeof kind, + ): void { + if (requestedKind !== "application-pepper") return; + throw new Error( + "Application-pepper rotation is unsupported until the runtime can authenticate existing client keys through a safe overlap window", + ); + } + const installation = InstallationSchema.parse( + await readProtectedJson(resolve(roots.stateRoot, "installation.json")), + ); + if (installation.status === "data-retained") + throw new Error("Service-secret rotation requires a running installation"); + const deployment = await deploymentAt(roots.stateRoot); + const ownership = await ownershipAt(roots.stateRoot); + const set = ServiceSecretSetSchema.parse( + await readProtectedJson( + resolve(roots.stateRoot, "service-secret-set.json"), + ), + ); + const currentReference = set.secrets.find((entry) => entry.kind === kind); + if (currentReference === undefined) + throw new Error("Owned service-secret reference is unavailable"); + const currentAsset = ownership.assets.find( + (asset) => + asset.kind === "service-secret" && + asset.client === null && + asset.locator === + `${installation.installationId}/${currentReference.relativePath}` && + asset.expectedIdentitySha256 === currentReference.identitySha256 && + (asset.disposition === "present" || asset.disposition === "retained"), + ); + if (currentAsset === undefined) + throw new Error("Matching service-secret ownership is unavailable"); + const rotationPreview = previewServiceSecretRotation({ + installationId: installation.installationId, + kind, + currentIdentitySha256: currentReference.identitySha256, + }); + const scope = { + installationId: installation.installationId, + kind, + targets: [...rotationPreview.targets], + currentIdentitySha256: currentReference.identitySha256, + ownershipRevision: ownership.recordRevision, + readiness: ["postgresql-17", "migration-010", "application"], + rollbackBoundary: "application-config", + }; + const preview = canonicalPreview("maintenance:rotate-service-secret", scope); + if (command.previewOnly) + return previewResult( + "maintenance:rotate-service-secret", + scope, + `Explicit ${kind} rotation preview`, + ); + confirmPreview(preview, command.confirmPreview); + const { lock, journal } = await acquireOperation({ + roots, + installationId: installation.installationId, + command: "rotate-service-secret", + }); + await journal.intent("rotate-service-secret", { + kind, + previewHash: preview.hash, + }); + const readValue = async (path: string): Promise => { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stats = await handle.stat(); + if ( + !stats.isFile() || + stats.nlink !== 1 || + stats.uid !== process.getuid?.() || + (stats.mode & 0o777) !== 0o600 || + stats.size !== 43 + ) + throw new Error("Service-secret file is unsafe"); + const value = await handle.readFile("ascii"); + if (!/^[A-Za-z0-9_-]{43}$/.test(value)) + throw new Error("Service-secret file format is invalid"); + return value; + } finally { + await handle.close(); + } + }; + const apply = async ( + secretPath: string, + operationSignal: AbortSignal = signal, + ): Promise => { + const databasePasswordFile = + kind === "database-password" + ? secretPath + : deployment.databasePasswordFile; + const applicationPepperFile = + kind === "application-pepper" + ? secretPath + : deployment.applicationPepperFile; + const dockerEnvironment = { + ...deploymentEnvironment(deployment, environment), + SKILLWIRE_DATABASE_PASSWORD_SECRET_FILE: databasePasswordFile, + SKILLWIRE_APPLICATION_PEPPER_SECRET_FILE: applicationPepperFile, + }; + if (kind === "database-password") { + const value = await readValue(secretPath); + try { + await runCommand({ + executable: "/usr/bin/docker", + args: [ + "compose", + "--project-name", + deployment.projectName, + "--file", + deployment.composePath, + "exec", + "-T", + "postgres", + "psql", + "--username=postgres", + "--dbname=postgres", + "--no-psqlrc", + "--set=ON_ERROR_STOP=1", + "--file=-", + ], + environment: dockerEnvironment, + stdin: `ALTER ROLE skillwire PASSWORD '${value}';\n`, + deadlineMilliseconds: 15_000, + maximumOutputBytes: 16 * 1024, + signal: operationSignal, + }); + } catch { + throw new Error("Database credential rotation command failed"); + } + } + const adapter = new DeploymentAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: deployment.composePath, + projectName: deployment.projectName, + volumeName: deployment.volumeName, + skillwireImage: deployment.skillwireImage, + postgresImage: deployment.postgresImage, + databasePasswordFile, + applicationPepperFile, + runtimeSocketDirectory: deployment.runtimeSocketDirectory, + socketPath: deployment.socketPath, + hostEnvironment: environment, + }); + await adapter.deploy(operationSignal); + }; + const readiness = async (): Promise => { + const database = new ServiceDatabase({ + dockerExecutable: "/usr/bin/docker", + projectName: deployment.projectName, + volumeName: deployment.volumeName, + composePath: deployment.composePath, + environment: deploymentEnvironment(deployment, environment), + }); + await database.verifySchemaAndReadiness(signal); + }; + try { + const lockedOwnership = await ownershipAt(roots.stateRoot); + const lockedSet = ServiceSecretSetSchema.parse( + await readProtectedJson( + resolve(roots.stateRoot, "service-secret-set.json"), + ), + ); + if ( + lockedOwnership.recordSha256 !== ownership.recordSha256 || + JSON.stringify(lockedSet) !== JSON.stringify(set) + ) + throw new Error("Service-secret state changed after preview"); + const rotated = await rotateServiceSecret({ + installationRoot: resolve( + roots.dataRoot, + "installations", + installation.installationId, + ), + stateRoot: roots.dataRoot, + kind, + confirmation: rotationPreview.previewHash, + preview: rotationPreview, + signal, + apply, + readiness, + rollback: async (path) => { + await apply(path, AbortSignal.timeout(60_000)); + await atomicWriteJson( + resolve(roots.stateRoot, "service-secret-set.json"), + set, + roots.stateRoot, + ); + await atomicWriteJson( + resolve(roots.stateRoot, "ownership.json"), + ownership, + roots.stateRoot, + ); + }, + journal, + publish: async ({ identitySha256 }) => { + let nextOwnership = replaceOwnedAssetIdentity( + ownership, + currentAsset.assetId, + { + locator: currentAsset.locator, + expectedIdentitySha256: identitySha256, + }, + ); + nextOwnership = recordOwnedAsset( + { record: nextOwnership, externalIntegrations: [] }, + { + kind: "service-secret", + client: null, + locator: `${installation.installationId}/secrets/${kind}.retained-${rotationPreview.operationId}`, + expectedIdentitySha256: currentReference.identitySha256, + createdByOperation: journal.operationId, + retention: "retain-by-default", + disposition: "present", + }, + ).record; + await atomicWriteJson( + resolve(roots.stateRoot, "service-secret-set.json"), + { + ...set, + state: "available", + secrets: set.secrets.map((entry) => + entry.kind === kind + ? { + ...entry, + identitySha256, + state: "created" as const, + } + : entry, + ), + }, + roots.stateRoot, + ); + await atomicWriteJson( + resolve(roots.stateRoot, "ownership.json"), + nextOwnership, + roots.stateRoot, + ); + }, + }); + await journal.commit({ status: "success" }); + return result({ + command: "maintenance:rotate-service-secret", + status: "success", + exitClass: "success", + previewHash: preview.hash, + previewScope: scope, + changed: true, + summary: `${kind} rotated after readiness and durable publication`, + components: [ + { + component: "service-secret", + state: "rotated", + changed: true, + owned: true, + identity: { kind, identitySha256: rotated.identitySha256 }, + }, + ], + findings: [], + recovery: { rollbackBoundary: "none", backupId: null, instructions: [] }, + }); + } catch (error) { + await journal + .cancel({ + status: journal.hasUnprovenEffect() ? "recovery-required" : "failed", + }) + .catch(() => undefined); + throw error; + } finally { + await lock.release(); + } +} + +async function backupOperation( + command: ParsedCommand, + signal: AbortSignal, + environment: NodeJS.ProcessEnv, +): Promise { + const roots = rootsFor(command, environment); + const installation = InstallationSchema.parse( + await readProtectedJson(resolve(roots.stateRoot, "installation.json")), + ); + const deployment = await deploymentAt(roots.stateRoot); + const secretSet = ServiceSecretSetSchema.parse( + await readProtectedJson( + resolve(roots.stateRoot, "service-secret-set.json"), + ), + ); + const references = CredentialReferencesStateSchema.parse( + await readProtectedJson( + resolve(roots.stateRoot, "credential-references.json"), + ), + ); + const ownership = await ownershipAt(roots.stateRoot); + const backupsRoot = resolve( + roots.dataRoot, + "backups", + installation.installationId, + ); + const scope = { + installationId: installation.installationId, + sourceReleaseId: installation.activeReleaseId, + archiveDirectory: backupsRoot, + format: "postgres-custom", + postgresImage: deployment.postgresImage, + serviceSecretReferenceCount: secretSet.secrets.length, + clientCredentialReferenceCount: references.credentials.filter( + ({ state }) => state === "available" || state === "retained", + ).length, + validation: [ + "sha256", + "isolated-postgresql-17-restore", + "migration-010", + "catalog-invariants", + "readiness", + ], + }; + const preview = canonicalPreview("backup", scope); + if (command.previewOnly) + return previewResult("backup", scope, "Restore-validated backup preview"); + confirmPreview(preview, command.confirmPreview); + const { lock, journal } = await acquireOperation({ + roots, + installationId: installation.installationId, + command: "backup", + }); + await journal.intent("backup", { previewHash: preview.hash }); + try { + const lockedInstallation = InstallationSchema.parse( + await readProtectedJson(resolve(roots.stateRoot, "installation.json")), + ); + const lockedOwnership = await ownershipAt(roots.stateRoot); + if ( + lockedInstallation.updatedAt !== installation.updatedAt || + lockedInstallation.installationId !== installation.installationId || + lockedOwnership.recordSha256 !== ownership.recordSha256 + ) + throw new Error("Backup prerequisites changed after preview"); + const adapter = new PostgresBackupAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: deployment.composePath, + projectName: deployment.projectName, + installationId: installation.installationId, + protectedRoot: roots.dataRoot, + backupsRoot, + postgresImage: deployment.postgresImage, + environment: deploymentEnvironment(deployment, environment), + validateRestoredDatabase: async (containerName, validationSignal) => { + const query = + "SELECT concat((SELECT max(version) FROM schema_migrations),'|',(to_regclass('public.accounts') IS NOT NULL)::text,'|',(to_regclass('public.external_skill_revisions') IS NOT NULL)::text,'|',(to_regclass('public.external_advisory_chain_head') IS NOT NULL)::text)"; + const inspected = await runCommand({ + executable: "/usr/bin/docker", + args: [ + "exec", + containerName, + "psql", + "--username=postgres", + "--dbname=postgres", + "--tuples-only", + "--no-align", + "--set=ON_ERROR_STOP=1", + "--command", + query, + ], + environment: deploymentEnvironment(deployment, environment), + deadlineMilliseconds: 15_000, + maximumOutputBytes: 16 * 1024, + signal: validationSignal, + }); + const [latestMigration, accounts, catalog, advisory] = inspected.stdout + .trim() + .split("|"); + return { + latestMigration: latestMigration ?? "", + invariantsValid: accounts === "true", + catalogValid: catalog === "true" && advisory === "true", + ready: latestMigration === "010", + }; + }, + }); + const backup = await createValidatedBackup({ + installationId: installation.installationId, + sourceReleaseId: installation.activeReleaseId, + serviceSecretReferences: secretSet.secrets, + clientCredentialReferences: references.credentials + .filter(({ state }) => state === "available" || state === "retained") + .map(({ credentialReferenceId }) => credentialReferenceId), + adapter, + signal, + journal, + }); + await journal.runEffect({ + step: "backup-ownership-publication", + intent: { backupId: backup.backupId }, + signal, + action: () => + atomicWriteJson( + resolve(roots.stateRoot, "ownership.json"), + recordOwnedAsset( + { + record: ownership, + externalIntegrations: [], + }, + { + kind: "backup", + client: null, + locator: backup.backupRoot, + expectedIdentitySha256: backup.backupIdentitySha256, + createdByOperation: journal.operationId, + retention: "retain-by-default", + disposition: "present", + }, + ).record, + roots.stateRoot, + ), + verification: () => ({ backupId: backup.backupId, owned: true }), + }); + await journal.commit({ status: "success" }); + return result({ + command: "backup", + status: "success", + exitClass: "success", + previewHash: preview.hash, + previewScope: scope, + changed: true, + summary: + "PostgreSQL backup passed checksum and isolated restore validation", + components: [ + { + component: "backup", + state: "validated", + changed: true, + owned: true, + identity: { + backupId: backup.backupId, + archiveSha256: backup.archiveSha256, + }, + }, + ], + findings: [], + recovery: { + rollbackBoundary: "none", + backupId: backup.backupId, + instructions: [], + }, + }); + } catch (error) { + await journal + .cancel({ + status: journal.hasUnprovenEffect() ? "recovery-required" : "failed", + }) + .catch(() => undefined); + throw error; + } finally { + await lock.release(); + } +} + +async function readVerifiedUpgradeManifest( + path: string, + expectedSha256: string, +) { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const before = await handle.stat({ bigint: true }); + if ( + !before.isFile() || + before.nlink !== 1n || + before.size < 2n || + before.size > BigInt(4 * 1024 * 1024) + ) + throw new Error("Upgrade manifest filesystem identity is unsafe"); + const bytes = await handle.readFile(); + const after = await handle.stat({ bigint: true }); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.size !== after.size || + before.mtimeNs !== after.mtimeNs || + createHash("sha256").update(bytes).digest("hex") !== expectedSha256 + ) + throw new Error("Upgrade manifest changed after signed verification"); + return ReleaseManifestSchema.parse( + JSON.parse(bytes.toString("utf8")) as unknown, + ); + } finally { + await handle.close(); + } +} + +async function verifiedUpgradeCandidate( + command: ParsedCommand, + environment: NodeJS.ProcessEnv, + signal: AbortSignal, +) { + if (command.release === undefined || !isAbsolute(command.release)) + throw new Error("Upgrade requires an absolute signed release archive"); + const archivePath = await realpath(command.release); + if (!archivePath.endsWith(".tar.zst")) + throw new Error("Upgrade archive filename is invalid"); + const releaseBase = basename(archivePath).slice(0, -".tar.zst".length); + const releaseRoot = await realpath( + resolve(dirname(archivePath), releaseBase), + ); + const candidateEnvironment = { + ...environment, + SKILLWIRE_RELEASE_ROOT: releaseRoot, + }; + const verified = await previewProductionSetup( + { clients: "none" }, + candidateEnvironment, + {}, + signal, + ); + const expectedArchive = resolve( + dirname(releaseRoot), + `${basename(releaseRoot)}.tar.zst`, + ); + if (archivePath !== expectedArchive) + throw new Error("Upgrade archive is not the verified payload sibling"); + const manifestPath = resolve( + dirname(releaseRoot), + `${basename(releaseRoot)}.release.json`, + ); + const manifest = await readVerifiedUpgradeManifest( + manifestPath, + verified.manifestSha256, + ); + const skillwireImage = manifest.images.find( + ({ role }) => role === "skillwire", + ); + const postgresImage = manifest.images.find(({ role }) => role === "postgres"); + if (skillwireImage === undefined || postgresImage === undefined) + throw new Error("Upgrade manifest has no complete digest-pinned image set"); + return { + archivePath, + releaseRoot, + manifestPath, + policyPath: resolve(dirname(releaseRoot), manifest.trustPolicy.path), + manifest, + manifestSha256: verified.manifestSha256, + archiveSha256: verified.archiveSha256, + skillwireImage: `${skillwireImage.repository}@${skillwireImage.digest}`, + postgresImage: `${postgresImage.repository}@${postgresImage.digest}`, + target: { + releaseId: `${String(manifest.releaseSequence)}-${manifest.architecture}`, + releaseSequence: manifest.releaseSequence, + trustPolicySequence: manifest.trustPolicySequence, + schemaMinimum: manifest.compatibility.schemaMinimum, + schemaMaximum: manifest.compatibility.schemaMaximum, + latestMigration: Number(manifest.components.migrations.latest), + manifestSha256: verified.manifestSha256, + imageDigest: skillwireImage.digest, + }, + }; +} + +async function readLiveMigration( + deployment: z.infer, + environment: NodeJS.ProcessEnv, + signal: AbortSignal, +): Promise { + const query = "SELECT max(version) FROM schema_migrations"; + const response = await runCommand({ + executable: "/usr/bin/docker", + args: [ + "compose", + "--project-name", + deployment.projectName, + "--file", + deployment.composePath, + "exec", + "-T", + "postgres", + "psql", + "--username=skillwire", + "--dbname=skillwire", + "--tuples-only", + "--no-align", + "--set=ON_ERROR_STOP=1", + "--command", + query, + ], + environment: deploymentEnvironment(deployment, environment), + deadlineMilliseconds: 15_000, + maximumOutputBytes: 16 * 1024, + signal, + }); + const value = response.stdout.trim(); + if (!/^\d{3}$/.test(value)) + throw new Error("Live migration identity is invalid"); + return Number(value); +} + +function stableLauncher(releaseRoot: string): string { + const quoted = (value: string): string => + `'${value.replaceAll("'", `'"'"'`)}'`; + return [ + "#!/bin/sh", + "set -eu", + `export SKILLWIRE_RELEASE_ROOT=${quoted(releaseRoot)}`, + `exec ${quoted(resolve(releaseRoot, "runtime/node"))} ${quoted(resolve(releaseRoot, "app/skillwire.mjs"))} "$@"`, + "", + ].join("\n"); +} + +async function restoreStableLauncher( + home: string, + releaseRoot: string, +): Promise { + const launcher = resolve(home, ".local/bin/skillwire"); + const handle = await open( + launcher, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const stats = await handle.stat(); + if ( + !stats.isFile() || + stats.nlink !== 1 || + stats.uid !== process.getuid?.() || + (stats.mode & 0o777) !== 0o700 + ) + throw new Error("Stable launcher rollback target is unsafe"); + } finally { + await handle.close(); + } + const staged = resolve(dirname(launcher), `.skillwire-${randomUUID()}.stage`); + const stagedHandle = await open( + staged, + constants.O_WRONLY | + constants.O_CREAT | + constants.O_EXCL | + constants.O_NOFOLLOW, + 0o700, + ); + try { + await stagedHandle.writeFile(stableLauncher(releaseRoot), "utf8"); + await stagedHandle.sync(); + } finally { + await stagedHandle.close(); + } + await rename(staged, launcher); + await chmod(launcher, 0o700); +} + +async function upgradeOperation( + command: ParsedCommand, + signal: AbortSignal, + environment: NodeJS.ProcessEnv, +): Promise { + const roots = rootsFor(command, environment); + const installation = InstallationSchema.parse( + await readProtectedJson(resolve(roots.stateRoot, "installation.json")), + ); + if ( + installation.status === "data-retained" || + installation.status === "purged" + ) + throw new Error( + "Upgrade requires a running retained-data-aware installation", + ); + const deployment = await deploymentAt(roots.stateRoot); + const ownership = await ownershipAt(roots.stateRoot); + const secretSet = ServiceSecretSetSchema.parse( + await readProtectedJson( + resolve(roots.stateRoot, "service-secret-set.json"), + ), + ); + const references = CredentialReferencesStateSchema.parse( + await readProtectedJson( + resolve(roots.stateRoot, "credential-references.json"), + ), + ); + const candidate = await verifiedUpgradeCandidate( + command, + environment, + signal, + ); + const liveSchema = await readLiveMigration(deployment, environment, signal); + const previewInput = { + installationId: installation.installationId, + currentReleaseSequence: installation.highestAcceptedReleaseSequence, + currentTrustPolicySequence: installation.activeTrustPolicySequence, + liveSchema, + target: candidate.target, + }; + const upgradePreview = previewUpgrade(previewInput); + if (command.previewOnly) + return previewResult( + "upgrade", + previewInput, + "Signed, restore-validated, forward-only upgrade preview", + ); + confirmPreview( + { command: "upgrade", json: "", hash: upgradePreview.previewHash }, + command.confirmPreview, + ); + const { lock, journal } = await acquireOperation({ + roots, + installationId: installation.installationId, + command: "upgrade", + }); + await journal.intent("upgrade", { previewHash: upgradePreview.previewHash }); + const targetComposePath = resolve( + candidate.releaseRoot, + "distribution/self-hosted/compose.yaml", + ); + const targetDeployment = DeploymentStateSchema.parse({ + ...deployment, + releaseRoot: resolve( + roots.dataRoot, + "releases", + basename(candidate.releaseRoot), + ), + composePath: resolve( + roots.dataRoot, + "releases", + basename(candidate.releaseRoot), + "distribution/self-hosted/compose.yaml", + ), + skillwireImage: candidate.skillwireImage, + postgresImage: candidate.postgresImage, + }); + const targetEnvironment = deploymentEnvironment( + targetDeployment, + environment, + ); + const targetAdapter = new DeploymentAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: targetDeployment.composePath, + projectName: targetDeployment.projectName, + volumeName: targetDeployment.volumeName, + skillwireImage: targetDeployment.skillwireImage, + postgresImage: targetDeployment.postgresImage, + databasePasswordFile: targetDeployment.databasePasswordFile, + applicationPepperFile: targetDeployment.applicationPepperFile, + runtimeSocketDirectory: targetDeployment.runtimeSocketDirectory, + socketPath: targetDeployment.socketPath, + hostEnvironment: environment, + }); + const composeCommand = ( + composePath: string, + args: readonly string[], + commandSignal = signal, + ) => + runCommand({ + executable: "/usr/bin/docker", + args: [ + "compose", + "--project-name", + deployment.projectName, + "--file", + composePath, + ...args, + ], + environment: targetEnvironment, + deadlineMilliseconds: 120_000, + maximumOutputBytes: 128 * 1024, + signal: commandSignal, + }); + let backup: Awaited> | undefined; + try { + const lockedInstallation = InstallationSchema.parse( + await readProtectedJson(resolve(roots.stateRoot, "installation.json")), + ); + const lockedOwnership = await ownershipAt(roots.stateRoot); + const lockedDeployment = await deploymentAt(roots.stateRoot); + const lockedSchema = await readLiveMigration( + lockedDeployment, + environment, + signal, + ); + if ( + lockedInstallation.updatedAt !== installation.updatedAt || + lockedOwnership.recordSha256 !== ownership.recordSha256 || + JSON.stringify(lockedDeployment) !== JSON.stringify(deployment) || + lockedSchema !== liveSchema + ) + throw new Error("Upgrade prerequisites changed after preview"); + const upgraded = await runUpgrade({ + preview: upgradePreview, + confirmation: command.confirmPreview, + signal, + journal, + verifyTarget: async () => + (await verifiedUpgradeCandidate(command, environment, signal)).target, + createBackup: async () => { + const backupsRoot = resolve( + roots.dataRoot, + "backups", + installation.installationId, + ); + const adapter = new PostgresBackupAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: deployment.composePath, + projectName: deployment.projectName, + installationId: installation.installationId, + protectedRoot: roots.dataRoot, + backupsRoot, + postgresImage: deployment.postgresImage, + expectedLatestMigration: String(liveSchema).padStart(3, "0"), + environment: deploymentEnvironment(deployment, environment), + validateRestoredDatabase: async (containerName, validationSignal) => { + const inspected = await runCommand({ + executable: "/usr/bin/docker", + args: [ + "exec", + containerName, + "psql", + "--username=postgres", + "--dbname=postgres", + "--tuples-only", + "--no-align", + "--set=ON_ERROR_STOP=1", + "--command", + "SELECT concat((SELECT max(version) FROM schema_migrations),'|',(to_regclass('public.accounts') IS NOT NULL)::text,'|',(to_regclass('public.external_skill_revisions') IS NOT NULL)::text,'|',(to_regclass('public.external_advisory_chain_head') IS NOT NULL)::text)", + ], + environment: deploymentEnvironment(deployment, environment), + deadlineMilliseconds: 15_000, + maximumOutputBytes: 16 * 1024, + signal: validationSignal, + }); + const [migration, accounts, catalog, advisory] = inspected.stdout + .trim() + .split("|"); + const expected = String(liveSchema).padStart(3, "0"); + return { + latestMigration: migration ?? "", + invariantsValid: accounts === "true", + catalogValid: catalog === "true" && advisory === "true", + ready: migration === expected, + }; + }, + }); + backup = await createValidatedBackup({ + installationId: installation.installationId, + sourceReleaseId: installation.activeReleaseId, + serviceSecretReferences: secretSet.secrets, + clientCredentialReferences: references.credentials + .filter( + ({ state }) => state === "available" || state === "retained", + ) + .map(({ credentialReferenceId }) => credentialReferenceId), + adapter, + signal, + }); + return { + backupId: backup.backupId, + validated: backup.status === "validated", + }; + }, + drainWriters: () => + drainWriters( + { + stopAdministration: () => Promise.resolve(), + stopIngestion: () => Promise.resolve(), + stopApplication: async (writerSignal) => { + await composeCommand( + deployment.composePath, + ["stop", "skillwire"], + writerSignal, + ); + }, + verifyNoWriters: async (writerSignal) => { + const inspected = await composeCommand( + deployment.composePath, + [ + "exec", + "-T", + "postgres", + "psql", + "--username=skillwire", + "--dbname=skillwire", + "--tuples-only", + "--no-align", + "--command", + "SELECT count(*) FROM pg_stat_activity WHERE datname='skillwire' AND pid <> pg_backend_pid()", + ], + writerSignal, + ); + return inspected.stdout.trim() === "0"; + }, + startApplication: () => Promise.resolve(), + startIngestion: () => Promise.resolve(), + startAdministration: () => Promise.resolve(), + }, + signal, + ), + installApplication: async () => { + await installVerifiedRelease({ + archivePath: candidate.archivePath, + manifest: candidate.manifest, + dataRoot: roots.dataRoot, + stateRoot: roots.stateRoot, + launcherRoot: roots.home, + launcherPath: resolve(roots.home, ".local/bin/skillwire"), + installationId: installation.installationId, + manifestSha256: candidate.manifestSha256, + trustPolicyPath: candidate.policyPath, + activate: false, + }); + await access(targetComposePath, constants.R_OK); + }, + migrate: async () => { + await composeCommand(targetDeployment.composePath, [ + "run", + "--rm", + "migrate", + ]); + }, + verifyLiveSchema: () => + readLiveMigration(targetDeployment, environment, signal), + readiness: async () => { + await targetAdapter.probe(signal); + await targetAdapter.deploy(signal); + }, + verifyClients: async () => { + for (const client of installation.selectedClients) { + const vendor = await executable(client, environment); + const adapter = + client === "codex" + ? new CodexClientAdapter(vendor, environment, signal) + : new ClaudeClientAdapter( + vendor, + environment, + undefined, + undefined, + signal, + ); + const mcp = await adapter.reconcileMcp( + resolve(roots.home, ".local/bin/skillwire"), + installation.installationId, + ); + const plugin = await adapter.reconcilePlugin( + resolve( + deployment.releaseRoot, + client === "codex" + ? "distribution/codex-release-marketplace" + : "distribution/claude-marketplace", + ), + ); + if ( + !["owned-equivalent", "external-equivalent"].includes( + mcp.classification, + ) || + !["owned-equivalent", "external-equivalent"].includes( + plugin.classification, + ) + ) + throw new Error(`${client} integration changed during upgrade`); + } + }, + commitSelection: () => + atomicWriteJson( + resolve(roots.stateRoot, "active-release.json"), + { + schemaVersion: "skillwire.active-release/v1", + releaseVersion: candidate.manifest.releaseVersion, + releaseSequence: candidate.manifest.releaseSequence, + trustPolicySequence: candidate.manifest.trustPolicySequence, + architecture: candidate.manifest.architecture, + manifestSha256: candidate.manifestSha256, + archiveSha256: candidate.archiveSha256, + trustPolicyPath: `trust/${candidate.manifest.trustPolicy.path}`, + }, + roots.stateRoot, + ), + rollbackApplication: async () => { + const recoverySignal = AbortSignal.timeout(60_000); + await restoreStableLauncher(roots.home, deployment.releaseRoot); + const prior = new DeploymentAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: deployment.composePath, + projectName: deployment.projectName, + volumeName: deployment.volumeName, + skillwireImage: deployment.skillwireImage, + postgresImage: deployment.postgresImage, + databasePasswordFile: deployment.databasePasswordFile, + applicationPepperFile: deployment.applicationPepperFile, + runtimeSocketDirectory: deployment.runtimeSocketDirectory, + socketPath: deployment.socketPath, + hostEnvironment: environment, + }); + await prior.deploy(recoverySignal); + }, + stopWriters: async () => { + await composeCommand( + targetDeployment.composePath, + ["stop", "skillwire"], + AbortSignal.timeout(60_000), + ); + }, + restartWriters: () => + restartWriters( + { + stopAdministration: () => Promise.resolve(), + stopIngestion: () => Promise.resolve(), + stopApplication: () => Promise.resolve(), + verifyNoWriters: () => Promise.resolve(true), + startApplication: () => Promise.resolve(), + startIngestion: () => Promise.resolve(), + startAdministration: () => Promise.resolve(), + }, + AbortSignal.timeout(60_000), + ), + }); + if (backup === undefined) + throw new Error("Upgrade completed without a restore-validated backup"); + const completedBackup = backup; + await journal.runEffect({ + step: "upgrade-state-publication", + intent: { releaseSequence: candidate.manifest.releaseSequence }, + signal, + action: async () => { + let nextOwnership = ownership; + const targetReleaseIdentity = await releaseDirectoryIdentity( + targetDeployment.releaseRoot, + ); + const launcherPath = resolve(roots.home, ".local/bin/skillwire"); + const launcherAsset = nextOwnership.assets.find( + (asset) => asset.kind === "path" && asset.locator === launcherPath, + ); + if (launcherAsset !== undefined) + nextOwnership = replaceOwnedAssetIdentity( + nextOwnership, + launcherAsset.assetId, + { + locator: launcherPath, + expectedIdentitySha256: await ownedLauncherIdentity(launcherPath), + }, + ); + for (const asset of [ + { + kind: "release" as const, + locator: targetDeployment.releaseRoot, + identity: targetReleaseIdentity, + }, + { + kind: "trust-policy" as const, + locator: `trust/${candidate.manifest.trustPolicy.path}`, + identity: candidate.manifest.trustPolicy.sha256, + }, + { + kind: "backup" as const, + locator: completedBackup.backupRoot, + identity: completedBackup.backupIdentitySha256, + }, + ]) + nextOwnership = recordOwnedAsset( + { record: nextOwnership, externalIntegrations: [] }, + { + kind: asset.kind, + client: null, + locator: asset.locator, + expectedIdentitySha256: asset.identity, + createdByOperation: journal.operationId, + retention: "retain-by-default", + disposition: "present", + }, + ).record; + const timestamp = new Date().toISOString(); + await atomicWriteJson( + resolve(roots.stateRoot, "deployment.json"), + targetDeployment, + roots.stateRoot, + ); + await atomicWriteJson( + resolve(roots.stateRoot, "ownership.json"), + nextOwnership, + roots.stateRoot, + ); + await atomicWriteJson( + resolve(roots.stateRoot, "installation.json"), + InstallationSchema.parse({ + ...installation, + activeReleaseId: candidate.target.releaseId, + highestAcceptedReleaseSequence: candidate.target.releaseSequence, + activeTrustPolicySequence: candidate.target.trustPolicySequence, + updatedAt: timestamp, + lastValidatedAt: timestamp, + }), + roots.stateRoot, + ); + }, + verification: () => ({ + releaseSequence: candidate.target.releaseSequence, + }), + }); + await journal.commit({ status: "success" }); + return result({ + command: "upgrade", + status: "success", + exitClass: "success", + previewHash: upgradePreview.previewHash, + previewScope: previewInput, + changed: true, + summary: + "Signed release selected after backup, schema and readiness gates", + components: [ + { + component: "release", + state: "active", + changed: true, + owned: true, + identity: { + releaseId: upgraded.releaseId, + releaseSequence: candidate.target.releaseSequence, + }, + }, + ], + findings: [], + recovery: { + rollbackBoundary: "none", + backupId: upgraded.backupId, + instructions: [], + }, + }); + } catch (error) { + const recovery = + error instanceof UpgradeRecoveryError + ? upgradeRecoveryGuidance(error) + : undefined; + if (recovery !== undefined && backup !== undefined) { + let retainedOwnership = ownership; + const retainedReleaseIdentity = await releaseDirectoryIdentity( + targetDeployment.releaseRoot, + ).catch(() => undefined); + const retainedCandidates = [ + { + kind: "backup" as const, + locator: backup.backupRoot, + identity: backup.backupIdentitySha256, + exists: true, + }, + { + kind: "release" as const, + locator: targetDeployment.releaseRoot, + identity: retainedReleaseIdentity ?? candidate.archiveSha256, + exists: retainedReleaseIdentity !== undefined, + }, + { + kind: "trust-policy" as const, + locator: `trust/${candidate.manifest.trustPolicy.path}`, + identity: candidate.manifest.trustPolicy.sha256, + exists: await access( + resolve( + roots.dataRoot, + "trust", + candidate.manifest.trustPolicy.path, + ), + constants.R_OK, + ) + .then(() => true) + .catch(() => false), + }, + ]; + for (const retained of retainedCandidates) { + if ( + retained.exists && + !retainedOwnership.assets.some( + ({ kind, locator, expectedIdentitySha256, disposition }) => + kind === retained.kind && + locator === retained.locator && + expectedIdentitySha256 === retained.identity && + disposition !== "removed", + ) + ) + retainedOwnership = recordOwnedAsset( + { record: retainedOwnership, externalIntegrations: [] }, + { + kind: retained.kind, + client: null, + locator: retained.locator, + expectedIdentitySha256: retained.identity, + createdByOperation: journal.operationId, + retention: "retain-by-default", + disposition: "present", + }, + ).record; + } + if (retainedOwnership.recordSha256 !== ownership.recordSha256) + await journal + .runEffect({ + step: "upgrade-retained-recovery-assets", + intent: { backupId: backup.backupId }, + signal: new AbortController().signal, + action: () => + atomicWriteJson( + resolve(roots.stateRoot, "ownership.json"), + retainedOwnership, + roots.stateRoot, + ), + verification: () => ({ + backupId: backup?.backupId ?? "unavailable", + }), + }) + .catch(() => undefined); + } + const restoreRequired = + recovery?.rollbackBoundary === "database-restore-required"; + const recoveryRequired = upgradeFailureRequiresRecovery( + restoreRequired, + journal.hasUnprovenEffect(), + ); + await journal + .cancel({ + status: recoveryRequired ? "recovery-required" : "failed", + }) + .catch(() => undefined); + if (recovery === undefined || !(error instanceof UpgradeRecoveryError)) + throw error; + return result({ + command: "upgrade", + status: restoreRequired ? "recovery-required" : "failure", + exitClass: restoreRequired ? "rollback-required" : "service-failure", + previewHash: upgradePreview.previewHash, + previewScope: previewInput, + changed: true, + summary: error.message, + components: [], + findings: [ + { + code: restoreRequired + ? "UPGRADE_RECOVERY_REQUIRED" + : "UPGRADE_AUTOMATIC_ROLLBACK_COMPLETED", + severity: restoreRequired ? "recovery-required" : "error", + component: "upgrade", + summary: error.message, + nextAction: recovery.instructions.join("; "), + }, + ], + recovery: { ...recovery, instructions: [...recovery.instructions] }, + }); + } finally { + await lock.release(); + } +} + +async function defaultUninstallOperation( + command: ParsedCommand, + signal: AbortSignal, + environment: NodeJS.ProcessEnv, +): Promise { + const roots = rootsFor(command, environment); + const installation = InstallationSchema.parse( + await readProtectedJson(resolve(roots.stateRoot, "installation.json")), + ); + const ownership = await ownershipAt(roots.stateRoot); + const deployment = await deploymentAt(roots.stateRoot); + const uninstallPreview = previewDefaultUninstall(ownership); + const scope = { + installationId: uninstallPreview.installationId, + ownershipRevision: uninstallPreview.ownershipRevision, + remove: uninstallPreview.remove.map( + ({ assetId, kind, client, locator, expectedIdentitySha256 }) => ({ + assetId, + kind, + client, + locator, + expectedIdentitySha256, + }), + ), + retain: uninstallPreview.retain.map( + ({ assetId, kind, client, locator }) => ({ + assetId, + kind, + client, + locator, + }), + ), + }; + const preview = canonicalPreview("uninstall", scope); + if (command.previewOnly) + return previewResult( + "uninstall", + scope, + "Owned client/runtime removal with retained recovery data", + ); + confirmPreview(preview, command.confirmPreview); + if (preview.hash !== uninstallPreview.previewHash) + throw new Error("Default-uninstall preview canonicalization changed"); + if (installation.status === "data-retained") + return result({ + command: "uninstall", + status: "success", + exitClass: "success", + previewHash: preview.hash, + previewScope: scope, + changed: false, + summary: + "Installation data is already retained and runtime state is removed", + components: [], + findings: [], + recovery: { rollbackBoundary: "none", backupId: null, instructions: [] }, + }); + const { lock, journal } = await acquireOperation({ + roots, + installationId: installation.installationId, + command: "uninstall", + }); + await journal.intent("uninstall", { previewHash: preview.hash }); + const adapters = new Map< + "codex" | "claude", + CodexClientAdapter | ClaudeClientAdapter + >(); + const adapterFor = async (client: "codex" | "claude") => { + const existing = adapters.get(client); + if (existing !== undefined) return existing; + const vendor = await executable(client, environment); + const adapter = + client === "codex" + ? new CodexClientAdapter(vendor, environment, signal) + : new ClaudeClientAdapter( + vendor, + environment, + undefined, + undefined, + signal, + ); + adapters.set(client, adapter); + return adapter; + }; + const marketplace = (client: "codex" | "claude") => + resolve( + deployment.releaseRoot, + client === "codex" + ? "distribution/codex-release-marketplace" + : "distribution/claude-marketplace", + ); + const removedPlugin = new Set<"codex" | "claude">(); + const observeIdentity = async (asset: OwnedAsset): Promise => { + if (asset.client !== null) { + const adapter = await adapterFor(asset.client); + if (asset.kind === "mcp-entry") { + const state = await adapter.reconcileMcp( + resolve(roots.home, ".local/bin/skillwire"), + installation.installationId, + asset.expectedIdentitySha256, + ); + if (state.classification !== "owned-equivalent") + throw new Error("Client MCP ownership is ambiguous or drifted"); + return asset.expectedIdentitySha256; + } + if (asset.kind === "plugin" || asset.kind === "marketplace") { + const state = await adapter.reconcilePlugin(marketplace(asset.client)); + if (state.classification !== "external-equivalent") + throw new Error("Client plugin ownership is ambiguous or drifted"); + return asset.expectedIdentitySha256; + } + } + if (asset.kind === "compose-project") { + if (asset.locator !== deployment.projectName) + throw new Error("Compose project ownership changed"); + if ( + !(await observeOwnedComposeService( + deployment, + "postgres", + environment, + signal, + )) || + !(await observeOwnedComposeService( + deployment, + "skillwire", + environment, + signal, + )) + ) + throw new Error("Owned Compose project is incomplete or missing"); + return clientComponentIdentity({ projectName: deployment.projectName }); + } + if (asset.kind === "container") { + const [, service] = asset.locator.split(":"); + if (service !== "skillwire" && service !== "postgres") + throw new Error("Container ownership changed"); + if ( + !(await observeOwnedComposeService( + deployment, + service, + environment, + signal, + )) + ) + throw new Error("Owned container is missing"); + return clientComponentIdentity({ + projectName: deployment.projectName, + service, + }); + } + return asset.expectedIdentitySha256; + }; + const removeAsset = async (asset: OwnedAsset): Promise => { + if ( + asset.client !== null && + (asset.kind === "plugin" || asset.kind === "marketplace") && + removedPlugin.has(asset.client) + ) + return; + await observeIdentity(asset); + if (asset.client === null) return; + const adapter = await adapterFor(asset.client); + if (asset.kind === "mcp-entry") await adapter.removeMcp(); + if ( + (asset.kind === "plugin" || asset.kind === "marketplace") && + !removedPlugin.has(asset.client) + ) { + if (asset.client === "codex") + await (adapter as CodexClientAdapter).removePlugin( + marketplace(asset.client), + ); + else await (adapter as ClaudeClientAdapter).removePlugin(); + removedPlugin.add(asset.client); + } + }; + try { + const lockedInstallation = InstallationSchema.parse( + await readProtectedJson(resolve(roots.stateRoot, "installation.json")), + ); + const lockedOwnership = await ownershipAt(roots.stateRoot); + if ( + lockedInstallation.updatedAt !== installation.updatedAt || + lockedOwnership.recordSha256 !== ownership.recordSha256 + ) + throw new Error("Uninstall prerequisites changed after preview"); + const uninstalled = await runDefaultUninstall({ + ownership, + preview: uninstallPreview, + confirmation: uninstallPreview.previewHash, + signal, + observeIdentity, + removeAsset, + journal, + stopOwnedService: async () => { + await runCommand({ + executable: "/usr/bin/docker", + args: [ + "compose", + "--project-name", + deployment.projectName, + "--file", + deployment.composePath, + "down", + "--remove-orphans", + ], + environment: deploymentEnvironment(deployment, environment), + deadlineMilliseconds: 60_000, + maximumOutputBytes: 64 * 1024, + signal, + }); + }, + publishRetained: async (nextOwnership) => { + await atomicWriteJson( + resolve(roots.stateRoot, "ownership.json"), + nextOwnership, + roots.stateRoot, + ); + await atomicWriteJson( + resolve(roots.stateRoot, "installation.json"), + transitionInstallation(installation, "data-retained"), + roots.stateRoot, + ); + }, + }); + await journal.commit({ status: "success" }); + return result({ + command: "uninstall", + status: "success", + exitClass: "success", + previewHash: preview.hash, + previewScope: scope, + changed: uninstalled.removed.length > 0, + summary: + "Owned clients and runtime were removed; recovery data is retained", + components: [ + { + component: "service", + state: "data-retained", + changed: true, + owned: true, + identity: { + installationId: installation.installationId, + retainedAssets: uninstalled.retained.length, + }, + }, + ], + findings: [], + recovery: { rollbackBoundary: "none", backupId: null, instructions: [] }, + }); + } catch (error) { + await journal + .cancel({ + status: journal.hasUnprovenEffect() ? "recovery-required" : "failed", + }) + .catch(() => undefined); + throw error; + } finally { + await lock.release(); + } +} + +async function clientUninstallOperation( + command: ParsedCommand, + signal: AbortSignal, + environment: NodeJS.ProcessEnv, +): Promise { + if (command.client === undefined) + throw new Error("Client uninstall requires an exact client"); + const client = command.client; + const roots = rootsFor(command, environment); + const installation = InstallationSchema.parse( + await readProtectedJson(resolve(roots.stateRoot, "installation.json")), + ); + const ownership = await ownershipAt(roots.stateRoot); + const deployment = await deploymentAt(roots.stateRoot); + const plan = planOwnedAssetDispositions( + ownership, + "client-uninstall", + client, + ); + const scope = { + installationId: installation.installationId, + client, + ownershipRevision: ownership.recordRevision, + remove: plan.remove.map( + ({ assetId, kind, locator, expectedIdentitySha256 }) => ({ + assetId, + kind, + locator, + expectedIdentitySha256, + }), + ), + siblingClient: client === "codex" ? "claude" : "codex", + sharedData: "retained", + }; + const preview = canonicalPreview("clients:uninstall", scope); + if (command.previewOnly) + return previewResult( + "clients:uninstall", + scope, + `Selective owned-only ${client} uninstall preview`, + ); + confirmPreview(preview, command.confirmPreview); + const currentOwnership = await ownershipAt(roots.stateRoot); + if (currentOwnership.recordSha256 !== ownership.recordSha256) + throw new Error("Client ownership changed after preview"); + const bridge = await bridgeAt(roots.stateRoot, installation.installationId); + const bridgeEntry = bridge.clients.find((entry) => entry.client === client); + const vendor = await executable(client, environment); + const adapter = + client === "codex" + ? new CodexClientAdapter(vendor, environment, signal) + : new ClaudeClientAdapter( + vendor, + environment, + undefined, + undefined, + signal, + ); + const marketplacePath = resolve( + deployment.releaseRoot, + client === "codex" + ? "distribution/codex-release-marketplace" + : "distribution/claude-marketplace", + ); + const observeRemovals = () => + Promise.all( + plan.remove.map(async (asset) => { + if (asset.kind === "mcp-entry") { + const state = await adapter.reconcileMcp( + resolve(roots.home, ".local/bin/skillwire"), + installation.installationId, + asset.expectedIdentitySha256, + ); + return { + component: "mcp-entry" as const, + classification: state.classification, + expectedIdentitySha256: asset.expectedIdentitySha256, + currentIdentitySha256: + state.classification === "owned-equivalent" + ? asset.expectedIdentitySha256 + : (state.observations[0]?.identitySha256 ?? null), + }; + } + if (asset.kind === "plugin" || asset.kind === "marketplace") { + const state = await adapter.reconcilePlugin(marketplacePath); + const equivalent = state.classification === "external-equivalent"; + return { + component: asset.kind, + classification: equivalent + ? ("owned-equivalent" as const) + : state.classification, + expectedIdentitySha256: asset.expectedIdentitySha256, + currentIdentitySha256: equivalent + ? asset.expectedIdentitySha256 + : (state.observations[0]?.identitySha256 ?? null), + }; + } + if (asset.kind === "credential") { + const matching = + bridgeEntry?.credentialReference === asset.locator && + bridgeEntry.keyId !== undefined; + return { + component: "credential" as const, + classification: matching + ? ("owned-equivalent" as const) + : ("ambiguous" as const), + expectedIdentitySha256: asset.expectedIdentitySha256, + currentIdentitySha256: matching + ? clientComponentIdentity({ reference: asset.locator }) + : null, + }; + } + throw new Error("Unexpected client-owned removal asset"); + }), + ); + const observations = await observeRemovals(); + const secretService = new SecretToolCredentialStore( + "/usr/bin/secret-tool", + environment, + ); + const fallback = new RestrictiveFileCredentialStore( + roots.dataRoot, + roots.dataRoot, + installation.installationId, + ); + const { lock, journal } = await acquireOperation({ + roots, + installationId: installation.installationId, + command: "clients-uninstall", + }); + await journal.intent("clients-uninstall", { + client, + previewHash: preview.hash, + }); + try { + const lockedOwnership = await ownershipAt(roots.stateRoot); + if (lockedOwnership.recordSha256 !== ownership.recordSha256) + throw new Error("Client ownership changed while acquiring the lock"); + const lockedObservations = await observeRemovals(); + if (JSON.stringify(lockedObservations) !== JSON.stringify(observations)) + throw new Error("Client profile state changed after preview"); + const removed = await journal.runEffect({ + step: `client-${client}-uninstall`, + intent: { client }, + signal, + action: () => + uninstallClientLifecycle( + client, + { + inspect: () => Promise.resolve(lockedObservations), + removeMcp: async () => { + const current = await adapter.reconcileMcp( + resolve(roots.home, ".local/bin/skillwire"), + installation.installationId, + plan.remove.find(({ kind }) => kind === "mcp-entry") + ?.expectedIdentitySha256, + ); + if (current.classification !== "owned-equivalent") + throw new Error("Client MCP changed before removal"); + await adapter.removeMcp(); + }, + removePlugin: async () => { + const current = await adapter.reconcilePlugin(marketplacePath); + if (current.classification !== "external-equivalent") + throw new Error("Client plugin changed before removal"); + if (client === "codex") + await (adapter as CodexClientAdapter).removePlugin( + marketplacePath, + ); + else await (adapter as ClaudeClientAdapter).removePlugin(); + }, + removeMarketplace: () => Promise.resolve(), + revokeCredential: async () => { + if (bridgeEntry?.keyId === undefined) + throw new Error("Client key identity is unavailable"); + await revokeClientKeyInAdminContainer({ + dockerExecutable: "/usr/bin/docker", + composePath: deployment.composePath, + projectName: deployment.projectName, + keyId: bridgeEntry.keyId, + environment: deploymentEnvironment(deployment, environment), + signal: AbortSignal.timeout(30_000), + }); + if (bridgeEntry.credentialReference.startsWith("secret-service:")) + await secretService.clear( + installation.installationId, + client, + bridgeEntry.credentialReference, + AbortSignal.timeout(30_000), + ); + else + await fallback.remove( + bridgeEntry.credentialReference as RestrictiveFileReference, + ); + }, + verifyAbsent: async (component) => { + if (component === "credential") return true; + if (component === "mcp-entry") + return ( + ( + await adapter.reconcileMcp( + resolve(roots.home, ".local/bin/skillwire"), + installation.installationId, + ) + ).classification === "absent" + ); + return ( + (await adapter.reconcilePlugin(marketplacePath)) + .classification === "absent" + ); + }, + }, + signal, + ), + verification: (value) => ({ client, status: value.status }), + }); + if (removed.status === "recovery-required") { + await journal.cancel({ status: "recovery-required" }); + return result({ + command: "clients:uninstall", + status: "recovery-required", + exitClass: "rollback-required", + previewHash: preview.hash, + previewScope: scope, + changed: removed.removed.length > 0, + summary: `${client} uninstall stopped at a recoverable boundary`, + components: [], + findings: [ + { + code: "CLIENT_UNINSTALL_RECOVERY_REQUIRED", + severity: "recovery-required", + component: client, + summary: "A client inverse operation could not be proven complete", + nextAction: + "Run repair to observe the client-specific journal boundary", + }, + ], + recovery: { + rollbackBoundary: "client-only", + backupId: null, + instructions: [], + }, + }); + } + await journal.runEffect({ + step: `client-${client}-uninstall-state`, + intent: { client }, + signal, + action: async () => { + let nextOwnership = currentOwnership; + for (const asset of plan.remove) + nextOwnership = recordAssetDisposition( + nextOwnership, + asset.assetId, + "removed", + ); + const references = CredentialReferencesStateSchema.parse( + await readProtectedJson( + resolve(roots.stateRoot, "credential-references.json"), + ), + ); + const integrations = ClientIntegrationsStateSchema.parse( + await readProtectedJson( + resolve(roots.stateRoot, "client-integrations.json"), + ), + ); + const nextIntegrations = integrations.integrations.map((entry) => { + if (entry.client !== client) return entry; + return transitionClientIntegration( + entry, + entry.state === "external-verified" + ? "retained-external" + : "removed", + ); + }); + const selectedClients = installation.selectedClients.filter( + (selected) => selected !== client, + ); + await atomicWriteJson( + resolve(roots.stateRoot, "ownership.json"), + nextOwnership, + roots.stateRoot, + ); + await atomicWriteJson( + resolve(roots.stateRoot, "credential-references.json"), + { + ...references, + credentials: references.credentials.map((entry) => + entry.client === client + ? { ...entry, state: "removed" as const } + : entry, + ), + }, + roots.stateRoot, + ); + await atomicWriteJson( + resolve(roots.stateRoot, "client-integrations.json"), + { ...integrations, integrations: nextIntegrations }, + roots.stateRoot, + ); + await atomicWriteJson( + resolve( + roots.stateRoot, + "installations", + installation.installationId, + "bridge-state.json", + ), + { + ...bridge, + clients: bridge.clients.filter((entry) => entry.client !== client), + }, + roots.stateRoot, + ); + await atomicWriteJson( + resolve(roots.stateRoot, "installation.json"), + InstallationSchema.parse({ + ...installation, + selectedClients, + clientIntegrationIds: { + ...installation.clientIntegrationIds, + [client]: null, + }, + status: selectedClients.length === 0 ? "service-ready" : "complete", + updatedAt: new Date().toISOString(), + }), + roots.stateRoot, + ); + }, + verification: () => ({ client, published: true }), + }); + await journal.commit({ status: "success" }); + return result({ + command: "clients:uninstall", + status: "success", + exitClass: "success", + previewHash: preview.hash, + previewScope: scope, + changed: removed.status === "removed", + summary: `${client} owned integration was removed independently`, + components: [ + { + component: client, + state: removed.status, + changed: removed.status === "removed", + owned: true, + identity: { siblingPreserved: true, sharedDataPreserved: true }, + }, + ], + findings: [], + recovery: { rollbackBoundary: "none", backupId: null, instructions: [] }, + }); + } catch (error) { + await journal + .cancel({ + status: journal.hasUnprovenEffect() ? "recovery-required" : "failed", + }) + .catch(() => undefined); + throw error; + } finally { + await lock.release(); + } +} + +function contained(root: string, candidate: string): boolean { + const child = relative(resolve(root), resolve(candidate)); + return child === "" || (!child.startsWith("..") && !isAbsolute(child)); +} + +async function purgeOperation( + command: ParsedCommand, + signal: AbortSignal, + environment: NodeJS.ProcessEnv, +): Promise { + const roots = rootsFor(command, environment); + const installation = InstallationSchema.parse( + await readProtectedJson(resolve(roots.stateRoot, "installation.json")), + ); + if (installation.status !== "data-retained") + throw new Error( + "Purge requires default-uninstall-equivalent retained state", + ); + const ownership = await ownershipAt(roots.stateRoot); + const purgePreview = previewPurge(ownership); + const scope = { + installationId: purgePreview.installationId, + ownershipRevision: purgePreview.ownershipRevision, + unrecoverable: purgePreview.unrecoverable.map( + ({ assetId, kind, client, locator, expectedIdentitySha256 }) => ({ + assetId, + kind, + client, + locator, + expectedIdentitySha256, + }), + ), + }; + const preview = canonicalPreview("purge", scope); + if (command.previewOnly) + return previewResult( + "purge", + scope, + "Permanent removal preview for exact owned retained assets", + ); + confirmPreview(preview, command.confirmPreview); + if (preview.hash !== purgePreview.previewHash) + throw new Error("Purge preview canonicalization changed"); + const deployment = await deploymentAt(roots.stateRoot); + const bridge = await bridgeAt(roots.stateRoot, installation.installationId); + const secretService = new SecretToolCredentialStore( + "/usr/bin/secret-tool", + environment, + ); + const fallback = new RestrictiveFileCredentialStore( + roots.dataRoot, + roots.dataRoot, + installation.installationId, + ); + const filesystemPath = (asset: OwnedAsset): string | undefined => { + if (asset.kind === "trust-policy") + return resolve(roots.dataRoot, asset.locator); + if (asset.kind === "service-secret") + return resolve(roots.dataRoot, "installations", asset.locator); + if ( + asset.kind === "path" || + asset.kind === "release" || + asset.kind === "backup" + ) + return resolve(asset.locator); + return undefined; + }; + const protectedRootFor = (path: string): string => { + if (contained(roots.dataRoot, path)) return roots.dataRoot; + if (contained(roots.stateRoot, path)) return roots.stateRoot; + if (contained(roots.home, path)) return roots.home; + throw new Error("Purge filesystem target is outside owned roots"); + }; + const allowedFilesFor = ( + asset: OwnedAsset, + path: string, + ): string[] | undefined => { + if (asset.kind !== "path") return undefined; + const allowed = new Set(); + allowed.add(path); + if ( + path === + resolve(roots.stateRoot, "installations", installation.installationId) + ) + allowed.add(resolve(path, "bridge-state.json")); + for (const candidate of ownership.assets) { + if ( + candidate.kind !== "service-secret" || + candidate.disposition === "removed" + ) + continue; + const candidatePath = resolve( + roots.dataRoot, + "installations", + candidate.locator, + ); + if (contained(path, candidatePath)) allowed.add(candidatePath); + } + return [...allowed]; + }; + const observeIdentity = async (asset: OwnedAsset): Promise => { + const path = filesystemPath(asset); + if (path !== undefined) { + await validateOwnedFilesystemTree( + path, + protectedRootFor(path), + allowedFilesFor(asset, path), + ); + if (asset.kind === "trust-policy" || asset.kind === "service-secret") { + const handle = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const bytes = await handle.readFile(); + return asset.kind === "service-secret" + ? createHash("sha256") + .update("skillwire-service-secret-identity-v1\0") + .update(bytes) + .digest("hex") + : createHash("sha256").update(bytes).digest("hex"); + } finally { + await handle.close(); + } + } + if (asset.kind === "backup") return backupDirectoryIdentity(path); + if (asset.kind === "path") { + const launcherIdentity = await ownedLauncherIdentity(path).catch( + () => undefined, + ); + if (launcherIdentity !== undefined) return launcherIdentity; + return clientComponentIdentity({ + [contained(roots.stateRoot, path) + ? "bridgeStateRoot" + : "installationRoot"]: path, + }); + } + if (asset.kind === "release") return releaseDirectoryIdentity(path); + return asset.expectedIdentitySha256; + } + if (asset.kind === "volume") { + const inspected = await runCommand({ + executable: "/usr/bin/docker", + args: [ + "volume", + "inspect", + asset.locator, + "--format", + '{{.Name}}|{{index .Labels "com.docker.compose.project"}}', + ], + environment: deploymentEnvironment(deployment, environment), + signal, + }); + if ( + inspected.stdout.trim() !== `${asset.locator}|${deployment.projectName}` + ) + throw new Error("PostgreSQL volume ownership is ambiguous"); + return clientComponentIdentity({ volumeName: asset.locator }); + } + if (asset.kind === "credential" && asset.client !== null) { + const entry = bridge.clients.find( + ({ client, credentialReference }) => + client === asset.client && credentialReference === asset.locator, + ); + if (entry === undefined) + throw new Error("Client credential ownership is ambiguous"); + if (asset.locator.startsWith("secret-service:")) + await secretService.lookup( + installation.installationId, + asset.client, + asset.locator, + signal, + ); + else await fallback.lookup(asset.locator as RestrictiveFileReference); + return clientComponentIdentity({ reference: asset.locator }); + } + throw new Error("Purge target kind is not safely removable"); + }; + const removeAsset = async (asset: OwnedAsset): Promise => { + const path = filesystemPath(asset); + if (path !== undefined) { + await removeOwnedFilesystemTree( + path, + protectedRootFor(path), + allowedFilesFor(asset, path), + ); + return; + } + if (asset.kind === "volume") { + await runCommand({ + executable: "/usr/bin/docker", + args: ["volume", "rm", asset.locator], + environment: deploymentEnvironment(deployment, environment), + signal, + }); + return; + } + if (asset.kind === "credential" && asset.client !== null) { + if (asset.locator.startsWith("secret-service:")) + await secretService.clear( + installation.installationId, + asset.client, + asset.locator, + signal, + ); + else await fallback.remove(asset.locator as RestrictiveFileReference); + return; + } + throw new Error("Purge target kind is not safely removable"); + }; + const { lock, journal } = await acquireOperation({ + roots, + installationId: installation.installationId, + command: "purge", + }); + await journal.intent("purge", { previewHash: preview.hash }); + try { + const lockedInstallation = InstallationSchema.parse( + await readProtectedJson(resolve(roots.stateRoot, "installation.json")), + ); + const lockedOwnership = await ownershipAt(roots.stateRoot); + if ( + lockedInstallation.updatedAt !== installation.updatedAt || + lockedOwnership.recordSha256 !== ownership.recordSha256 + ) + throw new Error("Purge prerequisites changed after preview"); + const purged = await runPurge({ + ownership, + preview: purgePreview, + confirmation: purgePreview.previewHash, + signal, + observeIdentity, + removeAsset, + journal, + }); + await journal.runEffect({ + step: "purge-state-publication", + intent: { installationId: installation.installationId }, + signal, + action: async () => { + await atomicWriteJson( + resolve(roots.stateRoot, "ownership.json"), + purged.ownership, + roots.stateRoot, + ); + await atomicWriteJson( + resolve(roots.stateRoot, "installation.json"), + transitionInstallation(installation, "purged"), + roots.stateRoot, + ); + }, + verification: () => ({ purged: true }), + }); + await journal.commit({ status: "success" }); + return result({ + command: "purge", + status: "success", + exitClass: "success", + previewHash: preview.hash, + previewScope: scope, + changed: purged.removed.length > 0, + summary: "Exact confirmed owned retained assets were permanently removed", + components: [ + { + component: "installation", + state: "purged", + changed: true, + owned: true, + identity: { + installationId: installation.installationId, + unrecoverableAssets: purged.removed.length, + }, + }, + ], + findings: [], + recovery: { rollbackBoundary: "none", backupId: null, instructions: [] }, + }); + } catch (error) { + await journal + .cancel({ + status: journal.hasUnprovenEffect() ? "recovery-required" : "failed", + }) + .catch(() => undefined); + throw error; + } finally { + await lock.release(); + } +} + +export function createProductionLifecycleOperations( + environment: NodeJS.ProcessEnv = process.env, +): AdministrativeOperations { + return { + status: (command, signal) => statusOperation(command, signal, environment), + doctor: (command, signal) => doctorOperation(command, signal, environment), + repair: (command, signal) => repairOperation(command, signal, environment), + "clients:rotate-key": (command, signal) => + rotateClientKeyOperation(command, signal, environment), + "maintenance:rotate-service-secret": (command, signal) => + rotateServiceSecretOperation(command, signal, environment), + backup: (command, signal) => backupOperation(command, signal, environment), + upgrade: (command, signal) => + upgradeOperation(command, signal, environment), + "clients:uninstall": (command, signal) => + clientUninstallOperation(command, signal, environment), + uninstall: (command, signal) => + defaultUninstallOperation(command, signal, environment), + purge: (command, signal) => purgeOperation(command, signal, environment), + }; +} diff --git a/src/onboarding/application/production-setup.ts b/src/onboarding/application/production-setup.ts index 9f06725..217301b 100644 --- a/src/onboarding/application/production-setup.ts +++ b/src/onboarding/application/production-setup.ts @@ -13,9 +13,13 @@ import { basename, dirname, isAbsolute, resolve } from "node:path"; import { z } from "zod"; import { atomicWriteJson } from "../adapters/filesystem/atomic-state.js"; -import { installVerifiedRelease } from "../adapters/filesystem/release-installer.js"; +import { + installVerifiedRelease, + releaseDirectoryIdentity, +} from "../adapters/filesystem/release-installer.js"; import { verifySelfHostedRelease } from "../adapters/filesystem/release-verifier.js"; import { DeploymentAdapter } from "../adapters/docker/deployment.js"; +import { dockerProcessEnvironment } from "../adapters/docker/environment.js"; import { ServiceDatabase } from "../adapters/postgres/service-database.js"; import { ClientKeyHandoffRecoveryError, @@ -37,6 +41,8 @@ import { canonicalPreview } from "../cli/confirmation.js"; import { ClientMutationNotStartedError } from "../domain/client-mutation.js"; import type { ReleaseManifest } from "../domain/release-manifest.js"; import { + ClientIntegrationSchema, + CredentialReferenceSchema, InstallationSchema, ServiceSecretSetSchema, } from "../domain/installation.js"; @@ -56,6 +62,8 @@ import { installClientLifecycle, } from "./client-lifecycle.js"; import { verifyClientIntegration } from "./client-verification.js"; +import { inspectInstalledStatus } from "./status.js"; +import { continueProductionSetup } from "./production-continuation.js"; import type { GuidedSetupOptions, GuidedSetupResult, @@ -94,6 +102,85 @@ const ActiveReleaseStateSchema = z }) .strict(); +const ClientIntegrationsStateSchema = z + .object({ + schemaVersion: z.literal("skillwire.client-integrations/v1"), + installationId: z.uuid(), + integrations: z.array(ClientIntegrationSchema).max(2), + }) + .strict(); + +export function unchangedSetupClientResults( + requested: readonly ClientName[], + installationId: string, + state: unknown, +): readonly SetupClientResult[] | undefined { + const integrations = ClientIntegrationsStateSchema.parse(state); + if (integrations.installationId !== installationId) + throw new Error( + "Repeated setup client state belongs to another installation", + ); + const results: SetupClientResult[] = []; + for (const client of requested) { + const integration = integrations.integrations.find( + (entry) => entry.client === client, + ); + if ( + integration === undefined || + (integration.state !== "verified" && + integration.state !== "external-verified") + ) + return undefined; + results.push({ + client, + status: integration.state, + compensated: false, + owned: integration.state !== "external-verified", + }); + } + return results; +} + +export async function ownedLauncherIdentity(path: string): Promise { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stats = await handle.stat(); + if ( + !stats.isFile() || + stats.nlink !== 1 || + stats.uid !== process.getuid?.() || + (stats.mode & 0o777) !== 0o700 || + stats.size < 1 || + stats.size > 64 * 1024 + ) + throw new Error("Stable launcher is not a protected owned file"); + return createHash("sha256") + .update("skillwire-owned-launcher-v1\0") + .update(await handle.readFile()) + .digest("hex"); + } finally { + await handle.close(); + } +} + +async function readProtectedSetupJson(path: string): Promise { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stats = await handle.stat(); + if ( + !stats.isFile() || + stats.nlink !== 1 || + stats.uid !== process.getuid?.() || + (stats.mode & 0o777) !== 0o600 || + stats.size > 1024 * 1024 + ) + throw new Error("Repeated setup state is unsafe"); + return JSON.parse(await handle.readFile("utf8")) as unknown; + } finally { + await handle.close(); + } +} + export interface ProductionTrustOverrides { readonly pinnedInitialPolicySha256?: string | undefined; } @@ -448,8 +535,7 @@ function composeEnvironment(options: { readonly applicationPepperFile: string; readonly runtimeSocketDirectory: string; }): NodeJS.ProcessEnv { - return { - ...options.environment, + return dockerProcessEnvironment(options.environment, { SKILLWIRE_COMPOSE_PROJECT: options.projectName, SKILLWIRE_POSTGRES_VOLUME: options.volumeName, SKILLWIRE_IMAGE: options.skillwireImage, @@ -459,7 +545,7 @@ function composeEnvironment(options: { SKILLWIRE_RUNTIME_SOCKET_DIRECTORY: options.runtimeSocketDirectory, SKILLWIRE_RUNTIME_UID: String(process.getuid?.() ?? 10001), SKILLWIRE_RUNTIME_GID: String(process.getgid?.() ?? 10001), - }; + }); } function image( @@ -614,6 +700,7 @@ async function runProductionSetupUnlocked( applicationPepperFile, runtimeSocketDirectory, socketPath, + hostEnvironment: environment, }); await runEffect({ step: "deployment", @@ -652,8 +739,11 @@ async function runProductionSetupUnlocked( installationId, ); await mkdir(bridgeStateRoot, { recursive: true, mode: 0o700 }); - const bridgeClients: { client: ClientName; credentialReference: string }[] = - []; + const bridgeClients: { + client: ClientName; + credentialReference: string; + keyId: string; + }[] = []; const persistBridgeState = (): Promise => atomicWriteJson( resolve(bridgeStateRoot, "bridge-state.json"), @@ -679,6 +769,7 @@ async function runProductionSetupUnlocked( readonly client: ClientName; readonly result: Awaited>; readonly credentialReference: string | undefined; + readonly keyId: string | undefined; readonly marketplacePath: string; }[] = []; for (const client of selectedClients(options.clients)) { @@ -692,6 +783,7 @@ async function runProductionSetupUnlocked( installationId, ); let currentReference: string | undefined; + let currentKeyId: string | undefined; const vendorExecutable = await executable(client, environment); const adapter = client === "codex" @@ -703,6 +795,20 @@ async function runProductionSetupUnlocked( undefined, signal, ); + const recoveryAdapter = () => + client === "codex" + ? new CodexClientAdapter( + vendorExecutable, + environment, + AbortSignal.timeout(30_000), + ) + : new ClaudeClientAdapter( + vendorExecutable, + environment, + undefined, + undefined, + AbortSignal.timeout(30_000), + ); const marketplacePath = resolve( installed.releaseRoot, client === "codex" @@ -785,6 +891,7 @@ async function runProductionSetupUnlocked( }), verification: (value) => ({ client, keyId: value.keyId }), }); + currentKeyId = key.keyId; try { return await runEffect({ step: `client-${client}-credential`, @@ -843,6 +950,7 @@ async function runProductionSetupUnlocked( bridgeClients.push({ client, credentialReference: currentReference, + keyId: key.keyId, }); await persistBridgeState(); return { keyId: key.keyId, reference: currentReference }; @@ -915,8 +1023,8 @@ async function runProductionSetupUnlocked( signal, }); }, - removePlugin: () => adapter.removePlugin(marketplacePath), - removeMcp: () => adapter.removeMcp(), + removePlugin: () => recoveryAdapter().removePlugin(marketplacePath), + removeMcp: () => recoveryAdapter().removeMcp(), revokeCredential: async (keyId, reference) => { const index = bridgeClients.findIndex( (entry) => entry.client === client, @@ -957,6 +1065,7 @@ async function runProductionSetupUnlocked( client, result, credentialReference: currentReference, + keyId: currentKeyId, marketplacePath, }); } @@ -973,6 +1082,82 @@ async function runProductionSetupUnlocked( : "success"; const timestamp = new Date().toISOString(); const selected = selectedClients(options.clients); + const credentialOperationId = randomUUID(); + const clientStateRecords = clientOwnership + .filter( + ({ result }) => + result.status === "verified" || result.status === "external-verified", + ) + .map((entry) => { + const clientIntegrationId = randomUUID(); + const credentialReferenceId = + entry.result.components.credential === "created" && + entry.credentialReference !== undefined && + entry.keyId !== undefined + ? randomUUID() + : null; + const mcpIdentitySha256 = clientComponentIdentity({ + command: installed.launcherPath, + args: [ + "bridge", + "--installation", + installationId, + "--client", + entry.client, + ], + scope: "user", + }); + const adapterIdentitySha256 = clientComponentIdentity({ + plugin: "skillwire-autonomous-activation@skillwire", + marketplacePath: entry.marketplacePath, + }); + return { + client: entry.client, + integration: ClientIntegrationSchema.parse({ + schemaVersion: "skillwire.client-integration/v1", + clientIntegrationId, + installationId, + client: entry.client, + clientVersion: entry.client === "codex" ? "0.147.0" : "2.1.229", + profileScope: "normal-user", + state: + entry.result.status === "external-verified" + ? "external-verified" + : "verified", + credentialReferenceId, + keyPublicIdHash: + entry.keyId === undefined + ? null + : createHash("sha256").update(entry.keyId).digest("hex"), + mcpIdentitySha256, + adapterIdentitySha256, + }), + credential: + credentialReferenceId === null || + entry.credentialReference === undefined || + entry.keyId === undefined + ? null + : CredentialReferenceSchema.parse({ + schemaVersion: "skillwire.credential-reference/v1", + credentialReferenceId, + installationId, + client: entry.client, + backend: entry.credentialReference.startsWith( + "secret-service:", + ) + ? "secret-service" + : "restrictive-file", + locator: entry.credentialReference, + keyPublicIdHash: createHash("sha256") + .update(entry.keyId) + .digest("hex"), + createdByOperation: credentialOperationId, + state: "available", + fallbackRiskConfirmed: + !entry.credentialReference.startsWith("secret-service:"), + }), + }; + }); const installation = InstallationSchema.parse({ schemaVersion: "skillwire.installation/v1", installationId, @@ -986,8 +1171,12 @@ async function runProductionSetupUnlocked( postgresVolume: volumeName, selectedClients: selected, clientIntegrationIds: { - codex: selected.includes("codex") ? randomUUID() : null, - claude: selected.includes("claude") ? randomUUID() : null, + codex: + clientStateRecords.find(({ client }) => client === "codex") + ?.integration.clientIntegrationId ?? null, + claude: + clientStateRecords.find(({ client }) => client === "claude") + ?.integration.clientIntegrationId ?? null, }, status: status === "recovery-required" @@ -1011,6 +1200,78 @@ async function runProductionSetupUnlocked( }); const ownershipOperationId = randomUUID(); let ownership = createOwnershipLedger(installationId); + const installedReleaseIdentity = await releaseDirectoryIdentity( + installed.releaseRoot, + ); + const installedLauncherIdentity = await ownedLauncherIdentity( + installed.launcherPath, + ); + for (const asset of [ + { + kind: "release" as const, + locator: installed.releaseRoot, + identity: installedReleaseIdentity, + retention: "retain-by-default" as const, + }, + { + kind: "trust-policy" as const, + locator: `trust/${manifest.trustPolicy.path}`, + identity: manifest.trustPolicy.sha256, + retention: "retain-by-default" as const, + }, + ...secretReferences.map((secret) => ({ + kind: "service-secret" as const, + locator: `${installationId}/${secret.relativePath}`, + identity: secret.identitySha256, + retention: "retain-by-default" as const, + })), + { + kind: "compose-project" as const, + locator: projectName, + identity: clientComponentIdentity({ projectName }), + retention: "remove-on-uninstall" as const, + }, + ...(["skillwire", "postgres"] as const).map((service) => ({ + kind: "container" as const, + locator: `${projectName}:${service}`, + identity: clientComponentIdentity({ projectName, service }), + retention: "remove-on-uninstall" as const, + })), + { + kind: "volume" as const, + locator: volumeName, + identity: clientComponentIdentity({ volumeName }), + retention: "retain-by-default" as const, + }, + { + kind: "path" as const, + locator: installed.launcherPath, + identity: installedLauncherIdentity, + retention: "remove-only-on-purge" as const, + }, + { + kind: "path" as const, + locator: installationRoot, + identity: clientComponentIdentity({ installationRoot }), + retention: "remove-only-on-purge" as const, + }, + { + kind: "path" as const, + locator: bridgeStateRoot, + identity: clientComponentIdentity({ bridgeStateRoot }), + retention: "remove-only-on-purge" as const, + }, + ]) { + ownership = recordOwnedAsset(ownership, { + kind: asset.kind, + client: null, + locator: asset.locator, + expectedIdentitySha256: asset.identity, + createdByOperation: ownershipOperationId, + retention: asset.retention, + disposition: "present", + }); + } for (const entry of clientOwnership) { if ( entry.result.status !== "verified" && @@ -1096,7 +1357,7 @@ async function runProductionSetupUnlocked( reference: entry.credentialReference, }), createdByOperation: ownershipOperationId, - retention: "remove-on-uninstall", + retention: "retain-by-default", disposition: "present", }); } @@ -1129,6 +1390,46 @@ async function runProductionSetupUnlocked( }, setupRoots.stateRoot, ); + await atomicWriteJson( + resolve(setupRoots.stateRoot, "client-integrations.json"), + { + schemaVersion: "skillwire.client-integrations/v1", + installationId, + integrations: clientStateRecords.map( + ({ integration }) => integration, + ), + }, + setupRoots.stateRoot, + ); + await atomicWriteJson( + resolve(setupRoots.stateRoot, "credential-references.json"), + { + schemaVersion: "skillwire.credential-references/v1", + installationId, + credentials: clientStateRecords + .map(({ credential }) => credential) + .filter((credential) => credential !== null), + }, + setupRoots.stateRoot, + ); + await atomicWriteJson( + resolve(setupRoots.stateRoot, "deployment.json"), + { + schemaVersion: "skillwire.deployment/v1", + installationId, + releaseRoot: installed.releaseRoot, + composePath, + skillwireImage, + postgresImage, + databasePasswordFile, + applicationPepperFile, + runtimeSocketDirectory, + socketPath, + projectName, + volumeName, + }, + setupRoots.stateRoot, + ); }, verification: () => ({ installationId, published: true }), }); @@ -1163,6 +1464,78 @@ export async function runProductionSetup( trustOverrides: ProductionTrustOverrides = {}, ): Promise { const setupRoots = roots(environment); + let existingInstallation: z.infer | undefined; + // Preserve the durable cancellation contract: even an already-aborted fresh + // setup records intent/cancel before returning. Installed-state inspection is + // therefore skipped until after the journal is available in that case. + if (!signal.aborted) { + try { + const existing = await inspectInstalledStatus({ + stateRoot: setupRoots.stateRoot, + signal, + }); + existingInstallation = existing.installation; + const requested = selectedClients(options.clients); + if ( + (existing.installation.status === "service-ready" || + existing.installation.status === "complete") && + requested.every((client) => + existing.installation.selectedClients.includes(client), + ) + ) { + const unchangedClients = unchangedSetupClientResults( + requested, + existing.installation.installationId, + await readProtectedSetupJson( + resolve(setupRoots.stateRoot, "client-integrations.json"), + ), + ); + if (unchangedClients === undefined) { + existingInstallation = existing.installation; + } else { + const candidate = await candidatePaths(environment); + const verified = await verifyCandidate( + candidate, + setupRoots, + trustOverrides, + signal, + ); + if ( + verified.releaseSequence !== + existing.installation.highestAcceptedReleaseSequence || + verified.trustPolicySequence !== + existing.installation.activeTrustPolicySequence + ) { + throw new Error( + "Unchanged setup candidate differs from the installed release state", + ); + } + return { + status: "success", + installationId: existing.installation.installationId, + serviceReady: true, + clients: unchangedClients, + changed: false, + }; + } + } + if (existing.installation.status === "recovery-required") + throw new Error( + "Setup cannot bypass an installation that requires journal recovery", + ); + } catch (error) { + if ( + !( + error instanceof Error && + "code" in error && + error.code === "ENOENT" + ) && + !(error instanceof Error && error.message.includes("ENOENT")) + ) { + throw error; + } + } + } await mkdir(setupRoots.stateRoot, { recursive: true, mode: 0o700 }); await mkdir(setupRoots.runtimeRoot, { recursive: true, mode: 0o700 }); const identity = await currentProcessIdentity(); @@ -1192,6 +1565,54 @@ export async function runProductionSetup( previewHash, }); try { + if ( + existingInstallation !== undefined && + existingInstallation.status !== "purged" + ) { + const current = ( + await inspectInstalledStatus({ + stateRoot: setupRoots.stateRoot, + signal, + }) + ).installation; + if ( + current.installationId !== existingInstallation.installationId || + current.updatedAt !== existingInstallation.updatedAt || + current.status !== existingInstallation.status + ) + throw new Error( + "Installed setup state changed before lock acquisition", + ); + const candidate = await candidatePaths(environment); + const verified = await verifyCandidate( + candidate, + setupRoots, + trustOverrides, + signal, + ); + if ( + verified.releaseSequence !== current.highestAcceptedReleaseSequence || + verified.trustPolicySequence !== current.activeTrustPolicySequence + ) + throw new Error( + "Repeated setup cannot replace the installed signed release; use upgrade", + ); + const continued = await continueProductionSetup({ + setup: options, + credentialBackend: options.credentialBackend, + installation: current, + home: setupRoots.home, + dataRoot: setupRoots.dataRoot, + stateRoot: setupRoots.stateRoot, + runtimeRoot: setupRoots.runtimeRoot, + launcherPath: setupRoots.launcherPath, + environment, + signal, + journal, + }); + await journal.commit({ status: "success" }); + return continued; + } const result = await runProductionSetupUnlocked( options, signal, diff --git a/src/onboarding/application/purge.ts b/src/onboarding/application/purge.ts new file mode 100644 index 0000000..a5b545b --- /dev/null +++ b/src/onboarding/application/purge.ts @@ -0,0 +1,163 @@ +import { constants } from "node:fs"; +import { lstat, open, readdir, rm } from "node:fs/promises"; +import { isAbsolute, relative, resolve } from "node:path"; + +import { canonicalPreview, confirmPreview } from "../cli/confirmation.js"; +import { + planOwnedAssetDispositions, + recordAssetDisposition, + requireCurrentOwnedAssetIdentity, +} from "../domain/ownership.js"; +import type { z } from "zod"; +import type { + OwnedAssetSchema, + OwnershipRecordSchema, +} from "../domain/ownership.js"; +import type { OperationJournal } from "../domain/operation-journal.js"; + +type OwnedAsset = z.infer; +type OwnershipRecord = z.infer; + +export interface PurgePreview { + readonly installationId: string; + readonly ownershipRevision: number; + readonly unrecoverable: readonly OwnedAsset[]; + readonly previewHash: string; +} + +export function previewPurge(ownership: unknown): PurgePreview { + const record = ownership as OwnershipRecord; + const plan = planOwnedAssetDispositions(ownership, "purge"); + const unrecoverable = [...plan.remove].sort((left, right) => { + const leftPriority = left.kind === "path" ? 1 : 0; + const rightPriority = right.kind === "path" ? 1 : 0; + return leftPriority - rightPriority; + }); + const scope = { + installationId: record.installationId, + ownershipRevision: record.recordRevision, + unrecoverable: unrecoverable.map( + ({ assetId, kind, client, locator, expectedIdentitySha256 }) => ({ + assetId, + kind, + client, + locator, + expectedIdentitySha256, + }), + ), + }; + return { + installationId: record.installationId, + ownershipRevision: record.recordRevision, + unrecoverable, + previewHash: canonicalPreview("purge", scope).hash, + }; +} + +export async function runPurge(options: { + readonly ownership: OwnershipRecord; + readonly preview: PurgePreview; + readonly confirmation: string | undefined; + readonly signal: AbortSignal; + readonly observeIdentity: (asset: OwnedAsset) => Promise; + readonly removeAsset: (asset: OwnedAsset) => Promise; + readonly journal?: OperationJournal | undefined; +}): Promise<{ + readonly removed: readonly string[]; + readonly ownership: OwnershipRecord; +}> { + confirmPreview( + { command: "purge", json: "", hash: options.preview.previewHash }, + options.confirmation, + ); + if ( + options.ownership.installationId !== options.preview.installationId || + options.ownership.recordRevision !== options.preview.ownershipRevision + ) + throw new Error("Purge ownership changed after preview"); + for (const asset of options.preview.unrecoverable) { + if (options.signal.aborted) throw new Error("Purge cancelled"); + requireCurrentOwnedAssetIdentity( + asset, + await options.observeIdentity(asset), + ); + } + let ownership = options.ownership; + const removed: string[] = []; + for (const asset of options.preview.unrecoverable) { + if (options.signal.aborted) + throw new Error("Purge stopped at a recoverable asset boundary"); + requireCurrentOwnedAssetIdentity( + asset, + await options.observeIdentity(asset), + ); + if (options.journal === undefined) await options.removeAsset(asset); + else + await options.journal.runEffect({ + step: `purge-${asset.kind}-${asset.assetId}`, + intent: { assetId: asset.assetId, kind: asset.kind }, + signal: options.signal, + action: () => options.removeAsset(asset), + verification: () => ({ assetId: asset.assetId, removed: true }), + }); + ownership = recordAssetDisposition(ownership, asset.assetId, "removed"); + removed.push(asset.assetId); + } + return { removed, ownership }; +} + +function contained(root: string, candidate: string): boolean { + const child = relative(root, candidate); + return child === "" || (!child.startsWith("..") && !isAbsolute(child)); +} + +async function validateTree( + path: string, + root: string, + allowedFiles?: ReadonlySet, +): Promise { + const target = resolve(path); + if (!contained(resolve(root), target) || target === resolve(root)) + throw new Error("Purge target escapes or equals its protected root"); + const stats = await lstat(target); + if ( + stats.isSymbolicLink() || + (!stats.isDirectory() && stats.nlink !== 1) || + stats.uid !== process.getuid?.() + ) + throw new Error("Purge target has an unsafe filesystem identity"); + if (stats.isDirectory()) { + const handle = await open( + target, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ); + await handle.close(); + for (const name of await readdir(target)) + await validateTree(resolve(target, name), root, allowedFiles); + } else if (!stats.isFile()) { + throw new Error("Purge target is not a regular owned filesystem object"); + } else if (allowedFiles !== undefined && !allowedFiles.has(target)) { + throw new Error("Purge target contains an unknown unowned file"); + } +} + +export async function validateOwnedFilesystemTree( + path: string, + protectedRoot: string, + allowedFiles?: readonly string[], +): Promise { + const allowed = + allowedFiles === undefined + ? undefined + : new Set(allowedFiles.map((candidate) => resolve(candidate))); + await validateTree(path, protectedRoot, allowed); +} + +export async function removeOwnedFilesystemTree( + path: string, + protectedRoot: string, + allowedFiles?: readonly string[], +): Promise { + await validateOwnedFilesystemTree(path, protectedRoot, allowedFiles); + await rm(path, { recursive: true }); +} diff --git a/src/onboarding/application/recovery.ts b/src/onboarding/application/recovery.ts new file mode 100644 index 0000000..8a1f721 --- /dev/null +++ b/src/onboarding/application/recovery.ts @@ -0,0 +1,179 @@ +import type { + JournalEntry, + OperationJournal, +} from "../domain/operation-journal.js"; + +export type RecoveryObservation = + "absent" | "matching" | "owned-mismatch" | "ambiguous"; + +export interface RecoveryResult { + readonly disposition: + "complete" | "safe-retry" | "resume" | "recovery-required"; + readonly changed: boolean; + readonly boundary: string | null; +} + +export function journalNeedsRecovery( + entries: readonly JournalEntry[], +): boolean { + const last = entries.at(-1); + if (last === undefined || last.phase === "commit") return false; + if (last.phase !== "cancel") return true; + return last.detail["status"] === "recovery-required"; +} + +function lastEffectBoundary( + entries: readonly JournalEntry[], +): JournalEntry | undefined { + return [...entries] + .reverse() + .find(({ phase }) => phase === "intent" || phase === "effect"); +} + +export async function recoverOperation(options: { + readonly journal: OperationJournal; + readonly signal: AbortSignal; + readonly observe: (step: string) => Promise; + readonly compensate: (step: string) => Promise; +}): Promise { + if (options.signal.aborted) throw new Error("Recovery cancelled"); + const last = options.journal.entries.at(-1); + if (last === undefined) + return { disposition: "safe-retry", changed: false, boundary: null }; + if (last.phase === "commit") + return { + disposition: "complete", + changed: false, + boundary: last.step, + }; + if ( + last.phase === "cancel" && + last.detail["status"] === "recovery-required" && + options.journal.hasUnprovenEffect() + ) { + const unresolved = [...options.journal.entries] + .reverse() + .find( + (entry) => + entry.phase === "compensate" && + entry.detail["completion"] === "unproven", + ); + if (unresolved === undefined) + return { + disposition: "recovery-required", + changed: false, + boundary: last.step, + }; + const observation = await options.observe(unresolved.step); + if (observation === "ambiguous") + return { + disposition: "recovery-required", + changed: false, + boundary: unresolved.step, + }; + if (observation === "owned-mismatch") + await options.compensate(unresolved.step); + await options.journal.compensate(unresolved.step, { + completion: observation === "absent" ? "not-started" : "recovered", + recoveryRequired: false, + }); + await options.journal.commit({ status: "recovered" }); + return { + disposition: observation === "absent" ? "safe-retry" : "resume", + changed: true, + boundary: unresolved.step, + }; + } + if (last.phase === "cancel" && !options.journal.hasUnprovenEffect()) + return { + disposition: "safe-retry", + changed: false, + boundary: last.step, + }; + if (last.phase === "intent") { + await options.journal.compensate(last.step, { + completion: "not-started", + recoveryRequired: false, + }); + await options.journal.commit({ status: "recovered" }); + return { + disposition: "safe-retry", + changed: true, + boundary: last.step, + }; + } + if (last.phase === "compensate" && options.journal.hasUnprovenEffect()) { + const observation = await options.observe(last.step); + if (observation === "ambiguous") + return { + disposition: "recovery-required", + changed: false, + boundary: last.step, + }; + if (observation === "owned-mismatch") await options.compensate(last.step); + await options.journal.compensate(last.step, { + completion: observation === "absent" ? "not-started" : "recovered", + recoveryRequired: false, + }); + await options.journal.commit({ status: "recovered" }); + return { + disposition: observation === "absent" ? "safe-retry" : "resume", + changed: true, + boundary: last.step, + }; + } + if (last.phase === "verify" || last.phase === "compensate") { + if (!options.journal.hasUnprovenEffect()) + await options.journal.commit({ status: "recovered" }); + return { + disposition: options.journal.hasUnprovenEffect() + ? "recovery-required" + : "resume", + changed: false, + boundary: last.step, + }; + } + + const boundary = lastEffectBoundary(options.journal.entries); + if (boundary === undefined) + return { disposition: "safe-retry", changed: false, boundary: null }; + const observation = await options.observe(boundary.step); + if (observation === "ambiguous") + return { + disposition: "recovery-required", + changed: false, + boundary: boundary.step, + }; + if (observation === "matching") { + await options.journal.verify(boundary.step, { recovered: true }); + await options.journal.commit({ status: "recovered" }); + return { + disposition: "resume", + changed: true, + boundary: boundary.step, + }; + } + if (observation === "owned-mismatch") { + await options.compensate(boundary.step); + await options.journal.compensate(boundary.step, { + completion: "recovered", + recoveryRequired: false, + }); + await options.journal.commit({ status: "recovered" }); + return { + disposition: "resume", + changed: true, + boundary: boundary.step, + }; + } + await options.journal.compensate(boundary.step, { + completion: "not-started", + recoveryRequired: false, + }); + await options.journal.commit({ status: "recovered" }); + return { + disposition: "safe-retry", + changed: true, + boundary: boundary.step, + }; +} diff --git a/src/onboarding/application/repair.ts b/src/onboarding/application/repair.ts new file mode 100644 index 0000000..225a583 --- /dev/null +++ b/src/onboarding/application/repair.ts @@ -0,0 +1,121 @@ +import { canonicalPreview, confirmPreview } from "../cli/confirmation.js"; +import type { ClientName } from "../cli/main.js"; + +export type RepairObservation = + "matching" | "missing" | "outdated" | "drifted" | "ambiguous" | "external"; + +export interface RepairAsset { + readonly assetId: string; + readonly kind: string; + readonly client: ClientName | null; + readonly locator: string; + readonly expectedIdentitySha256: string; + readonly observation: RepairObservation; + readonly ownershipProven?: boolean | undefined; +} + +export interface RepairBlock { + readonly code: + | "EXTERNAL_INTEGRATION_NOT_OWNED" + | "OWNED_ASSET_DRIFTED" + | "OWNED_ASSET_AMBIGUOUS" + | "SECRET_ROTATION_REQUIRES_EXPLICIT_COMMAND"; + readonly assetId: string; +} + +export interface RepairPlan { + readonly installationId: string; + readonly actions: readonly RepairAsset[]; + readonly blocked: readonly RepairBlock[]; + readonly previewHash: string; +} + +function blockFor(asset: RepairAsset): RepairBlock | undefined { + if (asset.kind === "credential" || asset.kind === "service-secret") + return { + code: "SECRET_ROTATION_REQUIRES_EXPLICIT_COMMAND", + assetId: asset.assetId, + }; + if (asset.observation === "external") + return { code: "EXTERNAL_INTEGRATION_NOT_OWNED", assetId: asset.assetId }; + if (asset.observation === "drifted" && asset.ownershipProven !== true) + return { code: "OWNED_ASSET_DRIFTED", assetId: asset.assetId }; + if (asset.observation === "ambiguous") + return { code: "OWNED_ASSET_AMBIGUOUS", assetId: asset.assetId }; + return undefined; +} + +export function planRepair(input: { + readonly installationId: string; + readonly assets: readonly RepairAsset[]; +}): RepairPlan { + const blocked = input.assets + .map(blockFor) + .filter((value): value is RepairBlock => value !== undefined); + const actions = input.assets.filter( + (asset) => + blockFor(asset) === undefined && + (asset.observation === "missing" || + asset.observation === "outdated" || + (asset.observation === "drifted" && asset.ownershipProven === true)), + ); + const scope = { + installationId: input.installationId, + actions: actions.map( + ({ assetId, kind, client, locator, expectedIdentitySha256 }) => ({ + assetId, + kind, + client, + locator, + expectedIdentitySha256, + }), + ), + blocked, + }; + return { + installationId: input.installationId, + actions, + blocked, + previewHash: canonicalPreview("repair", scope).hash, + }; +} + +export async function runRepair(options: { + readonly plan: RepairPlan; + readonly confirmation: string | undefined; + readonly signal: AbortSignal; + readonly observe: (asset: RepairAsset) => Promise<{ + readonly observation: RepairObservation; + readonly identitySha256: string; + readonly ownershipProven?: boolean | undefined; + }>; + readonly repair: (asset: RepairAsset) => Promise; + readonly rotate: (asset: RepairAsset) => Promise; +}): Promise<{ readonly changedAssets: readonly string[] }> { + confirmPreview( + { command: "repair", json: "", hash: options.plan.previewHash }, + options.confirmation, + ); + const changedAssets: string[] = []; + for (const asset of options.plan.actions) { + if (options.signal.aborted) throw new Error("Repair cancelled"); + const current = await options.observe(asset); + const repairableDrift = + asset.observation === "drifted" && + asset.ownershipProven === true && + current.observation === "drifted" && + current.ownershipProven === true; + if ( + (!repairableDrift && + current.identitySha256 !== asset.expectedIdentitySha256) || + (current.observation !== "missing" && + current.observation !== "outdated" && + !repairableDrift) + ) { + continue; + } + await options.repair(asset); + changedAssets.push(asset.assetId); + } + return { changedAssets }; +} diff --git a/src/onboarding/application/service-secret-rotation.ts b/src/onboarding/application/service-secret-rotation.ts new file mode 100644 index 0000000..fa32685 --- /dev/null +++ b/src/onboarding/application/service-secret-rotation.ts @@ -0,0 +1,263 @@ +import { createHash, randomBytes } from "node:crypto"; +import { constants } from "node:fs"; +import { open, rename, unlink } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { + validateOwnedDirectory, + validateOwnedPath, +} from "../adapters/filesystem/safe-paths.js"; +import { canonicalPreview, confirmPreview } from "../cli/confirmation.js"; +import type { ServiceSecretKind } from "../secrets/service-secrets.js"; +import type { OperationJournal } from "../domain/operation-journal.js"; + +export interface ServiceSecretRotationPreview { + readonly operationId: string; + readonly installationId: string; + readonly kind: ServiceSecretKind; + readonly targets: readonly [string, string]; + readonly previewHash: string; + readonly currentIdentitySha256?: string | undefined; +} + +function deterministicOperationId(value: string): string { + const hex = createHash("sha256").update(value).digest("hex").slice(0, 32); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-4${hex.slice(13, 16)}-8${hex.slice(17, 20)}-${hex.slice(20)}`; +} + +export function previewServiceSecretRotation(input: { + readonly installationId: string; + readonly kind: ServiceSecretKind; + readonly currentIdentitySha256?: string | undefined; +}): ServiceSecretRotationPreview { + const operationId = deterministicOperationId( + `${input.installationId}\0${input.kind}\0${input.currentIdentitySha256 ?? "unknown"}`, + ); + const targets = [ + `secrets/${input.kind}`, + `secrets/${input.kind}.retained-${operationId}`, + ] as const; + const scope = { + installationId: input.installationId, + kind: input.kind, + operationId, + targets, + ...(input.currentIdentitySha256 === undefined + ? {} + : { currentIdentitySha256: input.currentIdentitySha256 }), + }; + return { + ...scope, + previewHash: canonicalPreview("maintenance:rotate-service-secret", scope) + .hash, + }; +} + +async function validateSecret(path: string, root: string): Promise { + const target = await validateOwnedPath(path, root); + const handle = await open(target, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stats = await handle.stat(); + if ( + !stats.isFile() || + stats.nlink !== 1 || + stats.uid !== process.getuid?.() || + (stats.mode & 0o777) !== 0o600 || + stats.size !== 43 + ) { + throw new Error("Service-secret rotation target is unsafe"); + } + } finally { + await handle.close(); + } +} + +async function syncDirectory(path: string): Promise { + const handle = await open( + path, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +export async function rotateServiceSecret(options: { + readonly installationRoot: string; + readonly stateRoot: string; + readonly kind: ServiceSecretKind; + readonly confirmation: string | undefined; + readonly preview: ServiceSecretRotationPreview; + readonly signal: AbortSignal; + readonly apply: (path: string) => Promise; + readonly readiness: () => Promise; + readonly publish: (reference: { + readonly kind: ServiceSecretKind; + readonly identitySha256: string; + }) => Promise; + readonly rollback?: ((path: string) => Promise) | undefined; + readonly journal?: OperationJournal | undefined; +}): Promise<{ + readonly operationId: string; + readonly kind: ServiceSecretKind; + readonly retainedPath: string; + readonly identitySha256: string; +}> { + if (options.preview.kind !== options.kind) + throw new Error("Service-secret rotation preview changed"); + confirmPreview( + { + command: "maintenance:rotate-service-secret", + json: "", + hash: options.preview.previewHash, + }, + options.confirmation, + ); + if (options.signal.aborted) + throw new Error("Service-secret rotation cancelled"); + const installationRoot = await validateOwnedPath( + options.installationRoot, + options.stateRoot, + ); + const secretsRoot = resolve(installationRoot, "secrets"); + await validateOwnedDirectory(secretsRoot, installationRoot); + const currentPath = resolve(secretsRoot, options.kind); + const retainedPath = resolve( + secretsRoot, + `${options.kind}.retained-${options.preview.operationId}`, + ); + const candidatePath = resolve( + secretsRoot, + `${options.kind}.candidate-${options.preview.operationId}`, + ); + await validateSecret(currentPath, installationRoot); + const candidate = Buffer.from(randomBytes(32).toString("base64url"), "ascii"); + const effect = async ( + step: string, + action: () => Promise, + detail: Record, + ): Promise => { + if (options.journal === undefined) return action(); + await options.journal.runEffect({ + step, + intent: detail, + signal: options.signal, + action, + verification: () => ({ ...detail, completed: true }), + }); + }; + try { + await effect( + "service-secret-candidate", + async () => { + const candidateHandle = await open( + candidatePath, + constants.O_WRONLY | + constants.O_CREAT | + constants.O_EXCL | + constants.O_NOFOLLOW, + 0o600, + ); + try { + await candidateHandle.writeFile(candidate); + await candidateHandle.sync(); + } finally { + await candidateHandle.close(); + } + }, + { kind: options.kind }, + ); + } catch (error) { + candidate.fill(0); + await unlink(candidatePath).catch(() => undefined); + throw error; + } + const identitySha256 = createHash("sha256") + .update("skillwire-service-secret-identity-v1\0") + .update(candidate) + .digest("hex"); + candidate.fill(0); + const rotationState: { swapped: boolean } = { swapped: false }; + try { + await effect( + "service-secret-candidate-readiness", + async () => { + await options.apply(candidatePath); + if (options.signal.aborted) + throw new Error("Service-secret rotation cancelled before readiness"); + await options.readiness(); + }, + { kind: options.kind }, + ); + await effect( + "service-secret-atomic-swap", + async () => { + await rename(currentPath, retainedPath); + try { + await rename(candidatePath, currentPath); + rotationState.swapped = true; + await syncDirectory(secretsRoot); + } catch (error) { + if (rotationState.swapped) { + await rename(currentPath, candidatePath); + rotationState.swapped = false; + } + await rename(retainedPath, currentPath); + await syncDirectory(secretsRoot); + throw error; + } + }, + { kind: options.kind }, + ); + await effect( + "service-secret-active-readiness", + async () => { + await options.apply(currentPath); + await options.readiness(); + }, + { kind: options.kind }, + ); + await effect( + "service-secret-state-publication", + () => options.publish({ kind: options.kind, identitySha256 }), + { kind: options.kind, identitySha256 }, + ); + return { + operationId: options.preview.operationId, + kind: options.kind, + retainedPath, + identitySha256, + }; + } catch (error) { + try { + if (rotationState.swapped) { + await rename(currentPath, candidatePath); + await rename(retainedPath, currentPath); + await syncDirectory(secretsRoot); + } + await options.rollback?.(currentPath); + await unlink(candidatePath).catch((cleanupError: unknown) => { + if ( + !(cleanupError instanceof Error) || + !("code" in cleanupError) || + cleanupError.code !== "ENOENT" + ) + throw cleanupError; + }); + await options.journal?.compensate("service-secret-rollback", { + kind: options.kind, + restored: true, + }); + } catch (rollbackError) { + throw new Error( + "Service-secret rotation failed and application rollback requires recovery", + { cause: rollbackError }, + ); + } + throw new Error("Service-secret rotation failed before readiness commit", { + cause: error, + }); + } +} diff --git a/src/onboarding/application/setup.ts b/src/onboarding/application/setup.ts index d771ff9..75489b2 100644 --- a/src/onboarding/application/setup.ts +++ b/src/onboarding/application/setup.ts @@ -15,6 +15,20 @@ export interface SetupClientResult { } export interface GuidedSetupDependencies { + inspectExisting?( + options: GuidedSetupOptions, + ): Promise; + discoverRetained?( + options: GuidedSetupOptions, + ): Promise; + reactivateRetainedService?( + release: { readonly releaseSequence: number }, + retained: RetainedSetupState, + ): Promise<{ readonly ready: boolean }>; + reactivateClient?( + client: ClientName, + installationId: string, + ): Promise; verifyRelease(): Promise<{ readonly releaseSequence: number }>; installService(release: { readonly releaseSequence: number }): Promise<{ readonly installationId: string; @@ -26,11 +40,17 @@ export interface GuidedSetupDependencies { ): Promise; } +export interface RetainedSetupState { + readonly installationId: string; + readonly clients: readonly SetupClientResult[]; +} + export interface GuidedSetupResult { readonly status: "success" | "incomplete" | "recovery-required"; readonly installationId: string; readonly serviceReady: boolean; readonly clients: readonly SetupClientResult[]; + readonly changed?: boolean | undefined; } function selectedClients( @@ -45,7 +65,52 @@ export async function runGuidedSetup( options: GuidedSetupOptions, dependencies: GuidedSetupDependencies, ): Promise { + const existing = await dependencies.inspectExisting?.(options); + if (existing !== undefined) return existing; const release = await dependencies.verifyRelease(); + const retained = await dependencies.discoverRetained?.(options); + if (retained !== undefined) { + if (dependencies.reactivateRetainedService === undefined) + throw new Error("Retained installation reactivation is unavailable"); + const service = await dependencies.reactivateRetainedService( + release, + retained, + ); + if (!service.ready) + throw new Error("Retained service did not reach readiness"); + const clients: SetupClientResult[] = []; + for (const client of selectedClients(options.clients)) { + const current = retained.clients.find( + (entry) => + entry.client === client && + (entry.status === "verified" || entry.status === "external-verified"), + ); + clients.push( + current ?? + (await (dependencies.reactivateClient ?? dependencies.installClient)( + client, + retained.installationId, + )), + ); + } + const status = clients.some( + ({ status: clientStatus }) => clientStatus === "recovery-required", + ) + ? "recovery-required" + : clients.some( + ({ status: clientStatus }) => + clientStatus !== "verified" && + clientStatus !== "external-verified", + ) + ? "incomplete" + : "success"; + return { + status, + installationId: retained.installationId, + serviceReady: true, + clients, + }; + } const service = await dependencies.installService(release); if (!service.ready) throw new Error("Signed-release service did not reach readiness"); diff --git a/src/onboarding/application/status.ts b/src/onboarding/application/status.ts new file mode 100644 index 0000000..56c0aa2 --- /dev/null +++ b/src/onboarding/application/status.ts @@ -0,0 +1,64 @@ +import { constants } from "node:fs"; +import { open } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { + InstallationSchema, + type Installation, +} from "../domain/installation.js"; + +export interface StatusProbeResult { + readonly component: string; + readonly state: string; + readonly identity?: Readonly< + Record + >; +} + +export interface StatusProbe { + readonly component: string; + inspect(signal: AbortSignal): Promise; +} + +export interface InstalledStatus { + readonly installation: Installation; + readonly live: readonly StatusProbeResult[]; +} + +async function readProtectedJson(path: string): Promise { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stats = await handle.stat(); + if ( + !stats.isFile() || + stats.nlink !== 1 || + stats.uid !== process.getuid?.() || + (stats.mode & 0o777) !== 0o600 || + stats.size > 256 * 1024 + ) { + throw new Error("Installed state file is unsafe"); + } + return JSON.parse(await handle.readFile("utf8")) as unknown; + } finally { + await handle.close(); + } +} + +export async function inspectInstalledStatus(options: { + readonly stateRoot: string; + readonly probes?: readonly StatusProbe[]; + readonly signal: AbortSignal; +}): Promise { + if (options.signal.aborted) throw new Error("Status inspection cancelled"); + const installation = InstallationSchema.parse( + await readProtectedJson(resolve(options.stateRoot, "installation.json")), + ); + const live: StatusProbeResult[] = []; + for (const probe of options.probes ?? []) { + const result = await probe.inspect(options.signal); + if (result.component !== probe.component) + throw new Error("Status probe component identity changed"); + live.push(result); + } + return { installation, live }; +} diff --git a/src/onboarding/application/uninstall.ts b/src/onboarding/application/uninstall.ts new file mode 100644 index 0000000..79366a1 --- /dev/null +++ b/src/onboarding/application/uninstall.ts @@ -0,0 +1,141 @@ +import { canonicalPreview, confirmPreview } from "../cli/confirmation.js"; +import { + planOwnedAssetDispositions, + recordAssetDisposition, + requireCurrentOwnedAssetIdentity, + type OwnedAssetDispositionPlan, +} from "../domain/ownership.js"; +import type { z } from "zod"; +import type { + OwnedAssetSchema, + OwnershipRecordSchema, +} from "../domain/ownership.js"; +import type { OperationJournal } from "../domain/operation-journal.js"; + +type OwnedAsset = z.infer; +type OwnershipRecord = z.infer; + +export interface DefaultUninstallPreview { + readonly installationId: string; + readonly ownershipRevision: number; + readonly remove: readonly OwnedAsset[]; + readonly retain: readonly OwnedAsset[]; + readonly previewHash: string; +} + +export function previewDefaultUninstall( + ownership: unknown, +): DefaultUninstallPreview { + const plan: OwnedAssetDispositionPlan = planOwnedAssetDispositions( + ownership, + "uninstall", + ); + const record = ownership as OwnershipRecord; + const scope = { + installationId: record.installationId, + ownershipRevision: record.recordRevision, + remove: plan.remove.map( + ({ assetId, kind, client, locator, expectedIdentitySha256 }) => ({ + assetId, + kind, + client, + locator, + expectedIdentitySha256, + }), + ), + retain: plan.retain.map(({ assetId, kind, client, locator }) => ({ + assetId, + kind, + client, + locator, + })), + }; + return { + installationId: record.installationId, + ownershipRevision: record.recordRevision, + remove: plan.remove, + retain: plan.retain, + previewHash: canonicalPreview("uninstall", scope).hash, + }; +} + +export async function runDefaultUninstall(options: { + readonly ownership: OwnershipRecord; + readonly preview: DefaultUninstallPreview; + readonly confirmation: string | undefined; + readonly signal: AbortSignal; + readonly observeIdentity: (asset: OwnedAsset) => Promise; + readonly removeAsset: (asset: OwnedAsset) => Promise; + readonly stopOwnedService: () => Promise; + readonly publishRetained: (ownership: OwnershipRecord) => Promise; + readonly journal?: OperationJournal | undefined; +}): Promise<{ + readonly removed: readonly string[]; + readonly retained: readonly string[]; + readonly ownership: OwnershipRecord; +}> { + confirmPreview( + { command: "uninstall", json: "", hash: options.preview.previewHash }, + options.confirmation, + ); + if ( + options.ownership.installationId !== options.preview.installationId || + options.ownership.recordRevision !== options.preview.ownershipRevision + ) + throw new Error("Uninstall ownership changed after preview"); + for (const asset of options.preview.remove) { + if (options.signal.aborted) throw new Error("Uninstall cancelled"); + requireCurrentOwnedAssetIdentity( + asset, + await options.observeIdentity(asset), + ); + } + let ownership = options.ownership; + const removed: string[] = []; + const effect = async ( + step: string, + action: () => Promise, + detail: Record, + ): Promise => { + if (options.journal === undefined) return action(); + await options.journal.runEffect({ + step, + intent: detail, + signal: options.signal, + action, + verification: () => ({ ...detail, completed: true }), + }); + }; + for (const asset of options.preview.remove) { + if (options.signal.aborted) + throw new Error("Uninstall stopped at a recoverable asset boundary"); + requireCurrentOwnedAssetIdentity( + asset, + await options.observeIdentity(asset), + ); + await effect( + `uninstall-${asset.kind}-${asset.assetId}`, + () => options.removeAsset(asset), + { assetId: asset.assetId, kind: asset.kind }, + ); + ownership = recordAssetDisposition(ownership, asset.assetId, "removed"); + removed.push(asset.assetId); + } + await effect("uninstall-owned-service", options.stopOwnedService, { + installationId: options.preview.installationId, + }); + for (const asset of options.preview.retain) { + if (asset.disposition === "present") + ownership = recordAssetDisposition(ownership, asset.assetId, "retained"); + } + await effect( + "uninstall-retained-state", + () => options.publishRetained(ownership), + { ownershipRevision: ownership.recordRevision }, + ); + return { + removed, + retained: options.preview.retain.map(({ assetId }) => assetId), + ownership, + }; +} diff --git a/src/onboarding/application/upgrade-recovery.ts b/src/onboarding/application/upgrade-recovery.ts new file mode 100644 index 0000000..8a9a7aa --- /dev/null +++ b/src/onboarding/application/upgrade-recovery.ts @@ -0,0 +1,34 @@ +export class UpgradeRecoveryError extends Error { + public constructor( + message: string, + readonly rollbackBoundary: + "application-config" | "database-restore-required", + readonly backupId: string | null, + readonly dataLossBoundary: string, + options?: ErrorOptions, + ) { + super(message, options); + this.name = "UpgradeRecoveryError"; + } +} + +export function upgradeRecoveryGuidance(error: UpgradeRecoveryError): { + readonly rollbackBoundary: UpgradeRecoveryError["rollbackBoundary"]; + readonly backupId: string | null; + readonly instructions: readonly string[]; +} { + return { + rollbackBoundary: error.rollbackBoundary, + backupId: error.backupId, + instructions: + error.rollbackBoundary === "database-restore-required" + ? [ + "Keep all writers stopped", + "Restore the named validated backup before selecting an older executable", + "Confirm the erased-memory and data-loss boundary before restore", + ] + : [ + "The prior application and configuration were restored automatically", + ], + }; +} diff --git a/src/onboarding/application/upgrade.ts b/src/onboarding/application/upgrade.ts new file mode 100644 index 0000000..7f31f3f --- /dev/null +++ b/src/onboarding/application/upgrade.ts @@ -0,0 +1,226 @@ +import { canonicalPreview, confirmPreview } from "../cli/confirmation.js"; +import { classifySchemaUpgrade } from "../adapters/postgres/schema-compatibility.js"; +import { UpgradeRecoveryError } from "./upgrade-recovery.js"; +import type { OperationJournal } from "../domain/operation-journal.js"; + +export { UpgradeRecoveryError } from "./upgrade-recovery.js"; + +export function upgradeFailureRequiresRecovery( + restoreRequired: boolean, + hasUnprovenEffect: boolean, +): boolean { + return restoreRequired || hasUnprovenEffect; +} + +export interface UpgradeTarget { + readonly releaseId: string; + readonly releaseSequence: number; + readonly trustPolicySequence: number; + readonly schemaMinimum: number; + readonly schemaMaximum: number; + readonly latestMigration: number; + readonly manifestSha256: string; + readonly imageDigest: string; +} + +export interface UpgradePreview { + readonly installationId: string; + readonly currentReleaseSequence: number; + readonly currentTrustPolicySequence: number; + readonly liveSchema: number; + readonly target: UpgradeTarget; + readonly previewHash: string; +} + +export function previewUpgrade( + input: Omit, +): UpgradePreview { + return { + ...input, + previewHash: canonicalPreview("upgrade", input).hash, + }; +} + +function sameTarget(left: UpgradeTarget, right: UpgradeTarget): boolean { + return ( + left.releaseId === right.releaseId && + left.releaseSequence === right.releaseSequence && + left.trustPolicySequence === right.trustPolicySequence && + left.schemaMinimum === right.schemaMinimum && + left.schemaMaximum === right.schemaMaximum && + left.latestMigration === right.latestMigration && + left.manifestSha256 === right.manifestSha256 && + left.imageDigest === right.imageDigest + ); +} + +function ensureNotAborted(signal: AbortSignal, boundary: string): void { + if (signal.aborted) + throw new Error(`Upgrade cancelled at the ${boundary} boundary`); +} + +export async function runUpgrade(options: { + readonly preview: UpgradePreview; + readonly confirmation: string | undefined; + readonly signal: AbortSignal; + readonly verifyTarget: () => Promise; + readonly createBackup: () => Promise<{ + readonly backupId: string; + readonly validated: boolean; + }>; + readonly drainWriters: () => Promise; + readonly installApplication: () => Promise; + readonly migrate: () => Promise; + readonly verifyLiveSchema: () => Promise; + readonly readiness: () => Promise; + readonly verifyClients: () => Promise; + readonly commitSelection: () => Promise; + readonly rollbackApplication: () => Promise; + readonly stopWriters: () => Promise; + readonly restartWriters: () => Promise; + readonly journal?: OperationJournal | undefined; +}): Promise<{ readonly backupId: string; readonly releaseId: string }> { + confirmPreview( + { command: "upgrade", json: "", hash: options.preview.previewHash }, + options.confirmation, + ); + if (options.signal.aborted) throw new Error("Upgrade cancelled"); + const effect = async ( + step: string, + action: () => Promise, + verification: ( + value: T, + ) => Record, + ): Promise => + options.journal === undefined + ? action() + : options.journal.runEffect({ + step, + intent: { command: "upgrade" }, + signal: options.signal, + action, + verification, + }); + const target = await effect( + "upgrade-release-verification", + options.verifyTarget, + (value) => ({ + releaseSequence: value.releaseSequence, + trustPolicySequence: value.trustPolicySequence, + }), + ); + ensureNotAborted(options.signal, "release-verification"); + if (!sameTarget(target, options.preview.target)) + throw new Error("Verified upgrade target changed after preview"); + if (target.releaseSequence <= options.preview.currentReleaseSequence) + throw new Error( + "Release downgrade or equal-sequence replacement is forbidden", + ); + if (target.trustPolicySequence < options.preview.currentTrustPolicySequence) + throw new Error("Trust policy downgrade is forbidden"); + if (!/^sha256:[0-9a-f]{64}$/.test(target.imageDigest)) + throw new Error("Upgrade image is not digest-pinned"); + const decision = classifySchemaUpgrade({ + liveSchema: options.preview.liveSchema, + schemaMinimum: target.schemaMinimum, + schemaMaximum: target.schemaMaximum, + latestMigration: target.latestMigration, + forwardOnlyMigrations: [10], + }); + const backup = await effect( + "upgrade-backup", + options.createBackup, + (value) => ({ + backupId: value.backupId, + validated: value.validated, + }), + ); + if (!backup.validated) + throw new Error("Upgrade backup is not restore-validated"); + ensureNotAborted(options.signal, "backup"); + let applicationInstalled = false; + let writersDrained = false; + const migration = { started: false }; + try { + if (decision.requiresWriterDrain) { + await effect("upgrade-writer-drain", options.drainWriters, () => ({ + drained: true, + })); + writersDrained = true; + ensureNotAborted(options.signal, "writer-drain"); + } + await effect( + "upgrade-application-install", + options.installApplication, + () => ({ installed: true }), + ); + applicationInstalled = true; + ensureNotAborted(options.signal, "application-install"); + if (decision.kind === "forward-only") { + await effect( + "upgrade-migration", + async () => { + migration.started = true; + await options.migrate(); + }, + () => ({ schema: target.latestMigration }), + ); + ensureNotAborted(options.signal, "migration"); + } + const liveSchema = await effect( + "upgrade-schema-readback", + options.verifyLiveSchema, + (value) => ({ schema: value }), + ); + if (liveSchema !== target.latestMigration) + throw new Error("Live schema readback did not match the target"); + await effect("upgrade-readiness", options.readiness, () => ({ + ready: true, + })); + ensureNotAborted(options.signal, "readiness"); + await effect("upgrade-client-verification", options.verifyClients, () => ({ + clientsVerified: true, + })); + ensureNotAborted(options.signal, "client-verification"); + if (writersDrained) { + await effect("upgrade-writer-restart", options.restartWriters, () => ({ + restarted: true, + })); + writersDrained = false; + ensureNotAborted(options.signal, "writer-restart"); + } + await effect("upgrade-release-commit", options.commitSelection, () => ({ + releaseSequence: target.releaseSequence, + })); + return { backupId: backup.backupId, releaseId: target.releaseId }; + } catch (error) { + if (decision.kind === "forward-only" && migration.started) { + await options.stopWriters().catch(() => undefined); + throw new UpgradeRecoveryError( + "Forward-only upgrade stopped after migration may have begun", + "database-restore-required", + backup.backupId, + "Restore replaces all database changes after the named backup, including erased-memory records", + { cause: error }, + ); + } + if (applicationInstalled) await options.rollbackApplication(); + if (writersDrained) await options.restartWriters(); + const last = options.journal?.entries.at(-1); + if ( + last?.phase === "compensate" && + last.detail["completion"] === "unproven" + ) + await options.journal?.compensate(last.step, { + completion: "recovered", + recoveryRequired: false, + }); + throw new UpgradeRecoveryError( + "Upgrade failed before a forward-only database boundary and application rollback completed", + "application-config", + backup.backupId, + "No database restore is required", + { cause: error }, + ); + } +} diff --git a/src/onboarding/cli/command-router.ts b/src/onboarding/cli/command-router.ts index f66e5a3..71f2c04 100644 --- a/src/onboarding/cli/command-router.ts +++ b/src/onboarding/cli/command-router.ts @@ -12,6 +12,7 @@ import { previewProductionSetup, runProductionSetup, } from "../application/production-setup.js"; +import { inspectInstalledStatus } from "../application/status.js"; import type { AdminResult, ExitClass } from "./output.js"; function emit( @@ -25,6 +26,15 @@ function emit( return exitCodeForClass(result.exitClass); } +export type AdministrativeOperation = ( + command: ParsedCommand, + signal: AbortSignal, +) => Promise; + +export type AdministrativeOperations = Partial< + Readonly> +>; + function failureClass(error: unknown): ExitClass { const message = error instanceof Error ? error.message : ""; if (/preview hash confirmation/i.test(message)) return "invalid-invocation"; @@ -49,6 +59,18 @@ function failureClass(error: unknown): ExitClass { return "internal-failure"; } +function signalIsAborted(signal: AbortSignal): boolean { + return signal.aborted; +} + +function installationStateIsAbsent(error: unknown): boolean { + return ( + error instanceof Error && + (("code" in error && error.code === "ENOENT") || + error.message.includes("ENOENT")) + ); +} + export function setupFailureEnvelope(options: { readonly error: unknown; readonly operationId: string; @@ -219,7 +241,7 @@ async function routeSetup( status: result.status, exitClass, previewHash: preview.hash, - changed: true, + changed: result.changed !== false, summary: selection === "none" && result.status === "success" ? "Self-hosted service is ready; client integration remains pending" @@ -228,7 +250,7 @@ async function routeSetup( { component: "service", state: result.serviceReady ? "ready" : "failed", - changed: true, + changed: result.changed !== false, owned: true, identity: { installationId: result.installationId }, }, @@ -299,6 +321,7 @@ export async function routeAdministrativeCommand( command: ParsedCommand, io: DispatcherIo, signal: AbortSignal, + operations: AdministrativeOperations = {}, ): Promise { if (signal.aborted) return 11; if ( @@ -309,6 +332,177 @@ export async function routeAdministrativeCommand( return 2; } if (command.route === "setup") return routeSetup(command, io, signal); + if (command.route === "status" && operations.status !== undefined) { + try { + return emit(await operations.status(command, signal), command, io); + } catch (error) { + const absent = installationStateIsAbsent(error); + return emit( + AdminResultSchema.parse({ + schemaVersion: "skillwire.admin-result/v1", + command: "status", + operationId: randomUUID(), + status: signalIsAborted(signal) ? "cancelled" : "failure", + exitClass: signalIsAborted(signal) + ? "user-cancellation" + : absent + ? "unsupported-prerequisite" + : "degraded-or-incomplete", + previewHash: null, + changed: false, + summary: "Installed and live state could not be inspected safely", + components: [], + findings: [ + { + code: "STATUS_STATE_UNAVAILABLE", + severity: "error", + component: "installation", + summary: absent + ? "No SkillWire installation state exists in this profile" + : error instanceof Error + ? error.message.slice(0, 512) + : "Installed state is unavailable", + nextAction: absent + ? "Run setup to create a verified self-hosted installation" + : "Run doctor to classify the installed-state failure", + }, + ], + recovery: { + rollbackBoundary: "none", + backupId: null, + instructions: [], + }, + }), + command, + io, + ); + } + } + if (command.route === "status") { + const stateHome = + process.env["XDG_STATE_HOME"] ?? + `${process.env["HOME"] ?? ""}/.local/state`; + const stateRoot = command.stateRoot ?? `${stateHome}/skillwire`; + try { + const status = await inspectInstalledStatus({ stateRoot, signal }); + return emit( + AdminResultSchema.parse({ + schemaVersion: "skillwire.admin-result/v1", + command: "status", + operationId: randomUUID(), + status: "success", + exitClass: "success", + previewHash: null, + changed: false, + summary: `Installation state is ${status.installation.status}`, + components: [ + { + component: "installation", + state: status.installation.status, + changed: false, + owned: true, + identity: { + installationId: status.installation.installationId, + release: status.installation.activeReleaseId, + }, + }, + ...status.live.map((component) => ({ + component: component.component, + state: component.state, + changed: false, + owned: true, + identity: component.identity ?? {}, + })), + ], + findings: [], + recovery: { + rollbackBoundary: "none", + backupId: null, + instructions: [], + }, + }), + command, + io, + ); + } catch (error) { + return emit( + AdminResultSchema.parse({ + schemaVersion: "skillwire.admin-result/v1", + command: "status", + operationId: randomUUID(), + status: signalIsAborted(signal) ? "cancelled" : "failure", + exitClass: signalIsAborted(signal) + ? "user-cancellation" + : "degraded-or-incomplete", + previewHash: null, + changed: false, + summary: "Installed state could not be inspected safely", + components: [], + findings: [ + { + code: "STATUS_STATE_UNAVAILABLE", + severity: "error", + component: "installation", + summary: + error instanceof Error + ? error.message.slice(0, 512) + : "Installed state is unavailable", + nextAction: "Run doctor to classify the installed-state failure", + }, + ], + recovery: { + rollbackBoundary: "none", + backupId: null, + instructions: [], + }, + }), + command, + io, + ); + } + } + const operation = operations[command.route]; + if (operation !== undefined) { + try { + return emit(await operation(command, signal), command, io); + } catch (error) { + return emit( + AdminResultSchema.parse({ + schemaVersion: "skillwire.admin-result/v1", + command: command.route, + operationId: randomUUID(), + status: signalIsAborted(signal) ? "cancelled" : "failure", + exitClass: signalIsAborted(signal) + ? "user-cancellation" + : failureClass(error), + previewHash: null, + changed: false, + summary: `${command.route} stopped before successful completion`, + components: [], + findings: [ + { + code: "LIFECYCLE_OPERATION_FAILED", + severity: "error", + component: command.route, + summary: + error instanceof Error + ? error.message.slice(0, 512) + : "Lifecycle operation failed", + nextAction: + "Resolve the reported condition and generate a fresh preview", + }, + ], + recovery: { + rollbackBoundary: "none", + backupId: null, + instructions: [], + }, + }), + command, + io, + ); + } + } const result = AdminResultSchema.parse({ schemaVersion: "skillwire.admin-result/v1", command: command.route, diff --git a/src/onboarding/cli/main.ts b/src/onboarding/cli/main.ts index c5b363f..dd04cdd 100644 --- a/src/onboarding/cli/main.ts +++ b/src/onboarding/cli/main.ts @@ -5,6 +5,7 @@ import { pathToFileURL } from "node:url"; import { routeAdministrativeCommand } from "./command-router.js"; import { runBridgeCommand } from "../../credential-bridge/bridge-cli.js"; import { redactText } from "./output.js"; +import { createProductionLifecycleOperations } from "../application/production-lifecycle.js"; export type ClientName = "codex" | "claude"; export type CommandRoute = @@ -257,7 +258,13 @@ export function installCancellationSignals( } const defaultDependencies: DispatcherDependencies = { - admin: routeAdministrativeCommand, + admin: (command, io, signal) => + routeAdministrativeCommand( + command, + io, + signal, + createProductionLifecycleOperations(process.env), + ), bridge: runBridgeCommand, }; diff --git a/src/onboarding/cli/output.ts b/src/onboarding/cli/output.ts index b8ef0bf..a69ad81 100644 --- a/src/onboarding/cli/output.ts +++ b/src/onboarding/cli/output.ts @@ -78,6 +78,20 @@ const SetupPreviewScopeSchema = z }) .strict(); +const LifecyclePreviewScopeSchema = z + .record(z.string().min(1).max(64), z.json()) + .refine( + (value) => JSON.stringify(value).length <= 64 * 1024, + "preview scope is too large", + ) + .refine( + (value) => + !/(?:swk\.[A-Za-z0-9_-]{16}\.[A-Za-z0-9_-]{43}|bearer\s+\S+|password\s*[=:]\s*\S+|pepper\s*[=:]\s*\S+)/i.test( + JSON.stringify(value), + ), + "preview scope contains secret material", + ); + export const AdminResultSchema = z .object({ schemaVersion: z.literal("skillwire.admin-result/v1"), @@ -96,7 +110,9 @@ export const AdminResultSchema = z .string() .regex(/^[0-9a-f]{64}$/) .nullable(), - previewScope: SetupPreviewScopeSchema.optional(), + previewScope: z + .union([SetupPreviewScopeSchema, LifecyclePreviewScopeSchema]) + .optional(), changed: z.boolean(), summary: z.string().min(1).max(512), components: z.array(ComponentSchema).max(64), diff --git a/src/onboarding/domain/diagnostics.ts b/src/onboarding/domain/diagnostics.ts index 33d37ca..02d01d3 100644 --- a/src/onboarding/domain/diagnostics.ts +++ b/src/onboarding/domain/diagnostics.ts @@ -44,6 +44,11 @@ export const DiagnosticFindingSchema = z "backup", "journal", "setup", + "installation", + "ownership", + "concurrency", + "upgrade", + "uninstall", ]), summary: SafeTextSchema, nextAction: SafeTextSchema, diff --git a/src/onboarding/domain/operation-journal.ts b/src/onboarding/domain/operation-journal.ts index d1dc10a..c55212b 100644 --- a/src/onboarding/domain/operation-journal.ts +++ b/src/onboarding/domain/operation-journal.ts @@ -146,15 +146,28 @@ export class OperationJournal { detail: JournalEntry["detail"], ): Promise { const last = this.entries.at(-1); - if (last?.phase === "commit" || last?.phase === "cancel") { + const resumingRecoveryRequiredCancellation = + last?.phase === "cancel" && + last.detail["status"] === "recovery-required" && + phase === "compensate"; + if ( + last?.phase === "commit" || + (last?.phase === "cancel" && !resumingRecoveryRequiredCancellation) + ) { throw new Error("Operation journal is already terminal"); } if (phase === "effect" && (last?.phase !== "intent" || last.step !== step)) throw new Error("Effect must follow matching durable intent"); if (phase === "verify" && (last?.phase !== "effect" || last.step !== step)) throw new Error("Verification must follow matching effect"); + const recoveredCommit = + phase === "commit" && + detail["status"] === "recovered" && + last?.phase === "compensate" && + last.detail["completion"] !== "unproven"; if ( phase === "commit" && + !recoveredCommit && !this.entries.some(({ phase: entryPhase }) => entryPhase === "verify") ) throw new Error("Cannot report success without verification"); @@ -243,14 +256,12 @@ export class OperationJournal { } hasUnprovenEffect(): boolean { - const unproven = new Set( - this.entries - .filter( - ({ phase, detail }) => - phase === "compensate" && detail["completion"] === "unproven", - ) - .map(({ step }) => step), - ); + const unproven = new Set(); + for (const entry of this.entries) { + if (entry.phase !== "compensate") continue; + if (entry.detail["completion"] === "unproven") unproven.add(entry.step); + else unproven.delete(entry.step); + } return unproven.size > 0; } } diff --git a/src/onboarding/domain/ownership.ts b/src/onboarding/domain/ownership.ts index 5c6f6e0..83996c9 100644 --- a/src/onboarding/domain/ownership.ts +++ b/src/onboarding/domain/ownership.ts @@ -272,3 +272,121 @@ export function recordOwnedAsset( externalIntegrations: ledger.externalIntegrations, }; } + +export type OwnershipRemovalOperation = + "client-uninstall" | "uninstall" | "purge"; + +export interface OwnedAssetDispositionPlan { + readonly remove: readonly z.infer[]; + readonly retain: readonly z.infer[]; +} + +export function planOwnedAssetDispositions( + candidate: unknown, + operation: OwnershipRemovalOperation, + client?: "codex" | "claude", +): OwnedAssetDispositionPlan { + const record = verifyOwnershipRecord(candidate); + if (operation === "client-uninstall" && client === undefined) + throw new Error("Client uninstall requires an exact client"); + const remove = record.assets.filter((asset) => { + if (operation === "purge") + return ( + asset.disposition === "present" || asset.disposition === "retained" + ); + if (operation === "client-uninstall") + return ( + asset.client === client && + (asset.disposition === "present" || + (asset.kind === "credential" && asset.disposition === "retained")) + ); + if (asset.disposition !== "present") return false; + return ( + asset.kind === "mcp-entry" || + asset.kind === "plugin" || + asset.kind === "marketplace" || + asset.kind === "container" || + asset.kind === "compose-project" + ); + }); + const removeIds = new Set(remove.map(({ assetId }) => assetId)); + return { + remove, + retain: record.assets.filter(({ assetId }) => !removeIds.has(assetId)), + }; +} + +export function requireCurrentOwnedAssetIdentity( + asset: z.infer, + currentIdentitySha256: string, +): void { + if (asset.disposition === "ambiguous" || asset.disposition === "drifted") + throw new Error("Owned asset is ambiguous or drifted"); + if (asset.disposition !== "present" && asset.disposition !== "retained") + throw new Error("Owned asset is not currently removable"); + if (asset.expectedIdentitySha256 !== currentIdentitySha256) + throw new Error("Owned asset identity changed after preview"); +} + +export function recordAssetDisposition( + candidate: unknown, + assetId: string, + disposition: "removed" | "retained" | "drifted" | "ambiguous", +): z.infer { + const record = verifyOwnershipRecord(candidate); + if (!record.assets.some((asset) => asset.assetId === assetId)) + throw new Error("Owned asset is not recorded"); + return updateRecord(record, { + assets: record.assets.map((asset) => + asset.assetId === assetId ? { ...asset, disposition } : asset, + ), + }); +} + +export function reactivateOwnedAsset( + candidate: unknown, + assetId: string, + currentIdentitySha256: string, +): z.infer { + const record = verifyOwnershipRecord(candidate); + const asset = record.assets.find((entry) => entry.assetId === assetId); + if (asset === undefined) throw new Error("Owned asset is not recorded"); + if (asset.expectedIdentitySha256 !== currentIdentitySha256) + throw new Error("Retained owned asset identity changed"); + if (asset.disposition !== "retained" && asset.disposition !== "removed") + throw new Error("Owned asset is not eligible for reactivation"); + return updateRecord(record, { + assets: record.assets.map((entry) => + entry.assetId === assetId + ? { ...entry, disposition: "present" as const } + : entry, + ), + }); +} + +export function replaceOwnedAssetIdentity( + candidate: unknown, + assetId: string, + replacement: { + readonly locator: string; + readonly expectedIdentitySha256: string; + }, +): z.infer { + const record = verifyOwnershipRecord(candidate); + if (!/^[0-9a-f]{64}$/.test(replacement.expectedIdentitySha256)) + throw new Error("Replacement owned asset identity is invalid"); + if (!record.assets.some((asset) => asset.assetId === assetId)) + throw new Error("Owned asset is not recorded"); + return updateRecord(record, { + assets: record.assets.map((asset) => + asset.assetId === assetId + ? { + ...asset, + locator: replacement.locator, + expectedIdentitySha256: replacement.expectedIdentitySha256, + disposition: "present" as const, + } + : asset, + ), + }); +} diff --git a/tests/contract/cli/lifecycle-operations.test.ts b/tests/contract/cli/lifecycle-operations.test.ts new file mode 100644 index 0000000..8478861 --- /dev/null +++ b/tests/contract/cli/lifecycle-operations.test.ts @@ -0,0 +1,319 @@ +/* eslint-disable @typescript-eslint/require-await -- Async fakes mirror production lifecycle interfaces. */ +import { randomUUID } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + routeAdministrativeCommand, + type AdministrativeOperations, +} from "../../../src/onboarding/cli/command-router.js"; +import { createProductionLifecycleOperations } from "../../../src/onboarding/application/production-lifecycle.js"; +import { ensureServiceSecrets } from "../../../src/onboarding/secrets/service-secrets.js"; +import { createOwnershipLedger } from "../../../src/onboarding/domain/ownership.js"; +import type { ParsedCommand } from "../../../src/onboarding/cli/main.js"; +import { + createOnboardingEnvironment, + type OnboardingEnvironment, +} from "../../helpers/onboarding-environment.js"; + +describe("administrative lifecycle routes", () => { + let fixture: OnboardingEnvironment | undefined; + + afterEach(async () => { + vi.unstubAllEnvs(); + await fixture?.close(); + }); + + it("renders bounded installed status as JSON on stdout without mutation", async () => { + fixture = await createOnboardingEnvironment(); + const stateRoot = resolve(fixture.root, "admin-state"); + await mkdir(stateRoot, { recursive: true, mode: 0o700 }); + const installationId = randomUUID(); + await writeFile( + resolve(stateRoot, "installation.json"), + `${JSON.stringify({ + schemaVersion: "skillwire.installation/v1", + installationId, + ownerUid: process.getuid?.() ?? 0, + accountId: randomUUID(), + activeReleaseId: "7-amd64", + highestAcceptedReleaseSequence: 7, + activeTrustPolicySequence: 3, + endpoint: `unix://${resolve(fixture.runtimeRoot, "skillwire/mcp.sock")}`, + composeProject: "skillwire-test", + postgresVolume: "skillwire-test_postgres_data", + selectedClients: ["codex"], + clientIntegrationIds: { codex: randomUUID(), claude: null }, + status: "complete", + createdAt: "2026-08-14T08:00:00.000Z", + updatedAt: "2026-08-14T08:00:00.000Z", + lastValidatedAt: "2026-08-14T08:00:00.000Z", + })}\n`, + { mode: 0o600 }, + ); + const before = await fixtureSnapshot(stateRoot); + vi.stubEnv("SKILLWIRE_ALLOW_STATE_ROOT", "test"); + let stdout = ""; + let stderr = ""; + const command: ParsedCommand = { + route: "status", + output: "json", + previewOnly: false, + stateRoot, + }; + + const code = await routeAdministrativeCommand( + command, + { + stdout: (value) => (stdout += value), + stderr: (value) => (stderr += value), + }, + new AbortController().signal, + ); + + expect(code).toBe(0); + expect(JSON.parse(stdout)).toMatchObject({ + command: "status", + status: "success", + changed: false, + components: [ + { + component: "installation", + state: "complete", + owned: true, + identity: { installationId }, + }, + ], + }); + expect(stderr).toBe(""); + expect(await fixtureSnapshot(stateRoot)).toEqual(before); + }); + + it("uses the production live-status operation when one is supplied", async () => { + const operation = vi.fn(async () => ({ + schemaVersion: "skillwire.admin-result/v1" as const, + command: "status" as const, + operationId: randomUUID(), + status: "success" as const, + exitClass: "success" as const, + previewHash: null, + changed: false, + summary: "bounded live state", + components: [ + { + component: "postgres", + state: "ready", + changed: false, + owned: true, + identity: { migration: "010" }, + }, + ], + findings: [], + recovery: { + rollbackBoundary: "none" as const, + backupId: null, + instructions: [], + }, + })); + let stdout = ""; + const code = await routeAdministrativeCommand( + { + route: "status", + output: "json", + previewOnly: false, + }, + { stdout: (value) => (stdout += value), stderr: vi.fn() }, + new AbortController().signal, + { status: operation }, + ); + expect(code).toBe(0); + expect(operation).toHaveBeenCalledTimes(1); + expect(JSON.parse(stdout)).toMatchObject({ + components: [{ component: "postgres", state: "ready" }], + }); + }); + + it("rejects a missing absolute XDG runtime root before deriving any path from cwd", async () => { + fixture = await createOnboardingEnvironment(); + const operations = createProductionLifecycleOperations({ + ...fixture.environment, + XDG_RUNTIME_DIR: undefined, + }); + + await expect( + operations.status?.( + { + route: "status", + output: "json", + previewOnly: false, + }, + new AbortController().signal, + ), + ).rejects.toThrow( + "Absolute HOME and XDG data/state/runtime roots are required for lifecycle operations", + ); + }); + + it("does not report a complete installation healthy when its owned Docker service is unavailable", async () => { + fixture = await createOnboardingEnvironment(); + const stateRoot = resolve(fixture.xdgStateHome, "skillwire"); + const dataRoot = resolve(fixture.xdgDataHome, "skillwire"); + const installationId = randomUUID(); + const installationRoot = resolve(dataRoot, "installations", installationId); + const secretReferences = await ensureServiceSecrets( + installationRoot, + dataRoot, + ); + await mkdir(stateRoot, { recursive: true, mode: 0o700 }); + const writeProtected = (name: string, value: unknown) => + writeFile(resolve(stateRoot, name), `${JSON.stringify(value)}\n`, { + mode: 0o600, + }); + const timestamp = "2026-08-14T08:00:00.000Z"; + await Promise.all([ + writeProtected("installation.json", { + schemaVersion: "skillwire.installation/v1", + installationId, + ownerUid: process.getuid?.() ?? 0, + accountId: randomUUID(), + activeReleaseId: "7-amd64", + highestAcceptedReleaseSequence: 7, + activeTrustPolicySequence: 3, + endpoint: `unix://${resolve(fixture.runtimeRoot, "skillwire/mcp.sock")}`, + composeProject: fixture.composeProject, + postgresVolume: fixture.postgresVolume, + selectedClients: [], + clientIntegrationIds: { codex: null, claude: null }, + status: "complete", + createdAt: timestamp, + updatedAt: timestamp, + lastValidatedAt: timestamp, + }), + writeProtected( + "ownership.json", + createOwnershipLedger(installationId).record, + ), + writeProtected("service-secret-set.json", { + schemaVersion: "skillwire.service-secret-set/v1", + serviceSecretSetId: randomUUID(), + installationId, + createdByOperation: randomUUID(), + secrets: secretReferences, + state: "available", + }), + writeProtected("deployment.json", { + schemaVersion: "skillwire.deployment/v1", + installationId, + releaseRoot: resolve(dataRoot, "releases/skillwire-7-amd64"), + composePath: resolve( + dataRoot, + "releases/skillwire-7-amd64/compose.yaml", + ), + skillwireImage: `docker.io/skillwire/app@sha256:${"a".repeat(64)}`, + postgresImage: `docker.io/library/postgres@sha256:${"b".repeat(64)}`, + databasePasswordFile: resolve( + installationRoot, + "secrets/database-password", + ), + applicationPepperFile: resolve( + installationRoot, + "secrets/application-pepper", + ), + runtimeSocketDirectory: resolve(fixture.runtimeRoot, "skillwire"), + socketPath: resolve(fixture.runtimeRoot, "skillwire/mcp.sock"), + projectName: fixture.composeProject, + volumeName: fixture.postgresVolume, + }), + ]); + + const doctor = createProductionLifecycleOperations( + fixture.environment, + ).doctor; + const report = await doctor?.( + { route: "doctor", output: "json", previewOnly: false, stateRoot }, + new AbortController().signal, + ); + expect(report).toMatchObject({ status: "incomplete", changed: false }); + expect(report?.findings.map(({ code }) => code)).toContain( + "SERVICE_STOPPED", + ); + }); + + it.each([ + ["doctor", {}], + ["repair", { component: "codex" }], + ["clients:rotate-key", { client: "codex" }], + [ + "maintenance:rotate-service-secret", + { serviceSecret: "application-pepper" }, + ], + ["backup", {}], + ["upgrade", { release: "/tmp/verified-release.tar.zst" }], + ["clients:uninstall", { client: "claude" }], + ["uninstall", {}], + ["purge", {}], + ] as const)( + "routes %s with stable JSON/exit/stdout separation", + async (route, extra) => { + const command = { + route, + output: "json" as const, + previewOnly: route !== "doctor", + ...extra, + } satisfies ParsedCommand; + const previewHash = command.previewOnly ? "f".repeat(64) : null; + const operations: AdministrativeOperations = { + [route]: async () => ({ + schemaVersion: "skillwire.admin-result/v1" as const, + command: route, + operationId: randomUUID(), + status: command.previewOnly ? "preview" : "success", + exitClass: "success" as const, + previewHash, + changed: false, + summary: `${route} completed safely`, + components: [], + findings: [], + recovery: { + rollbackBoundary: "none" as const, + backupId: null, + instructions: [], + }, + }), + }; + let stdout = ""; + let stderr = ""; + const code = await routeAdministrativeCommand( + command, + { + stdout: (value) => (stdout += value), + stderr: (value) => (stderr += value), + }, + new AbortController().signal, + operations, + ); + expect(code).toBe(0); + expect(JSON.parse(stdout)).toMatchObject({ + command: route, + exitClass: "success", + changed: false, + }); + expect(stderr).toBe(""); + }, + ); +}); + +async function fixtureSnapshot(root: string): Promise { + const { readdir, readFile, stat } = await import("node:fs/promises"); + const names = (await readdir(root)).sort(); + const entries = await Promise.all( + names.map(async (name) => { + const path = resolve(root, name); + const metadata = await stat(path); + return `${name}:${String(metadata.mode & 0o777)}:${await readFile(path, "utf8")}`; + }), + ); + return entries.join("\n"); +} diff --git a/tests/e2e/self-hosted-onboarding/default-uninstall.test.ts b/tests/e2e/self-hosted-onboarding/default-uninstall.test.ts new file mode 100644 index 0000000..cda9281 --- /dev/null +++ b/tests/e2e/self-hosted-onboarding/default-uninstall.test.ts @@ -0,0 +1,174 @@ +/* eslint-disable @typescript-eslint/require-await -- Async fakes mirror production lifecycle interfaces. */ +import { randomUUID } from "node:crypto"; + +import { describe, expect, it, vi } from "vitest"; + +import { + previewDefaultUninstall, + runDefaultUninstall, +} from "../../../src/onboarding/application/uninstall.js"; +import { uninstallClientLifecycle } from "../../../src/onboarding/application/client-lifecycle.js"; +import { + createOwnershipLedger, + planOwnedAssetDispositions, + recordAssetDisposition, + recordOwnedAsset, +} from "../../../src/onboarding/domain/ownership.js"; + +describe("data-preserving default uninstall", () => { + it("removes only owned client/service runtime assets and retains recovery state", async () => { + const installationId = randomUUID(); + const operationId = randomUUID(); + let ledger = createOwnershipLedger(installationId); + const assets = [ + ["mcp-entry", "codex", "skillwire:user", "remove-on-uninstall"], + ["plugin", "codex", "skillwire-plugin", "remove-on-uninstall"], + ["marketplace", "claude", "skillwire-marketplace", "remove-on-uninstall"], + ["credential", "codex", "secret-service:codex", "retain-by-default"], + ["container", null, "skillwire-service", "remove-on-uninstall"], + ["compose-project", null, "skillwire-project", "remove-on-uninstall"], + ["volume", null, "skillwire-volume", "retain-by-default"], + ["backup", null, "backups/backup.dump", "retain-by-default"], + [ + "service-secret", + null, + "secrets/database-password", + "retain-by-default", + ], + ["release", null, "releases/skillwire-1", "retain-by-default"], + ["trust-policy", null, "trust/policy-v1.json", "retain-by-default"], + ["path", null, "state/installation.json", "remove-only-on-purge"], + ] as const; + for (const [kind, client, locator, retention] of assets) { + ledger = recordOwnedAsset(ledger, { + kind, + client, + locator, + expectedIdentitySha256: "a".repeat(64), + createdByOperation: operationId, + retention, + disposition: "present", + }); + } + const preview = previewDefaultUninstall(ledger.record); + const removed: string[] = []; + const unrelatedExternal = { + codexMcp: "external-command", + claudePlugin: "external-plugin", + profileSetting: "keep", + }; + const beforeExternal = structuredClone(unrelatedExternal); + const stopOwnedService = vi.fn(async () => undefined); + + const result = await runDefaultUninstall({ + ownership: ledger.record, + preview, + confirmation: preview.previewHash, + signal: new AbortController().signal, + observeIdentity: async (asset) => asset.expectedIdentitySha256, + removeAsset: async (asset) => { + removed.push(asset.kind); + }, + stopOwnedService, + publishRetained: async () => undefined, + }); + + expect(removed.sort()).toEqual( + [ + "compose-project", + "container", + "marketplace", + "mcp-entry", + "plugin", + ].sort(), + ); + expect(preview.retain.map(({ kind }) => kind).sort()).toEqual( + [ + "backup", + "credential", + "path", + "release", + "service-secret", + "trust-policy", + "volume", + ].sort(), + ); + expect(stopOwnedService).toHaveBeenCalledTimes(1); + expect(result.retained).toHaveLength(7); + expect(unrelatedExternal).toEqual(beforeExternal); + }); + + it("uninstalls one owned client including its key while preserving its sibling and external integrations", async () => { + const events: string[] = []; + const result = await uninstallClientLifecycle( + "codex", + { + inspect: async () => [ + { + component: "mcp-entry", + classification: "owned-equivalent", + expectedIdentitySha256: "b".repeat(64), + currentIdentitySha256: "b".repeat(64), + }, + { + component: "plugin", + classification: "external-equivalent", + expectedIdentitySha256: null, + currentIdentitySha256: "c".repeat(64), + }, + { + component: "credential", + classification: "owned-equivalent", + expectedIdentitySha256: "d".repeat(64), + currentIdentitySha256: "d".repeat(64), + }, + ], + removeMcp: async () => { + events.push("codex-mcp"); + }, + removePlugin: async () => { + events.push("codex-plugin"); + }, + removeMarketplace: async () => { + events.push("codex-marketplace"); + }, + revokeCredential: async () => { + events.push("codex-key-and-credential"); + }, + verifyAbsent: async () => true, + }, + new AbortController().signal, + ); + expect(result).toMatchObject({ + client: "codex", + status: "removed", + removed: ["mcp-entry", "credential"], + retainedExternal: ["plugin"], + }); + expect(events).toEqual(["codex-mcp", "codex-key-and-credential"]); + expect(events.some((event) => event.includes("claude"))).toBe(false); + }); + + it("still plans an independently revocable retained client credential after default uninstall", () => { + let ledger = createOwnershipLedger(randomUUID()); + ledger = recordOwnedAsset(ledger, { + kind: "credential", + client: "claude", + locator: "restrictive-file:claude:fixture", + expectedIdentitySha256: "e".repeat(64), + createdByOperation: randomUUID(), + retention: "retain-by-default", + disposition: "present", + }); + const asset = ledger.record.assets[0]; + if (asset === undefined) throw new Error("missing fixture asset"); + const retained = recordAssetDisposition( + ledger.record, + asset.assetId, + "retained", + ); + expect( + planOwnedAssetDispositions(retained, "client-uninstall", "claude").remove, + ).toEqual([expect.objectContaining({ assetId: asset.assetId })]); + }); +}); diff --git a/tests/e2e/self-hosted-onboarding/permanent-removal.test.ts b/tests/e2e/self-hosted-onboarding/permanent-removal.test.ts new file mode 100644 index 0000000..52d642f --- /dev/null +++ b/tests/e2e/self-hosted-onboarding/permanent-removal.test.ts @@ -0,0 +1,104 @@ +/* eslint-disable @typescript-eslint/require-await -- Async fakes mirror production lifecycle interfaces. */ +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + previewPurge, + removeOwnedFilesystemTree, + runPurge, +} from "../../../src/onboarding/application/purge.js"; +import { previewDefaultUninstall } from "../../../src/onboarding/application/uninstall.js"; +import { ownedLauncherIdentity } from "../../../src/onboarding/application/production-setup.js"; +import { + createOwnershipLedger, + recordOwnedAsset, +} from "../../../src/onboarding/domain/ownership.js"; +import { + createOnboardingEnvironment, + type OnboardingEnvironment, +} from "../../helpers/onboarding-environment.js"; + +describe("separately confirmed permanent removal", () => { + let fixture: OnboardingEnvironment | undefined; + afterEach(async () => fixture?.close()); + + it("binds a protected stable launcher to its exact owned bytes", async () => { + fixture = await createOnboardingEnvironment(); + const launcher = resolve(fixture.root, "bin/skillwire"); + await mkdir(resolve(launcher, ".."), { recursive: true, mode: 0o700 }); + await writeFile(launcher, "#!/bin/sh\nexit 0\n", { mode: 0o700 }); + + await expect(ownedLauncherIdentity(launcher)).resolves.toMatch( + /^[0-9a-f]{64}$/, + ); + }); + + it("binds confirmation to installation ID and exact owned targets", async () => { + fixture = await createOnboardingEnvironment(); + const installationId = randomUUID(); + const ownedRoot = resolve(fixture.root, "owned"); + const dataPath = resolve(ownedRoot, "data"); + const backupPath = resolve(ownedRoot, "backups/validated.dump"); + const unrelatedPath = resolve(fixture.root, "unrelated/keep.txt"); + await mkdir(dataPath, { recursive: true, mode: 0o700 }); + await mkdir(resolve(backupPath, ".."), { recursive: true, mode: 0o700 }); + await mkdir(resolve(unrelatedPath, ".."), { recursive: true, mode: 0o700 }); + await writeFile(resolve(dataPath, "state.json"), "owned", { mode: 0o600 }); + await writeFile(backupPath, "backup", { mode: 0o600 }); + await writeFile(unrelatedPath, "keep", { mode: 0o600 }); + let ledger = createOwnershipLedger(installationId); + for (const [kind, locator] of [ + ["path", dataPath], + ["backup", backupPath], + ] as const) { + ledger = recordOwnedAsset(ledger, { + kind, + client: null, + locator, + expectedIdentitySha256: "e".repeat(64), + createdByOperation: randomUUID(), + retention: "remove-only-on-purge", + disposition: "retained", + }); + } + const preview = previewPurge(ledger.record); + expect(preview.installationId).toBe(installationId); + expect(preview.unrecoverable.map(({ locator }) => locator).sort()).toEqual( + [backupPath, dataPath].sort(), + ); + const uninstallPreview = previewDefaultUninstall(ledger.record); + const removeAsset = vi.fn(async (asset: { locator: string }) => + removeOwnedFilesystemTree(asset.locator, fixture?.root ?? ""), + ); + + await expect( + runPurge({ + ownership: ledger.record, + preview, + confirmation: uninstallPreview.previewHash, + signal: new AbortController().signal, + observeIdentity: async (asset) => asset.expectedIdentitySha256, + removeAsset, + }), + ).rejects.toThrow(/preview/i); + expect(removeAsset).not.toHaveBeenCalled(); + + const result = await runPurge({ + ownership: ledger.record, + preview, + confirmation: preview.previewHash, + signal: new AbortController().signal, + observeIdentity: async (asset) => asset.expectedIdentitySha256, + removeAsset, + }); + expect(result.removed).toHaveLength(2); + await expect(readFile(dataPath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(readFile(backupPath)).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(readFile(unrelatedPath, "utf8")).resolves.toBe("keep"); + }); +}); diff --git a/tests/e2e/self-hosted-onboarding/reinstall-retained-data.test.ts b/tests/e2e/self-hosted-onboarding/reinstall-retained-data.test.ts new file mode 100644 index 0000000..bee454b --- /dev/null +++ b/tests/e2e/self-hosted-onboarding/reinstall-retained-data.test.ts @@ -0,0 +1,71 @@ +/* eslint-disable @typescript-eslint/require-await, prefer-const -- Async fakes mirror production interfaces; the closure-backed result is assigned after dependency construction. */ +import { describe, expect, it, vi } from "vitest"; + +import { + runGuidedSetup, + type GuidedSetupResult, +} from "../../../src/onboarding/application/setup.js"; + +describe("retained-data reinstall", () => { + it("reuses installation, account, volume, secrets, and keys while restoring each client registration once", async () => { + const installationId = "00000000-0000-4000-8000-000000000129"; + const writes = { + account: 0, + volume: 0, + serviceSecrets: 0, + keys: 0, + mcp: 0, + plugins: 0, + }; + let completed: GuidedSetupResult | undefined; + const dependencies = { + inspectExisting: vi.fn(async () => completed), + verifyRelease: vi.fn(async () => ({ releaseSequence: 12 })), + discoverRetained: vi.fn(async () => + completed === undefined ? { installationId, clients: [] } : undefined, + ), + reactivateRetainedService: vi.fn(async () => ({ ready: true })), + reactivateClient: vi.fn(async (client: "codex" | "claude") => { + writes.mcp += 1; + writes.plugins += 1; + return { + client, + status: "verified" as const, + compensated: false, + owned: true, + }; + }), + installService: vi.fn(async () => { + writes.account += 1; + writes.volume += 1; + writes.serviceSecrets += 1; + return { installationId: "new-installation", ready: true }; + }), + installClient: vi.fn(async (client: "codex" | "claude") => { + writes.keys += 1; + return { client, status: "verified" as const, compensated: false }; + }), + }; + + completed = await runGuidedSetup({ clients: "codex,claude" }, dependencies); + const repeated = await runGuidedSetup( + { clients: "codex,claude" }, + dependencies, + ); + + expect(completed.installationId).toBe(installationId); + expect(repeated).toEqual(completed); + expect(writes).toEqual({ + account: 0, + volume: 0, + serviceSecrets: 0, + keys: 0, + mcp: 2, + plugins: 2, + }); + expect(dependencies.reactivateRetainedService).toHaveBeenCalledTimes(1); + expect(dependencies.reactivateClient).toHaveBeenCalledTimes(2); + expect(dependencies.installService).not.toHaveBeenCalled(); + expect(dependencies.installClient).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/e2e/self-hosted-onboarding/repeated-setup.test.ts b/tests/e2e/self-hosted-onboarding/repeated-setup.test.ts new file mode 100644 index 0000000..ed81713 --- /dev/null +++ b/tests/e2e/self-hosted-onboarding/repeated-setup.test.ts @@ -0,0 +1,108 @@ +/* eslint-disable @typescript-eslint/require-await -- Async fakes mirror production lifecycle interfaces. */ +import { describe, expect, it, vi } from "vitest"; + +import { + runGuidedSetup, + type GuidedSetupResult, +} from "../../../src/onboarding/application/setup.js"; +import { unchangedSetupClientResults } from "../../../src/onboarding/application/production-setup.js"; + +describe("unchanged guided setup", () => { + it("preserves external ownership classification in the production no-op result", () => { + const installationId = "00000000-0000-4000-8000-000000000009"; + expect( + unchangedSetupClientResults(["codex", "claude"], installationId, { + schemaVersion: "skillwire.client-integrations/v1", + installationId, + integrations: [ + integration(installationId, "codex", "external-verified"), + integration(installationId, "claude", "verified"), + ], + }), + ).toMatchObject([ + { client: "codex", status: "external-verified", owned: false }, + { client: "claude", status: "verified", owned: true }, + ]); + }); + + it("is a byte-for-byte no-op for ten repeated executions", async () => { + let installed: GuidedSetupResult | undefined; + const writes = { + account: 0, + key: 0, + volume: 0, + serviceSecret: 0, + source: 0, + plugin: 0, + mcp: 0, + }; + const dependencies = { + inspectExisting: vi.fn(async () => installed), + verifyRelease: vi.fn(async () => ({ releaseSequence: 9 })), + installService: vi.fn(async () => { + writes.account += 1; + writes.volume += 1; + writes.serviceSecret += 1; + return { + installationId: "00000000-0000-4000-8000-000000000009", + ready: true, + }; + }), + installClient: vi.fn(async (client: "codex" | "claude") => { + writes.key += 1; + writes.plugin += 1; + writes.mcp += 1; + return { + client, + status: "verified" as const, + compensated: false, + }; + }), + }; + + for (let run = 0; run < 10; run += 1) { + const result = await runGuidedSetup( + { clients: "codex,claude" }, + dependencies, + ); + installed ??= result; + expect(result).toEqual(installed); + } + + expect(writes).toEqual({ + account: 1, + key: 2, + volume: 1, + serviceSecret: 1, + source: 0, + plugin: 2, + mcp: 2, + }); + expect(dependencies.verifyRelease).toHaveBeenCalledTimes(1); + expect(dependencies.installService).toHaveBeenCalledTimes(1); + expect(dependencies.installClient).toHaveBeenCalledTimes(2); + }); +}); + +function integration( + installationId: string, + client: "codex" | "claude", + state: "verified" | "external-verified", +) { + return { + schemaVersion: "skillwire.client-integration/v1", + clientIntegrationId: + client === "codex" + ? "00000000-0000-4000-8000-000000000010" + : "00000000-0000-4000-8000-000000000011", + installationId, + client, + clientVersion: "0.147.0", + profileScope: "normal-user", + state, + credentialReferenceId: null, + keyPublicIdHash: null, + mcpIdentitySha256: "a".repeat(64), + adapterIdentitySha256: "b".repeat(64), + }; +} diff --git a/tests/e2e/self-hosted-onboarding/upgrade-preservation.test.ts b/tests/e2e/self-hosted-onboarding/upgrade-preservation.test.ts new file mode 100644 index 0000000..eda51fa --- /dev/null +++ b/tests/e2e/self-hosted-onboarding/upgrade-preservation.test.ts @@ -0,0 +1,64 @@ +/* eslint-disable @typescript-eslint/require-await -- Async fakes mirror production lifecycle interfaces. */ +import { randomUUID } from "node:crypto"; + +import { describe, expect, it } from "vitest"; + +import { + previewUpgrade, + runUpgrade, +} from "../../../src/onboarding/application/upgrade.js"; + +describe("upgrade state preservation", () => { + it("preserves repository memory, client/service references, ownership, volume, backup, source, and unrelated profile state", async () => { + const preserved = { + repositoryMemory: { skill: "kept", erased: false }, + clientCredentials: [randomUUID(), randomUUID()], + serviceSecretIdentities: ["a".repeat(64), "b".repeat(64)], + ownershipRevision: 7, + volume: "skillwire-install_postgres_data", + sources: ["external-source-id"], + codexUnknown: { theme: "dark", otherMcp: { command: "other" } }, + claudeUnknown: { model: "existing", otherPlugin: true }, + }; + const before = structuredClone(preserved); + const target = { + releaseId: "12-amd64", + releaseSequence: 12, + trustPolicySequence: 6, + schemaMinimum: 10, + schemaMaximum: 10, + latestMigration: 10, + manifestSha256: "c".repeat(64), + imageDigest: `sha256:${"d".repeat(64)}`, + }; + const preview = previewUpgrade({ + installationId: randomUUID(), + currentReleaseSequence: 11, + currentTrustPolicySequence: 6, + liveSchema: 10, + target, + }); + const backupId = randomUUID(); + + await expect( + runUpgrade({ + preview, + confirmation: preview.previewHash, + signal: new AbortController().signal, + verifyTarget: async () => target, + createBackup: async () => ({ backupId, validated: true }), + drainWriters: async () => undefined, + installApplication: async () => undefined, + migrate: async () => undefined, + verifyLiveSchema: async () => 10, + readiness: async () => undefined, + verifyClients: async () => undefined, + commitSelection: async () => undefined, + rollbackApplication: async () => undefined, + stopWriters: async () => undefined, + restartWriters: async () => undefined, + }), + ).resolves.toEqual({ backupId, releaseId: "12-amd64" }); + expect(preserved).toEqual(before); + }); +}); diff --git a/tests/integration/onboarding/backup-restore-validation.test.ts b/tests/integration/onboarding/backup-restore-validation.test.ts new file mode 100644 index 0000000..0daf6ac --- /dev/null +++ b/tests/integration/onboarding/backup-restore-validation.test.ts @@ -0,0 +1,445 @@ +/* eslint-disable @typescript-eslint/require-await -- Async fakes mirror production validation interfaces. */ +import { randomUUID } from "node:crypto"; +import { + access, + chmod, + lstat, + mkdir, + readdir, + symlink, + writeFile, +} from "node:fs/promises"; +import { resolve } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { createValidatedBackup } from "../../../src/onboarding/application/backup.js"; +import { PostgresBackupAdapter } from "../../../src/onboarding/adapters/postgres/backup.js"; +import { + runCommand, + type CommandOptions, +} from "../../../src/onboarding/adapters/process/command-runner.js"; +import { + createOnboardingEnvironment, + type OnboardingEnvironment, +} from "../../helpers/onboarding-environment.js"; + +describe("restore-validated PostgreSQL backup", () => { + let fixture: OnboardingEnvironment | undefined; + afterEach(async () => fixture?.close()); + + it("creates a protected custom dump and validates it through an isolated restore", async () => { + fixture = await createOnboardingEnvironment(); + const commands: CommandOptions[] = []; + const run = vi.fn(async (options: CommandOptions) => { + commands.push(options); + if ( + options.args.includes("compose") && + options.args.includes("cp") && + options.args.at(-1)?.endsWith(".dump") + ) { + const target = options.args.at(-1); + if (target === undefined) throw new Error("missing target"); + await writeFile(target, "PGDMP\0fixture-custom-archive", { + mode: 0o600, + }); + await chmod(target, 0o600); + } + return { code: 0, stdout: "", stderr: "", durationMilliseconds: 1 }; + }); + const installationId = randomUUID(); + const adapter = new PostgresBackupAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: resolve("distribution/self-hosted/compose.yaml"), + projectName: "skillwire-test", + installationId, + protectedRoot: fixture.root, + backupsRoot: resolve(fixture.root, "backups"), + postgresImage: `docker.io/library/postgres@sha256:${"a".repeat(64)}`, + environment: { + HOME: fixture.home, + DOCKER_HOST: `unix://${fixture.runtimeRoot}/docker.sock`, + GH_TOKEN: "ambient-canary", + }, + run, + validateRestoredDatabase: async () => ({ + latestMigration: "010", + invariantsValid: true, + catalogValid: true, + ready: true, + }), + }); + + const record = await createValidatedBackup({ + installationId, + sourceReleaseId: "9-amd64", + serviceSecretReferences: [ + { + kind: "database-password", + relativePath: "secrets/database-password", + identitySha256: "b".repeat(64), + state: "reused", + }, + ], + clientCredentialReferences: [randomUUID()], + adapter, + signal: new AbortController().signal, + }); + + expect(record.status).toBe("validated"); + expect(record.archiveSha256).toMatch(/^[0-9a-f]{64}$/); + expect((await lstat(record.archivePath)).mode & 0o777).toBe(0o600); + const backupRoot = resolve(record.archivePath, ".."); + expect(record.archivePath).toBe(resolve(backupRoot, "database.dump")); + expect((await readdir(backupRoot)).sort()).toEqual([ + "checksums.json", + "database.dump", + "recovery-manifest.json", + "validation.json", + ]); + expect(commands.some(({ args }) => args.includes("--format=custom"))).toBe( + true, + ); + const restore = commands.find(({ args }) => args.includes("pg_restore")); + expect(restore?.args).toEqual( + expect.arrayContaining([ + "--exit-on-error", + "--single-transaction", + "--no-owner", + "--no-acl", + ]), + ); + expect(JSON.stringify(record)).not.toMatch( + /swk\.|password\s*[=:]|pepper\s*[=:]/i, + ); + expect( + commands.every( + ({ environment }) => + !Object.keys(environment ?? {}).some((key) => + /TOKEN|SECRET|PASSWORD|PEPPER|CREDENTIAL/i.test(key), + ), + ), + ).toBe(true); + expect( + commands.every( + ({ environment }) => + environment?.["DOCKER_HOST"] === + `unix://${fixture?.runtimeRoot ?? ""}/docker.sock` && + environment["GH_TOKEN"] === undefined, + ), + ).toBe(true); + expect(commands.at(-1)?.args).toEqual( + expect.arrayContaining(["volume", "rm"]), + ); + }); + + it("rejects an invalid archive and still cleans validation resources", async () => { + fixture = await createOnboardingEnvironment(); + const commands: CommandOptions[] = []; + const run = vi.fn(async (options: CommandOptions) => { + commands.push(options); + if ( + options.args.includes("compose") && + options.args.includes("cp") && + options.args.at(-1)?.endsWith(".dump") + ) { + const target = options.args.at(-1); + if (target !== undefined) + await writeFile(target, "invalid-archive", { mode: 0o600 }); + } + if (options.args.includes("pg_restore")) throw new Error("invalid dump"); + return { code: 0, stdout: "", stderr: "", durationMilliseconds: 1 }; + }); + const adapter = new PostgresBackupAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: resolve("distribution/self-hosted/compose.yaml"), + projectName: "skillwire-test", + installationId: randomUUID(), + protectedRoot: fixture.root, + backupsRoot: resolve(fixture.root, "backups"), + postgresImage: `docker.io/library/postgres@sha256:${"c".repeat(64)}`, + run, + validateRestoredDatabase: vi.fn(), + }); + await expect( + adapter.createAndValidate(new AbortController().signal), + ).rejects.toThrow(/archive|dump|restore/i); + expect( + commands.some( + ({ args }) => args.includes("volume") && args.includes("rm"), + ), + ).toBe(true); + }); + + it("removes the incomplete backup set when pg_dump fails before copy", async () => { + fixture = await createOnboardingEnvironment(); + const backupsRoot = resolve(fixture.root, "backups"); + const adapter = new PostgresBackupAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: resolve("distribution/self-hosted/compose.yaml"), + projectName: "skillwire-test", + installationId: randomUUID(), + protectedRoot: fixture.root, + backupsRoot, + postgresImage: `docker.io/library/postgres@sha256:${"c".repeat(64)}`, + run: async (options) => { + if (options.args.includes("pg_dump")) throw new Error("dump failed"); + return { code: 0, stdout: "", stderr: "", durationMilliseconds: 1 }; + }, + validateRestoredDatabase: vi.fn(), + }); + + await expect( + adapter.createAndValidate(new AbortController().signal), + ).rejects.toThrow(/dump/i); + expect(await readdir(backupsRoot)).toEqual([]); + }); + + it("restore-validates the exact pre-upgrade schema instead of assuming 010", async () => { + fixture = await createOnboardingEnvironment(); + const run = vi.fn(async (options: CommandOptions) => { + if ( + options.args.includes("compose") && + options.args.includes("cp") && + options.args.at(-1)?.endsWith(".dump") + ) { + const target = options.args.at(-1); + if (target !== undefined) + await writeFile(target, "PGDMP\0schema-009", { mode: 0o600 }); + } + return { code: 0, stdout: "", stderr: "", durationMilliseconds: 1 }; + }); + const adapter = new PostgresBackupAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: resolve("distribution/self-hosted/compose.yaml"), + projectName: "skillwire-test", + installationId: randomUUID(), + protectedRoot: fixture.root, + backupsRoot: resolve(fixture.root, "backups"), + postgresImage: `docker.io/library/postgres@sha256:${"d".repeat(64)}`, + expectedLatestMigration: "009", + run, + validateRestoredDatabase: async () => ({ + latestMigration: "009", + invariantsValid: true, + catalogValid: true, + ready: true, + }), + }); + + await expect( + adapter.createAndValidate(new AbortController().signal), + ).resolves.toMatchObject({ validation: { latestMigration: "009" } }); + }); + + it("uses an independent bounded signal to remove validation resources after cancellation", async () => { + fixture = await createOnboardingEnvironment(); + const controller = new AbortController(); + const cleanupSignals: AbortSignal[] = []; + const run = vi.fn(async (options: CommandOptions) => { + if ( + options.args.includes("compose") && + options.args.includes("cp") && + options.args.at(-1)?.endsWith(".dump") + ) { + const target = options.args.at(-1); + if (target !== undefined) + await writeFile(target, "PGDMP\0cancelled", { mode: 0o600 }); + } + if (options.args[0] === "run") controller.abort(); + if ( + (options.args.includes("container") && options.args.includes("rm")) || + (options.args.includes("volume") && options.args.includes("rm")) + ) + cleanupSignals.push(options.signal ?? AbortSignal.abort()); + return { code: 0, stdout: "", stderr: "", durationMilliseconds: 1 }; + }); + const adapter = new PostgresBackupAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: resolve("distribution/self-hosted/compose.yaml"), + projectName: "skillwire-test", + installationId: randomUUID(), + protectedRoot: fixture.root, + backupsRoot: resolve(fixture.root, "backups"), + postgresImage: `docker.io/library/postgres@sha256:${"e".repeat(64)}`, + run, + validateRestoredDatabase: vi.fn(), + }); + + await expect(adapter.createAndValidate(controller.signal)).rejects.toThrow( + /cancel|validation/i, + ); + expect(cleanupSignals).toHaveLength(2); + expect(cleanupSignals.every((signal) => !signal.aborted)).toBe(true); + }); + + it("rejects a symlinked backup ancestor before creating anything outside the protected root", async () => { + fixture = await createOnboardingEnvironment(); + const protectedRoot = resolve(fixture.root, "protected-data"); + const outside = resolve(fixture.root, "outside-data"); + await Promise.all([ + mkdir(protectedRoot, { mode: 0o700 }), + mkdir(outside, { mode: 0o700 }), + ]); + await symlink(outside, resolve(protectedRoot, "redirect")); + const escaped = resolve(outside, "generated-backups"); + const run = vi.fn(); + const adapter = new PostgresBackupAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: resolve("distribution/self-hosted/compose.yaml"), + projectName: "skillwire-test", + installationId: randomUUID(), + protectedRoot, + backupsRoot: resolve(protectedRoot, "redirect", "generated-backups"), + postgresImage: `docker.io/library/postgres@sha256:${"f".repeat(64)}`, + run, + validateRestoredDatabase: vi.fn(), + }); + + await expect( + adapter.createAndValidate(new AbortController().signal), + ).rejects.toThrow(/symbolic link|owned root|unsafe/i); + await expect(access(escaped)).rejects.toMatchObject({ code: "ENOENT" }); + expect(run).not.toHaveBeenCalled(); + }); + + const realPostgresIt = + process.env["SKILLWIRE_RUN_POSTGRES_BACKUP_INTEGRATION"] === "1" + ? it + : it.skip; + + realPostgresIt( + "creates and restore-validates a real PostgreSQL custom archive in disposable Docker resources", + async () => { + fixture = await createOnboardingEnvironment(); + const digest = + "742f40ea20b9ff2ff31db5458d127452988a2164df9e17441e191f3b72252193"; + const image = `docker.io/library/postgres@sha256:${digest}`; + const projectName = `skillwire-real-${randomUUID().slice(0, 12)}`; + const composePath = resolve(fixture.root, "compose.yaml"); + const dockerEnvironment: NodeJS.ProcessEnv = { + PATH: "/usr/bin:/bin", + LANG: "C.UTF-8", + DOCKER_HOST: process.env["DOCKER_HOST"], + }; + await writeFile( + composePath, + [ + "services:", + " postgres:", + ` image: ${image}`, + " environment:", + " POSTGRES_HOST_AUTH_METHOD: trust", + " POSTGRES_USER: skillwire", + " POSTGRES_DB: skillwire", + " healthcheck:", + " test: [CMD-SHELL, pg_isready -U skillwire -d skillwire]", + " interval: 1s", + " timeout: 2s", + " retries: 30", + "", + ].join("\n"), + { mode: 0o600 }, + ); + const composeArgs = [ + "compose", + "--project-name", + projectName, + "--file", + composePath, + ]; + try { + await runCommand({ + executable: "/usr/bin/docker", + args: [...composeArgs, "up", "--detach", "--wait"], + environment: dockerEnvironment, + deadlineMilliseconds: 120_000, + }); + await runCommand({ + executable: "/usr/bin/docker", + args: [ + ...composeArgs, + "exec", + "-T", + "postgres", + "psql", + "--username=skillwire", + "--dbname=skillwire", + "--set=ON_ERROR_STOP=1", + "--file=-", + ], + environment: dockerEnvironment, + stdin: [ + "CREATE TABLE schema_migrations (version text PRIMARY KEY);", + "INSERT INTO schema_migrations(version) VALUES ('010');", + "CREATE TABLE accounts (id uuid PRIMARY KEY);", + "CREATE TABLE external_skill_revisions (id uuid PRIMARY KEY);", + "CREATE TABLE external_advisory_chain_head (singleton boolean PRIMARY KEY);", + ].join("\n"), + deadlineMilliseconds: 30_000, + }); + const adapter = new PostgresBackupAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath, + projectName, + installationId: randomUUID(), + protectedRoot: fixture.root, + backupsRoot: resolve(fixture.root, "backups"), + postgresImage: image, + environment: dockerEnvironment, + validateRestoredDatabase: async (containerName, validationSignal) => { + const inspected = await runCommand({ + executable: "/usr/bin/docker", + args: [ + "exec", + containerName, + "psql", + "--username=postgres", + "--dbname=postgres", + "--tuples-only", + "--no-align", + "--set=ON_ERROR_STOP=1", + "--command", + "SELECT concat((SELECT max(version) FROM schema_migrations),'|',(to_regclass('public.accounts') IS NOT NULL)::text,'|',(to_regclass('public.external_skill_revisions') IS NOT NULL)::text,'|',(to_regclass('public.external_advisory_chain_head') IS NOT NULL)::text)", + ], + environment: dockerEnvironment, + signal: validationSignal, + deadlineMilliseconds: 30_000, + }); + const [migration, accounts, catalog, advisory] = inspected.stdout + .trim() + .split("|"); + return { + latestMigration: migration ?? "", + invariantsValid: accounts === "true", + catalogValid: catalog === "true" && advisory === "true", + ready: migration === "010", + }; + }, + }); + + const backup = await adapter.createAndValidate( + AbortSignal.timeout(120_000), + ); + expect(backup.archiveSha256).toMatch(/^[0-9a-f]{64}$/); + expect(backup).toMatchObject({ + validation: { + latestMigration: "010", + invariantsValid: true, + catalogValid: true, + ready: true, + }, + }); + } finally { + await runCommand({ + executable: "/usr/bin/docker", + args: [...composeArgs, "down", "--volumes", "--remove-orphans"], + environment: dockerEnvironment, + acceptExitCodes: [0, 1], + deadlineMilliseconds: 60_000, + }).catch(() => undefined); + } + }, + 180_000, + ); +}); diff --git a/tests/integration/onboarding/concurrent-mutator.test.ts b/tests/integration/onboarding/concurrent-mutator.test.ts new file mode 100644 index 0000000..96f86bb --- /dev/null +++ b/tests/integration/onboarding/concurrent-mutator.test.ts @@ -0,0 +1,64 @@ +import { resolve } from "node:path"; +import { mkdir, writeFile } from "node:fs/promises"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + currentProcessIdentity, + InstallationLock, +} from "../../../src/onboarding/domain/operation-journal.js"; +import { + createOnboardingEnvironment, + type OnboardingEnvironment, +} from "../../helpers/onboarding-environment.js"; + +describe("single installation mutator", () => { + let fixture: OnboardingEnvironment | undefined; + afterEach(async () => fixture?.close()); + + it("allows exactly one live holder and permits a later proven successor", async () => { + fixture = await createOnboardingEnvironment(); + const root = resolve(fixture.runtimeRoot, "locks"); + const identity = await currentProcessIdentity(); + const outcomes = await Promise.allSettled([ + InstallationLock.acquire(root, "installation", identity), + InstallationLock.acquire(root, "installation", identity), + ]); + const holders = outcomes.filter( + (outcome): outcome is PromiseFulfilledResult => + outcome.status === "fulfilled", + ); + expect(holders).toHaveLength(1); + expect(outcomes.filter(({ status }) => status === "rejected")).toHaveLength( + 1, + ); + await holders[0]?.value.release(); + const successor = await InstallationLock.acquire( + root, + "installation", + identity, + ); + await successor.release(); + }); + + it("does not trust stale metadata when the kernel lock is no longer held", async () => { + fixture = await createOnboardingEnvironment(); + const root = resolve(fixture.runtimeRoot, "stale-locks"); + await mkdir(root, { recursive: true, mode: 0o700 }); + await writeFile( + resolve(root, "installation.lock"), + `${JSON.stringify({ + pid: 999_999_999, + bootId: "00000000-0000-4000-8000-000000000001", + processStart: "1", + })}\n`, + { mode: 0o600 }, + ); + const holder = await InstallationLock.acquire( + root, + "installation", + await currentProcessIdentity(), + ); + await holder.release(); + }); +}); diff --git a/tests/integration/onboarding/doctor-classification.test.ts b/tests/integration/onboarding/doctor-classification.test.ts new file mode 100644 index 0000000..d83bf85 --- /dev/null +++ b/tests/integration/onboarding/doctor-classification.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; + +import { + diagnosticProbe, + runDiagnosticProbes, + type DiagnosticCondition, +} from "../../../src/onboarding/application/diagnostic-probes.js"; +import { runDoctor } from "../../../src/onboarding/application/doctor.js"; + +describe("layered doctor classification", () => { + it("classifies every FR-061 condition with stable redacted guidance", async () => { + const fixtures: readonly [DiagnosticCondition, string][] = [ + ["service-stopped", "SERVICE_STOPPED"], + ["postgres-unavailable", "POSTGRES_UNAVAILABLE"], + ["migration-pending", "MIGRATION_PENDING"], + ["schema-incompatible", "SCHEMA_INCOMPATIBLE"], + ["schema-drifted", "SCHEMA_DRIFTED"], + ["catalog-invalid", "CATALOG_INTEGRITY_INVALID"], + ["advisory-invalid", "ADVISORY_INTEGRITY_INVALID"], + ["client-missing", "CLIENT_MISSING"], + ["client-version-unsupported", "CLIENT_VERSION_UNSUPPORTED"], + ["plugin-missing", "PLUGIN_MISSING"], + ["plugin-outdated", "PLUGIN_OUTDATED"], + ["mcp-absent", "MCP_CONFIGURATION_ABSENT"], + ["mcp-conflicting", "MCP_CONFIGURATION_CONFLICTING"], + ["mcp-duplicate", "MCP_CONFIGURATION_DUPLICATE"], + ["credential-unavailable", "CREDENTIAL_UNAVAILABLE"], + ["authentication-rejected", "AUTHENTICATION_REJECTED"], + ["endpoint-unreachable", "ENDPOINT_UNREACHABLE"], + ["tool-contract-mismatch", "TOOL_CONTRACT_MISMATCH"], + ["activation-adapter-unavailable", "ACTIVATION_ADAPTER_UNAVAILABLE"], + ["source-degraded", "SOURCE_SYNCHRONIZATION_DEGRADED"], + ["release-invalid", "RELEASE_INTEGRITY_INVALID"], + ["trust-policy-invalid", "TRUST_POLICY_INVALID"], + ["service-secret-unsafe", "SERVICE_SECRET_UNSAFE"], + ["ownership-drifted", "OWNERSHIP_DRIFTED"], + ["operation-locked", "OPERATION_LOCKED"], + ["backup-invalid", "BACKUP_INVALID"], + ["journal-recovery-required", "JOURNAL_RECOVERY_REQUIRED"], + ]; + const probes = fixtures.map(([condition]) => + diagnosticProbe(condition, { + observed: "categorical", + unsafeInput: + "swk.AAAAAAAAAAAAAAAA.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + }), + ); + + const report = await runDoctor( + await runDiagnosticProbes(probes, new AbortController().signal), + ); + + expect(report.map(({ code }) => code)).toEqual( + fixtures.map(([, code]) => code).sort(), + ); + expect(JSON.stringify(report)).not.toMatch(/swk\.|Bearer|password|pepper/i); + expect(report.every(({ nextAction }) => nextAction.length > 0)).toBe(true); + }); +}); diff --git a/tests/integration/onboarding/interruption-recovery.test.ts b/tests/integration/onboarding/interruption-recovery.test.ts new file mode 100644 index 0000000..86ae057 --- /dev/null +++ b/tests/integration/onboarding/interruption-recovery.test.ts @@ -0,0 +1,278 @@ +/* eslint-disable @typescript-eslint/require-await -- Async fakes mirror production recovery interfaces. */ +import { randomUUID } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + journalNeedsRecovery, + recoverOperation, +} from "../../../src/onboarding/application/recovery.js"; +import { createProductionLifecycleOperations } from "../../../src/onboarding/application/production-lifecycle.js"; +import { OperationJournal } from "../../../src/onboarding/domain/operation-journal.js"; +import { createOwnershipLedger } from "../../../src/onboarding/domain/ownership.js"; +import { + createOnboardingEnvironment, + type OnboardingEnvironment, +} from "../../helpers/onboarding-environment.js"; + +describe("observation-based interruption recovery", () => { + let fixture: OnboardingEnvironment | undefined; + afterEach(async () => fixture?.close()); + + it.each([ + ["intent", "safe-retry"], + ["effect", "resume"], + ["verify", "resume"], + ["compensate", "resume"], + ["commit", "complete"], + ] as const)( + "recovers interruption after %s without inventing completion", + async (boundary, expected) => { + fixture = await createOnboardingEnvironment(); + const root = resolve(fixture.root, "journals"); + const journal = await OperationJournal.create( + root, + randomUUID(), + "repair", + ); + await journal.intent("owned-plugin", { client: "codex" }); + if (boundary !== "intent") + await journal.effect("owned-plugin", { completion: "recorded" }); + if (["verify", "compensate", "commit"].includes(boundary)) + await journal.verify("owned-plugin", { identityMatches: true }); + if (boundary === "compensate") + await journal.compensate("owned-plugin", { + completion: "recorded", + }); + if (boundary === "commit") await journal.commit({ status: "success" }); + const compensate = vi.fn(async () => undefined); + + const result = await recoverOperation({ + journal, + signal: new AbortController().signal, + observe: async () => "matching", + compensate, + }); + + expect(result.disposition).toBe(expected); + expect(result.disposition === "complete" ? result.changed : false).toBe( + false, + ); + expect(compensate).not.toHaveBeenCalled(); + }, + ); + + it("blocks ambiguous effects and compensates only a proven mismatching owned effect", async () => { + fixture = await createOnboardingEnvironment(); + const root = resolve(fixture.root, "journals"); + const ambiguous = await OperationJournal.create( + root, + randomUUID(), + "uninstall", + ); + await ambiguous.intent("client-codex-plugin", { client: "codex" }); + await ambiguous.effect("client-codex-plugin", { completion: "recorded" }); + const compensate = vi.fn(async () => undefined); + await expect( + recoverOperation({ + journal: ambiguous, + signal: new AbortController().signal, + observe: async () => "ambiguous", + compensate, + }), + ).resolves.toMatchObject({ disposition: "recovery-required" }); + expect(compensate).not.toHaveBeenCalled(); + + const mismatching = await OperationJournal.create( + root, + randomUUID(), + "repair", + ); + await mismatching.intent("owned-plugin", { client: "claude" }); + await mismatching.effect("owned-plugin", { completion: "recorded" }); + await recoverOperation({ + journal: mismatching, + signal: new AbortController().signal, + observe: async () => "owned-mismatch", + compensate, + }); + expect(compensate).toHaveBeenCalledWith("owned-plugin"); + }); + + it("re-observes and clears a journaled unproven compensation boundary", async () => { + fixture = await createOnboardingEnvironment(); + const root = resolve(fixture.root, "journals"); + const journal = await OperationJournal.create( + root, + randomUUID(), + "uninstall", + ); + await journal.intent("client-codex-mcp", { client: "codex" }); + await journal.compensate("client-codex-mcp", { + completion: "unproven", + recoveryRequired: true, + }); + + await expect( + recoverOperation({ + journal, + signal: new AbortController().signal, + observe: async () => "absent", + compensate: vi.fn(), + }), + ).resolves.toMatchObject({ disposition: "safe-retry", changed: true }); + expect(journal.hasUnprovenEffect()).toBe(false); + expect(journal.entries.at(-1)).toMatchObject({ + phase: "commit", + detail: { status: "recovered" }, + }); + }); + + it("can finish a recovery-required cancellation after fresh observation", async () => { + fixture = await createOnboardingEnvironment(); + const root = resolve(fixture.root, "journals"); + const journal = await OperationJournal.create( + root, + randomUUID(), + "uninstall", + ); + await journal.intent("client-codex-mcp", { client: "codex" }); + await journal.compensate("client-codex-mcp", { + completion: "unproven", + recoveryRequired: true, + }); + await journal.cancel({ status: "recovery-required" }); + + await expect( + recoverOperation({ + journal, + signal: new AbortController().signal, + observe: async () => "absent", + compensate: vi.fn(), + }), + ).resolves.toMatchObject({ disposition: "safe-retry", changed: true }); + expect(journal.hasUnprovenEffect()).toBe(false); + expect(journal.entries.at(-1)).toMatchObject({ + phase: "commit", + detail: { status: "recovered" }, + }); + }); + + it("does not mistake a recovery-required cancellation for a terminal safe state", async () => { + fixture = await createOnboardingEnvironment(); + const root = resolve(fixture.root, "journals"); + const journal = await OperationJournal.create( + root, + randomUUID(), + "upgrade", + ); + await journal.intent("upgrade-migration", { schema: 10 }); + await journal.compensate("upgrade-migration", { + completion: "unproven", + recoveryRequired: true, + }); + await journal.cancel({ status: "recovery-required" }); + + expect(journalNeedsRecovery(journal.entries)).toBe(true); + await expect( + recoverOperation({ + journal, + signal: new AbortController().signal, + observe: async () => "ambiguous", + compensate: vi.fn(), + }), + ).resolves.toMatchObject({ disposition: "recovery-required" }); + }); + + it("returns a stable recovery result when production repair finds an ambiguous interrupted effect", async () => { + fixture = await createOnboardingEnvironment(); + const stateRoot = resolve(fixture.xdgStateHome, "skillwire"); + const installationId = randomUUID(); + await mkdir(resolve(stateRoot, "operations"), { + recursive: true, + mode: 0o700, + }); + const writeProtected = async (name: string, value: unknown) => + writeFile(resolve(stateRoot, name), `${JSON.stringify(value)}\n`, { + mode: 0o600, + }); + const timestamp = "2026-08-14T08:00:00.000Z"; + await writeProtected("installation.json", { + schemaVersion: "skillwire.installation/v1", + installationId, + ownerUid: process.getuid?.() ?? 0, + accountId: randomUUID(), + activeReleaseId: "7-amd64", + highestAcceptedReleaseSequence: 7, + activeTrustPolicySequence: 3, + endpoint: `unix://${resolve(fixture.runtimeRoot, "skillwire/mcp.sock")}`, + composeProject: fixture.composeProject, + postgresVolume: fixture.postgresVolume, + selectedClients: [], + clientIntegrationIds: { codex: null, claude: null }, + status: "complete", + createdAt: timestamp, + updatedAt: timestamp, + lastValidatedAt: timestamp, + }); + await writeProtected("deployment.json", { + schemaVersion: "skillwire.deployment/v1", + installationId, + releaseRoot: resolve(fixture.root, "release"), + composePath: resolve(fixture.root, "release/compose.yaml"), + skillwireImage: `skillwire@sha256:${"a".repeat(64)}`, + postgresImage: `postgres@sha256:${"b".repeat(64)}`, + databasePasswordFile: resolve(fixture.root, "database-password"), + applicationPepperFile: resolve(fixture.root, "application-pepper"), + runtimeSocketDirectory: resolve(fixture.runtimeRoot, "skillwire"), + socketPath: resolve(fixture.runtimeRoot, "skillwire/mcp.sock"), + projectName: fixture.composeProject, + volumeName: fixture.postgresVolume, + }); + await writeProtected( + "ownership.json", + createOwnershipLedger(installationId).record, + ); + const interrupted = await OperationJournal.create( + resolve(stateRoot, "operations"), + randomUUID(), + "upgrade", + ); + await interrupted.intent("upgrade-migration", { schema: 10 }); + await interrupted.effect("upgrade-migration", { completion: "recorded" }); + + const repair = createProductionLifecycleOperations( + fixture.environment, + ).repair; + expect(repair).toBeDefined(); + const preview = await repair?.( + { + route: "repair", + output: "json", + previewOnly: true, + stateRoot, + }, + new AbortController().signal, + ); + + await expect( + repair?.( + { + route: "repair", + output: "json", + previewOnly: false, + confirmPreview: preview?.previewHash ?? undefined, + stateRoot, + }, + new AbortController().signal, + ), + ).resolves.toMatchObject({ + status: "recovery-required", + exitClass: "rollback-required", + changed: false, + findings: [{ code: "JOURNAL_RECOVERY_REQUIRED" }], + }); + }); +}); diff --git a/tests/integration/onboarding/repair.test.ts b/tests/integration/onboarding/repair.test.ts new file mode 100644 index 0000000..c87f43c --- /dev/null +++ b/tests/integration/onboarding/repair.test.ts @@ -0,0 +1,132 @@ +/* eslint-disable @typescript-eslint/require-await -- Async fakes mirror production repair interfaces. */ +import { randomUUID } from "node:crypto"; + +import { describe, expect, it, vi } from "vitest"; + +import { + planRepair, + runRepair, +} from "../../../src/onboarding/application/repair.js"; + +describe("ownership-proven data-preserving repair", () => { + it("repairs only a matching owned missing component and never rotates secrets", async () => { + const installationId = randomUUID(); + const expectedIdentitySha256 = "a".repeat(64); + const plan = planRepair({ + installationId, + assets: [ + { + assetId: randomUUID(), + kind: "plugin", + client: "codex", + locator: "skillwire-autonomous-activation@skillwire", + expectedIdentitySha256, + observation: "missing", + }, + { + assetId: randomUUID(), + kind: "mcp-entry", + client: "claude", + locator: "external-equivalent", + expectedIdentitySha256: "b".repeat(64), + observation: "external", + }, + { + assetId: randomUUID(), + kind: "service-secret", + client: null, + locator: "secrets/database-password", + expectedIdentitySha256: "c".repeat(64), + observation: "missing", + }, + ], + }); + const repair = vi.fn(async () => undefined); + const rotate = vi.fn(async () => undefined); + + const result = await runRepair({ + plan, + confirmation: plan.previewHash, + signal: new AbortController().signal, + observe: async (asset) => ({ + observation: "missing", + identitySha256: asset.expectedIdentitySha256, + }), + repair, + rotate, + }); + + expect(result.changedAssets).toHaveLength(1); + expect(repair).toHaveBeenCalledWith( + expect.objectContaining({ kind: "plugin", client: "codex" }), + ); + expect(rotate).not.toHaveBeenCalled(); + expect(plan.blocked.map(({ code }) => code).sort()).toEqual([ + "EXTERNAL_INTEGRATION_NOT_OWNED", + "SECRET_ROTATION_REQUIRES_EXPLICIT_COMMAND", + ]); + }); + + it.each(["drifted", "ambiguous"] as const)( + "blocks %s owned state without mutation", + async (observation) => { + const plan = planRepair({ + installationId: randomUUID(), + assets: [ + { + assetId: randomUUID(), + kind: "mcp-entry", + client: "codex", + locator: "skillwire:user", + expectedIdentitySha256: "d".repeat(64), + observation, + }, + ], + }); + const repair = vi.fn(async () => undefined); + await expect( + runRepair({ + plan, + confirmation: plan.previewHash, + signal: new AbortController().signal, + observe: async () => ({ + observation, + identitySha256: "e".repeat(64), + }), + repair, + rotate: vi.fn(), + }), + ).resolves.toMatchObject({ changedAssets: [] }); + expect(repair).not.toHaveBeenCalled(); + }, + ); + + it("repairs drift only when current ownership remains independently proven", async () => { + const asset = { + assetId: randomUUID(), + kind: "container", + client: null, + locator: "skillwire-owned-service", + expectedIdentitySha256: "f".repeat(64), + observation: "drifted" as const, + ownershipProven: true, + }; + const plan = planRepair({ installationId: randomUUID(), assets: [asset] }); + const repair = vi.fn(async () => undefined); + await expect( + runRepair({ + plan, + confirmation: plan.previewHash, + signal: new AbortController().signal, + observe: async () => ({ + observation: "drifted", + identitySha256: "0".repeat(64), + ownershipProven: true, + }), + repair, + rotate: vi.fn(), + }), + ).resolves.toEqual({ changedAssets: [asset.assetId] }); + expect(repair).toHaveBeenCalledWith(asset); + }); +}); diff --git a/tests/integration/onboarding/service-secret-rotation.test.ts b/tests/integration/onboarding/service-secret-rotation.test.ts new file mode 100644 index 0000000..c2246d4 --- /dev/null +++ b/tests/integration/onboarding/service-secret-rotation.test.ts @@ -0,0 +1,190 @@ +/* eslint-disable @typescript-eslint/require-await, @typescript-eslint/no-confusing-void-expression -- Async fakes mirror production rotation interfaces. */ +import { randomUUID } from "node:crypto"; +import { lstat, readFile, readdir } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + previewServiceSecretRotation, + rotateServiceSecret, +} from "../../../src/onboarding/application/service-secret-rotation.js"; +import { createProductionLifecycleOperations } from "../../../src/onboarding/application/production-lifecycle.js"; +import { ensureServiceSecrets } from "../../../src/onboarding/secrets/service-secrets.js"; +import { snapshotTree } from "../../helpers/filesystem-snapshot.js"; +import { + createOnboardingEnvironment, + type OnboardingEnvironment, +} from "../../helpers/onboarding-environment.js"; + +describe("explicit service-secret rotation", () => { + let fixture: OnboardingEnvironment | undefined; + afterEach(async () => fixture?.close()); + + it.each(["database-password", "application-pepper"] as const)( + "rotates %s independently, retains the old file, and commits only after readiness", + async (kind) => { + fixture = await createOnboardingEnvironment(); + const installationId = randomUUID(); + const installationRoot = resolve( + fixture.stateRoot, + "installations", + installationId, + ); + await ensureServiceSecrets(installationRoot, fixture.stateRoot); + const currentPath = resolve(installationRoot, "secrets", kind); + const before = await readFile(currentPath, "utf8"); + const siblingKind = + kind === "database-password" + ? "application-pepper" + : "database-password"; + const siblingPath = resolve(installationRoot, "secrets", siblingKind); + const siblingBefore = await readFile(siblingPath, "utf8"); + const preview = previewServiceSecretRotation({ installationId, kind }); + const events: string[] = []; + + const result = await rotateServiceSecret({ + installationRoot, + stateRoot: fixture.stateRoot, + kind, + confirmation: preview.previewHash, + preview, + signal: new AbortController().signal, + apply: async (path) => { + events.push(`apply:${path}`); + }, + readiness: async () => { + events.push("ready"); + }, + publish: async () => { + events.push("publish"); + }, + }); + + const after = await readFile(currentPath, "utf8"); + expect(after).not.toBe(before); + expect(await readFile(result.retainedPath, "utf8")).toBe(before); + expect(await readFile(siblingPath, "utf8")).toBe(siblingBefore); + expect((await lstat(currentPath)).mode & 0o777).toBe(0o600); + expect(events.at(-1)).toBe("publish"); + expect(JSON.stringify(result)).not.toContain(before); + expect(JSON.stringify(result)).not.toContain(after); + }, + ); + + it("blocks application-pepper rotation before mutation when the runtime has no safe overlap support", async () => { + fixture = await createOnboardingEnvironment(); + const operations = createProductionLifecycleOperations(fixture.environment); + const before = await snapshotTree(fixture.root); + + await expect( + operations["maintenance:rotate-service-secret"]?.( + { + route: "maintenance:rotate-service-secret", + output: "json", + previewOnly: true, + serviceSecret: "application-pepper", + }, + new AbortController().signal, + ), + ).rejects.toThrow(/overlap|existing client keys|unsupported/i); + expect(await snapshotTree(fixture.root)).toEqual(before); + }); + + it("rolls application configuration back at a failed readiness boundary", async () => { + fixture = await createOnboardingEnvironment(); + const installationId = randomUUID(); + const installationRoot = resolve( + fixture.stateRoot, + "installations", + installationId, + ); + await ensureServiceSecrets(installationRoot, fixture.stateRoot); + const currentPath = resolve(installationRoot, "secrets/database-password"); + const before = await readFile(currentPath, "utf8"); + const preview = previewServiceSecretRotation({ + installationId, + kind: "database-password", + }); + const rollback = vi.fn(async () => undefined); + + await expect( + rotateServiceSecret({ + installationRoot, + stateRoot: fixture.stateRoot, + kind: "database-password", + confirmation: preview.previewHash, + preview, + signal: new AbortController().signal, + apply: async () => undefined, + readiness: async () => { + throw new Error("not ready"); + }, + publish: async () => undefined, + rollback, + }), + ).rejects.toThrow(/ready|readiness/i); + expect(await readFile(currentPath, "utf8")).toBe(before); + expect(rollback).toHaveBeenCalledWith(currentPath); + expect( + (await readdir(resolve(installationRoot, "secrets"))).some((name) => + name.includes("candidate"), + ), + ).toBe(false); + }); + + it.each([ + "apply-candidate", + "candidate-readiness", + "apply-current", + "current-readiness", + "publish", + ] as const)("restores the old value after a %s failure", async (boundary) => { + fixture = await createOnboardingEnvironment(); + const installationId = randomUUID(); + const installationRoot = resolve( + fixture.stateRoot, + "installations", + installationId, + ); + await ensureServiceSecrets(installationRoot, fixture.stateRoot); + const currentPath = resolve(installationRoot, "secrets/application-pepper"); + const before = await readFile(currentPath, "utf8"); + const preview = previewServiceSecretRotation({ + installationId, + kind: "application-pepper", + }); + let applyCount = 0; + let readinessCount = 0; + const fail = (name: typeof boundary): void => { + if (boundary === name) throw new Error(`${name} failed`); + }; + await expect( + rotateServiceSecret({ + installationRoot, + stateRoot: fixture.stateRoot, + kind: "application-pepper", + confirmation: preview.previewHash, + preview, + signal: new AbortController().signal, + apply: async () => { + applyCount += 1; + fail(applyCount === 1 ? "apply-candidate" : "apply-current"); + }, + readiness: async () => { + readinessCount += 1; + fail( + readinessCount === 1 ? "candidate-readiness" : "current-readiness", + ); + }, + publish: async () => fail("publish"), + }), + ).rejects.toThrow(/rotation|readiness|failed/i); + expect(await readFile(currentPath, "utf8")).toBe(before); + expect( + (await readdir(resolve(installationRoot, "secrets"))).some((name) => + /candidate|retained/.test(name), + ), + ).toBe(false); + }); +}); diff --git a/tests/integration/onboarding/service-setup.test.ts b/tests/integration/onboarding/service-setup.test.ts index 8490678..30c9b33 100644 --- a/tests/integration/onboarding/service-setup.test.ts +++ b/tests/integration/onboarding/service-setup.test.ts @@ -257,6 +257,39 @@ describe("service-only deployment boundary", () => { ); expect(run).not.toHaveBeenCalled(); }); + + it("observes one owned Compose service without inserting a second compose subcommand", async () => { + const calls: CommandOptions[] = []; + const run = vi.fn((options: CommandOptions) => { + calls.push(options); + return Promise.resolve( + options.args.includes("ps") + ? result("container-id\n") + : result( + `skillwire-test-0123456789abcdef|skillwire|localhost:5000/skillwire@sha256:${"1".repeat(64)}\n`, + ), + ); + }); + + await expect( + deployment(run).observeOwnedService( + "skillwire", + new AbortController().signal, + ), + ).resolves.toBe(true); + expect(calls[0]?.args).toEqual([ + "compose", + "--project-name", + "skillwire-test-0123456789abcdef", + "--file", + "/tmp/disposable/compose.yaml", + "ps", + "--all", + "--quiet", + "skillwire", + ]); + expect(calls[0]?.args.filter((arg) => arg === "compose")).toHaveLength(1); + }); }); function result(stdout: string): CommandResult { diff --git a/tests/integration/onboarding/upgrade-compatible.test.ts b/tests/integration/onboarding/upgrade-compatible.test.ts new file mode 100644 index 0000000..c84a772 --- /dev/null +++ b/tests/integration/onboarding/upgrade-compatible.test.ts @@ -0,0 +1,72 @@ +/* eslint-disable @typescript-eslint/require-await -- Async fakes mirror production upgrade interfaces. */ +import { randomUUID } from "node:crypto"; + +import { describe, expect, it, vi } from "vitest"; + +import { + previewUpgrade, + runUpgrade, +} from "../../../src/onboarding/application/upgrade.js"; +import type { UpgradeRecoveryError } from "../../../src/onboarding/application/upgrade.js"; + +describe("same-schema signed upgrade", () => { + it("automatically rolls application/config back when readiness fails", async () => { + const target = { + releaseId: "11-amd64", + releaseSequence: 11, + trustPolicySequence: 5, + schemaMinimum: 10, + schemaMaximum: 10, + latestMigration: 10, + manifestSha256: "a".repeat(64), + imageDigest: `sha256:${"b".repeat(64)}`, + }; + const preview = previewUpgrade({ + installationId: randomUUID(), + currentReleaseSequence: 10, + currentTrustPolicySequence: 5, + liveSchema: 10, + target, + }); + const rollbackApplication = vi.fn(async () => undefined); + const migrate = vi.fn(async () => undefined); + const drain = vi.fn(async () => undefined); + const events: string[] = []; + + await expect( + runUpgrade({ + preview, + confirmation: preview.previewHash, + signal: new AbortController().signal, + verifyTarget: async () => { + events.push("verified"); + return target; + }, + createBackup: async () => { + events.push("backup"); + return { backupId: randomUUID(), validated: true }; + }, + drainWriters: drain, + installApplication: async () => { + events.push("installed"); + }, + migrate, + verifyLiveSchema: async () => 10, + readiness: async () => { + throw new Error("not ready"); + }, + verifyClients: async () => undefined, + commitSelection: async () => undefined, + rollbackApplication, + stopWriters: async () => undefined, + restartWriters: async () => undefined, + }), + ).rejects.toMatchObject({ + rollbackBoundary: "application-config", + } satisfies Partial); + expect(events).toEqual(["verified", "backup", "installed"]); + expect(rollbackApplication).toHaveBeenCalledTimes(1); + expect(migrate).not.toHaveBeenCalled(); + expect(drain).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/integration/onboarding/upgrade-forward-only-010.test.ts b/tests/integration/onboarding/upgrade-forward-only-010.test.ts new file mode 100644 index 0000000..8bd680f --- /dev/null +++ b/tests/integration/onboarding/upgrade-forward-only-010.test.ts @@ -0,0 +1,84 @@ +/* eslint-disable @typescript-eslint/require-await -- Async fakes mirror production upgrade interfaces. */ +import { randomUUID } from "node:crypto"; + +import { describe, expect, it, vi } from "vitest"; + +import { + previewUpgrade, + runUpgrade, +} from "../../../src/onboarding/application/upgrade.js"; + +describe("forward-only migration 010 upgrade", () => { + it("restore-validates first, drains writers, reads schema 010, and refuses image-only rollback", async () => { + const target = { + releaseId: "10-amd64", + releaseSequence: 10, + trustPolicySequence: 4, + schemaMinimum: 9, + schemaMaximum: 10, + latestMigration: 10, + manifestSha256: "1".repeat(64), + imageDigest: `sha256:${"2".repeat(64)}`, + }; + const preview = previewUpgrade({ + installationId: randomUUID(), + currentReleaseSequence: 9, + currentTrustPolicySequence: 4, + liveSchema: 9, + target, + }); + const backupId = randomUUID(); + const events: string[] = []; + const rollbackApplication = vi.fn(async () => undefined); + const stopWriters = vi.fn(async () => { + events.push("writers-stopped"); + }); + + await expect( + runUpgrade({ + preview, + confirmation: preview.previewHash, + signal: new AbortController().signal, + verifyTarget: async () => target, + createBackup: async () => { + events.push("backup-validated"); + return { backupId, validated: true }; + }, + drainWriters: async () => { + events.push("writers-drained"); + }, + installApplication: async () => { + events.push("application-installed"); + }, + migrate: async () => { + events.push("migration-010"); + }, + verifyLiveSchema: async () => { + events.push("schema-readback-010"); + return 10; + }, + readiness: async () => { + throw new Error("service failed after migration"); + }, + verifyClients: async () => undefined, + commitSelection: async () => undefined, + rollbackApplication, + stopWriters, + restartWriters: async () => undefined, + }), + ).rejects.toMatchObject({ + rollbackBoundary: "database-restore-required", + backupId, + }); + expect(events).toEqual([ + "backup-validated", + "writers-drained", + "application-installed", + "migration-010", + "schema-readback-010", + "writers-stopped", + ]); + expect(stopWriters).toHaveBeenCalledTimes(1); + expect(rollbackApplication).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/integration/onboarding/upgrade-interruption.test.ts b/tests/integration/onboarding/upgrade-interruption.test.ts new file mode 100644 index 0000000..80d27a5 --- /dev/null +++ b/tests/integration/onboarding/upgrade-interruption.test.ts @@ -0,0 +1,176 @@ +/* eslint-disable @typescript-eslint/require-await, @typescript-eslint/no-confusing-void-expression -- Async fakes mirror production upgrade interfaces. */ +import { randomUUID } from "node:crypto"; +import { readFile, mkdir } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +import { + upgradeFailureRequiresRecovery, + previewUpgrade, + runUpgrade, +} from "../../../src/onboarding/application/upgrade.js"; +import { commitActiveReleaseSelection } from "../../../src/onboarding/adapters/filesystem/release-installer.js"; +import { OperationJournal } from "../../../src/onboarding/domain/operation-journal.js"; +import { createOnboardingEnvironment } from "../../helpers/onboarding-environment.js"; + +describe("upgrade interruption boundaries", () => { + it("keeps a failed final publication journal recoverable after an unproven effect", () => { + expect(upgradeFailureRequiresRecovery(false, true)).toBe(true); + expect(upgradeFailureRequiresRecovery(false, false)).toBe(false); + expect(upgradeFailureRequiresRecovery(true, false)).toBe(true); + }); + + it.each([ + "release-verification", + "backup", + "drain", + "migration", + "readiness", + "clients", + ] as const)("stops safely after %s cancellation", async (boundary) => { + const target = { + releaseId: "10-amd64", + releaseSequence: 10, + trustPolicySequence: 3, + schemaMinimum: 9, + schemaMaximum: 10, + latestMigration: 10, + manifestSha256: "3".repeat(64), + imageDigest: `sha256:${"4".repeat(64)}`, + }; + const preview = previewUpgrade({ + installationId: randomUUID(), + currentReleaseSequence: 9, + currentTrustPolicySequence: 3, + liveSchema: 9, + target, + }); + const controller = new AbortController(); + const reached: string[] = []; + const at = async (name: string, value: T): Promise => { + reached.push(name); + if (boundary === name) controller.abort(); + return value; + }; + + await expect( + runUpgrade({ + preview, + confirmation: preview.previewHash, + signal: controller.signal, + verifyTarget: () => at("release-verification", target), + createBackup: () => + at("backup", { backupId: randomUUID(), validated: true }), + drainWriters: () => at("drain", undefined), + installApplication: () => at("application", undefined), + migrate: () => at("migration", undefined), + verifyLiveSchema: async () => 10, + readiness: () => at("readiness", undefined), + verifyClients: () => at("clients", undefined), + commitSelection: () => at("release-commit", undefined), + rollbackApplication: vi.fn(async () => undefined), + stopWriters: vi.fn(async () => undefined), + restartWriters: vi.fn(async () => undefined), + }), + ).rejects.toThrow(/cancel|upgrade|recovery|boundary/i); + expect(reached).toContain(boundary); + const order = [ + "release-verification", + "backup", + "drain", + "application", + "migration", + "readiness", + "clients", + "release-commit", + ]; + const boundaryIndex = order.indexOf(boundary); + expect(reached.some((name) => order.indexOf(name) > boundaryIndex)).toBe( + false, + ); + }); + + it("treats a completed atomic release commit as the terminal success boundary", async () => { + const target = { + releaseId: "10-amd64", + releaseSequence: 10, + trustPolicySequence: 3, + schemaMinimum: 10, + schemaMaximum: 10, + latestMigration: 10, + manifestSha256: "3".repeat(64), + imageDigest: `sha256:${"4".repeat(64)}`, + }; + const preview = previewUpgrade({ + installationId: randomUUID(), + currentReleaseSequence: 9, + currentTrustPolicySequence: 3, + liveSchema: 10, + target, + }); + const controller = new AbortController(); + await expect( + runUpgrade({ + preview, + confirmation: preview.previewHash, + signal: controller.signal, + verifyTarget: async () => target, + createBackup: async () => ({ + backupId: randomUUID(), + validated: true, + }), + drainWriters: async () => undefined, + installApplication: async () => undefined, + migrate: async () => undefined, + verifyLiveSchema: async () => 10, + readiness: async () => undefined, + verifyClients: async () => undefined, + commitSelection: async () => controller.abort(), + rollbackApplication: vi.fn(), + stopWriters: vi.fn(), + restartWriters: vi.fn(), + }), + ).resolves.toMatchObject({ releaseId: "10-amd64" }); + }); + + it("publishes active release/trust selection as one journaled atomic effect", async () => { + const fixture = await createOnboardingEnvironment(); + try { + const stateRoot = resolve(fixture.root, "state"); + await mkdir(stateRoot, { mode: 0o700 }); + const journal = await OperationJournal.create( + resolve(stateRoot, "operations"), + randomUUID(), + "upgrade", + ); + await commitActiveReleaseSelection({ + stateRoot, + journal, + signal: new AbortController().signal, + selection: { + schemaVersion: "skillwire.active-release/v1", + releaseVersion: "1.2.0", + releaseSequence: 12, + trustPolicySequence: 6, + architecture: "amd64", + manifestSha256: "7".repeat(64), + archiveSha256: "8".repeat(64), + trustPolicyPath: "trust/skillwire-trust-policy-v6.json", + }, + }); + expect(journal.entries.map(({ phase }) => phase)).toEqual([ + "intent", + "effect", + "verify", + ]); + expect( + JSON.parse( + await readFile(resolve(stateRoot, "active-release.json"), "utf8"), + ), + ).toMatchObject({ releaseSequence: 12, trustPolicySequence: 6 }); + } finally { + await fixture.close(); + } + }); +}); diff --git a/tests/security/onboarding/docker-environment.test.ts b/tests/security/onboarding/docker-environment.test.ts new file mode 100644 index 0000000..33ef9ae --- /dev/null +++ b/tests/security/onboarding/docker-environment.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; + +import { dockerProcessEnvironment } from "../../../src/onboarding/adapters/docker/environment.js"; + +describe("Docker subprocess environment isolation", () => { + it("keeps only runtime routing and explicit non-secret Compose values", () => { + const environment = dockerProcessEnvironment( + { + HOME: "/tmp/disposable-home", + XDG_RUNTIME_DIR: "/tmp/disposable-runtime", + DOCKER_HOST: "unix:///tmp/disposable-runtime/docker.sock", + DOCKER_CONTEXT: "rootless", + LANG: "it_IT.UTF-8", + GH_TOKEN: "ambient-github-canary", + OPENAI_API_KEY: "ambient-openai-canary", + DATABASE_URL: "postgres://ambient-secret", + }, + { + SKILLWIRE_COMPOSE_PROJECT: "skillwire-test", + SKILLWIRE_DATABASE_PASSWORD_SECRET_FILE: + "/tmp/disposable/secrets/database-password", + }, + ); + + expect(environment).toMatchObject({ + PATH: "/usr/bin:/bin", + LANG: "C.UTF-8", + HOME: "/tmp/disposable-home", + XDG_RUNTIME_DIR: "/tmp/disposable-runtime", + DOCKER_HOST: "unix:///tmp/disposable-runtime/docker.sock", + DOCKER_CONTEXT: "rootless", + SKILLWIRE_COMPOSE_PROJECT: "skillwire-test", + SKILLWIRE_DATABASE_PASSWORD_SECRET_FILE: + "/tmp/disposable/secrets/database-password", + }); + expect(environment).not.toHaveProperty("GH_TOKEN"); + expect(environment).not.toHaveProperty("OPENAI_API_KEY"); + expect(environment).not.toHaveProperty("DATABASE_URL"); + }); +}); diff --git a/tests/security/onboarding/key-rotation.test.ts b/tests/security/onboarding/key-rotation.test.ts new file mode 100644 index 0000000..578742a --- /dev/null +++ b/tests/security/onboarding/key-rotation.test.ts @@ -0,0 +1,124 @@ +/* eslint-disable @typescript-eslint/require-await, @typescript-eslint/restrict-template-expressions -- Async fakes mirror production credential interfaces. */ +import { describe, expect, it, vi } from "vitest"; + +import { createApiKeyToken } from "../../../src/authentication/api-key-token.js"; +import { ClientCredentialService } from "../../../src/onboarding/application/client-credentials.js"; + +describe("independent client credential rotation", () => { + it("verifies the replacement before revoking the old key", async () => { + const replacement = createApiKeyToken().token; + const events: string[] = []; + const service = new ClientCredentialService( + { + create: vi.fn(async (client) => { + events.push(`create:${client}`); + return { keyId: "new-codex", token: replacement }; + }), + revoke: vi.fn(async (keyId) => { + events.push(`revoke:${keyId}`); + }), + }, + { + store: vi.fn(async (client) => { + events.push(`store:${client}`); + return "secret-service:new-codex"; + }), + lookup: vi.fn(async (client) => { + events.push(`lookup:${client}`); + return replacement; + }), + remove: vi.fn(async () => undefined), + }, + ); + + await expect( + service.rotate({ + client: "codex", + keyId: "old-codex", + reference: "secret-service:old-codex", + }), + ).resolves.toEqual({ + client: "codex", + keyId: "new-codex", + reference: "secret-service:new-codex", + }); + expect(events).toEqual([ + "create:codex", + "store:codex", + "lookup:codex", + "revoke:old-codex", + ]); + }); + + it("retains the old key and removes only an unverifiable replacement", async () => { + const replacement = createApiKeyToken().token; + const revoke = vi.fn(async () => undefined); + const remove = vi.fn(async () => undefined); + const service = new ClientCredentialService( + { + create: vi.fn().mockResolvedValue({ + keyId: "new-claude", + token: replacement, + }), + revoke, + }, + { + store: vi.fn().mockResolvedValue("secret-service:new-claude"), + lookup: vi.fn().mockResolvedValue(createApiKeyToken().token), + remove, + }, + ); + + await expect( + service.rotate({ + client: "claude", + keyId: "old-claude", + reference: "secret-service:old-claude", + }), + ).rejects.toThrow(/verify/i); + expect(remove).toHaveBeenCalledWith("claude", "secret-service:new-claude"); + expect(revoke).toHaveBeenCalledTimes(1); + expect(revoke).toHaveBeenCalledWith("new-claude"); + expect(revoke).not.toHaveBeenCalledWith("old-claude"); + }); + + it("rotates one client without reading, replacing, or revoking its sibling", async () => { + const replacement = createApiKeyToken().token; + const sibling = { + keyId: "claude-stable", + reference: "secret-service:claude-stable", + }; + const revoke = vi.fn(async () => undefined); + const store = vi.fn(async () => "secret-service:codex-replacement"); + const lookup = vi.fn(async () => replacement); + const service = new ClientCredentialService( + { + create: vi.fn(async (client) => ({ + keyId: `${client}-replacement`, + token: replacement, + })), + revoke, + }, + { store, lookup, remove: vi.fn(async () => undefined) }, + ); + + await service.rotate({ + client: "codex", + keyId: "codex-old", + reference: "secret-service:codex-old", + }); + + expect(store).toHaveBeenCalledWith("codex", replacement); + expect(lookup).toHaveBeenCalledWith( + "codex", + "secret-service:codex-replacement", + ); + expect(revoke).toHaveBeenCalledWith("codex-old"); + expect( + JSON.stringify([store.mock.calls, lookup.mock.calls, revoke.mock.calls]), + ).not.toContain(sibling.keyId); + expect( + JSON.stringify([store.mock.calls, lookup.mock.calls, revoke.mock.calls]), + ).not.toContain(sibling.reference); + }); +}); diff --git a/tests/security/onboarding/removal-boundaries.test.ts b/tests/security/onboarding/removal-boundaries.test.ts new file mode 100644 index 0000000..aed67d7 --- /dev/null +++ b/tests/security/onboarding/removal-boundaries.test.ts @@ -0,0 +1,245 @@ +/* eslint-disable @typescript-eslint/require-await -- Async fakes mirror production removal interfaces. */ +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, symlink, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + previewPurge, + removeOwnedFilesystemTree, + runPurge, + validateOwnedFilesystemTree, +} from "../../../src/onboarding/application/purge.js"; +import { + createOwnershipLedger, + reactivateOwnedAsset, + recordAssetDisposition, + recordOwnedAsset, +} from "../../../src/onboarding/domain/ownership.js"; +import { + createOnboardingEnvironment, + type OnboardingEnvironment, +} from "../../helpers/onboarding-environment.js"; + +describe("removal ownership and filesystem boundaries", () => { + let fixture: OnboardingEnvironment | undefined; + afterEach(async () => fixture?.close()); + + it("orders owned child assets before their containing path during purge", () => { + let ledger = createOwnershipLedger(randomUUID()); + ledger = recordOwnedAsset(ledger, { + kind: "path", + client: null, + locator: "/tmp/disposable/installations/example", + expectedIdentitySha256: "1".repeat(64), + createdByOperation: randomUUID(), + retention: "remove-only-on-purge", + disposition: "present", + }); + ledger = recordOwnedAsset(ledger, { + kind: "service-secret", + client: null, + locator: "example/secrets/database-password", + expectedIdentitySha256: "2".repeat(64), + createdByOperation: randomUUID(), + retention: "retain-by-default", + disposition: "present", + }); + + expect( + previewPurge(ledger.record).unrecoverable.map(({ kind }) => kind), + ).toEqual(["service-secret", "path"]); + }); + + it("refuses an unknown regular file inside an owned purge directory", async () => { + fixture = await createOnboardingEnvironment(); + const owned = resolve(fixture.root, "owned"); + await mkdir(owned, { mode: 0o700 }); + const expected = resolve(owned, "expected.json"); + await writeFile(expected, "{}\n", { mode: 0o600 }); + await writeFile(resolve(owned, "unrelated.txt"), "do not remove", { + mode: 0o600, + }); + + await expect( + validateOwnedFilesystemTree(owned, fixture.root, [expected]), + ).rejects.toThrow(/unknown|owned/i); + }); + + it.each(["drifted", "ambiguous"] as const)( + "does not plan a %s owned asset", + (disposition) => { + let ledger = createOwnershipLedger(randomUUID()); + ledger = recordOwnedAsset(ledger, { + kind: "path", + client: null, + locator: "owned/path", + expectedIdentitySha256: "1".repeat(64), + createdByOperation: randomUUID(), + retention: "remove-only-on-purge", + disposition: "present", + }); + const assetId = ledger.record.assets[0]?.assetId; + if (assetId === undefined) throw new Error("missing fixture asset"); + const changed = recordAssetDisposition( + ledger.record, + assetId, + disposition, + ); + expect(previewPurge(changed).unrecoverable).toEqual([]); + }, + ); + + it("revalidates identity and ownership revision before any removal", async () => { + let ledger = createOwnershipLedger(randomUUID()); + ledger = recordOwnedAsset(ledger, { + kind: "volume", + client: null, + locator: "skillwire-owned-volume", + expectedIdentitySha256: "2".repeat(64), + createdByOperation: randomUUID(), + retention: "retain-by-default", + disposition: "retained", + }); + const preview = previewPurge(ledger.record); + const removeAsset = vi.fn(async () => undefined); + await expect( + runPurge({ + ownership: ledger.record, + preview, + confirmation: preview.previewHash, + signal: new AbortController().signal, + observeIdentity: async () => "3".repeat(64), + removeAsset, + }), + ).rejects.toThrow(/identity/i); + expect(removeAsset).not.toHaveBeenCalled(); + + const concurrent = recordAssetDisposition( + ledger.record, + ledger.record.assets[0]?.assetId ?? "", + "retained", + ); + await expect( + runPurge({ + ownership: concurrent, + preview, + confirmation: preview.previewHash, + signal: new AbortController().signal, + observeIdentity: async (asset) => asset.expectedIdentitySha256, + removeAsset, + }), + ).rejects.toThrow(/changed/i); + expect(removeAsset).not.toHaveBeenCalled(); + }); + + it("revalidates each purge asset immediately before deleting it", async () => { + let ledger = createOwnershipLedger(randomUUID()); + for (const locator of ["skillwire-owned-one", "skillwire-owned-two"]) { + ledger = recordOwnedAsset(ledger, { + kind: "volume", + client: null, + locator, + expectedIdentitySha256: "7".repeat(64), + createdByOperation: randomUUID(), + retention: "retain-by-default", + disposition: "retained", + }); + } + const preview = previewPurge(ledger.record); + let observations = 0; + const removeAsset = vi.fn(async () => undefined); + + await expect( + runPurge({ + ownership: ledger.record, + preview, + confirmation: preview.previewHash, + signal: new AbortController().signal, + observeIdentity: async (asset) => { + observations += 1; + return observations <= preview.unrecoverable.length || + asset.locator === "skillwire-owned-one" + ? asset.expectedIdentitySha256 + : "8".repeat(64); + }, + removeAsset, + }), + ).rejects.toThrow(/identity/i); + expect(removeAsset).toHaveBeenCalledTimes(1); + }); + + it("rejects symlink traversal and leaves external targets unchanged", async () => { + fixture = await createOnboardingEnvironment(); + const protectedRoot = resolve(fixture.root, "purge-root"); + const external = resolve(fixture.root, "external/keep.txt"); + await mkdir(protectedRoot, { mode: 0o700 }); + await mkdir(resolve(external, ".."), { recursive: true, mode: 0o700 }); + await writeFile(external, "keep", { mode: 0o600 }); + const link = resolve(protectedRoot, "owned-looking-link"); + await symlink(external, link); + await expect( + removeOwnedFilesystemTree(link, protectedRoot), + ).rejects.toThrow(/unsafe|link/i); + await expect(readFile(external, "utf8")).resolves.toBe("keep"); + }); + + it("stops an interrupted purge before the first asset", async () => { + let ledger = createOwnershipLedger(randomUUID()); + ledger = recordOwnedAsset(ledger, { + kind: "backup", + client: null, + locator: "backups/one.dump", + expectedIdentitySha256: "4".repeat(64), + createdByOperation: randomUUID(), + retention: "retain-by-default", + disposition: "retained", + }); + const preview = previewPurge(ledger.record); + const controller = new AbortController(); + controller.abort(); + const removeAsset = vi.fn(async () => undefined); + await expect( + runPurge({ + ownership: ledger.record, + preview, + confirmation: preview.previewHash, + signal: controller.signal, + observeIdentity: async (asset) => asset.expectedIdentitySha256, + removeAsset, + }), + ).rejects.toThrow(/cancel/i); + expect(removeAsset).not.toHaveBeenCalled(); + }); + + it("reactivates only an exact retained or removed owned asset", () => { + let ledger = createOwnershipLedger(randomUUID()); + ledger = recordOwnedAsset(ledger, { + kind: "credential", + client: "codex", + locator: "restrictive-file:codex:fixture", + expectedIdentitySha256: "5".repeat(64), + createdByOperation: randomUUID(), + retention: "retain-by-default", + disposition: "present", + }); + const asset = ledger.record.assets[0]; + if (asset === undefined) throw new Error("missing fixture asset"); + const retained = recordAssetDisposition( + ledger.record, + asset.assetId, + "retained", + ); + expect( + reactivateOwnedAsset( + retained, + asset.assetId, + asset.expectedIdentitySha256, + ).assets[0]?.disposition, + ).toBe("present"); + expect(() => + reactivateOwnedAsset(retained, asset.assetId, "6".repeat(64)), + ).toThrow(/identity/i); + }); +}); diff --git a/tests/security/onboarding/upgrade-trust-downgrade.test.ts b/tests/security/onboarding/upgrade-trust-downgrade.test.ts new file mode 100644 index 0000000..5d64e4c --- /dev/null +++ b/tests/security/onboarding/upgrade-trust-downgrade.test.ts @@ -0,0 +1,98 @@ +/* eslint-disable @typescript-eslint/require-await -- Async fakes mirror production upgrade interfaces. */ +import { randomUUID } from "node:crypto"; + +import { describe, expect, it, vi } from "vitest"; + +import { + previewUpgrade, + runUpgrade, + type UpgradeTarget, +} from "../../../src/onboarding/application/upgrade.js"; + +function target(overrides: Partial = {}): UpgradeTarget { + return { + releaseId: "12-amd64", + releaseSequence: 12, + trustPolicySequence: 6, + schemaMinimum: 10, + schemaMaximum: 10, + latestMigration: 10, + manifestSha256: "5".repeat(64), + imageDigest: `sha256:${"6".repeat(64)}`, + ...overrides, + }; +} + +describe("upgrade trust and downgrade boundary", () => { + it.each([ + ["lower release", target({ releaseSequence: 9 })], + ["lower policy", target({ trustPolicySequence: 4 })], + ["unpinned image", target({ imageDigest: "postgres:17" })], + ])("rejects %s before backup or mutation", async (_name, candidate) => { + const preview = previewUpgrade({ + installationId: randomUUID(), + currentReleaseSequence: 10, + currentTrustPolicySequence: 5, + liveSchema: 10, + target: candidate, + }); + const createBackup = vi.fn(); + await expect( + runUpgrade({ + preview, + confirmation: preview.previewHash, + signal: new AbortController().signal, + verifyTarget: async () => candidate, + createBackup, + drainWriters: vi.fn(), + installApplication: vi.fn(), + migrate: vi.fn(), + verifyLiveSchema: vi.fn(), + readiness: vi.fn(), + verifyClients: vi.fn(), + commitSelection: vi.fn(), + rollbackApplication: vi.fn(), + stopWriters: vi.fn(), + restartWriters: vi.fn(), + }), + ).rejects.toThrow(/downgrade|digest|pinned|sequence/i); + expect(createBackup).not.toHaveBeenCalled(); + }); + + it.each(["stale policy", "bad overlap", "denied signer/material"])( + "propagates signed-release verifier rejection for %s without effects", + async (reason) => { + const candidate = target(); + const preview = previewUpgrade({ + installationId: randomUUID(), + currentReleaseSequence: 10, + currentTrustPolicySequence: 5, + liveSchema: 10, + target: candidate, + }); + const createBackup = vi.fn(); + await expect( + runUpgrade({ + preview, + confirmation: preview.previewHash, + signal: new AbortController().signal, + verifyTarget: async () => { + throw new Error(reason); + }, + createBackup, + drainWriters: vi.fn(), + installApplication: vi.fn(), + migrate: vi.fn(), + verifyLiveSchema: vi.fn(), + readiness: vi.fn(), + verifyClients: vi.fn(), + commitSelection: vi.fn(), + rollbackApplication: vi.fn(), + stopWriters: vi.fn(), + restartWriters: vi.fn(), + }), + ).rejects.toThrow(reason); + expect(createBackup).not.toHaveBeenCalled(); + }, + ); +}); diff --git a/tests/unit/onboarding/release-installer.test.ts b/tests/unit/onboarding/release-installer.test.ts index d448c8a..2dea788 100644 --- a/tests/unit/onboarding/release-installer.test.ts +++ b/tests/unit/onboarding/release-installer.test.ts @@ -10,7 +10,10 @@ import { dirname, resolve } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { buildSelfHostedRelease } from "../../../scripts/build-self-hosted-release.js"; -import { installVerifiedRelease } from "../../../src/onboarding/adapters/filesystem/release-installer.js"; +import { + installVerifiedRelease, + releaseDirectoryIdentity, +} from "../../../src/onboarding/adapters/filesystem/release-installer.js"; import { createOnboardingEnvironment, type OnboardingEnvironment, @@ -122,6 +125,13 @@ describe("immutable self-hosted release installer", () => { trustPolicyPath, }); expect(repeated.changed).toBe(false); + const installedIdentity = await releaseDirectoryIdentity( + result.releaseRoot, + ); + await appendFile(resolve(result.releaseRoot, "app/skillwire.mjs"), "drift"); + await expect( + releaseDirectoryIdentity(result.releaseRoot), + ).resolves.not.toBe(installedIdentity); }); }); import { createHash } from "node:crypto"; From 0d5ceb8f5db124c955cd431c6fda20a6df1915c0 Mon Sep 17 00:00:00 2001 From: Lucenx9 Date: Fri, 14 Aug 2026 16:53:21 +0200 Subject: [PATCH 2/7] fix: recover lifecycle operations at owned boundaries --- src/onboarding/application/purge.ts | 178 ++++++++++++++- src/onboarding/application/recovery.ts | 208 +++++++++--------- src/onboarding/application/uninstall.ts | 71 +++++- src/onboarding/domain/operation-journal.ts | 7 +- .../default-uninstall.test.ts | 77 ++++++- .../onboarding/interruption-recovery.test.ts | 143 +++++++++++- .../onboarding/removal-boundaries.test.ts | 72 +++++- 7 files changed, 624 insertions(+), 132 deletions(-) diff --git a/src/onboarding/application/purge.ts b/src/onboarding/application/purge.ts index a5b545b..47725c0 100644 --- a/src/onboarding/application/purge.ts +++ b/src/onboarding/application/purge.ts @@ -1,6 +1,15 @@ import { constants } from "node:fs"; -import { lstat, open, readdir, rm } from "node:fs/promises"; -import { isAbsolute, relative, resolve } from "node:path"; +import { randomBytes } from "node:crypto"; +import { + lstat, + mkdir, + open, + readdir, + rename, + rm, + rmdir, +} from "node:fs/promises"; +import { basename, dirname, isAbsolute, relative, resolve } from "node:path"; import { canonicalPreview, confirmPreview } from "../cli/confirmation.js"; import { @@ -123,7 +132,8 @@ async function validateTree( if ( stats.isSymbolicLink() || (!stats.isDirectory() && stats.nlink !== 1) || - stats.uid !== process.getuid?.() + stats.uid !== process.getuid?.() || + (stats.isDirectory() && (stats.mode & 0o022) !== 0) ) throw new Error("Purge target has an unsafe filesystem identity"); if (stats.isDirectory()) { @@ -158,6 +168,164 @@ export async function removeOwnedFilesystemTree( protectedRoot: string, allowedFiles?: readonly string[], ): Promise { - await validateOwnedFilesystemTree(path, protectedRoot, allowedFiles); - await rm(path, { recursive: true }); + const target = resolve(path); + const root = resolve(protectedRoot); + const parent = dirname(target); + await validateProtectedDirectoryChain(parent, root); + await validateOwnedFilesystemTree(target, root, allowedFiles); + const validated = await lstat(target, { bigint: true }); + const parentIdentity = await lstat(parent, { bigint: true }); + if (validated.dev !== parentIdentity.dev) + throw new Error("Purge target cannot be quarantined on another filesystem"); + + const quarantine = await createProtectedQuarantine(parent); + const staged = resolve(quarantine, basename(target)); + let renamed = false; + let stagedIdentityVerified = false; + try { + await rename(target, staged); + renamed = true; + const stagedIdentity = await lstat(staged, { bigint: true }); + if ( + stagedIdentity.dev !== validated.dev || + stagedIdentity.ino !== validated.ino + ) + throw new Error( + "Purge target identity changed while entering protected quarantine", + ); + stagedIdentityVerified = true; + const stagedAllowed = allowedFiles?.map((candidate) => { + const current = resolve(candidate); + if (!contained(target, current)) + throw new Error("Purge allowed-file identity escapes its owned tree"); + return resolve(staged, relative(target, current)); + }); + await validateOwnedFilesystemTree(staged, quarantine, stagedAllowed); + const beforeRemoval = await lstat(staged, { bigint: true }); + if ( + beforeRemoval.dev !== validated.dev || + beforeRemoval.ino !== validated.ino + ) + throw new Error( + "Quarantined purge target identity changed before removal", + ); + await rm(staged, { recursive: true }); + await rmdir(quarantine); + } catch (error) { + if (!renamed) { + await rmdir(quarantine).catch(() => undefined); + throw error; + } + let restored = false; + let restoredAtOriginalPath = false; + if (stagedIdentityVerified) { + try { + const current = await lstat(staged, { bigint: true }); + if (current.dev === validated.dev && current.ino === validated.ino) { + let originalAbsent = false; + try { + await lstat(target); + } catch (targetError) { + if ( + targetError instanceof Error && + "code" in targetError && + targetError.code === "ENOENT" + ) + originalAbsent = true; + else throw targetError; + } + if (originalAbsent) { + await rename(staged, target); + const restoredIdentity = await lstat(target, { bigint: true }); + if ( + restoredIdentity.dev === validated.dev && + restoredIdentity.ino === validated.ino + ) { + restoredAtOriginalPath = true; + await rmdir(quarantine); + restored = true; + } + } + } + } catch { + restored = false; + } + } + if (restored) + throw new Error( + "Purge removal failed; the identity-proven owned target was restored for a safe retry", + { cause: error }, + ); + if (restoredAtOriginalPath) + throw new Error( + `Purge removal failed; the retained target was restored but quarantine cleanup requires recovery at ${quarantine}`, + { cause: error }, + ); + throw new Error( + `Purge removal requires recovery; the retained target is isolated at ${quarantine}`, + { cause: error }, + ); + } +} + +async function validateProtectedDirectoryChain( + directory: string, + protectedRoot: string, +): Promise { + const root = resolve(protectedRoot); + const target = resolve(directory); + if (!contained(root, target)) + throw new Error("Purge quarantine parent escapes its protected root"); + const suffix = relative(root, target); + const directories = [ + root, + ...suffix + .split(/[/\\]/u) + .filter(Boolean) + .map((_, index, segments) => + resolve(root, ...segments.slice(0, index + 1)), + ), + ]; + for (const candidate of directories) { + const handle = await open( + candidate, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ); + try { + const stats = await handle.stat(); + if ( + !stats.isDirectory() || + stats.uid !== process.getuid?.() || + (stats.mode & 0o022) !== 0 + ) + throw new Error("Purge quarantine directory is unsafe"); + } finally { + await handle.close(); + } + } +} + +async function createProtectedQuarantine(parent: string): Promise { + for (let attempt = 0; attempt < 8; attempt += 1) { + const candidate = resolve( + parent, + `.skillwire-purge-${randomBytes(16).toString("hex")}`, + ); + try { + await mkdir(candidate, { mode: 0o700 }); + const stats = await lstat(candidate); + if ( + !stats.isDirectory() || + stats.uid !== process.getuid?.() || + (stats.mode & 0o777) !== 0o700 + ) + throw new Error("Purge quarantine directory is unsafe"); + return candidate; + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "EEXIST") + continue; + throw error; + } + } + throw new Error("Unable to create a unique purge quarantine directory"); } diff --git a/src/onboarding/application/recovery.ts b/src/onboarding/application/recovery.ts index 8a1f721..fd3a464 100644 --- a/src/onboarding/application/recovery.ts +++ b/src/onboarding/application/recovery.ts @@ -13,107 +13,140 @@ export interface RecoveryResult { readonly boundary: string | null; } +function requireActiveRecovery(signal: AbortSignal): void { + if (signal.aborted) throw new Error("Recovery cancelled"); +} + +function unresolvedEffectBoundaries( + entries: readonly JournalEntry[], +): readonly JournalEntry[] { + const unresolved = new Map(); + for (const entry of entries) { + if (entry.phase === "effect") unresolved.set(entry.step, entry); + else if (entry.phase === "compensate") { + if (entry.detail["completion"] === "unproven") + unresolved.set(entry.step, entry); + else unresolved.delete(entry.step); + } else if (entry.phase === "commit") unresolved.clear(); + } + return [...unresolved.values()].sort( + (left, right) => left.sequence - right.sequence, + ); +} + export function journalNeedsRecovery( entries: readonly JournalEntry[], ): boolean { const last = entries.at(-1); if (last === undefined || last.phase === "commit") return false; if (last.phase !== "cancel") return true; - return last.detail["status"] === "recovery-required"; + if (last.detail["status"] === "recovery-required") return true; + return unresolvedEffectBoundaries(entries).length > 0; } -function lastEffectBoundary( - entries: readonly JournalEntry[], -): JournalEntry | undefined { - return [...entries] - .reverse() - .find(({ phase }) => phase === "intent" || phase === "effect"); -} - -export async function recoverOperation(options: { +async function reconcileEffectBoundaries(options: { readonly journal: OperationJournal; readonly signal: AbortSignal; readonly observe: (step: string) => Promise; readonly compensate: (step: string) => Promise; + readonly boundaries: readonly JournalEntry[]; }): Promise { - if (options.signal.aborted) throw new Error("Recovery cancelled"); - const last = options.journal.entries.at(-1); - if (last === undefined) - return { disposition: "safe-retry", changed: false, boundary: null }; - if (last.phase === "commit") - return { - disposition: "complete", - changed: false, - boundary: last.step, - }; - if ( - last.phase === "cancel" && - last.detail["status"] === "recovery-required" && - options.journal.hasUnprovenEffect() - ) { - const unresolved = [...options.journal.entries] - .reverse() - .find( - (entry) => - entry.phase === "compensate" && - entry.detail["completion"] === "unproven", - ); - if (unresolved === undefined) - return { - disposition: "recovery-required", - changed: false, - boundary: last.step, - }; - const observation = await options.observe(unresolved.step); + const observations: { + readonly boundary: JournalEntry; + readonly observation: Exclude; + }[] = []; + for (const boundary of [...options.boundaries].reverse()) { + const observation = await options.observe(boundary.step); + requireActiveRecovery(options.signal); if (observation === "ambiguous") return { disposition: "recovery-required", changed: false, - boundary: unresolved.step, + boundary: boundary.step, }; + observations.push({ boundary, observation }); + } + + for (const { boundary, observation } of observations) { if (observation === "owned-mismatch") - await options.compensate(unresolved.step); - await options.journal.compensate(unresolved.step, { + await options.compensate(boundary.step); + await options.journal.compensate(boundary.step, { completion: observation === "absent" ? "not-started" : "recovered", recoveryRequired: false, + observation, }); - await options.journal.commit({ status: "recovered" }); - return { - disposition: observation === "absent" ? "safe-retry" : "resume", - changed: true, - boundary: unresolved.step, - }; } - if (last.phase === "cancel" && !options.journal.hasUnprovenEffect()) + await options.journal.commit({ status: "recovered" }); + const mostRecent = options.boundaries.at(-1); + return { + disposition: observations.every( + ({ observation }) => observation === "absent", + ) + ? "safe-retry" + : "resume", + changed: true, + boundary: mostRecent?.step ?? null, + }; +} + +export async function recoverOperation(options: { + readonly journal: OperationJournal; + readonly signal: AbortSignal; + readonly observe: (step: string) => Promise; + readonly compensate: (step: string) => Promise; +}): Promise { + requireActiveRecovery(options.signal); + const last = options.journal.entries.at(-1); + if (last === undefined) + return { disposition: "safe-retry", changed: false, boundary: null }; + if (last.phase === "commit") return { - disposition: "safe-retry", + disposition: "complete", changed: false, boundary: last.step, }; + + const unresolved = unresolvedEffectBoundaries(options.journal.entries); + if (unresolved.length > 0) + return reconcileEffectBoundaries({ ...options, boundaries: unresolved }); + + if (last.phase === "cancel") + return last.detail["status"] === "recovery-required" + ? { + disposition: "recovery-required", + changed: false, + boundary: last.step, + } + : { + disposition: "safe-retry", + changed: false, + boundary: last.step, + }; + if (last.phase === "intent") { - await options.journal.compensate(last.step, { - completion: "not-started", - recoveryRequired: false, - }); - await options.journal.commit({ status: "recovered" }); - return { - disposition: "safe-retry", - changed: true, - boundary: last.step, - }; - } - if (last.phase === "compensate" && options.journal.hasUnprovenEffect()) { const observation = await options.observe(last.step); + requireActiveRecovery(options.signal); if (observation === "ambiguous") return { disposition: "recovery-required", changed: false, boundary: last.step, }; + if (observation === "matching") { + await options.journal.effect(last.step, { completion: "recovered" }); + await options.journal.verify(last.step, { recovered: true }); + await options.journal.commit({ status: "recovered" }); + return { + disposition: "resume", + changed: true, + boundary: last.step, + }; + } if (observation === "owned-mismatch") await options.compensate(last.step); await options.journal.compensate(last.step, { completion: observation === "absent" ? "not-started" : "recovered", recoveryRequired: false, + observation, }); await options.journal.commit({ status: "recovered" }); return { @@ -122,58 +155,15 @@ export async function recoverOperation(options: { boundary: last.step, }; } + if (last.phase === "verify" || last.phase === "compensate") { - if (!options.journal.hasUnprovenEffect()) - await options.journal.commit({ status: "recovered" }); + await options.journal.commit({ status: "recovered" }); return { - disposition: options.journal.hasUnprovenEffect() - ? "recovery-required" - : "resume", + disposition: "resume", changed: false, boundary: last.step, }; } - const boundary = lastEffectBoundary(options.journal.entries); - if (boundary === undefined) - return { disposition: "safe-retry", changed: false, boundary: null }; - const observation = await options.observe(boundary.step); - if (observation === "ambiguous") - return { - disposition: "recovery-required", - changed: false, - boundary: boundary.step, - }; - if (observation === "matching") { - await options.journal.verify(boundary.step, { recovered: true }); - await options.journal.commit({ status: "recovered" }); - return { - disposition: "resume", - changed: true, - boundary: boundary.step, - }; - } - if (observation === "owned-mismatch") { - await options.compensate(boundary.step); - await options.journal.compensate(boundary.step, { - completion: "recovered", - recoveryRequired: false, - }); - await options.journal.commit({ status: "recovered" }); - return { - disposition: "resume", - changed: true, - boundary: boundary.step, - }; - } - await options.journal.compensate(boundary.step, { - completion: "not-started", - recoveryRequired: false, - }); - await options.journal.commit({ status: "recovered" }); - return { - disposition: "safe-retry", - changed: true, - boundary: boundary.step, - }; + return { disposition: "safe-retry", changed: false, boundary: null }; } diff --git a/src/onboarding/application/uninstall.ts b/src/onboarding/application/uninstall.ts index 79366a1..9f0cfd0 100644 --- a/src/onboarding/application/uninstall.ts +++ b/src/onboarding/application/uninstall.ts @@ -23,6 +23,46 @@ export interface DefaultUninstallPreview { readonly previewHash: string; } +interface UninstallEffectGroup { + readonly step: string; + readonly assets: readonly OwnedAsset[]; +} + +function groupUninstallEffects( + assets: readonly OwnedAsset[], +): readonly UninstallEffectGroup[] { + const groups: UninstallEffectGroup[] = []; + const clientIntegrations = new Map<"codex" | "claude", OwnedAsset[]>(); + for (const asset of assets) { + if ( + asset.client !== null && + (asset.kind === "marketplace" || asset.kind === "plugin") + ) { + const existing = clientIntegrations.get(asset.client); + if (existing === undefined) { + const grouped = [asset]; + clientIntegrations.set(asset.client, grouped); + groups.push({ + step: `uninstall-client-integration-${asset.client}`, + assets: grouped, + }); + } else { + if (existing.some(({ kind }) => kind === asset.kind)) + throw new Error( + `Uninstall ${asset.client} integration ownership is ambiguous`, + ); + existing.push(asset); + } + continue; + } + groups.push({ + step: `uninstall-${asset.kind}-${asset.assetId}`, + assets: [asset], + }); + } + return groups; +} + export function previewDefaultUninstall( ownership: unknown, ): DefaultUninstallPreview { @@ -106,20 +146,27 @@ export async function runDefaultUninstall(options: { verification: () => ({ ...detail, completed: true }), }); }; - for (const asset of options.preview.remove) { + for (const group of groupUninstallEffects(options.preview.remove)) { if (options.signal.aborted) throw new Error("Uninstall stopped at a recoverable asset boundary"); - requireCurrentOwnedAssetIdentity( - asset, - await options.observeIdentity(asset), - ); - await effect( - `uninstall-${asset.kind}-${asset.assetId}`, - () => options.removeAsset(asset), - { assetId: asset.assetId, kind: asset.kind }, - ); - ownership = recordAssetDisposition(ownership, asset.assetId, "removed"); - removed.push(asset.assetId); + for (const asset of group.assets) + requireCurrentOwnedAssetIdentity( + asset, + await options.observeIdentity(asset), + ); + const representative = group.assets[0]; + if (representative === undefined) + throw new Error("Uninstall effect group is empty"); + await effect(group.step, () => options.removeAsset(representative), { + assetId: representative.assetId, + groupedAssets: group.assets.length, + kind: + group.assets.length > 1 ? "client-integration" : representative.kind, + }); + for (const asset of group.assets) { + ownership = recordAssetDisposition(ownership, asset.assetId, "removed"); + removed.push(asset.assetId); + } } await effect("uninstall-owned-service", options.stopOwnedService, { installationId: options.preview.installationId, diff --git a/src/onboarding/domain/operation-journal.ts b/src/onboarding/domain/operation-journal.ts index c55212b..3359aed 100644 --- a/src/onboarding/domain/operation-journal.ts +++ b/src/onboarding/domain/operation-journal.ts @@ -146,13 +146,14 @@ export class OperationJournal { detail: JournalEntry["detail"], ): Promise { const last = this.entries.at(-1); - const resumingRecoveryRequiredCancellation = + const resumingCancelledOperation = last?.phase === "cancel" && - last.detail["status"] === "recovery-required" && + (last.detail["status"] === "failed" || + last.detail["status"] === "recovery-required") && phase === "compensate"; if ( last?.phase === "commit" || - (last?.phase === "cancel" && !resumingRecoveryRequiredCancellation) + (last?.phase === "cancel" && !resumingCancelledOperation) ) { throw new Error("Operation journal is already terminal"); } diff --git a/tests/e2e/self-hosted-onboarding/default-uninstall.test.ts b/tests/e2e/self-hosted-onboarding/default-uninstall.test.ts index cda9281..d8dbed7 100644 --- a/tests/e2e/self-hosted-onboarding/default-uninstall.test.ts +++ b/tests/e2e/self-hosted-onboarding/default-uninstall.test.ts @@ -1,21 +1,29 @@ /* eslint-disable @typescript-eslint/require-await -- Async fakes mirror production lifecycle interfaces. */ import { randomUUID } from "node:crypto"; -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { previewDefaultUninstall, runDefaultUninstall, } from "../../../src/onboarding/application/uninstall.js"; import { uninstallClientLifecycle } from "../../../src/onboarding/application/client-lifecycle.js"; +import { OperationJournal } from "../../../src/onboarding/domain/operation-journal.js"; import { createOwnershipLedger, planOwnedAssetDispositions, recordAssetDisposition, recordOwnedAsset, } from "../../../src/onboarding/domain/ownership.js"; +import { + createOnboardingEnvironment, + type OnboardingEnvironment, +} from "../../helpers/onboarding-environment.js"; describe("data-preserving default uninstall", () => { + let fixture: OnboardingEnvironment | undefined; + afterEach(async () => fixture?.close()); + it("removes only owned client/service runtime assets and retains recovery state", async () => { const installationId = randomUUID(); const operationId = randomUUID(); @@ -98,6 +106,73 @@ describe("data-preserving default uninstall", () => { expect(unrelatedExternal).toEqual(beforeExternal); }); + it("removes same-client marketplace and plugin aliases as one journaled inverse", async () => { + fixture = await createOnboardingEnvironment(); + const operationId = randomUUID(); + let ledger = createOwnershipLedger(randomUUID()); + for (const client of ["codex", "claude"] as const) + for (const kind of ["marketplace", "plugin"] as const) + ledger = recordOwnedAsset(ledger, { + kind, + client, + locator: `${client}-${kind}`, + expectedIdentitySha256: (client === "codex" ? "c" : "d").repeat(64), + createdByOperation: operationId, + retention: "remove-on-uninstall", + disposition: "present", + }); + const preview = previewDefaultUninstall(ledger.record); + const present = new Map([ + ["codex", true], + ["claude", true], + ]); + const removeAsset = vi.fn( + async (asset: (typeof preview.remove)[number]) => { + if (asset.client === null) throw new Error("missing client fixture"); + present.set(asset.client, false); + }, + ); + const journal = await OperationJournal.create( + fixture.root, + randomUUID(), + "uninstall", + ); + + const result = await runDefaultUninstall({ + ownership: ledger.record, + preview, + confirmation: preview.previewHash, + signal: new AbortController().signal, + observeIdentity: async (asset) => + asset.client !== null && present.get(asset.client) === true + ? asset.expectedIdentitySha256 + : "0".repeat(64), + removeAsset, + stopOwnedService: async () => undefined, + publishRetained: async () => undefined, + journal, + }); + + expect(removeAsset).toHaveBeenCalledTimes(2); + expect( + removeAsset.mock.calls.map(([asset]) => asset.client).sort(), + ).toEqual(["claude", "codex"]); + expect(result.removed).toHaveLength(4); + expect( + journal.entries + .filter( + ({ phase, step }) => + phase === "effect" && + step.startsWith("uninstall-client-integration-"), + ) + .map(({ step }) => step) + .sort(), + ).toEqual([ + "uninstall-client-integration-claude", + "uninstall-client-integration-codex", + ]); + }); + it("uninstalls one owned client including its key while preserving its sibling and external integrations", async () => { const events: string[] = []; const result = await uninstallClientLifecycle( diff --git a/tests/integration/onboarding/interruption-recovery.test.ts b/tests/integration/onboarding/interruption-recovery.test.ts index 86ae057..a6d283c 100644 --- a/tests/integration/onboarding/interruption-recovery.test.ts +++ b/tests/integration/onboarding/interruption-recovery.test.ts @@ -22,7 +22,7 @@ describe("observation-based interruption recovery", () => { afterEach(async () => fixture?.close()); it.each([ - ["intent", "safe-retry"], + ["intent", "resume"], ["effect", "resume"], ["verify", "resume"], ["compensate", "resume"], @@ -64,6 +64,42 @@ describe("observation-based interruption recovery", () => { }, ); + it.each([ + ["matching", "resume", ["intent", "effect", "verify", "commit"], 0], + ["absent", "safe-retry", ["intent", "compensate", "commit"], 0], + ["owned-mismatch", "resume", ["intent", "compensate", "commit"], 1], + ["ambiguous", "recovery-required", ["intent"], 0], + ] as const)( + "observes an intent-only %s boundary before deciding recovery", + async (observation, disposition, phases, compensations) => { + fixture = await createOnboardingEnvironment(); + const journal = await OperationJournal.create( + resolve(fixture.root, "journals"), + randomUUID(), + "repair", + ); + await journal.intent("owned-plugin", { client: "codex" }); + const observe = vi.fn(async () => observation); + const compensate = vi.fn(async () => undefined); + + await expect( + recoverOperation({ + journal, + signal: new AbortController().signal, + observe, + compensate, + }), + ).resolves.toMatchObject({ + disposition, + boundary: "owned-plugin", + }); + + expect(observe).toHaveBeenCalledExactlyOnceWith("owned-plugin"); + expect(compensate).toHaveBeenCalledTimes(compensations); + expect(journal.entries.map(({ phase }) => phase)).toEqual(phases); + }, + ); + it("blocks ambiguous effects and compensates only a proven mismatching owned effect", async () => { fixture = await createOnboardingEnvironment(); const root = resolve(fixture.root, "journals"); @@ -186,6 +222,111 @@ describe("observation-based interruption recovery", () => { ).resolves.toMatchObject({ disposition: "recovery-required" }); }); + it("retains and consumes a verified effect boundary after a failed cancellation", async () => { + fixture = await createOnboardingEnvironment(); + const journal = await OperationJournal.create( + resolve(fixture.root, "journals"), + randomUUID(), + "uninstall", + ); + await journal.intent("client-codex-plugin", { client: "codex" }); + await journal.effect("client-codex-plugin", { completion: "recorded" }); + await journal.verify("client-codex-plugin", { removed: true }); + await journal.cancel({ status: "failed" }); + const observe = vi.fn(async () => "matching" as const); + + expect(journalNeedsRecovery(journal.entries)).toBe(true); + await expect( + recoverOperation({ + journal, + signal: new AbortController().signal, + observe, + compensate: vi.fn(), + }), + ).resolves.toMatchObject({ + disposition: "resume", + changed: true, + boundary: "client-codex-plugin", + }); + expect(observe).toHaveBeenCalledExactlyOnceWith("client-codex-plugin"); + expect(journal.entries.at(-2)).toMatchObject({ + phase: "compensate", + step: "client-codex-plugin", + detail: { completion: "recovered" }, + }); + expect(journal.entries.at(-1)).toMatchObject({ + phase: "commit", + detail: { status: "recovered" }, + }); + }); + + it("does not hide an earlier unresolved effect behind a later compensation", async () => { + fixture = await createOnboardingEnvironment(); + const journal = await OperationJournal.create( + resolve(fixture.root, "journals"), + randomUUID(), + "uninstall", + ); + await journal.intent("client-codex-plugin", { client: "codex" }); + await journal.effect("client-codex-plugin", { completion: "recorded" }); + await journal.verify("client-codex-plugin", { removed: true }); + await journal.intent("client-claude-plugin", { client: "claude" }); + await journal.effect("client-claude-plugin", { completion: "recorded" }); + await journal.verify("client-claude-plugin", { removed: true }); + await journal.compensate("client-claude-plugin", { + completion: "recovered", + recoveryRequired: false, + }); + await journal.cancel({ status: "failed" }); + const observe = vi.fn(async () => "matching" as const); + + expect(journalNeedsRecovery(journal.entries)).toBe(true); + await expect( + recoverOperation({ + journal, + signal: new AbortController().signal, + observe, + compensate: vi.fn(), + }), + ).resolves.toMatchObject({ + disposition: "resume", + boundary: "client-codex-plugin", + }); + expect(observe).toHaveBeenCalledExactlyOnceWith("client-codex-plugin"); + }); + + it("treats a fully compensated failed cancellation as terminally safe", async () => { + fixture = await createOnboardingEnvironment(); + const journal = await OperationJournal.create( + resolve(fixture.root, "journals"), + randomUUID(), + "uninstall", + ); + await journal.intent("client-codex-plugin", { client: "codex" }); + await journal.effect("client-codex-plugin", { completion: "recorded" }); + await journal.verify("client-codex-plugin", { removed: true }); + await journal.compensate("client-codex-plugin", { + completion: "recovered", + recoveryRequired: false, + }); + await journal.cancel({ status: "failed" }); + const observe = vi.fn(async () => "ambiguous" as const); + + expect(journalNeedsRecovery(journal.entries)).toBe(false); + await expect( + recoverOperation({ + journal, + signal: new AbortController().signal, + observe, + compensate: vi.fn(), + }), + ).resolves.toMatchObject({ + disposition: "safe-retry", + changed: false, + }); + expect(observe).not.toHaveBeenCalled(); + }); + it("returns a stable recovery result when production repair finds an ambiguous interrupted effect", async () => { fixture = await createOnboardingEnvironment(); const stateRoot = resolve(fixture.xdgStateHome, "skillwire"); diff --git a/tests/security/onboarding/removal-boundaries.test.ts b/tests/security/onboarding/removal-boundaries.test.ts index aed67d7..bda4523 100644 --- a/tests/security/onboarding/removal-boundaries.test.ts +++ b/tests/security/onboarding/removal-boundaries.test.ts @@ -1,10 +1,25 @@ /* eslint-disable @typescript-eslint/require-await -- Async fakes mirror production removal interfaces. */ import { randomUUID } from "node:crypto"; -import { mkdir, readFile, symlink, writeFile } from "node:fs/promises"; +import { + lstat, + mkdir, + readFile, + readdir, + rm as filesystemRm, + symlink, + unlink, + writeFile, +} from "node:fs/promises"; import { resolve } from "node:path"; +import type * as FileSystemPromises from "node:fs/promises"; import { afterEach, describe, expect, it, vi } from "vitest"; +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, rm: vi.fn(actual.rm) }; +}); + import { previewPurge, removeOwnedFilesystemTree, @@ -185,6 +200,61 @@ describe("removal ownership and filesystem boundaries", () => { await expect(readFile(external, "utf8")).resolves.toBe("keep"); }); + it("binds recursive deletion to a quarantined inode before a target-path substitution", async () => { + fixture = await createOnboardingEnvironment(); + const protectedRoot = resolve(fixture.root, "purge-root"); + const owned = resolve(protectedRoot, "owned"); + const externalRoot = resolve(fixture.root, "external"); + const external = resolve(externalRoot, "keep.txt"); + await mkdir(owned, { recursive: true, mode: 0o700 }); + await mkdir(externalRoot, { mode: 0o700 }); + await writeFile(resolve(owned, "state.json"), "owned", { mode: 0o600 }); + await writeFile(external, "keep", { mode: 0o600 }); + const before = await lstat(owned, { bigint: true }); + const actual = + await vi.importActual("node:fs/promises"); + vi.mocked(filesystemRm).mockImplementationOnce( + async (candidate, options) => { + if (typeof candidate !== "string") + throw new Error("expected a string purge target"); + expect(resolve(candidate)).not.toBe(owned); + const quarantined = await lstat(candidate, { bigint: true }); + expect({ dev: quarantined.dev, ino: quarantined.ino }).toEqual({ + dev: before.dev, + ino: before.ino, + }); + await symlink(externalRoot, owned); + await actual.rm(candidate, options); + }, + ); + + await removeOwnedFilesystemTree(owned, protectedRoot, [ + resolve(owned, "state.json"), + ]); + + await expect(readFile(external, "utf8")).resolves.toBe("keep"); + await unlink(owned); + }); + + it("restores an identity-proven quarantine when recursive deletion fails", async () => { + fixture = await createOnboardingEnvironment(); + const protectedRoot = resolve(fixture.root, "purge-root"); + const owned = resolve(protectedRoot, "owned"); + const ownedFile = resolve(owned, "state.json"); + await mkdir(owned, { recursive: true, mode: 0o700 }); + await writeFile(ownedFile, "owned", { mode: 0o600 }); + vi.mocked(filesystemRm).mockRejectedValueOnce( + new Error("simulated recursive removal failure"), + ); + + await expect( + removeOwnedFilesystemTree(owned, protectedRoot, [ownedFile]), + ).rejects.toThrow(/restored|retry/i); + + await expect(readFile(ownedFile, "utf8")).resolves.toBe("owned"); + await expect(readdir(protectedRoot)).resolves.toEqual(["owned"]); + }); + it("stops an interrupted purge before the first asset", async () => { let ledger = createOwnershipLedger(randomUUID()); ledger = recordOwnedAsset(ledger, { From 26f724ea66bfa77479d96d5483952b6ef0c6a3c3 Mon Sep 17 00:00:00 2001 From: Lucenx9 Date: Fri, 14 Aug 2026 16:58:16 +0200 Subject: [PATCH 3/7] fix: verify repeated setup and secret identity --- .../application/production-setup.ts | 492 +++++++++++++++--- .../application/service-secret-rotation.ts | 66 ++- .../repeated-setup.test.ts | 18 +- .../onboarding/production-setup.test.ts | 59 +++ .../repeated-production-setup.test.ts | 215 ++++++++ .../service-secret-rotation.test.ts | 194 ++++++- 6 files changed, 947 insertions(+), 97 deletions(-) create mode 100644 tests/integration/onboarding/repeated-production-setup.test.ts diff --git a/src/onboarding/application/production-setup.ts b/src/onboarding/application/production-setup.ts index 217301b..fd6e4b7 100644 --- a/src/onboarding/application/production-setup.ts +++ b/src/onboarding/application/production-setup.ts @@ -1,5 +1,6 @@ import { createHash, randomUUID, timingSafeEqual } from "node:crypto"; import { constants } from "node:fs"; +import { request as httpRequest } from "node:http"; import { access, chmod, @@ -12,6 +13,7 @@ import { import { basename, dirname, isAbsolute, resolve } from "node:path"; import { z } from "zod"; +import { CredentialResolver } from "../../credential-bridge/credential-resolver.js"; import { atomicWriteJson } from "../adapters/filesystem/atomic-state.js"; import { installVerifiedRelease, @@ -19,7 +21,10 @@ import { } from "../adapters/filesystem/release-installer.js"; import { verifySelfHostedRelease } from "../adapters/filesystem/release-verifier.js"; import { DeploymentAdapter } from "../adapters/docker/deployment.js"; -import { dockerProcessEnvironment } from "../adapters/docker/environment.js"; +import { + assertLocalDockerContext, + dockerProcessEnvironment, +} from "../adapters/docker/environment.js"; import { ServiceDatabase } from "../adapters/postgres/service-database.js"; import { ClientKeyHandoffRecoveryError, @@ -53,8 +58,10 @@ import { } from "../domain/operation-journal.js"; import { createOwnershipLedger, + ExternalIntegrationDependencySchema, recordExternalIntegration, recordOwnedAsset, + verifyOwnershipRecord, } from "../domain/ownership.js"; import { clientConflictFinding, @@ -110,6 +117,31 @@ const ClientIntegrationsStateSchema = z }) .strict(); +const ExternalIntegrationsStateSchema = z + .object({ + schemaVersion: z.literal("skillwire.external-integrations/v1"), + installationId: z.uuid(), + dependencies: z.array(ExternalIntegrationDependencySchema).max(6), + }) + .strict(); + +const DeploymentStateSchema = z + .object({ + schemaVersion: z.literal("skillwire.deployment/v1"), + installationId: z.uuid(), + releaseRoot: z.string().refine(isAbsolute), + composePath: z.string().refine(isAbsolute), + skillwireImage: z.string().min(1), + postgresImage: z.string().min(1), + databasePasswordFile: z.string().refine(isAbsolute), + applicationPepperFile: z.string().refine(isAbsolute), + runtimeSocketDirectory: z.string().refine(isAbsolute), + socketPath: z.string().refine(isAbsolute), + projectName: z.string().min(1), + volumeName: z.string().min(1), + }) + .strict(); + export function unchangedSetupClientResults( requested: readonly ClientName[], installationId: string, @@ -141,6 +173,24 @@ export function unchangedSetupClientResults( return results; } +function repeatedSetupRequiresManagedCredential( + state: "verified" | "external-verified", +): boolean { + return state === "verified"; +} + +export async function runRepeatedSetupClientVerification( + state: "verified" | "external-verified", + verification: { + readonly verifyManaged: () => Promise; + readonly verifyExternal: () => Promise; + }, +): Promise { + await (repeatedSetupRequiresManagedCredential(state) + ? verification.verifyManaged() + : verification.verifyExternal()); +} + export async function ownedLauncherIdentity(path: string): Promise { const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); try { @@ -181,6 +231,282 @@ async function readProtectedSetupJson(path: string): Promise { } } +async function requireLiveReadiness( + socketPath: string, + signal: AbortSignal, +): Promise { + const boundedSignal = AbortSignal.any([signal, AbortSignal.timeout(5_000)]); + const ready = await new Promise((done, reject) => { + const request = httpRequest( + { + socketPath, + path: "/health/ready", + method: "GET", + headers: { host: "localhost" }, + signal: boundedSignal, + }, + (response) => { + response.resume(); + done(response.statusCode === 200); + }, + ); + request.once("error", reject); + request.end(); + }); + if (!ready) throw new Error("Installed SkillWire service is not ready"); +} + +async function verifyUnchangedProductionSetup(options: { + readonly installation: z.infer; + readonly integrations: z.infer; + readonly clientsToVerify: readonly ClientName[]; + readonly roots: SetupRoots; + readonly environment: NodeJS.ProcessEnv; + readonly signal: AbortSignal; +}): Promise { + const { installation, clientsToVerify, environment, signal } = options; + const deployment = DeploymentStateSchema.parse( + await readProtectedSetupJson( + resolve(options.roots.stateRoot, "deployment.json"), + ), + ); + const ownership = verifyOwnershipRecord( + await readProtectedSetupJson( + resolve(options.roots.stateRoot, "ownership.json"), + ), + ); + const externalIntegrations = ExternalIntegrationsStateSchema.parse( + await readProtectedSetupJson( + resolve(options.roots.stateRoot, "external-integrations.json"), + ), + ); + if ( + deployment.installationId !== installation.installationId || + ownership.installationId !== installation.installationId || + externalIntegrations.installationId !== installation.installationId || + deployment.projectName !== installation.composeProject || + deployment.volumeName !== installation.postgresVolume || + installation.endpoint !== `unix://${deployment.socketPath}` + ) { + throw new Error("Repeated setup service ownership state is inconsistent"); + } + if ( + ownership.externalDependencies.length !== + externalIntegrations.dependencies.length || + externalIntegrations.dependencies.some( + ({ externalDependencyId }) => + !ownership.externalDependencies.includes(externalDependencyId), + ) + ) + throw new Error( + "Repeated setup external integration ownership state is inconsistent", + ); + + const deploymentAdapter = new DeploymentAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: deployment.composePath, + projectName: deployment.projectName, + volumeName: deployment.volumeName, + skillwireImage: deployment.skillwireImage, + postgresImage: deployment.postgresImage, + databasePasswordFile: deployment.databasePasswordFile, + applicationPepperFile: deployment.applicationPepperFile, + runtimeSocketDirectory: deployment.runtimeSocketDirectory, + socketPath: deployment.socketPath, + hostEnvironment: environment, + }); + await assertLocalDockerContext({ + dockerExecutable: "/usr/bin/docker", + environment, + signal, + }); + const [skillwirePresent, postgresPresent] = await Promise.all([ + deploymentAdapter.observeOwnedService("skillwire", signal), + deploymentAdapter.observeOwnedService("postgres", signal), + ]); + if (!skillwirePresent || !postgresPresent) + throw new Error("Installed SkillWire services are not live"); + const database = new ServiceDatabase({ + dockerExecutable: "/usr/bin/docker", + projectName: deployment.projectName, + volumeName: deployment.volumeName, + composePath: deployment.composePath, + environment: composeEnvironment({ + environment, + projectName: deployment.projectName, + volumeName: deployment.volumeName, + skillwireImage: deployment.skillwireImage, + postgresImage: deployment.postgresImage, + databasePasswordFile: deployment.databasePasswordFile, + applicationPepperFile: deployment.applicationPepperFile, + runtimeSocketDirectory: deployment.runtimeSocketDirectory, + }), + }); + await database.verifyVolume(signal); + await database.verifySchemaAndReadiness(signal); + await requireLiveReadiness(deployment.socketPath, signal); + + const launcherIdentity = await ownedLauncherIdentity( + options.roots.launcherPath, + ); + const launcherAsset = ownership.assets.find( + ({ kind, locator, disposition }) => + kind === "path" && + locator === options.roots.launcherPath && + disposition === "present", + ); + if (launcherAsset?.expectedIdentitySha256 !== launcherIdentity) + throw new Error("Repeated setup launcher ownership identity drifted"); + const installedReleaseIdentity = await releaseDirectoryIdentity( + deployment.releaseRoot, + ); + const releaseAsset = ownership.assets.find( + ({ kind, locator, disposition }) => + kind === "release" && + locator === deployment.releaseRoot && + disposition === "present", + ); + if (releaseAsset?.expectedIdentitySha256 !== installedReleaseIdentity) + throw new Error("Repeated setup release ownership identity drifted"); + + for (const client of clientsToVerify) { + const integration = options.integrations.integrations.find( + (entry) => entry.client === client, + ); + if (integration === undefined) + throw new Error(`Repeated setup ${client} integration state is absent`); + if ( + integration.state !== "verified" && + integration.state !== "external-verified" + ) + throw new Error(`Repeated setup ${client} integration is not verified`); + const vendorExecutable = await executable(client, environment); + const adapter = + client === "codex" + ? new CodexClientAdapter(vendorExecutable, environment, signal) + : new ClaudeClientAdapter( + vendorExecutable, + environment, + undefined, + undefined, + signal, + ); + const marketplacePath = resolve( + deployment.releaseRoot, + client === "codex" + ? "distribution/codex-release-marketplace" + : "distribution/claude-marketplace", + ); + const mcpIdentity = clientComponentIdentity({ + command: options.roots.launcherPath, + args: [ + "bridge", + "--installation", + installation.installationId, + "--client", + client, + ], + scope: "user", + }); + const pluginIdentity = clientComponentIdentity({ + plugin: "skillwire-autonomous-activation@skillwire", + marketplacePath, + }); + if ( + integration.mcpIdentitySha256 !== mcpIdentity || + integration.adapterIdentitySha256 !== pluginIdentity + ) + throw new Error(`Repeated setup ${client} integration identity drifted`); + for (const [kind, identity] of [ + ["mcp-entry", mcpIdentity], + ["marketplace", pluginIdentity], + ["plugin", pluginIdentity], + ] as const) { + const owned = ownership.assets.filter( + (candidate) => + candidate.kind === kind && + candidate.client === client && + candidate.disposition === "present" && + candidate.expectedIdentitySha256 === identity, + ); + const external = externalIntegrations.dependencies.filter( + (candidate) => + candidate.kind === kind && + candidate.client === client && + candidate.verification === "equivalent" && + candidate.observedIdentitySha256 === identity, + ); + if (owned.length + external.length !== 1) + throw new Error( + `Repeated setup ${client} ${kind} ownership identity drifted`, + ); + } + if ( + integration.credentialReferenceId !== null && + !ownership.assets.some( + ({ kind, client: assetClient, disposition }) => + kind === "credential" && + assetClient === client && + disposition === "present", + ) + ) + throw new Error( + `Repeated setup ${client} credential ownership is absent`, + ); + const registration = await adapter.readMcp(); + await runRepeatedSetupClientVerification(integration.state, { + verifyManaged: async () => { + const resolvedCredential = await new CredentialResolver( + options.roots.stateRoot, + options.roots.dataRoot, + new SecretToolCredentialStore("/usr/bin/secret-tool", environment), + ).resolve(installation.installationId, client, signal); + if ( + resolvedCredential.socketPath !== deployment.socketPath || + resolvedCredential.endpoint.href !== "http://localhost/mcp" + ) + throw new Error( + `Repeated setup ${client} credential bridge identity drifted`, + ); + await verifyClientIntegration({ + client, + vendorExecutable, + installationId: installation.installationId, + registration, + expectedLauncher: options.roots.launcherPath, + environment, + inventory: () => adapter.readInventory(marketplacePath), + signal, + }); + }, + verifyExternal: async () => { + const expectedArguments = [ + "bridge", + "--installation", + installation.installationId, + "--client", + client, + ]; + if ( + registration.command !== options.roots.launcherPath || + registration.args.join("\0") !== expectedArguments.join("\0") + ) + throw new Error( + `Repeated setup ${client} external MCP identity drifted`, + ); + const inventory = await adapter.readInventory(marketplacePath); + if ( + inventory.mcp.command !== registration.command || + inventory.mcp.args.join("\0") !== registration.args.join("\0") + ) + throw new Error( + `Repeated setup ${client} external profile inventory drifted`, + ); + }, + }); + } +} + export interface ProductionTrustOverrides { readonly pinnedInitialPolicySha256?: string | undefined; } @@ -1464,35 +1790,59 @@ export async function runProductionSetup( trustOverrides: ProductionTrustOverrides = {}, ): Promise { const setupRoots = roots(environment); - let existingInstallation: z.infer | undefined; - // Preserve the durable cancellation contract: even an already-aborted fresh - // setup records intent/cancel before returning. Installed-state inspection is - // therefore skipped until after the journal is available in that case. - if (!signal.aborted) { - try { - const existing = await inspectInstalledStatus({ - stateRoot: setupRoots.stateRoot, - signal, - }); - existingInstallation = existing.installation; + await mkdir(setupRoots.stateRoot, { recursive: true, mode: 0o700 }); + await mkdir(setupRoots.runtimeRoot, { recursive: true, mode: 0o700 }); + const identity = await currentProcessIdentity(); + const lock = await InstallationLock.acquire( + resolve(setupRoots.runtimeRoot, "locks"), + "installation", + identity, + ); + let journal: OperationJournal | undefined; + try { + let existingInstallation: z.infer | undefined; + // Preserve the durable cancellation contract: even an already-aborted + // fresh setup records intent/cancel before returning. Live installed-state + // inspection is skipped until after that journal exists in this case. + if (!signal.aborted) { + try { + existingInstallation = ( + await inspectInstalledStatus({ + stateRoot: setupRoots.stateRoot, + signal, + }) + ).installation; + } catch (error) { + if (!( + error instanceof Error && + (("code" in error && error.code === "ENOENT") || + error.message.includes("ENOENT")) + )) + throw error; + } + if (existingInstallation?.status === "recovery-required") + throw new Error( + "Setup cannot bypass an installation that requires journal recovery", + ); const requested = selectedClients(options.clients); + const installed = existingInstallation; if ( - (existing.installation.status === "service-ready" || - existing.installation.status === "complete") && - requested.every((client) => - existing.installation.selectedClients.includes(client), - ) + installed !== undefined && + (installed.status === "service-ready" || + installed.status === "complete") && + requested.every((client) => installed.selectedClients.includes(client)) ) { - const unchangedClients = unchangedSetupClientResults( - requested, - existing.installation.installationId, + const integrations = ClientIntegrationsStateSchema.parse( await readProtectedSetupJson( resolve(setupRoots.stateRoot, "client-integrations.json"), ), ); - if (unchangedClients === undefined) { - existingInstallation = existing.installation; - } else { + const unchangedClients = unchangedSetupClientResults( + requested, + installed.installationId, + integrations, + ); + if (unchangedClients !== undefined) { const candidate = await candidatePaths(environment); const verified = await verifyCandidate( candidate, @@ -1502,69 +1852,51 @@ export async function runProductionSetup( ); if ( verified.releaseSequence !== - existing.installation.highestAcceptedReleaseSequence || - verified.trustPolicySequence !== - existing.installation.activeTrustPolicySequence + installed.highestAcceptedReleaseSequence || + verified.trustPolicySequence !== installed.activeTrustPolicySequence ) { throw new Error( "Unchanged setup candidate differs from the installed release state", ); } + await verifyUnchangedProductionSetup({ + installation: installed, + integrations, + clientsToVerify: installed.selectedClients, + roots: setupRoots, + environment, + signal, + }); return { status: "success", - installationId: existing.installation.installationId, + installationId: installed.installationId, serviceReady: true, clients: unchangedClients, changed: false, }; } } - if (existing.installation.status === "recovery-required") - throw new Error( - "Setup cannot bypass an installation that requires journal recovery", - ); - } catch (error) { - if ( - !( - error instanceof Error && - "code" in error && - error.code === "ENOENT" - ) && - !(error instanceof Error && error.message.includes("ENOENT")) - ) { - throw error; - } } - } - await mkdir(setupRoots.stateRoot, { recursive: true, mode: 0o700 }); - await mkdir(setupRoots.runtimeRoot, { recursive: true, mode: 0o700 }); - const identity = await currentProcessIdentity(); - const lock = await InstallationLock.acquire( - resolve(setupRoots.runtimeRoot, "locks"), - "installation", - identity, - ); - const operationId = randomUUID(); - const journal = await OperationJournal.create( - resolve(setupRoots.stateRoot, "operations"), - operationId, - "setup", - ); - const previewHash = - options.previewHash ?? - createHash("sha256") - .update( - JSON.stringify({ - clients: options.clients, - credentialBackend: options.credentialBackend, - }), - ) - .digest("hex"); - await journal.intent("setup", { - clients: options.clients, - previewHash, - }); - try { + + journal = await OperationJournal.create( + resolve(setupRoots.stateRoot, "operations"), + randomUUID(), + "setup", + ); + const previewHash = + options.previewHash ?? + createHash("sha256") + .update( + JSON.stringify({ + clients: options.clients, + credentialBackend: options.credentialBackend, + }), + ) + .digest("hex"); + await journal.intent("setup", { + clients: options.clients, + previewHash, + }); if ( existingInstallation !== undefined && existingInstallation.status !== "purged" @@ -1623,12 +1955,16 @@ export async function runProductionSetup( await journal.commit({ status: result.status }); return result; } catch (error) { - if (signal.aborted) { - await journal.cancel({ - status: journal.hasUnprovenEffect() ? "recovery-required" : "cancelled", - }); - } else { - await journal.compensate("setup", { status: "recovery-required" }); + if (journal !== undefined) { + if (signal.aborted) { + await journal.cancel({ + status: journal.hasUnprovenEffect() + ? "recovery-required" + : "cancelled", + }); + } else { + await journal.compensate("setup", { status: "recovery-required" }); + } } throw error; } finally { diff --git a/src/onboarding/application/service-secret-rotation.ts b/src/onboarding/application/service-secret-rotation.ts index fa32685..35da453 100644 --- a/src/onboarding/application/service-secret-rotation.ts +++ b/src/onboarding/application/service-secret-rotation.ts @@ -1,6 +1,6 @@ import { createHash, randomBytes } from "node:crypto"; import { constants } from "node:fs"; -import { open, rename, unlink } from "node:fs/promises"; +import { link, lstat, open, rename, unlink } from "node:fs/promises"; import { resolve } from "node:path"; import { @@ -17,7 +17,7 @@ export interface ServiceSecretRotationPreview { readonly kind: ServiceSecretKind; readonly targets: readonly [string, string]; readonly previewHash: string; - readonly currentIdentitySha256?: string | undefined; + readonly currentIdentitySha256: string; } function deterministicOperationId(value: string): string { @@ -28,10 +28,10 @@ function deterministicOperationId(value: string): string { export function previewServiceSecretRotation(input: { readonly installationId: string; readonly kind: ServiceSecretKind; - readonly currentIdentitySha256?: string | undefined; + readonly currentIdentitySha256: string; }): ServiceSecretRotationPreview { const operationId = deterministicOperationId( - `${input.installationId}\0${input.kind}\0${input.currentIdentitySha256 ?? "unknown"}`, + `${input.installationId}\0${input.kind}\0${input.currentIdentitySha256}`, ); const targets = [ `secrets/${input.kind}`, @@ -42,9 +42,7 @@ export function previewServiceSecretRotation(input: { kind: input.kind, operationId, targets, - ...(input.currentIdentitySha256 === undefined - ? {} - : { currentIdentitySha256: input.currentIdentitySha256 }), + currentIdentitySha256: input.currentIdentitySha256, }; return { ...scope, @@ -53,7 +51,7 @@ export function previewServiceSecretRotation(input: { }; } -async function validateSecret(path: string, root: string): Promise { +async function validateSecret(path: string, root: string): Promise { const target = await validateOwnedPath(path, root); const handle = await open(target, constants.O_RDONLY | constants.O_NOFOLLOW); try { @@ -67,6 +65,17 @@ async function validateSecret(path: string, root: string): Promise { ) { throw new Error("Service-secret rotation target is unsafe"); } + const bytes = await handle.readFile(); + try { + if (!/^[A-Za-z0-9_-]{43}$/.test(bytes.toString("ascii"))) + throw new Error("Service-secret rotation target has an invalid format"); + return createHash("sha256") + .update("skillwire-service-secret-identity-v1\0") + .update(bytes) + .digest("hex"); + } finally { + bytes.fill(0); + } } finally { await handle.close(); } @@ -84,6 +93,17 @@ async function syncDirectory(path: string): Promise { } } +async function requirePathAbsent(path: string): Promise { + try { + await lstat(path); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") + return; + throw error; + } + throw new Error("Service-secret rotation retained target already exists"); +} + export async function rotateServiceSecret(options: { readonly installationRoot: string; readonly stateRoot: string; @@ -132,7 +152,13 @@ export async function rotateServiceSecret(options: { secretsRoot, `${options.kind}.candidate-${options.preview.operationId}`, ); - await validateSecret(currentPath, installationRoot); + const currentIdentitySha256 = await validateSecret( + currentPath, + installationRoot, + ); + if (currentIdentitySha256 !== options.preview.currentIdentitySha256) + throw new Error("Service-secret rotation target identity changed"); + await requirePathAbsent(retainedPath); const candidate = Buffer.from(randomBytes(32).toString("base64url"), "ascii"); const effect = async ( step: string, @@ -148,6 +174,7 @@ export async function rotateServiceSecret(options: { verification: () => ({ ...detail, completed: true }), }); }; + const ownedCandidatePaths = new Set(); try { await effect( "service-secret-candidate", @@ -160,9 +187,14 @@ export async function rotateServiceSecret(options: { constants.O_NOFOLLOW, 0o600, ); + ownedCandidatePaths.add(candidatePath); try { await candidateHandle.writeFile(candidate); await candidateHandle.sync(); + } catch (error) { + await unlink(candidatePath).catch(() => undefined); + ownedCandidatePaths.delete(candidatePath); + throw error; } finally { await candidateHandle.close(); } @@ -171,7 +203,8 @@ export async function rotateServiceSecret(options: { ); } catch (error) { candidate.fill(0); - await unlink(candidatePath).catch(() => undefined); + if (ownedCandidatePaths.delete(candidatePath)) + await unlink(candidatePath).catch(() => undefined); throw error; } const identitySha256 = createHash("sha256") @@ -179,7 +212,10 @@ export async function rotateServiceSecret(options: { .update(candidate) .digest("hex"); candidate.fill(0); - const rotationState: { swapped: boolean } = { swapped: false }; + const rotationState: { retainedLinked: boolean; swapped: boolean } = { + retainedLinked: false, + swapped: false, + }; try { await effect( "service-secret-candidate-readiness", @@ -194,7 +230,8 @@ export async function rotateServiceSecret(options: { await effect( "service-secret-atomic-swap", async () => { - await rename(currentPath, retainedPath); + await link(currentPath, retainedPath); + rotationState.retainedLinked = true; try { await rename(candidatePath, currentPath); rotationState.swapped = true; @@ -204,7 +241,8 @@ export async function rotateServiceSecret(options: { await rename(currentPath, candidatePath); rotationState.swapped = false; } - await rename(retainedPath, currentPath); + await unlink(retainedPath); + rotationState.retainedLinked = false; await syncDirectory(secretsRoot); throw error; } @@ -235,6 +273,8 @@ export async function rotateServiceSecret(options: { if (rotationState.swapped) { await rename(currentPath, candidatePath); await rename(retainedPath, currentPath); + rotationState.swapped = false; + rotationState.retainedLinked = false; await syncDirectory(secretsRoot); } await options.rollback?.(currentPath); diff --git a/tests/e2e/self-hosted-onboarding/repeated-setup.test.ts b/tests/e2e/self-hosted-onboarding/repeated-setup.test.ts index ed81713..fcedab0 100644 --- a/tests/e2e/self-hosted-onboarding/repeated-setup.test.ts +++ b/tests/e2e/self-hosted-onboarding/repeated-setup.test.ts @@ -5,7 +5,10 @@ import { runGuidedSetup, type GuidedSetupResult, } from "../../../src/onboarding/application/setup.js"; -import { unchangedSetupClientResults } from "../../../src/onboarding/application/production-setup.js"; +import { + runRepeatedSetupClientVerification, + unchangedSetupClientResults, +} from "../../../src/onboarding/application/production-setup.js"; describe("unchanged guided setup", () => { it("preserves external ownership classification in the production no-op result", () => { @@ -25,6 +28,19 @@ describe("unchanged guided setup", () => { ]); }); + it("does not adopt a credential for an equivalent external integration", async () => { + const verifyManaged = vi.fn(async () => undefined); + const verifyExternal = vi.fn(async () => undefined); + + await runRepeatedSetupClientVerification("external-verified", { + verifyManaged, + verifyExternal, + }); + + expect(verifyManaged).not.toHaveBeenCalled(); + expect(verifyExternal).toHaveBeenCalledOnce(); + }); + it("is a byte-for-byte no-op for ten repeated executions", async () => { let installed: GuidedSetupResult | undefined; const writes = { diff --git a/tests/integration/onboarding/production-setup.test.ts b/tests/integration/onboarding/production-setup.test.ts index 7299449..cb9afc3 100644 --- a/tests/integration/onboarding/production-setup.test.ts +++ b/tests/integration/onboarding/production-setup.test.ts @@ -7,6 +7,7 @@ import { copyFile, mkdir, readFile, + unlink, writeFile, } from "node:fs/promises"; import { dirname, resolve } from "node:path"; @@ -24,6 +25,7 @@ import { createOnboardingEnvironment, type OnboardingEnvironment, } from "../../helpers/onboarding-environment.js"; +import { snapshotTree } from "../../helpers/filesystem-snapshot.js"; import { createFakeExecutables } from "../../helpers/onboarding-executables.js"; import { bundleV03Fixture, @@ -358,6 +360,63 @@ describe("real disposable production setup", () => { expect( await readFile(resolve(fixture.home, ".claude.json"), "utf8"), ).toContain('"skillwire"'); + + const stateBeforeRepeat = await snapshotTree( + resolve(fixture.xdgStateHome, "skillwire"), + ); + const profilesBeforeRepeat = await snapshotTree(fixture.home); + const unchanged = await runProductionSetup( + { clients: "none", credentialBackend: "not-selected" }, + new AbortController().signal, + clientEnvironment, + { pinnedInitialPolicySha256: sha256(policyBytes) }, + ); + expect(unchanged).toMatchObject({ + status: "success", + serviceReady: true, + clients: [], + changed: false, + }); + expect( + await snapshotTree(resolve(fixture.xdgStateHome, "skillwire")), + ).toEqual(stateBeforeRepeat); + expect(await snapshotTree(fixture.home)).toEqual(profilesBeforeRepeat); + + const claudeProfilePath = resolve(fixture.home, ".claude.json"); + const claudeProfile = await readFile(claudeProfilePath); + await writeFile(claudeProfilePath, "{}", { mode: 0o600 }); + await expect( + runProductionSetup( + { clients: "none", credentialBackend: "not-selected" }, + new AbortController().signal, + clientEnvironment, + { pinnedInitialPolicySha256: sha256(policyBytes) }, + ), + ).rejects.toThrow(/claude|integration|profile|registration/i); + expect( + await snapshotTree(resolve(fixture.xdgStateHome, "skillwire")), + ).toEqual(stateBeforeRepeat); + await writeFile(claudeProfilePath, claudeProfile, { mode: 0o600 }); + + await unlink( + resolve( + fixture.stateRoot, + "credentials", + clientResult.installationId, + "claude.key", + ), + ); + await expect( + runProductionSetup( + { clients: "none", credentialBackend: "not-selected" }, + new AbortController().signal, + clientEnvironment, + { pinnedInitialPolicySha256: sha256(policyBytes) }, + ), + ).rejects.toThrow(/credential|bridge|unavailable/i); + expect( + await snapshotTree(resolve(fixture.xdgStateHome, "skillwire")), + ).toEqual(stateBeforeRepeat); }, 360_000, ); diff --git a/tests/integration/onboarding/repeated-production-setup.test.ts b/tests/integration/onboarding/repeated-production-setup.test.ts new file mode 100644 index 0000000..de16b61 --- /dev/null +++ b/tests/integration/onboarding/repeated-production-setup.test.ts @@ -0,0 +1,215 @@ +import { randomUUID } from "node:crypto"; +import { chmod, copyFile, mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { buildSelfHostedRelease } from "../../../scripts/build-self-hosted-release.js"; +import { runProductionSetup } from "../../../src/onboarding/application/production-setup.js"; +import { + currentProcessIdentity, + InstallationLock, +} from "../../../src/onboarding/domain/operation-journal.js"; +import { snapshotTree } from "../../helpers/filesystem-snapshot.js"; +import { + createOnboardingEnvironment, + type OnboardingEnvironment, +} from "../../helpers/onboarding-environment.js"; +import { createFakeExecutables } from "../../helpers/onboarding-executables.js"; +import { + bundleV03Fixture, + canonicalJson, + RELEASE_PAYLOAD_FILES, + releasePayloadMode, + sha256, + trustedRootFixture, + trustPolicyFixture, +} from "../../helpers/self-hosted-release-fixtures.js"; + +async function preparePersistedSetup(fixture: OnboardingEnvironment): Promise<{ + readonly environment: NodeJS.ProcessEnv; + readonly installationId: string; + readonly pinnedInitialPolicySha256: string; + readonly stateRoot: string; +}> { + const candidateDirectory = resolve(fixture.root, "candidate"); + const candidateRoot = resolve( + candidateDirectory, + "skillwire-0.1.0-test.1-linux-amd64", + ); + await mkdir(candidateRoot, { recursive: true, mode: 0o700 }); + for (const [path, contents] of Object.entries(RELEASE_PAYLOAD_FILES)) { + const target = resolve(candidateRoot, path); + await mkdir(dirname(target), { recursive: true, mode: 0o700 }); + await writeFile(target, contents, { mode: releasePayloadMode(path) }); + await chmod(target, releasePayloadMode(path)); + } + + const trustedRootText = canonicalJson(trustedRootFixture()); + await writeFile( + resolve(candidateRoot, "distribution/self-hosted/trusted-root.v1.json"), + trustedRootText, + { mode: 0o644 }, + ); + const fake = await createFakeExecutables(fixture.root); + const cosignPath = resolve(candidateRoot, "tools/cosign"); + await mkdir(dirname(cosignPath), { recursive: true, mode: 0o700 }); + await copyFile(fake.cosign, cosignPath); + await chmod(cosignPath, 0o755); + const policy = trustPolicyFixture({ + trustedRoot: { + path: "trusted-root.v1.json", + sha256: sha256(trustedRootText), + mediaType: "application/vnd.dev.sigstore.trustedroot+json;version=0.1", + }, + cosign: { + version: "3.1.3", + binaries: { + amd64: sha256(await readFile(cosignPath)), + arm64: "a".repeat(64), + }, + }, + }); + const policyPath = resolve( + candidateDirectory, + "skillwire-trust-policy-v1.json", + ); + const policyBytes = canonicalJson(policy); + await writeFile(policyPath, policyBytes, { mode: 0o600 }); + const built = await buildSelfHostedRelease({ + payloadRoot: candidateRoot, + outputDirectory: candidateDirectory, + architecture: "amd64", + releaseVersion: "0.1.0-test.1", + releaseSequence: 1, + publishedAt: "2026-08-13T00:00:00.000Z", + sourceCommit: "1".repeat(40), + trustPolicySequence: 1, + trustPolicyPath: policyPath, + images: [ + { + role: "skillwire", + repository: "ghcr.io/lucenx9/skillwire", + digest: `sha256:${sha256("image")}`, + platform: "linux/amd64", + }, + { + role: "postgres", + repository: "docker.io/library/postgres", + digest: `sha256:${sha256("postgres-image")}`, + platform: "linux/amd64", + }, + ], + }); + await writeFile( + resolve( + candidateDirectory, + "skillwire-0.1.0-test.1-linux-amd64.release.sigstore.json", + ), + canonicalJson(bundleV03Fixture(built.manifest)), + { mode: 0o600 }, + ); + + const stateRoot = resolve(fixture.xdgStateHome, "skillwire"); + await mkdir(stateRoot, { recursive: true, mode: 0o700 }); + const installationId = randomUUID(); + const now = new Date().toISOString(); + const socketPath = resolve(fixture.runtimeRoot, "skillwire/absent/mcp.sock"); + await writeFile( + resolve(stateRoot, "installation.json"), + JSON.stringify({ + schemaVersion: "skillwire.installation/v1", + installationId, + ownerUid: process.getuid?.() ?? 1000, + accountId: randomUUID(), + activeReleaseId: "1-amd64", + highestAcceptedReleaseSequence: 1, + activeTrustPolicySequence: 1, + endpoint: `unix://${socketPath}`, + composeProject: fixture.composeProject, + postgresVolume: fixture.postgresVolume, + selectedClients: [], + clientIntegrationIds: { codex: null, claude: null }, + status: "service-ready", + createdAt: now, + updatedAt: now, + lastValidatedAt: now, + }), + { mode: 0o600 }, + ); + await writeFile( + resolve(stateRoot, "client-integrations.json"), + JSON.stringify({ + schemaVersion: "skillwire.client-integrations/v1", + installationId, + integrations: [], + }), + { mode: 0o600 }, + ); + return { + environment: { + ...fixture.environment, + SKILLWIRE_RELEASE_ROOT: candidateRoot, + }, + installationId, + pinnedInitialPolicySha256: sha256(policyBytes), + stateRoot, + }; +} + +describe("repeated production setup live verification", () => { + let fixture: OnboardingEnvironment | undefined; + + afterEach(async () => fixture?.close()); + + it("refuses persisted service-ready metadata when the live service is absent", async () => { + fixture = await createOnboardingEnvironment(); + const setup = await preparePersistedSetup(fixture); + const stateBefore = await snapshotTree(setup.stateRoot); + + await expect( + runProductionSetup( + { clients: "none", credentialBackend: "not-selected" }, + new AbortController().signal, + setup.environment, + { + pinnedInitialPolicySha256: setup.pinnedInitialPolicySha256, + }, + ), + ).rejects.toThrow(/deployment|live|ready|service|state/i); + + expect(await snapshotTree(setup.stateRoot)).toEqual(stateBefore); + await expect( + readFile(resolve(fixture.home, ".codex/config.toml"), "utf8"), + ).rejects.toMatchObject({ code: "ENOENT" }); + await expect( + readFile(resolve(fixture.home, ".claude.json"), "utf8"), + ).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("acquires the installation lock before deciding a setup is unchanged", async () => { + fixture = await createOnboardingEnvironment(); + const setup = await preparePersistedSetup(fixture); + const lockRoot = resolve(fixture.runtimeRoot, "skillwire/locks"); + await mkdir(lockRoot, { recursive: true, mode: 0o700 }); + const held = await InstallationLock.acquire( + lockRoot, + "installation", + await currentProcessIdentity(), + ); + try { + await expect( + runProductionSetup( + { clients: "none", credentialBackend: "not-selected" }, + new AbortController().signal, + setup.environment, + { + pinnedInitialPolicySha256: setup.pinnedInitialPolicySha256, + }, + ), + ).rejects.toThrow(/locked|operation/i); + } finally { + await held.release(); + } + }); +}); diff --git a/tests/integration/onboarding/service-secret-rotation.test.ts b/tests/integration/onboarding/service-secret-rotation.test.ts index c2246d4..aec6143 100644 --- a/tests/integration/onboarding/service-secret-rotation.test.ts +++ b/tests/integration/onboarding/service-secret-rotation.test.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/require-await, @typescript-eslint/no-confusing-void-expression -- Async fakes mirror production rotation interfaces. */ import { randomUUID } from "node:crypto"; -import { lstat, readFile, readdir } from "node:fs/promises"; +import { lstat, readFile, readdir, writeFile } from "node:fs/promises"; import { resolve } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -31,7 +31,12 @@ describe("explicit service-secret rotation", () => { "installations", installationId, ); - await ensureServiceSecrets(installationRoot, fixture.stateRoot); + const references = await ensureServiceSecrets( + installationRoot, + fixture.stateRoot, + ); + const reference = references.find((entry) => entry.kind === kind); + if (reference === undefined) throw new Error("Missing service secret"); const currentPath = resolve(installationRoot, "secrets", kind); const before = await readFile(currentPath, "utf8"); const siblingKind = @@ -40,7 +45,11 @@ describe("explicit service-secret rotation", () => { : "database-password"; const siblingPath = resolve(installationRoot, "secrets", siblingKind); const siblingBefore = await readFile(siblingPath, "utf8"); - const preview = previewServiceSecretRotation({ installationId, kind }); + const preview = previewServiceSecretRotation({ + installationId, + kind, + currentIdentitySha256: reference.identitySha256, + }); const events: string[] = []; const result = await rotateServiceSecret({ @@ -91,6 +100,165 @@ describe("explicit service-secret rotation", () => { expect(await snapshotTree(fixture.root)).toEqual(before); }); + it("rejects current-secret identity drift before any rotation effect", async () => { + fixture = await createOnboardingEnvironment(); + const installationId = randomUUID(); + const installationRoot = resolve( + fixture.stateRoot, + "installations", + installationId, + ); + const references = await ensureServiceSecrets( + installationRoot, + fixture.stateRoot, + ); + const reference = references.find( + ({ kind }) => kind === "database-password", + ); + if (reference === undefined) throw new Error("Missing database secret"); + const currentPath = resolve(installationRoot, "secrets/database-password"); + const driftedValue = "D".repeat(43); + await writeFile(currentPath, driftedValue, "ascii"); + const beforeNames = await readdir(resolve(installationRoot, "secrets")); + const preview = previewServiceSecretRotation({ + installationId, + kind: "database-password", + currentIdentitySha256: reference.identitySha256, + }); + const apply = vi.fn(async () => undefined); + const publish = vi.fn(async () => undefined); + + await expect( + rotateServiceSecret({ + installationRoot, + stateRoot: fixture.stateRoot, + kind: "database-password", + confirmation: preview.previewHash, + preview, + signal: new AbortController().signal, + apply, + readiness: async () => undefined, + publish, + }), + ).rejects.toThrow(/identity|drift|changed/i); + + expect(apply).not.toHaveBeenCalled(); + expect(publish).not.toHaveBeenCalled(); + expect(await readFile(currentPath, "ascii")).toBe(driftedValue); + expect(await readdir(resolve(installationRoot, "secrets"))).toEqual( + beforeNames, + ); + }); + + it("preserves a pre-existing candidate residue when exclusive creation fails", async () => { + fixture = await createOnboardingEnvironment(); + const installationId = randomUUID(); + const installationRoot = resolve( + fixture.stateRoot, + "installations", + installationId, + ); + const references = await ensureServiceSecrets( + installationRoot, + fixture.stateRoot, + ); + const reference = references.find( + ({ kind }) => kind === "database-password", + ); + if (reference === undefined) throw new Error("Missing database secret"); + const currentPath = resolve(installationRoot, "secrets/database-password"); + const currentBefore = await readFile(currentPath, "ascii"); + const preview = previewServiceSecretRotation({ + installationId, + kind: "database-password", + currentIdentitySha256: reference.identitySha256, + }); + const candidatePath = resolve( + installationRoot, + "secrets", + `database-password.candidate-${preview.operationId}`, + ); + const residue = "C".repeat(43); + await writeFile(candidatePath, residue, { encoding: "ascii", mode: 0o600 }); + const apply = vi.fn(async () => undefined); + const publish = vi.fn(async () => undefined); + + await expect( + rotateServiceSecret({ + installationRoot, + stateRoot: fixture.stateRoot, + kind: "database-password", + confirmation: preview.previewHash, + preview, + signal: new AbortController().signal, + apply, + readiness: async () => undefined, + publish, + }), + ).rejects.toThrow(/exist|candidate|rotation/i); + + expect(await readFile(candidatePath, "ascii")).toBe(residue); + expect(await readFile(currentPath, "ascii")).toBe(currentBefore); + expect(apply).not.toHaveBeenCalled(); + expect(publish).not.toHaveBeenCalled(); + }); + + it("fails closed without replacing a pre-existing retained target", async () => { + fixture = await createOnboardingEnvironment(); + const installationId = randomUUID(); + const installationRoot = resolve( + fixture.stateRoot, + "installations", + installationId, + ); + const references = await ensureServiceSecrets( + installationRoot, + fixture.stateRoot, + ); + const reference = references.find( + ({ kind }) => kind === "application-pepper", + ); + if (reference === undefined) throw new Error("Missing application secret"); + const currentPath = resolve(installationRoot, "secrets/application-pepper"); + const currentBefore = await readFile(currentPath, "ascii"); + const preview = previewServiceSecretRotation({ + installationId, + kind: "application-pepper", + currentIdentitySha256: reference.identitySha256, + }); + const retainedPath = resolve( + installationRoot, + "secrets", + `application-pepper.retained-${preview.operationId}`, + ); + const retainedBefore = "R".repeat(43); + await writeFile(retainedPath, retainedBefore, { + encoding: "ascii", + mode: 0o600, + }); + const apply = vi.fn(async () => undefined); + const publish = vi.fn(async () => undefined); + + await expect( + rotateServiceSecret({ + installationRoot, + stateRoot: fixture.stateRoot, + kind: "application-pepper", + confirmation: preview.previewHash, + preview, + signal: new AbortController().signal, + apply, + readiness: async () => undefined, + publish, + }), + ).rejects.toThrow(/exist|retained|rotation/i); + + expect(await readFile(retainedPath, "ascii")).toBe(retainedBefore); + expect(await readFile(currentPath, "ascii")).toBe(currentBefore); + expect(apply).not.toHaveBeenCalled(); + expect(publish).not.toHaveBeenCalled(); + }); + it("rolls application configuration back at a failed readiness boundary", async () => { fixture = await createOnboardingEnvironment(); const installationId = randomUUID(); @@ -99,12 +267,20 @@ describe("explicit service-secret rotation", () => { "installations", installationId, ); - await ensureServiceSecrets(installationRoot, fixture.stateRoot); + const references = await ensureServiceSecrets( + installationRoot, + fixture.stateRoot, + ); + const reference = references.find( + ({ kind }) => kind === "database-password", + ); + if (reference === undefined) throw new Error("Missing database secret"); const currentPath = resolve(installationRoot, "secrets/database-password"); const before = await readFile(currentPath, "utf8"); const preview = previewServiceSecretRotation({ installationId, kind: "database-password", + currentIdentitySha256: reference.identitySha256, }); const rollback = vi.fn(async () => undefined); @@ -147,12 +323,20 @@ describe("explicit service-secret rotation", () => { "installations", installationId, ); - await ensureServiceSecrets(installationRoot, fixture.stateRoot); + const references = await ensureServiceSecrets( + installationRoot, + fixture.stateRoot, + ); + const reference = references.find( + ({ kind: secretKind }) => secretKind === "application-pepper", + ); + if (reference === undefined) throw new Error("Missing application secret"); const currentPath = resolve(installationRoot, "secrets/application-pepper"); const before = await readFile(currentPath, "utf8"); const preview = previewServiceSecretRotation({ installationId, kind: "application-pepper", + currentIdentitySha256: reference.identitySha256, }); let applyCount = 0; let readinessCount = 0; From cf99d85af9f850d7c2c259b68b3c3d6e2fe572d6 Mon Sep 17 00:00:00 2001 From: Lucenx9 Date: Fri, 14 Aug 2026 17:06:22 +0200 Subject: [PATCH 4/7] fix: enforce restore and upgrade trust boundaries --- src/onboarding/adapters/docker/deployment.ts | 36 +- src/onboarding/adapters/docker/environment.ts | 64 +++ src/onboarding/adapters/postgres/backup.ts | 64 ++- .../adapters/postgres/restore-validation.ts | 406 ++++++++++++++++++ .../application/production-lifecycle.ts | 354 ++++++++++----- .../application/upgrade-recovery.ts | 19 +- src/onboarding/application/upgrade.ts | 45 +- .../upgrade-preservation.test.ts | 4 +- .../backup-restore-validation.test.ts | 163 ++++++- .../onboarding/service-setup.test.ts | 38 ++ .../onboarding/upgrade-compatible.test.ts | 4 +- .../upgrade-forward-only-010.test.ts | 132 +++++- .../onboarding/upgrade-interruption.test.ts | 19 +- .../onboarding/docker-environment.test.ts | 85 +++- .../upgrade-trust-downgrade.test.ts | 8 +- .../restored-database-validation.test.ts | 326 ++++++++++++++ 16 files changed, 1582 insertions(+), 185 deletions(-) create mode 100644 src/onboarding/adapters/postgres/restore-validation.ts create mode 100644 tests/unit/onboarding/restored-database-validation.test.ts diff --git a/src/onboarding/adapters/docker/deployment.ts b/src/onboarding/adapters/docker/deployment.ts index 663c9e7..bdd7f70 100644 --- a/src/onboarding/adapters/docker/deployment.ts +++ b/src/onboarding/adapters/docker/deployment.ts @@ -152,21 +152,27 @@ export class DeploymentAdapter { ); if (composeVersion === null || Number(composeVersion[1]) < 2) throw new Error("Unsupported Docker Compose version"); - const context = ( - await this.command(["context", "show"], signal) - ).stdout.trim(); - const endpoint = ( - await this.command( - [ - "context", - "inspect", - context, - "--format", - "{{.Endpoints.docker.Host}}", - ], - signal, - ) - ).stdout.trim(); + const routedEnvironment = dockerProcessEnvironment( + this.options.hostEnvironment ?? {}, + ); + const pinnedEndpoint = + this.options.hostEnvironment?.["DOCKER_CONTEXT"] === undefined + ? routedEnvironment["DOCKER_HOST"] + : undefined; + const endpoint = + pinnedEndpoint ?? + ( + await this.command( + [ + "context", + "inspect", + (await this.command(["context", "show"], signal)).stdout.trim(), + "--format", + "{{.Endpoints.docker.Host}}", + ], + signal, + ) + ).stdout.trim(); if (!endpoint.startsWith("unix://") && !endpoint.startsWith("npipe://")) throw new Error( "A local Docker context is required; remote contexts are refused", diff --git a/src/onboarding/adapters/docker/environment.ts b/src/onboarding/adapters/docker/environment.ts index 3b4a082..6704415 100644 --- a/src/onboarding/adapters/docker/environment.ts +++ b/src/onboarding/adapters/docker/environment.ts @@ -1,5 +1,11 @@ import { isAbsolute } from "node:path"; +import { + runCommand, + type CommandOptions, + type CommandResult, +} from "../process/command-runner.js"; + const ROUTING_KEYS = [ "HOME", "XDG_CONFIG_HOME", @@ -63,3 +69,61 @@ export function dockerProcessEnvironment( } return result; } + +export async function assertLocalDockerContext(options: { + readonly dockerExecutable: string; + readonly environment: NodeJS.ProcessEnv; + readonly signal: AbortSignal; + readonly run?: + ((options: CommandOptions) => Promise) | undefined; +}): Promise { + if (!isAbsolute(options.dockerExecutable)) + throw new Error("Docker executable must be absolute"); + if (options.signal.aborted) throw new Error("Docker context check cancelled"); + const routedEnvironment = dockerProcessEnvironment(options.environment); + if (options.environment["DOCKER_CONTEXT"] === undefined) { + const explicitHost = routedEnvironment["DOCKER_HOST"]; + if (explicitHost !== undefined) return explicitHost; + } + const run = options.run ?? runCommand; + const command = (args: readonly string[]) => + run({ + executable: options.dockerExecutable, + args, + environment: routedEnvironment, + deadlineMilliseconds: 10_000, + maximumOutputBytes: 16 * 1024, + signal: options.signal, + }); + const context = (await command(["context", "show"])).stdout.trim(); + if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/.test(context)) + throw new Error("Effective Docker context identity is invalid"); + const endpoint = ( + await command([ + "context", + "inspect", + context, + "--format", + "{{.Endpoints.docker.Host}}", + ]) + ).stdout.trim(); + if ( + endpoint.length === 0 || + endpoint.length > 4096 || + /[\0\r\n]/.test(endpoint) || + (!endpoint.startsWith("unix://") && !endpoint.startsWith("npipe://")) + ) + throw new Error( + "A local Docker context is required; remote contexts are refused", + ); + return endpoint; +} + +export function pinLocalDockerEndpoint( + environment: NodeJS.ProcessEnv, + endpoint: string, +): NodeJS.ProcessEnv { + const pinned: NodeJS.ProcessEnv = { ...environment, DOCKER_HOST: endpoint }; + delete pinned["DOCKER_CONTEXT"]; + return pinned; +} diff --git a/src/onboarding/adapters/postgres/backup.ts b/src/onboarding/adapters/postgres/backup.ts index 0e8ef57..94e3ff1 100644 --- a/src/onboarding/adapters/postgres/backup.ts +++ b/src/onboarding/adapters/postgres/backup.ts @@ -12,7 +12,14 @@ import { validateOwnedDirectory, validateOwnedPath, } from "../filesystem/safe-paths.js"; -import { dockerProcessEnvironment } from "../docker/environment.js"; +import { + assertLocalDockerContext, + dockerProcessEnvironment, + pinLocalDockerEndpoint, +} from "../docker/environment.js"; +import type { RestoredDatabaseValidation } from "./restore-validation.js"; + +export type { RestoredDatabaseValidation } from "./restore-validation.js"; const COMPOSE_KEYS = [ "SKILLWIRE_COMPOSE_PROJECT", @@ -26,13 +33,6 @@ const COMPOSE_KEYS = [ "SKILLWIRE_RUNTIME_GID", ] as const; -export interface RestoredDatabaseValidation { - readonly latestMigration: string; - readonly invariantsValid: boolean; - readonly catalogValid: boolean; - readonly ready: boolean; -} - export interface PostgresBackupOptions { readonly dockerExecutable: string; readonly composePath: string; @@ -87,8 +87,8 @@ export class PostgresBackupAdapter { args: readonly string[], signal: AbortSignal, deadlineMilliseconds = 120_000, + ambient: NodeJS.ProcessEnv = this.options.environment ?? {}, ): Promise { - const ambient = this.options.environment ?? {}; const explicit: Record = {}; for (const key of COMPOSE_KEYS) { const value = ambient[key]; @@ -107,6 +107,7 @@ export class PostgresBackupAdapter { private async waitForValidationDatabase( containerName: string, signal: AbortSignal, + environment: NodeJS.ProcessEnv, ): Promise { let lastError: unknown; let consecutiveReadyChecks = 0; @@ -123,6 +124,7 @@ export class PostgresBackupAdapter { ], signal, 2_000, + environment, ); consecutiveReadyChecks += 1; if (consecutiveReadyChecks >= 2) return; @@ -168,6 +170,16 @@ export class PostgresBackupAdapter { resolve(this.options.composePath), ]; try { + const endpoint = await assertLocalDockerContext({ + dockerExecutable: this.options.dockerExecutable, + environment: this.options.environment ?? {}, + signal, + run: this.run, + }); + const operationEnvironment = pinLocalDockerEndpoint( + this.options.environment ?? {}, + endpoint, + ); await this.command( [ ...compose, @@ -183,16 +195,22 @@ export class PostgresBackupAdapter { `--file=${containerArchive}`, ], signal, + 120_000, + operationEnvironment, ); try { await this.command( [...compose, "cp", `postgres:${containerArchive}`, archivePath], signal, + 120_000, + operationEnvironment, ); } finally { await this.command( [...compose, "exec", "-T", "postgres", "rm", "-f", containerArchive], AbortSignal.timeout(30_000), + 120_000, + operationEnvironment, ).catch(() => undefined); } await chmod(archivePath, 0o600); @@ -227,7 +245,12 @@ export class PostgresBackupAdapter { const validationVolume = `${validationContainer}_data`; let validation: RestoredDatabaseValidation | undefined; try { - await this.command(["volume", "create", validationVolume], signal); + await this.command( + ["volume", "create", validationVolume], + signal, + 120_000, + operationEnvironment, + ); await this.command( [ "run", @@ -243,8 +266,14 @@ export class PostgresBackupAdapter { this.options.postgresImage, ], signal, + 120_000, + operationEnvironment, + ); + await this.waitForValidationDatabase( + validationContainer, + signal, + operationEnvironment, ); - await this.waitForValidationDatabase(validationContainer, signal); await this.command( [ "cp", @@ -252,6 +281,8 @@ export class PostgresBackupAdapter { `${validationContainer}:/tmp/${basename(archivePath)}`, ], signal, + 120_000, + operationEnvironment, ); await this.command( [ @@ -267,6 +298,8 @@ export class PostgresBackupAdapter { `/tmp/${basename(archivePath)}`, ], signal, + 120_000, + operationEnvironment, ); validation = await this.options.validateRestoredDatabase( validationContainer, @@ -275,8 +308,11 @@ export class PostgresBackupAdapter { if ( validation.latestMigration !== (this.options.expectedLatestMigration ?? "010") || - !validation.invariantsValid || + !validation.migrationInventoryValid || + !validation.constraintsValid || !validation.catalogValid || + !validation.advisoryValid || + !validation.authoritativeStateValid || !validation.ready ) throw new Error("Restored backup did not pass readiness invariants"); @@ -289,10 +325,14 @@ export class PostgresBackupAdapter { await this.command( ["container", "rm", "--force", validationContainer], cleanupSignal, + 120_000, + operationEnvironment, ).catch(() => undefined); await this.command( ["volume", "rm", validationVolume], cleanupSignal, + 120_000, + operationEnvironment, ).catch(() => undefined); } return { backupId, archivePath, archiveSha256, validation }; diff --git a/src/onboarding/adapters/postgres/restore-validation.ts b/src/onboarding/adapters/postgres/restore-validation.ts new file mode 100644 index 0000000..21e8e58 --- /dev/null +++ b/src/onboarding/adapters/postgres/restore-validation.ts @@ -0,0 +1,406 @@ +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { open, readdir } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { z } from "zod"; + +import { verifyExternalAdvisoryChain } from "../../../domain/external-catalog/external-advisory-chain.js"; +import { + runCommand, + type CommandOptions, + type CommandResult, +} from "../process/command-runner.js"; +import { dockerProcessEnvironment } from "../docker/environment.js"; + +const MigrationSchema = z.object({ + version: z.string().regex(/^\d{3}$/), + checksum: z.string().regex(/^[0-9a-f]{64}$/), +}); + +export interface ExpectedMigration { + readonly version: string; + readonly checksum: string; +} + +const AdvisoryEventSchema = z.object({ + sequence: z.string().regex(/^[1-9]\d*$/), + previousEventSha256: z.string().regex(/^[0-9a-f]{64}$/), + revisionId: z.uuid(), + kind: z.enum(["availability", "security"]), + status: z.enum(["available", "unavailable", "revoked"]), + reasonCode: z.string().min(1).max(80), + effectiveAt: z.iso.datetime({ offset: true }), + eventSha256: z.string().regex(/^[0-9a-f]{64}$/), +}); + +const EvidenceSchema = z.object({ + currentDatabase: z.string(), + inRecovery: z.boolean(), + transactionReadOnly: z.string(), + migrations: z.array(MigrationSchema).max(999), + constraints: z.array(z.string().min(1).max(128)).max(4096), + triggers: z.array(z.string().min(1).max(128)).max(4096), + catalog: z.object({ + snapshotCount: z.number().int().nonnegative(), + revisionCount: z.number().int().nonnegative(), + resourceCount: z.number().int().nonnegative(), + dependencyCount: z.number().int().nonnegative(), + contentObjectCount: z.number().int().nonnegative(), + identitySha256: z.string().regex(/^[0-9a-f]{64}$/), + invalidSnapshotCounts: z.number().int().nonnegative(), + invalidPublishedPointers: z.number().int().nonnegative(), + invalidContentLengths: z.number().int().nonnegative(), + invalidContentHashes: z.number().int().nonnegative(), + invalidSnapshotAdvisoryHeads: z.number().int().nonnegative(), + }), + advisory: z.object({ + lastSequence: z.string().regex(/^\d+$/), + lastEventSha256: z.string().regex(/^[0-9a-f]{64}$/), + events: z.array(AdvisoryEventSchema).max(100_000), + }), + authoritativeState: z.object({ + installationAccountStatus: z.enum(["active", "disabled"]).nullable(), + activeApiKeyCount: z.number().int().nonnegative(), + repositoryUsageRows: z.number().int().nonnegative(), + repositoryErasureRows: z.number().int().nonnegative(), + }), +}); + +export type RestoredDatabaseEvidence = z.infer; + +export interface DatabaseStateExpectation { + readonly catalog: { + readonly snapshotCount: number; + readonly revisionCount: number; + readonly resourceCount: number; + readonly dependencyCount: number; + readonly contentObjectCount: number; + readonly identitySha256: string; + }; + readonly advisory: { + readonly lastSequence: string; + readonly lastEventSha256: string; + }; + readonly authoritativeState: RestoredDatabaseEvidence["authoritativeState"]; +} + +export interface RestoredDatabaseValidation { + readonly latestMigration: string; + readonly migrationInventoryValid: boolean; + readonly constraintsValid: boolean; + readonly catalogValid: boolean; + readonly advisoryValid: boolean; + readonly authoritativeStateValid: boolean; + readonly ready: boolean; +} + +const REQUIRED_CONSTRAINTS = [ + "accounts_pkey", + "accounts_status_check", + "api_keys_account_id_fkey", + "api_keys_pkey", + "api_keys_public_id_key", + "external_advisory_chain_head_pkey", + "external_revision_advisory_events_pkey", + "external_revision_dependencies_pkey", + "external_revision_resources_pkey", + "external_skill_revisions_pkey", + "schema_migrations_checksum_check", + "schema_migrations_pkey", +] as const; + +const REQUIRED_TRIGGERS = [ + "external_advisory_append_valid", + "external_advisory_events_immutable", + "external_dependencies_immutable", + "external_resources_immutable", + "external_revisions_immutable", + "external_snapshots_immutable", +] as const; + +export async function expectedMigrationInventory( + directory: string, + latestMigration: string, +): Promise { + if (!/^\d{3}$/.test(latestMigration) || latestMigration === "000") + throw new Error("Expected latest migration identity is invalid"); + const resolvedDirectory = resolve(directory); + const directoryHandle = await open( + resolvedDirectory, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ).catch((error: unknown) => { + throw new Error("Expected migration directory is unsafe", { + cause: error, + }); + }); + try { + const directoryStats = await directoryHandle.stat(); + if ( + !directoryStats.isDirectory() || + directoryStats.uid !== process.getuid?.() || + (directoryStats.mode & 0o022) !== 0 + ) + throw new Error("Expected migration directory is unsafe"); + const names = (await readdir(resolvedDirectory)) + .filter((name) => /^\d{3}_[a-z0-9_]+\.sql$/.test(name)) + .filter((name) => name.slice(0, 3) <= latestMigration) + .toSorted(); + const expectedVersions = Array.from( + { length: Number(latestMigration) }, + (_, index) => String(index + 1).padStart(3, "0"), + ); + if ( + names.length !== expectedVersions.length || + names.some((name, index) => name.slice(0, 3) !== expectedVersions[index]) + ) + throw new Error( + "Expected migration inventory is incomplete or ambiguous", + ); + return await Promise.all( + names.map(async (name) => { + const handle = await open( + resolve(directory, name), + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const stats = await handle.stat(); + if (!stats.isFile() || stats.nlink !== 1) + throw new Error( + "Expected migration is not a protected regular file", + ); + return { + version: name.slice(0, 3), + checksum: createHash("sha256") + .update(await handle.readFile()) + .digest("hex"), + }; + } finally { + await handle.close(); + } + }), + ); + } finally { + await directoryHandle.close(); + } +} + +export function assessRestoredDatabaseEvidence( + input: unknown, + expectations: { + readonly expectedMigrations: readonly ExpectedMigration[]; + readonly installationAccountId: string; + readonly expectedActiveApiKeys: number; + readonly expectedDatabase: "postgres" | "skillwire"; + readonly expectedState: DatabaseStateExpectation; + }, +): RestoredDatabaseValidation { + z.uuid().parse(expectations.installationAccountId); + if ( + !Number.isInteger(expectations.expectedActiveApiKeys) || + expectations.expectedActiveApiKeys < 0 + ) + throw new Error("Restore validation expectation is invalid"); + const evidence = EvidenceSchema.parse(input); + const expectedMigrations = z + .array(MigrationSchema) + .parse(expectations.expectedMigrations); + if ( + expectedMigrations.length === 0 || + JSON.stringify(evidence.migrations) !== JSON.stringify(expectedMigrations) + ) + throw new Error("Restored migration inventory or checksum is invalid"); + const constraints = new Set(evidence.constraints); + const triggers = new Set(evidence.triggers); + const constraintsValid = REQUIRED_CONSTRAINTS.every((name) => + constraints.has(name), + ); + const triggersValid = REQUIRED_TRIGGERS.every((name) => triggers.has(name)); + const catalogValid = + evidence.catalog.invalidSnapshotCounts === 0 && + evidence.catalog.invalidPublishedPointers === 0 && + evidence.catalog.invalidContentLengths === 0 && + evidence.catalog.invalidContentHashes === 0 && + evidence.catalog.invalidSnapshotAdvisoryHeads === 0 && + evidence.catalog.snapshotCount === + expectations.expectedState.catalog.snapshotCount && + evidence.catalog.revisionCount === + expectations.expectedState.catalog.revisionCount && + evidence.catalog.resourceCount === + expectations.expectedState.catalog.resourceCount && + evidence.catalog.dependencyCount === + expectations.expectedState.catalog.dependencyCount && + evidence.catalog.contentObjectCount === + expectations.expectedState.catalog.contentObjectCount && + evidence.catalog.identitySha256 === + expectations.expectedState.catalog.identitySha256; + let advisoryValid: boolean; + try { + verifyExternalAdvisoryChain( + evidence.advisory.events, + evidence.advisory.lastEventSha256, + ); + advisoryValid = + evidence.advisory.lastSequence === + String(evidence.advisory.events.length) && + evidence.advisory.lastSequence === + expectations.expectedState.advisory.lastSequence && + evidence.advisory.lastEventSha256 === + expectations.expectedState.advisory.lastEventSha256; + } catch { + advisoryValid = false; + } + const authoritativeStateValid = + evidence.authoritativeState.installationAccountStatus === "active" && + evidence.authoritativeState.activeApiKeyCount === + expectations.expectedActiveApiKeys && + JSON.stringify(evidence.authoritativeState) === + JSON.stringify(expectations.expectedState.authoritativeState); + const ready = + evidence.currentDatabase === expectations.expectedDatabase && + !evidence.inRecovery && + evidence.transactionReadOnly === "off"; + if ( + !constraintsValid || + !triggersValid || + !catalogValid || + !advisoryValid || + !authoritativeStateValid || + !ready + ) + throw new Error("Restored database failed production restore validation"); + return { + latestMigration: expectedMigrations.at(-1)?.version ?? "", + migrationInventoryValid: true, + constraintsValid: true, + catalogValid: true, + advisoryValid: true, + authoritativeStateValid: true, + ready: true, + }; +} + +export function databaseStateExpectation( + input: unknown, +): DatabaseStateExpectation { + const evidence = EvidenceSchema.parse(input); + return { + catalog: { + snapshotCount: evidence.catalog.snapshotCount, + revisionCount: evidence.catalog.revisionCount, + resourceCount: evidence.catalog.resourceCount, + dependencyCount: evidence.catalog.dependencyCount, + contentObjectCount: evidence.catalog.contentObjectCount, + identitySha256: evidence.catalog.identitySha256, + }, + advisory: { + lastSequence: evidence.advisory.lastSequence, + lastEventSha256: evidence.advisory.lastEventSha256, + }, + authoritativeState: evidence.authoritativeState, + }; +} + +function restoredDatabaseEvidenceQuery(installationAccountId: string): string { + const accountId = z.uuid().parse(installationAccountId); + return `SELECT json_build_object( + 'currentDatabase', current_database(), + 'inRecovery', pg_is_in_recovery(), + 'transactionReadOnly', current_setting('transaction_read_only'), + 'migrations', (SELECT COALESCE(json_agg(json_build_object('version',version,'checksum',checksum) ORDER BY version),'[]'::json) FROM schema_migrations), + 'constraints', (SELECT COALESCE(json_agg(conname ORDER BY conname),'[]'::json) FROM pg_constraint JOIN pg_namespace ON pg_namespace.oid=pg_constraint.connamespace WHERE nspname='public'), + 'triggers', (SELECT COALESCE(json_agg(tgname ORDER BY tgname),'[]'::json) FROM pg_trigger JOIN pg_class ON pg_class.oid=pg_trigger.tgrelid JOIN pg_namespace ON pg_namespace.oid=pg_class.relnamespace WHERE nspname='public' AND NOT tgisinternal), + 'catalog', json_build_object( + 'snapshotCount', (SELECT count(*) FROM external_source_snapshots), + 'revisionCount', (SELECT count(*) FROM external_skill_revisions), + 'resourceCount', (SELECT count(*) FROM external_revision_resources), + 'dependencyCount', (SELECT count(*) FROM external_revision_dependencies), + 'contentObjectCount', (SELECT count(*) FROM external_content_objects), + 'identitySha256', (SELECT encode(sha256(convert_to(COALESCE(string_agg(identity,'\n' ORDER BY identity),''),'UTF8')),'hex') FROM (SELECT 'snapshot:'||id::text||':'||source_id::text||':'||commit_sha||':'||tree_sha||':'||COALESCE(advisory_chain_head_sha256,'null') AS identity FROM external_source_snapshots UNION ALL SELECT 'revision:'||id::text||':'||snapshot_id::text||':'||bundle_sha256||':'||content_identity_sha256 FROM external_skill_revisions UNION ALL SELECT 'resource:'||revision_id::text||':'||resource_path||':'||content_sha256 FROM external_revision_resources UNION ALL SELECT 'dependency:'||revision_id::text||':'||target_revision_id::text||':'||evidence_source_sha256 FROM external_revision_dependencies UNION ALL SELECT 'content:'||sha256||':'||byte_length::text FROM external_content_objects) catalog_identity), + 'invalidSnapshotCounts', (SELECT count(*) FROM external_source_snapshots snapshot WHERE snapshot.revision_count<>(SELECT count(*) FROM external_skill_revisions revision WHERE revision.snapshot_id=snapshot.id) OR snapshot.candidate_count<>(SELECT count(*) FROM external_import_candidates candidate WHERE candidate.snapshot_id=snapshot.id) OR snapshot.quarantine_count<>(SELECT count(*) FROM external_import_candidates candidate JOIN external_current_classifications current ON current.candidate_id=candidate.id WHERE candidate.snapshot_id=snapshot.id AND current.classification='quarantined') OR snapshot.resource_count<>(SELECT count(*) FROM external_revision_resources resource JOIN external_skill_revisions revision ON revision.id=resource.revision_id WHERE revision.snapshot_id=snapshot.id) OR snapshot.dependency_count<>(SELECT count(*) FROM external_revision_dependencies dependency JOIN external_skill_revisions revision ON revision.id=dependency.revision_id WHERE revision.snapshot_id=snapshot.id) OR snapshot.decoded_bytes<>(SELECT COALESCE(sum(octet_length(revision.canonical_bytes)),0) FROM external_skill_revisions revision WHERE revision.snapshot_id=snapshot.id)), + 'invalidPublishedPointers', (SELECT count(*) FROM github_sources source JOIN external_source_snapshots snapshot ON snapshot.id=source.current_published_snapshot_id WHERE snapshot.source_id<>source.id), + 'invalidContentLengths', (SELECT count(*) FROM external_content_objects WHERE byte_length<>octet_length(content)), + 'invalidContentHashes', (SELECT count(*) FROM external_content_objects WHERE sha256<>encode(sha256(convert_to(content,'UTF8')),'hex')), + 'invalidSnapshotAdvisoryHeads', (SELECT count(*) FROM external_source_snapshots snapshot WHERE snapshot.advisory_chain_head_sha256 IS NULL OR (snapshot.advisory_chain_head_sha256<>repeat('0',64) AND NOT EXISTS (SELECT 1 FROM external_revision_advisory_events event WHERE event.event_sha256=snapshot.advisory_chain_head_sha256))) + ), + 'advisory', json_build_object( + 'lastSequence', (SELECT last_sequence::text FROM external_advisory_chain_head WHERE singleton), + 'lastEventSha256', (SELECT last_event_sha256 FROM external_advisory_chain_head WHERE singleton), + 'events', (SELECT COALESCE(json_agg(json_build_object('sequence',sequence::text,'previousEventSha256',previous_event_sha256,'revisionId',revision_id::text,'kind',advisory_kind,'status',advisory_status,'reasonCode',reason_code,'effectiveAt',to_char(effective_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'),'eventSha256',event_sha256) ORDER BY sequence),'[]'::json) FROM external_revision_advisory_events) + ), + 'authoritativeState', json_build_object( + 'installationAccountStatus', (SELECT status FROM accounts WHERE id='${accountId}'::uuid), + 'activeApiKeyCount', (SELECT count(*) FROM api_keys WHERE account_id='${accountId}'::uuid AND revoked_at IS NULL AND (expires_at IS NULL OR expires_at>statement_timestamp())), + 'repositoryUsageRows', (SELECT count(*) FROM repository_skill_usage WHERE account_id='${accountId}'::uuid), + 'repositoryErasureRows', (SELECT count(*) FROM repository_erasure_audit WHERE account_id='${accountId}'::uuid) + ) + )::text`; +} + +export async function validateRestoredDatabaseContainer(options: { + readonly dockerExecutable: string; + readonly containerName: string; + readonly environment: NodeJS.ProcessEnv; + readonly signal: AbortSignal; + readonly expectedMigrations: readonly ExpectedMigration[]; + readonly installationAccountId: string; + readonly expectedActiveApiKeys: number; + readonly expectedState: DatabaseStateExpectation; + readonly run?: + ((options: CommandOptions) => Promise) | undefined; +}): Promise { + if (!/^skillwire-backup-validate-[0-9a-f]{16}$/.test(options.containerName)) + throw new Error("Restore-validation container identity is invalid"); + const evidence = await readDatabaseEvidence({ + dockerExecutable: options.dockerExecutable, + dockerArgs: ["exec", options.containerName], + databaseName: "postgres", + databaseUser: "postgres", + environment: options.environment, + signal: options.signal, + installationAccountId: options.installationAccountId, + run: options.run, + }); + return assessRestoredDatabaseEvidence(evidence, { + ...options, + expectedDatabase: "postgres", + }); +} + +export async function readDatabaseEvidence(options: { + readonly dockerExecutable: string; + readonly dockerArgs: readonly string[]; + readonly databaseName: "postgres" | "skillwire"; + readonly databaseUser: "postgres" | "skillwire"; + readonly environment: NodeJS.ProcessEnv; + readonly signal: AbortSignal; + readonly installationAccountId: string; + readonly run?: + ((options: CommandOptions) => Promise) | undefined; +}): Promise { + const result = await (options.run ?? runCommand)({ + executable: resolve(options.dockerExecutable), + args: [ + ...options.dockerArgs, + "psql", + `--username=${options.databaseUser}`, + `--dbname=${options.databaseName}`, + "--tuples-only", + "--no-align", + "--set=ON_ERROR_STOP=1", + "--command", + restoredDatabaseEvidenceQuery(options.installationAccountId), + ], + environment: dockerProcessEnvironment(options.environment), + deadlineMilliseconds: 30_000, + maximumOutputBytes: 16 * 1024 * 1024, + signal: options.signal, + }); + let evidence: unknown; + try { + evidence = JSON.parse(result.stdout.trim()) as unknown; + } catch { + throw new Error("Restored database returned malformed validation evidence"); + } + return EvidenceSchema.parse(evidence); +} diff --git a/src/onboarding/application/production-lifecycle.ts b/src/onboarding/application/production-lifecycle.ts index d35ac66..8032b86 100644 --- a/src/onboarding/application/production-lifecycle.ts +++ b/src/onboarding/application/production-lifecycle.ts @@ -3,10 +3,13 @@ import { constants } from "node:fs"; import { access, chmod, + lstat, + mkdir, open, readdir, realpath, rename, + rm, } from "node:fs/promises"; import { request as httpRequest } from "node:http"; import { basename, dirname, isAbsolute, relative, resolve } from "node:path"; @@ -19,7 +22,11 @@ import { canonicalPreview, confirmPreview } from "../cli/confirmation.js"; import { atomicWriteJson } from "../adapters/filesystem/atomic-state.js"; import { clientComponentIdentity } from "../adapters/clients/client-state.js"; import { DeploymentAdapter } from "../adapters/docker/deployment.js"; -import { dockerProcessEnvironment } from "../adapters/docker/environment.js"; +import { + assertLocalDockerContext, + dockerProcessEnvironment, + pinLocalDockerEndpoint, +} from "../adapters/docker/environment.js"; import { CodexClientAdapter } from "../adapters/clients/codex.js"; import { ClaudeClientAdapter } from "../adapters/clients/claude.js"; import { SecretToolCredentialStore } from "../adapters/credentials/secret-tool.js"; @@ -56,6 +63,13 @@ import { OperationJournal, } from "../domain/operation-journal.js"; import { PostgresBackupAdapter } from "../adapters/postgres/backup.js"; +import { + assessRestoredDatabaseEvidence, + databaseStateExpectation, + expectedMigrationInventory, + readDatabaseEvidence, + validateRestoredDatabaseContainer, +} from "../adapters/postgres/restore-validation.js"; import { installVerifiedRelease, releaseDirectoryIdentity, @@ -82,10 +96,7 @@ import { rotateServiceSecret, } from "./service-secret-rotation.js"; import { backupDirectoryIdentity, createValidatedBackup } from "./backup.js"; -import { - drainWriters, - restartWriters, -} from "../adapters/docker/writer-drain.js"; +import { drainWriters } from "../adapters/docker/writer-drain.js"; import { previewUpgrade, runUpgrade, @@ -1978,6 +1989,54 @@ async function backupOperation( lockedOwnership.recordSha256 !== ownership.recordSha256 ) throw new Error("Backup prerequisites changed after preview"); + const localDockerEndpoint = await assertLocalDockerContext({ + dockerExecutable: "/usr/bin/docker", + environment, + signal, + }); + const operationEnvironment = pinLocalDockerEndpoint( + environment, + localDockerEndpoint, + ); + const liveSchema = await readLiveMigration( + deployment, + operationEnvironment, + signal, + ); + const expectedLatestMigration = String(liveSchema).padStart(3, "0"); + const expectedMigrations = await expectedMigrationInventory( + resolve(deployment.releaseRoot, "migrations"), + expectedLatestMigration, + ); + const activeCredentialReferenceCount = references.credentials.filter( + ({ state }) => state === "available" || state === "retained", + ).length; + const sourceEvidence = await readDatabaseEvidence({ + dockerExecutable: "/usr/bin/docker", + dockerArgs: [ + "compose", + "--project-name", + deployment.projectName, + "--file", + deployment.composePath, + "exec", + "-T", + "postgres", + ], + databaseName: "skillwire", + databaseUser: "skillwire", + environment: deploymentEnvironment(deployment, operationEnvironment), + signal, + installationAccountId: installation.accountId, + }); + const expectedState = databaseStateExpectation(sourceEvidence); + assessRestoredDatabaseEvidence(sourceEvidence, { + expectedMigrations, + installationAccountId: installation.accountId, + expectedActiveApiKeys: activeCredentialReferenceCount, + expectedDatabase: "skillwire", + expectedState, + }); const adapter = new PostgresBackupAdapter({ dockerExecutable: "/usr/bin/docker", composePath: deployment.composePath, @@ -1986,39 +2045,19 @@ async function backupOperation( protectedRoot: roots.dataRoot, backupsRoot, postgresImage: deployment.postgresImage, - environment: deploymentEnvironment(deployment, environment), - validateRestoredDatabase: async (containerName, validationSignal) => { - const query = - "SELECT concat((SELECT max(version) FROM schema_migrations),'|',(to_regclass('public.accounts') IS NOT NULL)::text,'|',(to_regclass('public.external_skill_revisions') IS NOT NULL)::text,'|',(to_regclass('public.external_advisory_chain_head') IS NOT NULL)::text)"; - const inspected = await runCommand({ - executable: "/usr/bin/docker", - args: [ - "exec", - containerName, - "psql", - "--username=postgres", - "--dbname=postgres", - "--tuples-only", - "--no-align", - "--set=ON_ERROR_STOP=1", - "--command", - query, - ], - environment: deploymentEnvironment(deployment, environment), - deadlineMilliseconds: 15_000, - maximumOutputBytes: 16 * 1024, + expectedLatestMigration, + environment: deploymentEnvironment(deployment, operationEnvironment), + validateRestoredDatabase: (containerName, validationSignal) => + validateRestoredDatabaseContainer({ + dockerExecutable: "/usr/bin/docker", + containerName, + environment: deploymentEnvironment(deployment, operationEnvironment), signal: validationSignal, - }); - const [latestMigration, accounts, catalog, advisory] = inspected.stdout - .trim() - .split("|"); - return { - latestMigration: latestMigration ?? "", - invariantsValid: accounts === "true", - catalogValid: catalog === "true" && advisory === "true", - ready: latestMigration === "010", - }; - }, + expectedMigrations, + installationAccountId: installation.accountId, + expectedActiveApiKeys: activeCredentialReferenceCount, + expectedState, + }), }); const backup = await createValidatedBackup({ installationId: installation.installationId, @@ -2314,12 +2353,25 @@ async function upgradeOperation( resolve(roots.stateRoot, "credential-references.json"), ), ); + const localDockerEndpoint = await assertLocalDockerContext({ + dockerExecutable: "/usr/bin/docker", + environment, + signal, + }); + const operationEnvironment = pinLocalDockerEndpoint( + environment, + localDockerEndpoint, + ); const candidate = await verifiedUpgradeCandidate( command, - environment, + operationEnvironment, + signal, + ); + const liveSchema = await readLiveMigration( + deployment, + operationEnvironment, signal, ); - const liveSchema = await readLiveMigration(deployment, environment, signal); const previewInput = { installationId: installation.installationId, currentReleaseSequence: installation.highestAcceptedReleaseSequence, @@ -2366,9 +2418,31 @@ async function upgradeOperation( }); const targetEnvironment = deploymentEnvironment( targetDeployment, - environment, + operationEnvironment, ); - const targetAdapter = new DeploymentAdapter({ + const privateRuntimeSocketDirectory = resolve( + roots.runtimeRoot, + `upgrade-${journal.operationId}`, + ); + const privateTargetDeployment = DeploymentStateSchema.parse({ + ...targetDeployment, + runtimeSocketDirectory: privateRuntimeSocketDirectory, + socketPath: resolve(privateRuntimeSocketDirectory, "mcp.sock"), + }); + const privateTargetAdapter = new DeploymentAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: privateTargetDeployment.composePath, + projectName: privateTargetDeployment.projectName, + volumeName: privateTargetDeployment.volumeName, + skillwireImage: privateTargetDeployment.skillwireImage, + postgresImage: privateTargetDeployment.postgresImage, + databasePasswordFile: privateTargetDeployment.databasePasswordFile, + applicationPepperFile: privateTargetDeployment.applicationPepperFile, + runtimeSocketDirectory: privateTargetDeployment.runtimeSocketDirectory, + socketPath: privateTargetDeployment.socketPath, + hostEnvironment: operationEnvironment, + }); + const publicTargetAdapter = new DeploymentAdapter({ dockerExecutable: "/usr/bin/docker", composePath: targetDeployment.composePath, projectName: targetDeployment.projectName, @@ -2379,7 +2453,7 @@ async function upgradeOperation( applicationPepperFile: targetDeployment.applicationPepperFile, runtimeSocketDirectory: targetDeployment.runtimeSocketDirectory, socketPath: targetDeployment.socketPath, - hostEnvironment: environment, + hostEnvironment: operationEnvironment, }); const composeCommand = ( composePath: string, @@ -2410,7 +2484,7 @@ async function upgradeOperation( const lockedDeployment = await deploymentAt(roots.stateRoot); const lockedSchema = await readLiveMigration( lockedDeployment, - environment, + operationEnvironment, signal, ); if ( @@ -2420,13 +2494,51 @@ async function upgradeOperation( lockedSchema !== liveSchema ) throw new Error("Upgrade prerequisites changed after preview"); + const expectedMigrations = await expectedMigrationInventory( + resolve(deployment.releaseRoot, "migrations"), + String(liveSchema).padStart(3, "0"), + ); + const activeCredentialReferenceCount = references.credentials.filter( + ({ state }) => state === "available" || state === "retained", + ).length; + const sourceEvidence = await readDatabaseEvidence({ + dockerExecutable: "/usr/bin/docker", + dockerArgs: [ + "compose", + "--project-name", + deployment.projectName, + "--file", + deployment.composePath, + "exec", + "-T", + "postgres", + ], + databaseName: "skillwire", + databaseUser: "skillwire", + environment: deploymentEnvironment(deployment, operationEnvironment), + signal, + installationAccountId: installation.accountId, + }); + const expectedState = databaseStateExpectation(sourceEvidence); + assessRestoredDatabaseEvidence(sourceEvidence, { + expectedMigrations, + installationAccountId: installation.accountId, + expectedActiveApiKeys: activeCredentialReferenceCount, + expectedDatabase: "skillwire", + expectedState, + }); + const expectedTargetMigrations = await expectedMigrationInventory( + resolve(candidate.releaseRoot, "migrations"), + String(candidate.target.latestMigration).padStart(3, "0"), + ); const upgraded = await runUpgrade({ preview: upgradePreview, confirmation: command.confirmPreview, signal, journal, verifyTarget: async () => - (await verifiedUpgradeCandidate(command, environment, signal)).target, + (await verifiedUpgradeCandidate(command, operationEnvironment, signal)) + .target, createBackup: async () => { const backupsRoot = resolve( roots.dataRoot, @@ -2442,38 +2554,21 @@ async function upgradeOperation( backupsRoot, postgresImage: deployment.postgresImage, expectedLatestMigration: String(liveSchema).padStart(3, "0"), - environment: deploymentEnvironment(deployment, environment), - validateRestoredDatabase: async (containerName, validationSignal) => { - const inspected = await runCommand({ - executable: "/usr/bin/docker", - args: [ - "exec", - containerName, - "psql", - "--username=postgres", - "--dbname=postgres", - "--tuples-only", - "--no-align", - "--set=ON_ERROR_STOP=1", - "--command", - "SELECT concat((SELECT max(version) FROM schema_migrations),'|',(to_regclass('public.accounts') IS NOT NULL)::text,'|',(to_regclass('public.external_skill_revisions') IS NOT NULL)::text,'|',(to_regclass('public.external_advisory_chain_head') IS NOT NULL)::text)", - ], - environment: deploymentEnvironment(deployment, environment), - deadlineMilliseconds: 15_000, - maximumOutputBytes: 16 * 1024, + environment: deploymentEnvironment(deployment, operationEnvironment), + validateRestoredDatabase: (containerName, validationSignal) => + validateRestoredDatabaseContainer({ + dockerExecutable: "/usr/bin/docker", + containerName, + environment: deploymentEnvironment( + deployment, + operationEnvironment, + ), signal: validationSignal, - }); - const [migration, accounts, catalog, advisory] = inspected.stdout - .trim() - .split("|"); - const expected = String(liveSchema).padStart(3, "0"); - return { - latestMigration: migration ?? "", - invariantsValid: accounts === "true", - catalogValid: catalog === "true" && advisory === "true", - ready: migration === expected, - }; - }, + expectedMigrations, + installationAccountId: installation.accountId, + expectedActiveApiKeys: activeCredentialReferenceCount, + expectedState, + }), }); backup = await createValidatedBackup({ installationId: installation.installationId, @@ -2552,20 +2647,46 @@ async function upgradeOperation( ]); }, verifyLiveSchema: () => - readLiveMigration(targetDeployment, environment, signal), - readiness: async () => { - await targetAdapter.probe(signal); - await targetAdapter.deploy(signal); + readLiveMigration(targetDeployment, operationEnvironment, signal), + preActivationReadiness: async () => { + await privateTargetAdapter.probe(signal); + await mkdir(privateRuntimeSocketDirectory, { mode: 0o700 }); + await privateTargetAdapter.deploy(signal); + const targetEvidence = await readDatabaseEvidence({ + dockerExecutable: "/usr/bin/docker", + dockerArgs: [ + "compose", + "--project-name", + targetDeployment.projectName, + "--file", + targetDeployment.composePath, + "exec", + "-T", + "postgres", + ], + databaseName: "skillwire", + databaseUser: "skillwire", + environment: targetEnvironment, + signal, + installationAccountId: installation.accountId, + }); + assessRestoredDatabaseEvidence(targetEvidence, { + expectedMigrations: expectedTargetMigrations, + installationAccountId: installation.accountId, + expectedActiveApiKeys: activeCredentialReferenceCount, + expectedDatabase: "skillwire", + expectedState, + }); }, verifyClients: async () => { for (const client of installation.selectedClients) { - const vendor = await executable(client, environment); + const vendor = await executable(client, operationEnvironment); const adapter = client === "codex" - ? new CodexClientAdapter(vendor, environment, signal) + ? new CodexClientAdapter(vendor, operationEnvironment, signal) : new ClaudeClientAdapter( vendor, - environment, + operationEnvironment, undefined, undefined, signal, @@ -2593,6 +2714,7 @@ async function upgradeOperation( throw new Error(`${client} integration changed during upgrade`); } }, + activateApplication: () => publicTargetAdapter.deploy(signal), commitSelection: () => atomicWriteJson( resolve(roots.stateRoot, "active-release.json"), @@ -2622,7 +2744,7 @@ async function upgradeOperation( applicationPepperFile: deployment.applicationPepperFile, runtimeSocketDirectory: deployment.runtimeSocketDirectory, socketPath: deployment.socketPath, - hostEnvironment: environment, + hostEnvironment: operationEnvironment, }); await prior.deploy(recoverySignal); }, @@ -2633,19 +2755,6 @@ async function upgradeOperation( AbortSignal.timeout(60_000), ); }, - restartWriters: () => - restartWriters( - { - stopAdministration: () => Promise.resolve(), - stopIngestion: () => Promise.resolve(), - stopApplication: () => Promise.resolve(), - verifyNoWriters: () => Promise.resolve(true), - startApplication: () => Promise.resolve(), - startIngestion: () => Promise.resolve(), - startAdministration: () => Promise.resolve(), - }, - AbortSignal.timeout(60_000), - ), }); if (backup === undefined) throw new Error("Upgrade completed without a restore-validated backup"); @@ -2841,8 +2950,12 @@ async function upgradeOperation( } const restoreRequired = recovery?.rollbackBoundary === "database-restore-required"; + const targetActivationRequired = + error instanceof UpgradeRecoveryError && + error.dataLossBoundary === + "Retry target activation; do not restore the pre-upgrade backup"; const recoveryRequired = upgradeFailureRequiresRecovery( - restoreRequired, + restoreRequired || targetActivationRequired, journal.hasUnprovenEffect(), ); await journal @@ -2854,8 +2967,14 @@ async function upgradeOperation( throw error; return result({ command: "upgrade", - status: restoreRequired ? "recovery-required" : "failure", - exitClass: restoreRequired ? "rollback-required" : "service-failure", + status: + restoreRequired || targetActivationRequired + ? "recovery-required" + : "failure", + exitClass: + restoreRequired || targetActivationRequired + ? "rollback-required" + : "service-failure", previewHash: upgradePreview.previewHash, previewScope: previewInput, changed: true, @@ -2863,10 +2982,14 @@ async function upgradeOperation( components: [], findings: [ { - code: restoreRequired - ? "UPGRADE_RECOVERY_REQUIRED" - : "UPGRADE_AUTOMATIC_ROLLBACK_COMPLETED", - severity: restoreRequired ? "recovery-required" : "error", + code: + restoreRequired || targetActivationRequired + ? "UPGRADE_RECOVERY_REQUIRED" + : "UPGRADE_AUTOMATIC_ROLLBACK_COMPLETED", + severity: + restoreRequired || targetActivationRequired + ? "recovery-required" + : "error", component: "upgrade", summary: error.message, nextAction: recovery.instructions.join("; "), @@ -2875,7 +2998,30 @@ async function upgradeOperation( recovery: { ...recovery, instructions: [...recovery.instructions] }, }); } finally { - await lock.release(); + try { + const privateRuntime = await lstat(privateRuntimeSocketDirectory).catch( + (error: unknown) => { + if ((error as NodeJS.ErrnoException).code === "ENOENT") + return undefined; + throw error; + }, + ); + if (privateRuntime !== undefined) { + if ( + !privateRuntime.isDirectory() || + privateRuntime.isSymbolicLink() || + privateRuntime.uid !== process.getuid?.() || + (privateRuntime.mode & 0o777) !== 0o700 + ) + // The exact private directory cannot be ignored or removed after + // ownership drift; surfacing this intentionally overrides success. + // eslint-disable-next-line no-unsafe-finally + throw new Error("Private upgrade runtime directory is unsafe"); + await rm(privateRuntimeSocketDirectory, { recursive: true }); + } + } finally { + await lock.release(); + } } } diff --git a/src/onboarding/application/upgrade-recovery.ts b/src/onboarding/application/upgrade-recovery.ts index 8a9a7aa..fa40037 100644 --- a/src/onboarding/application/upgrade-recovery.ts +++ b/src/onboarding/application/upgrade-recovery.ts @@ -21,14 +21,21 @@ export function upgradeRecoveryGuidance(error: UpgradeRecoveryError): { rollbackBoundary: error.rollbackBoundary, backupId: error.backupId, instructions: - error.rollbackBoundary === "database-restore-required" + error.dataLossBoundary === + "Retry target activation; do not restore the pre-upgrade backup" ? [ "Keep all writers stopped", - "Restore the named validated backup before selecting an older executable", - "Confirm the erased-memory and data-loss boundary before restore", + "Retry activation of the already committed target release", + "Do not restore the pre-upgrade backup", ] - : [ - "The prior application and configuration were restored automatically", - ], + : error.rollbackBoundary === "database-restore-required" + ? [ + "Keep all writers stopped", + "Restore the named validated backup before selecting an older executable", + "Confirm the erased-memory and data-loss boundary before restore", + ] + : [ + "The prior application and configuration were restored automatically", + ], }; } diff --git a/src/onboarding/application/upgrade.ts b/src/onboarding/application/upgrade.ts index 7f31f3f..9b2849c 100644 --- a/src/onboarding/application/upgrade.ts +++ b/src/onboarding/application/upgrade.ts @@ -72,12 +72,12 @@ export async function runUpgrade(options: { readonly installApplication: () => Promise; readonly migrate: () => Promise; readonly verifyLiveSchema: () => Promise; - readonly readiness: () => Promise; + readonly preActivationReadiness: () => Promise; readonly verifyClients: () => Promise; + readonly activateApplication: () => Promise; readonly commitSelection: () => Promise; readonly rollbackApplication: () => Promise; readonly stopWriters: () => Promise; - readonly restartWriters: () => Promise; readonly journal?: OperationJournal | undefined; }): Promise<{ readonly backupId: string; readonly releaseId: string }> { confirmPreview( @@ -140,6 +140,7 @@ export async function runUpgrade(options: { ensureNotAborted(options.signal, "backup"); let applicationInstalled = false; let writersDrained = false; + let releaseCommitted = false; const migration = { started: false }; try { if (decision.requiresWriterDrain) { @@ -174,26 +175,42 @@ export async function runUpgrade(options: { ); if (liveSchema !== target.latestMigration) throw new Error("Live schema readback did not match the target"); - await effect("upgrade-readiness", options.readiness, () => ({ - ready: true, - })); + await effect( + "upgrade-preactivation-readiness", + options.preActivationReadiness, + () => ({ + ready: true, + }), + ); ensureNotAborted(options.signal, "readiness"); await effect("upgrade-client-verification", options.verifyClients, () => ({ clientsVerified: true, })); ensureNotAborted(options.signal, "client-verification"); - if (writersDrained) { - await effect("upgrade-writer-restart", options.restartWriters, () => ({ - restarted: true, - })); - writersDrained = false; - ensureNotAborted(options.signal, "writer-restart"); - } await effect("upgrade-release-commit", options.commitSelection, () => ({ releaseSequence: target.releaseSequence, })); + releaseCommitted = true; + ensureNotAborted(options.signal, "release-commit"); + await effect( + "upgrade-application-activation", + options.activateApplication, + () => ({ activated: true }), + ); + writersDrained = false; + ensureNotAborted(options.signal, "application-activation"); return { backupId: backup.backupId, releaseId: target.releaseId }; } catch (error) { + if (releaseCommitted) { + await options.stopWriters().catch(() => undefined); + throw new UpgradeRecoveryError( + "Upgrade committed the target release but public activation did not complete", + "application-config", + backup.backupId, + "Retry target activation; do not restore the pre-upgrade backup", + { cause: error }, + ); + } if (decision.kind === "forward-only" && migration.started) { await options.stopWriters().catch(() => undefined); throw new UpgradeRecoveryError( @@ -204,8 +221,8 @@ export async function runUpgrade(options: { { cause: error }, ); } - if (applicationInstalled) await options.rollbackApplication(); - if (writersDrained) await options.restartWriters(); + if (applicationInstalled || writersDrained) + await options.rollbackApplication(); const last = options.journal?.entries.at(-1); if ( last?.phase === "compensate" && diff --git a/tests/e2e/self-hosted-onboarding/upgrade-preservation.test.ts b/tests/e2e/self-hosted-onboarding/upgrade-preservation.test.ts index eda51fa..ce57358 100644 --- a/tests/e2e/self-hosted-onboarding/upgrade-preservation.test.ts +++ b/tests/e2e/self-hosted-onboarding/upgrade-preservation.test.ts @@ -51,12 +51,12 @@ describe("upgrade state preservation", () => { installApplication: async () => undefined, migrate: async () => undefined, verifyLiveSchema: async () => 10, - readiness: async () => undefined, + preActivationReadiness: async () => undefined, verifyClients: async () => undefined, + activateApplication: async () => undefined, commitSelection: async () => undefined, rollbackApplication: async () => undefined, stopWriters: async () => undefined, - restartWriters: async () => undefined, }), ).resolves.toEqual({ backupId, releaseId: "12-amd64" }); expect(preserved).toEqual(before); diff --git a/tests/integration/onboarding/backup-restore-validation.test.ts b/tests/integration/onboarding/backup-restore-validation.test.ts index 0daf6ac..3d391e6 100644 --- a/tests/integration/onboarding/backup-restore-validation.test.ts +++ b/tests/integration/onboarding/backup-restore-validation.test.ts @@ -18,12 +18,38 @@ import { PostgresBackupAdapter } from "../../../src/onboarding/adapters/postgres import { runCommand, type CommandOptions, + type CommandResult, } from "../../../src/onboarding/adapters/process/command-runner.js"; import { createOnboardingEnvironment, type OnboardingEnvironment, } from "../../helpers/onboarding-environment.js"; +function localDockerContext( + options: CommandOptions, +): CommandResult | undefined { + if (options.args[0] !== "context") return undefined; + return { + code: 0, + stdout: + options.args[1] === "show" + ? "rootless\n" + : `${options.environment?.["DOCKER_HOST"] ?? "unix:///run/user/1000/docker.sock"}\n`, + stderr: "", + durationMilliseconds: 1, + }; +} + +const completeValidation = (latestMigration = "010") => ({ + latestMigration, + migrationInventoryValid: true, + constraintsValid: true, + catalogValid: true, + advisoryValid: true, + authoritativeStateValid: true, + ready: true, +}); + describe("restore-validated PostgreSQL backup", () => { let fixture: OnboardingEnvironment | undefined; afterEach(async () => fixture?.close()); @@ -33,6 +59,8 @@ describe("restore-validated PostgreSQL backup", () => { const commands: CommandOptions[] = []; const run = vi.fn(async (options: CommandOptions) => { commands.push(options); + const context = localDockerContext(options); + if (context !== undefined) return context; if ( options.args.includes("compose") && options.args.includes("cp") && @@ -62,12 +90,7 @@ describe("restore-validated PostgreSQL backup", () => { GH_TOKEN: "ambient-canary", }, run, - validateRestoredDatabase: async () => ({ - latestMigration: "010", - invariantsValid: true, - catalogValid: true, - ready: true, - }), + validateRestoredDatabase: async () => completeValidation(), }); const record = await createValidatedBackup({ @@ -138,6 +161,8 @@ describe("restore-validated PostgreSQL backup", () => { const commands: CommandOptions[] = []; const run = vi.fn(async (options: CommandOptions) => { commands.push(options); + const context = localDockerContext(options); + if (context !== undefined) return context; if ( options.args.includes("compose") && options.args.includes("cp") && @@ -183,6 +208,8 @@ describe("restore-validated PostgreSQL backup", () => { backupsRoot, postgresImage: `docker.io/library/postgres@sha256:${"c".repeat(64)}`, run: async (options) => { + const context = localDockerContext(options); + if (context !== undefined) return context; if (options.args.includes("pg_dump")) throw new Error("dump failed"); return { code: 0, stdout: "", stderr: "", durationMilliseconds: 1 }; }, @@ -198,6 +225,8 @@ describe("restore-validated PostgreSQL backup", () => { it("restore-validates the exact pre-upgrade schema instead of assuming 010", async () => { fixture = await createOnboardingEnvironment(); const run = vi.fn(async (options: CommandOptions) => { + const context = localDockerContext(options); + if (context !== undefined) return context; if ( options.args.includes("compose") && options.args.includes("cp") && @@ -219,12 +248,7 @@ describe("restore-validated PostgreSQL backup", () => { postgresImage: `docker.io/library/postgres@sha256:${"d".repeat(64)}`, expectedLatestMigration: "009", run, - validateRestoredDatabase: async () => ({ - latestMigration: "009", - invariantsValid: true, - catalogValid: true, - ready: true, - }), + validateRestoredDatabase: async () => completeValidation("009"), }); await expect( @@ -237,6 +261,8 @@ describe("restore-validated PostgreSQL backup", () => { const controller = new AbortController(); const cleanupSignals: AbortSignal[] = []; const run = vi.fn(async (options: CommandOptions) => { + const context = localDockerContext(options); + if (context !== undefined) return context; if ( options.args.includes("compose") && options.args.includes("cp") && @@ -303,6 +329,110 @@ describe("restore-validated PostgreSQL backup", () => { expect(run).not.toHaveBeenCalled(); }); + it("rejects a named remote Docker context before the first backup workload command", async () => { + fixture = await createOnboardingEnvironment(); + const commands: CommandOptions[] = []; + const run = vi.fn(async (options: CommandOptions) => { + commands.push(options); + if (options.args[0] === "context" && options.args[1] === "show") + return { + code: 0, + stdout: "remote-proof\n", + stderr: "", + durationMilliseconds: 1, + }; + if (options.args[0] === "context" && options.args[1] === "inspect") + return { + code: 0, + stdout: "ssh://builder@example.test\n", + stderr: "", + durationMilliseconds: 1, + }; + return { code: 0, stdout: "", stderr: "", durationMilliseconds: 1 }; + }); + const adapter = new PostgresBackupAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: resolve("distribution/self-hosted/compose.yaml"), + projectName: "skillwire-test", + installationId: randomUUID(), + protectedRoot: fixture.root, + backupsRoot: resolve(fixture.root, "backups"), + postgresImage: `docker.io/library/postgres@sha256:${"a".repeat(64)}`, + environment: { + DOCKER_CONTEXT: "remote-proof", + DOCKER_CONFIG: resolve(fixture.root, "docker-config"), + }, + run, + validateRestoredDatabase: vi.fn(), + }); + + await expect( + adapter.createAndValidate(new AbortController().signal), + ).rejects.toThrow(/local Docker context|remote/i); + expect(commands.some(({ args }) => args.includes("pg_dump"))).toBe(false); + expect(commands.map(({ args }) => args).slice(0, 2)).toEqual([ + ["context", "show"], + [ + "context", + "inspect", + "remote-proof", + "--format", + "{{.Endpoints.docker.Host}}", + ], + ]); + }); + + it("pins an accepted named context endpoint for every backup workload command", async () => { + fixture = await createOnboardingEnvironment(); + const commands: CommandOptions[] = []; + const endpoint = `unix://${fixture.runtimeRoot}/docker.sock`; + const run = vi.fn(async (options: CommandOptions) => { + commands.push(options); + if (options.args[0] === "context") + return { + code: 0, + stdout: options.args[1] === "show" ? "rootless\n" : `${endpoint}\n`, + stderr: "", + durationMilliseconds: 1, + }; + if ( + options.args.includes("compose") && + options.args.includes("cp") && + options.args.at(-1)?.endsWith(".dump") + ) { + const target = options.args.at(-1); + if (target !== undefined) + await writeFile(target, "PGDMP\0pinned-context", { mode: 0o600 }); + } + return { code: 0, stdout: "", stderr: "", durationMilliseconds: 1 }; + }); + const adapter = new PostgresBackupAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: resolve("distribution/self-hosted/compose.yaml"), + projectName: "skillwire-test", + installationId: randomUUID(), + protectedRoot: fixture.root, + backupsRoot: resolve(fixture.root, "backups"), + postgresImage: `docker.io/library/postgres@sha256:${"a".repeat(64)}`, + environment: { DOCKER_CONTEXT: "rootless" }, + run, + validateRestoredDatabase: async () => completeValidation(), + }); + + await expect( + adapter.createAndValidate(new AbortController().signal), + ).resolves.toMatchObject({ validation: { ready: true } }); + const workload = commands.filter(({ args }) => args[0] !== "context"); + expect(workload.length).toBeGreaterThan(0); + expect( + workload.every( + ({ environment }) => + environment?.["DOCKER_HOST"] === endpoint && + environment["DOCKER_CONTEXT"] === undefined, + ), + ).toBe(true); + }); + const realPostgresIt = process.env["SKILLWIRE_RUN_POSTGRES_BACKUP_INTEGRATION"] === "1" ? it @@ -410,8 +540,8 @@ describe("restore-validated PostgreSQL backup", () => { .trim() .split("|"); return { - latestMigration: migration ?? "", - invariantsValid: accounts === "true", + ...completeValidation(migration ?? ""), + constraintsValid: accounts === "true", catalogValid: catalog === "true" && advisory === "true", ready: migration === "010", }; @@ -425,8 +555,11 @@ describe("restore-validated PostgreSQL backup", () => { expect(backup).toMatchObject({ validation: { latestMigration: "010", - invariantsValid: true, + migrationInventoryValid: true, + constraintsValid: true, catalogValid: true, + advisoryValid: true, + authoritativeStateValid: true, ready: true, }, }); diff --git a/tests/integration/onboarding/service-setup.test.ts b/tests/integration/onboarding/service-setup.test.ts index 30c9b33..3c88dda 100644 --- a/tests/integration/onboarding/service-setup.test.ts +++ b/tests/integration/onboarding/service-setup.test.ts @@ -235,6 +235,44 @@ describe("service-only deployment boundary", () => { ).toBe(false); }); + it("uses an already resolved local endpoint without re-reading another context", async () => { + const run = vi.fn(async (options: CommandOptions) => { + await Promise.resolve(); + const joined = options.args.join(" "); + if (joined.startsWith("context ")) + throw new Error("the default context must not replace a pinned host"); + if (joined === "--version") return result("Docker version 29.7.2\n"); + if (joined === "compose version") + return result("Docker Compose version v5.4.0\n"); + if (joined.includes("image inspect")) + return result(`${JSON.stringify([options.args.at(-1) ?? ""])}\n`); + return result(""); + }); + const adapter = new DeploymentAdapter({ + dockerExecutable: "/usr/bin/docker", + composePath: "/tmp/disposable/compose.yaml", + projectName: "skillwire-test-0123456789abcdef", + volumeName: "skillwire-test-0123456789abcdef_postgres_data", + skillwireImage: `localhost:5000/skillwire@sha256:${"1".repeat(64)}`, + postgresImage: `docker.io/library/postgres@sha256:${"2".repeat(64)}`, + databasePasswordFile: "/tmp/disposable/database-password", + applicationPepperFile: "/tmp/disposable/application-pepper", + runtimeSocketDirectory: runtimeDirectory, + socketPath: resolve(runtimeDirectory, "mcp.sock"), + hostEnvironment: { DOCKER_HOST: "unix:///run/user/1000/docker.sock" }, + run, + }); + + await expect(adapter.probe()).resolves.toBeUndefined(); + expect( + run.mock.calls.every( + ([options]) => + options.environment?.["DOCKER_HOST"] === + "unix:///run/user/1000/docker.sock", + ), + ).toBe(true); + }); + it("rejects an unsafe socket directory before Compose mutation", async () => { const run = vi.fn<(options: CommandOptions) => Promise>(); const unsafeDirectory = resolve(runtimeDirectory, "unsafe"); diff --git a/tests/integration/onboarding/upgrade-compatible.test.ts b/tests/integration/onboarding/upgrade-compatible.test.ts index c84a772..8e5c265 100644 --- a/tests/integration/onboarding/upgrade-compatible.test.ts +++ b/tests/integration/onboarding/upgrade-compatible.test.ts @@ -52,14 +52,14 @@ describe("same-schema signed upgrade", () => { }, migrate, verifyLiveSchema: async () => 10, - readiness: async () => { + preActivationReadiness: async () => { throw new Error("not ready"); }, verifyClients: async () => undefined, + activateApplication: async () => undefined, commitSelection: async () => undefined, rollbackApplication, stopWriters: async () => undefined, - restartWriters: async () => undefined, }), ).rejects.toMatchObject({ rollbackBoundary: "application-config", diff --git a/tests/integration/onboarding/upgrade-forward-only-010.test.ts b/tests/integration/onboarding/upgrade-forward-only-010.test.ts index 8bd680f..691a62c 100644 --- a/tests/integration/onboarding/upgrade-forward-only-010.test.ts +++ b/tests/integration/onboarding/upgrade-forward-only-010.test.ts @@ -7,8 +7,136 @@ import { previewUpgrade, runUpgrade, } from "../../../src/onboarding/application/upgrade.js"; +import type { UpgradeRecoveryError } from "../../../src/onboarding/application/upgrade.js"; describe("forward-only migration 010 upgrade", () => { + it("keeps the public writer stopped through preactivation and client gates", async () => { + const target = { + releaseId: "10-amd64", + releaseSequence: 10, + trustPolicySequence: 4, + schemaMinimum: 9, + schemaMaximum: 10, + latestMigration: 10, + manifestSha256: "1".repeat(64), + imageDigest: `sha256:${"2".repeat(64)}`, + }; + const preview = previewUpgrade({ + installationId: randomUUID(), + currentReleaseSequence: 9, + currentTrustPolicySequence: 4, + liveSchema: 9, + target, + }); + const events: string[] = []; + let publicWriterRunning = true; + + await expect( + runUpgrade({ + preview, + confirmation: preview.previewHash, + signal: new AbortController().signal, + verifyTarget: async () => target, + createBackup: async () => ({ + backupId: randomUUID(), + validated: true, + }), + drainWriters: async () => { + publicWriterRunning = false; + events.push("writers-drained"); + }, + installApplication: async () => { + events.push("application-staged"); + }, + migrate: async () => { + events.push("migration-010"); + }, + verifyLiveSchema: async () => 10, + preActivationReadiness: async () => { + expect(publicWriterRunning).toBe(false); + events.push("preactivation-ready"); + }, + verifyClients: async () => { + expect(publicWriterRunning).toBe(false); + events.push("clients-verified"); + }, + activateApplication: async () => { + publicWriterRunning = true; + events.push("target-activated"); + }, + commitSelection: async () => { + events.push("release-committed"); + }, + rollbackApplication: vi.fn(), + stopWriters: vi.fn(), + }), + ).resolves.toMatchObject({ releaseId: target.releaseId }); + expect(events).toEqual([ + "writers-drained", + "application-staged", + "migration-010", + "preactivation-ready", + "clients-verified", + "release-committed", + "target-activated", + ]); + }); + + it("keeps writers stopped and recovery actionable when activation fails after release commit", async () => { + const target = { + releaseId: "10-amd64", + releaseSequence: 10, + trustPolicySequence: 4, + schemaMinimum: 9, + schemaMaximum: 10, + latestMigration: 10, + manifestSha256: "1".repeat(64), + imageDigest: `sha256:${"2".repeat(64)}`, + }; + const preview = previewUpgrade({ + installationId: randomUUID(), + currentReleaseSequence: 9, + currentTrustPolicySequence: 4, + liveSchema: 9, + target, + }); + const stopWriters = vi.fn(async () => undefined); + const rollbackApplication = vi.fn(async () => undefined); + const commitSelection = vi.fn(async () => undefined); + + await expect( + runUpgrade({ + preview, + confirmation: preview.previewHash, + signal: new AbortController().signal, + verifyTarget: async () => target, + createBackup: async () => ({ + backupId: randomUUID(), + validated: true, + }), + drainWriters: async () => undefined, + installApplication: async () => undefined, + migrate: async () => undefined, + verifyLiveSchema: async () => 10, + preActivationReadiness: async () => undefined, + verifyClients: async () => undefined, + commitSelection, + activateApplication: async () => { + throw new Error("public socket unavailable"); + }, + rollbackApplication, + stopWriters, + }), + ).rejects.toMatchObject({ + rollbackBoundary: "application-config", + dataLossBoundary: + "Retry target activation; do not restore the pre-upgrade backup", + } satisfies Partial); + expect(commitSelection).toHaveBeenCalledTimes(1); + expect(stopWriters).toHaveBeenCalledTimes(1); + expect(rollbackApplication).not.toHaveBeenCalled(); + }); + it("restore-validates first, drains writers, reads schema 010, and refuses image-only rollback", async () => { const target = { releaseId: "10-amd64", @@ -57,14 +185,14 @@ describe("forward-only migration 010 upgrade", () => { events.push("schema-readback-010"); return 10; }, - readiness: async () => { + preActivationReadiness: async () => { throw new Error("service failed after migration"); }, verifyClients: async () => undefined, + activateApplication: async () => undefined, commitSelection: async () => undefined, rollbackApplication, stopWriters, - restartWriters: async () => undefined, }), ).rejects.toMatchObject({ rollbackBoundary: "database-restore-required", diff --git a/tests/integration/onboarding/upgrade-interruption.test.ts b/tests/integration/onboarding/upgrade-interruption.test.ts index 80d27a5..6252f3e 100644 --- a/tests/integration/onboarding/upgrade-interruption.test.ts +++ b/tests/integration/onboarding/upgrade-interruption.test.ts @@ -66,12 +66,12 @@ describe("upgrade interruption boundaries", () => { installApplication: () => at("application", undefined), migrate: () => at("migration", undefined), verifyLiveSchema: async () => 10, - readiness: () => at("readiness", undefined), + preActivationReadiness: () => at("readiness", undefined), verifyClients: () => at("clients", undefined), + activateApplication: () => at("activation", undefined), commitSelection: () => at("release-commit", undefined), rollbackApplication: vi.fn(async () => undefined), stopWriters: vi.fn(async () => undefined), - restartWriters: vi.fn(async () => undefined), }), ).rejects.toThrow(/cancel|upgrade|recovery|boundary/i); expect(reached).toContain(boundary); @@ -83,6 +83,7 @@ describe("upgrade interruption boundaries", () => { "migration", "readiness", "clients", + "activation", "release-commit", ]; const boundaryIndex = order.indexOf(boundary); @@ -91,7 +92,7 @@ describe("upgrade interruption boundaries", () => { ); }); - it("treats a completed atomic release commit as the terminal success boundary", async () => { + it("keeps a committed release recoverable when cancellation prevents public activation", async () => { const target = { releaseId: "10-amd64", releaseSequence: 10, @@ -110,6 +111,8 @@ describe("upgrade interruption boundaries", () => { target, }); const controller = new AbortController(); + const activateApplication = vi.fn(async () => undefined); + const stopWriters = vi.fn(async () => undefined); await expect( runUpgrade({ preview, @@ -124,14 +127,16 @@ describe("upgrade interruption boundaries", () => { installApplication: async () => undefined, migrate: async () => undefined, verifyLiveSchema: async () => 10, - readiness: async () => undefined, + preActivationReadiness: async () => undefined, verifyClients: async () => undefined, + activateApplication, commitSelection: async () => controller.abort(), rollbackApplication: vi.fn(), - stopWriters: vi.fn(), - restartWriters: vi.fn(), + stopWriters, }), - ).resolves.toMatchObject({ releaseId: "10-amd64" }); + ).rejects.toMatchObject({ rollbackBoundary: "application-config" }); + expect(activateApplication).not.toHaveBeenCalled(); + expect(stopWriters).toHaveBeenCalledTimes(1); }); it("publishes active release/trust selection as one journaled atomic effect", async () => { diff --git a/tests/security/onboarding/docker-environment.test.ts b/tests/security/onboarding/docker-environment.test.ts index 33ef9ae..878ce56 100644 --- a/tests/security/onboarding/docker-environment.test.ts +++ b/tests/security/onboarding/docker-environment.test.ts @@ -1,6 +1,11 @@ -import { describe, expect, it } from "vitest"; +/* eslint-disable @typescript-eslint/require-await -- Async fakes mirror the production command runner. */ +import { describe, expect, it, vi } from "vitest"; -import { dockerProcessEnvironment } from "../../../src/onboarding/adapters/docker/environment.js"; +import { + assertLocalDockerContext, + dockerProcessEnvironment, +} from "../../../src/onboarding/adapters/docker/environment.js"; +import type { CommandOptions } from "../../../src/onboarding/adapters/process/command-runner.js"; describe("Docker subprocess environment isolation", () => { it("keeps only runtime routing and explicit non-secret Compose values", () => { @@ -37,4 +42,80 @@ describe("Docker subprocess environment isolation", () => { expect(environment).not.toHaveProperty("OPENAI_API_KEY"); expect(environment).not.toHaveProperty("DATABASE_URL"); }); + + it.each(["tcp://docker.example.test:2376", "ssh://builder@example.test"])( + "rejects a named Docker context resolving to %s before any workload command", + async (endpoint) => { + const commands: CommandOptions[] = []; + const run = vi.fn(async (options: CommandOptions) => { + commands.push(options); + return { + code: 0, + stdout: + options.args[1] === "show" ? "remote-proof\n" : `${endpoint}\n`, + stderr: "", + durationMilliseconds: 1, + }; + }); + + await expect( + assertLocalDockerContext({ + dockerExecutable: "/usr/bin/docker", + environment: { + DOCKER_CONTEXT: "remote-proof", + DOCKER_CONFIG: "/tmp/disposable-docker-config", + }, + signal: new AbortController().signal, + run, + }), + ).rejects.toThrow(/local Docker context|remote/i); + expect(commands.map(({ args }) => args)).toEqual([ + ["context", "show"], + [ + "context", + "inspect", + "remote-proof", + "--format", + "{{.Endpoints.docker.Host}}", + ], + ]); + }, + ); + + it.each([ + "unix:///run/user/1000/docker.sock", + "npipe:////./pipe/docker_engine", + ])( + "preserves a named Docker context resolving to local endpoint %s", + async (endpoint) => { + const run = vi.fn(async (options: CommandOptions) => ({ + code: 0, + stdout: options.args[1] === "show" ? "rootless\n" : `${endpoint}\n`, + stderr: "", + durationMilliseconds: 1, + })); + + await expect( + assertLocalDockerContext({ + dockerExecutable: "/usr/bin/docker", + environment: { DOCKER_CONTEXT: "rootless" }, + signal: new AbortController().signal, + run, + }), + ).resolves.toBe(endpoint); + }, + ); + + it("accepts an explicit local Docker host without resolving an unrelated context", async () => { + const run = vi.fn(); + await expect( + assertLocalDockerContext({ + dockerExecutable: "/usr/bin/docker", + environment: { DOCKER_HOST: "unix:///run/user/1000/docker.sock" }, + signal: new AbortController().signal, + run, + }), + ).resolves.toBe("unix:///run/user/1000/docker.sock"); + expect(run).not.toHaveBeenCalled(); + }); }); diff --git a/tests/security/onboarding/upgrade-trust-downgrade.test.ts b/tests/security/onboarding/upgrade-trust-downgrade.test.ts index 5d64e4c..9add1b8 100644 --- a/tests/security/onboarding/upgrade-trust-downgrade.test.ts +++ b/tests/security/onboarding/upgrade-trust-downgrade.test.ts @@ -48,12 +48,12 @@ describe("upgrade trust and downgrade boundary", () => { installApplication: vi.fn(), migrate: vi.fn(), verifyLiveSchema: vi.fn(), - readiness: vi.fn(), + preActivationReadiness: vi.fn(), verifyClients: vi.fn(), + activateApplication: vi.fn(), commitSelection: vi.fn(), rollbackApplication: vi.fn(), stopWriters: vi.fn(), - restartWriters: vi.fn(), }), ).rejects.toThrow(/downgrade|digest|pinned|sequence/i); expect(createBackup).not.toHaveBeenCalled(); @@ -84,12 +84,12 @@ describe("upgrade trust and downgrade boundary", () => { installApplication: vi.fn(), migrate: vi.fn(), verifyLiveSchema: vi.fn(), - readiness: vi.fn(), + preActivationReadiness: vi.fn(), verifyClients: vi.fn(), + activateApplication: vi.fn(), commitSelection: vi.fn(), rollbackApplication: vi.fn(), stopWriters: vi.fn(), - restartWriters: vi.fn(), }), ).rejects.toThrow(reason); expect(createBackup).not.toHaveBeenCalled(); diff --git a/tests/unit/onboarding/restored-database-validation.test.ts b/tests/unit/onboarding/restored-database-validation.test.ts new file mode 100644 index 0000000..8c83f3a --- /dev/null +++ b/tests/unit/onboarding/restored-database-validation.test.ts @@ -0,0 +1,326 @@ +/* eslint-disable @typescript-eslint/require-await -- Async fake mirrors the production command runner. */ +import { createHash, randomUUID } from "node:crypto"; +import { mkdir, symlink, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +import { + assessRestoredDatabaseEvidence, + databaseStateExpectation, + expectedMigrationInventory, + validateRestoredDatabaseContainer, + type RestoredDatabaseEvidence, +} from "../../../src/onboarding/adapters/postgres/restore-validation.js"; +import { hashExternalAdvisoryEvent } from "../../../src/domain/external-catalog/external-advisory-chain.js"; +import type { CommandOptions } from "../../../src/onboarding/adapters/process/command-runner.js"; +import { createOnboardingEnvironment } from "../../helpers/onboarding-environment.js"; + +const REQUIRED_CONSTRAINTS = [ + "accounts_pkey", + "accounts_status_check", + "api_keys_account_id_fkey", + "api_keys_pkey", + "api_keys_public_id_key", + "external_advisory_chain_head_pkey", + "external_revision_advisory_events_pkey", + "external_revision_dependencies_pkey", + "external_revision_resources_pkey", + "external_skill_revisions_pkey", + "schema_migrations_checksum_check", + "schema_migrations_pkey", +]; + +const REQUIRED_TRIGGERS = [ + "external_advisory_append_valid", + "external_advisory_events_immutable", + "external_dependencies_immutable", + "external_resources_immutable", + "external_revisions_immutable", + "external_snapshots_immutable", +]; + +function evidence(options: { + readonly accountId: string; + readonly checksums: readonly string[]; +}): RestoredDatabaseEvidence { + const advisoryInput = { + sequence: "1", + previousEventSha256: "0".repeat(64), + revisionId: randomUUID(), + kind: "security" as const, + status: "available" as const, + reasonCode: "initial-publication", + effectiveAt: "2026-01-01T00:00:00.000Z", + }; + const eventSha256 = hashExternalAdvisoryEvent(advisoryInput); + return { + currentDatabase: "postgres", + inRecovery: false, + transactionReadOnly: "off", + migrations: options.checksums.map((checksum, index) => ({ + version: String(index + 1).padStart(3, "0"), + checksum, + })), + constraints: REQUIRED_CONSTRAINTS, + triggers: REQUIRED_TRIGGERS, + catalog: { + snapshotCount: 1, + revisionCount: 2, + resourceCount: 3, + dependencyCount: 1, + contentObjectCount: 5, + identitySha256: "c".repeat(64), + invalidSnapshotCounts: 0, + invalidPublishedPointers: 0, + invalidContentLengths: 0, + invalidContentHashes: 0, + invalidSnapshotAdvisoryHeads: 0, + }, + advisory: { + lastSequence: "1", + lastEventSha256: eventSha256, + events: [{ ...advisoryInput, eventSha256 }], + }, + authoritativeState: { + installationAccountStatus: "active", + activeApiKeyCount: 2, + repositoryUsageRows: 3, + repositoryErasureRows: 1, + }, + }; +} + +describe("production restored-database validation", () => { + it("requires the complete immutable migration inventory and checksums", async () => { + const fixture = await createOnboardingEnvironment(); + try { + const migrations = resolve(fixture.root, "migrations"); + await mkdir(migrations, { mode: 0o700 }); + await writeFile(resolve(migrations, "001_first.sql"), "SELECT 1;\n"); + await writeFile(resolve(migrations, "002_second.sql"), "SELECT 2;\n"); + + const expected = await expectedMigrationInventory(migrations, "002"); + expect(expected).toEqual([ + { + version: "001", + checksum: createHash("sha256").update("SELECT 1;\n").digest("hex"), + }, + { + version: "002", + checksum: createHash("sha256").update("SELECT 2;\n").digest("hex"), + }, + ]); + const accountId = randomUUID(); + const valid = evidence({ + accountId, + checksums: expected.map(({ checksum }) => checksum), + }); + expect( + assessRestoredDatabaseEvidence(valid, { + expectedMigrations: expected, + installationAccountId: accountId, + expectedActiveApiKeys: 2, + expectedDatabase: "postgres", + expectedState: databaseStateExpectation(valid), + }), + ).toMatchObject({ + latestMigration: "002", + migrationInventoryValid: true, + constraintsValid: true, + catalogValid: true, + advisoryValid: true, + authoritativeStateValid: true, + ready: true, + }); + + expect(() => + assessRestoredDatabaseEvidence( + { + ...valid, + migrations: valid.migrations.slice(0, 1), + }, + { + expectedMigrations: expected, + installationAccountId: accountId, + expectedActiveApiKeys: 2, + expectedDatabase: "postgres", + expectedState: databaseStateExpectation(valid), + }, + ), + ).toThrow(/migration inventory/i); + expect(() => + assessRestoredDatabaseEvidence( + { + ...valid, + migrations: valid.migrations.map((migration, index) => + index === 1 + ? { ...migration, checksum: "f".repeat(64) } + : migration, + ), + }, + { + expectedMigrations: expected, + installationAccountId: accountId, + expectedActiveApiKeys: 2, + expectedDatabase: "postgres", + expectedState: databaseStateExpectation(valid), + }, + ), + ).toThrow(/migration inventory/i); + } finally { + await fixture.close(); + } + }); + + it("rejects a symlinked expected-migration directory", async () => { + const fixture = await createOnboardingEnvironment(); + try { + const outside = resolve(fixture.root, "outside-migrations"); + await mkdir(outside, { mode: 0o700 }); + await writeFile(resolve(outside, "001_first.sql"), "SELECT 1;\n"); + const linked = resolve(fixture.root, "linked-migrations"); + await symlink(outside, linked); + + await expect(expectedMigrationInventory(linked, "001")).rejects.toThrow( + /migration.*directory|symbolic link|unsafe/i, + ); + } finally { + await fixture.close(); + } + }); + + it.each([ + [ + "required constraint", + (value: RestoredDatabaseEvidence) => ({ + ...value, + constraints: value.constraints.slice(1), + }), + ], + [ + "catalog integrity", + (value: RestoredDatabaseEvidence) => ({ + ...value, + catalog: { ...value.catalog, invalidContentLengths: 1 }, + }), + ], + [ + "advisory integrity", + (value: RestoredDatabaseEvidence) => ({ + ...value, + advisory: { ...value.advisory, lastEventSha256: "f".repeat(64) }, + }), + ], + [ + "authoritative account", + (value: RestoredDatabaseEvidence) => ({ + ...value, + authoritativeState: { + ...value.authoritativeState, + installationAccountStatus: null, + }, + }), + ], + [ + "active API keys", + (value: RestoredDatabaseEvidence) => ({ + ...value, + authoritativeState: { + ...value.authoritativeState, + activeApiKeyCount: 1, + }, + }), + ], + [ + "duplicated authoritative rows", + (value: RestoredDatabaseEvidence) => ({ + ...value, + authoritativeState: { + ...value.authoritativeState, + repositoryUsageRows: value.authoritativeState.repositoryUsageRows + 1, + }, + }), + ], + [ + "catalog identity drift", + (value: RestoredDatabaseEvidence) => ({ + ...value, + catalog: { + ...value.catalog, + revisionCount: value.catalog.revisionCount + 1, + }, + }), + ], + [ + "database readiness", + (value: RestoredDatabaseEvidence) => ({ ...value, inRecovery: true }), + ], + ] as const)( + "rejects restored data with invalid %s evidence", + (_name, corrupt) => { + const accountId = randomUUID(); + const checksum = "a".repeat(64); + const valid = evidence({ accountId, checksums: [checksum] }); + expect(() => + assessRestoredDatabaseEvidence(corrupt(valid), { + expectedMigrations: [{ version: "001", checksum }], + installationAccountId: accountId, + expectedActiveApiKeys: 2, + expectedDatabase: "postgres", + expectedState: databaseStateExpectation(valid), + }), + ).toThrow(/restore validation/i); + }, + ); + + it("queries raw production evidence and rejects corrupt callback output", async () => { + const accountId = randomUUID(); + const checksum = "b".repeat(64); + const commands: CommandOptions[] = []; + const run = vi.fn(async (options: CommandOptions) => { + commands.push(options); + return { + code: 0, + stdout: `${JSON.stringify({ + ...evidence({ accountId, checksums: [checksum] }), + catalog: { + ...evidence({ accountId, checksums: [checksum] }).catalog, + invalidPublishedPointers: 1, + }, + })}\n`, + stderr: "", + durationMilliseconds: 1, + }; + }); + + await expect( + validateRestoredDatabaseContainer({ + dockerExecutable: "/usr/bin/docker", + containerName: "skillwire-backup-validate-deadbeefdeadbeef", + environment: { HOME: "/tmp/disposable-home" }, + signal: new AbortController().signal, + expectedMigrations: [{ version: "001", checksum }], + installationAccountId: accountId, + expectedActiveApiKeys: 2, + expectedState: databaseStateExpectation( + evidence({ accountId, checksums: [checksum] }), + ), + run, + }), + ).rejects.toThrow(/restore validation/i); + expect(commands).toHaveLength(1); + expect(commands[0]?.args.slice(0, 3)).toEqual([ + "exec", + "skillwire-backup-validate-deadbeefdeadbeef", + "psql", + ]); + const query = commands[0]?.args.at(-1) ?? ""; + expect(query).toContain("schema_migrations"); + expect(query).toContain("pg_constraint"); + expect(query).toContain("external_revision_advisory_events"); + expect(query).toContain("external_content_objects"); + expect(query).toContain("repository_skill_usage"); + expect(query).not.toMatch(/AS\s+(?:invariants|catalog|advisory)_valid/i); + }); +}); From 39881128f7db175787113703191f2b11ee8f84c8 Mon Sep 17 00:00:00 2001 From: Lucenx9 Date: Fri, 14 Aug 2026 17:06:37 +0200 Subject: [PATCH 5/7] fix: pin repeated setup Docker endpoint --- .../application/production-setup.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/onboarding/application/production-setup.ts b/src/onboarding/application/production-setup.ts index fd6e4b7..6695557 100644 --- a/src/onboarding/application/production-setup.ts +++ b/src/onboarding/application/production-setup.ts @@ -24,6 +24,7 @@ import { DeploymentAdapter } from "../adapters/docker/deployment.js"; import { assertLocalDockerContext, dockerProcessEnvironment, + pinLocalDockerEndpoint, } from "../adapters/docker/environment.js"; import { ServiceDatabase } from "../adapters/postgres/service-database.js"; import { @@ -302,6 +303,15 @@ async function verifyUnchangedProductionSetup(options: { "Repeated setup external integration ownership state is inconsistent", ); + const localDockerEndpoint = await assertLocalDockerContext({ + dockerExecutable: "/usr/bin/docker", + environment, + signal, + }); + const dockerEnvironment = pinLocalDockerEndpoint( + environment, + localDockerEndpoint, + ); const deploymentAdapter = new DeploymentAdapter({ dockerExecutable: "/usr/bin/docker", composePath: deployment.composePath, @@ -313,12 +323,7 @@ async function verifyUnchangedProductionSetup(options: { applicationPepperFile: deployment.applicationPepperFile, runtimeSocketDirectory: deployment.runtimeSocketDirectory, socketPath: deployment.socketPath, - hostEnvironment: environment, - }); - await assertLocalDockerContext({ - dockerExecutable: "/usr/bin/docker", - environment, - signal, + hostEnvironment: dockerEnvironment, }); const [skillwirePresent, postgresPresent] = await Promise.all([ deploymentAdapter.observeOwnedService("skillwire", signal), @@ -332,7 +337,7 @@ async function verifyUnchangedProductionSetup(options: { volumeName: deployment.volumeName, composePath: deployment.composePath, environment: composeEnvironment({ - environment, + environment: dockerEnvironment, projectName: deployment.projectName, volumeName: deployment.volumeName, skillwireImage: deployment.skillwireImage, From 83aad4b79d1454511d3fe2edf2b9078d8137edf1 Mon Sep 17 00:00:00 2001 From: Lucenx9 Date: Fri, 14 Aug 2026 17:41:54 +0200 Subject: [PATCH 6/7] fix: enforce operations recovery trust boundaries --- .../adapters/postgres/restore-validation.ts | 321 ++++++++++++- .../application/production-continuation.ts | 45 +- .../application/production-lifecycle.ts | 84 +++- .../application/production-setup.ts | 18 + src/onboarding/cli/command-router.ts | 34 +- src/onboarding/domain/operation-journal.ts | 47 ++ .../contract/cli/lifecycle-operations.test.ts | 85 ++++ .../production-continuation.test.ts | 448 ++++++++++++++++++ .../unit/onboarding/operation-journal.test.ts | 28 ++ .../restored-database-validation.test.ts | 402 +++++++++++++++- 10 files changed, 1448 insertions(+), 64 deletions(-) create mode 100644 tests/integration/onboarding/production-continuation.test.ts diff --git a/src/onboarding/adapters/postgres/restore-validation.ts b/src/onboarding/adapters/postgres/restore-validation.ts index 21e8e58..32bdd24 100644 --- a/src/onboarding/adapters/postgres/restore-validation.ts +++ b/src/onboarding/adapters/postgres/restore-validation.ts @@ -34,13 +34,63 @@ const AdvisoryEventSchema = z.object({ eventSha256: z.string().regex(/^[0-9a-f]{64}$/), }); +const SqlIdentifierSchema = z.string().regex(/^[a-z_][a-z0-9_$]{0,127}$/); + +const ConstraintEvidenceSchema = z + .object({ + schemaName: SqlIdentifierSchema, + tableName: SqlIdentifierSchema, + constraintName: SqlIdentifierSchema, + constraintType: z.enum([ + "check", + "foreign-key", + "primary-key", + "unique", + "constraint-trigger", + "not-null", + "exclusion", + ]), + definition: z + .string() + .min(1) + .max(16 * 1024), + validated: z.boolean(), + }) + .strict(); + +const TriggerEventSchema = z.enum(["INSERT", "DELETE", "UPDATE", "TRUNCATE"]); + +const TriggerEvidenceSchema = z + .object({ + schemaName: SqlIdentifierSchema, + tableName: SqlIdentifierSchema, + triggerName: SqlIdentifierSchema, + functionSchema: SqlIdentifierSchema, + functionName: SqlIdentifierSchema, + functionArguments: z.string().max(4096), + functionDefinition: z + .string() + .min(1) + .max(64 * 1024), + functionBodySha256: z.string().regex(/^[0-9a-f]{64}$/), + timing: z.enum(["BEFORE", "AFTER", "INSTEAD OF"]), + level: z.enum(["ROW", "STATEMENT"]), + events: z.array(TriggerEventSchema).min(1).max(4), + enabled: z.enum(["origin", "disabled", "replica", "always"]), + definition: z + .string() + .min(1) + .max(16 * 1024), + }) + .strict(); + const EvidenceSchema = z.object({ currentDatabase: z.string(), inRecovery: z.boolean(), transactionReadOnly: z.string(), migrations: z.array(MigrationSchema).max(999), - constraints: z.array(z.string().min(1).max(128)).max(4096), - triggers: z.array(z.string().min(1).max(128)).max(4096), + constraints: z.array(ConstraintEvidenceSchema).max(4096), + triggers: z.array(TriggerEvidenceSchema).max(4096), catalog: z.object({ snapshotCount: z.number().int().nonnegative(), revisionCount: z.number().int().nonnegative(), @@ -70,6 +120,10 @@ const EvidenceSchema = z.object({ export type RestoredDatabaseEvidence = z.infer; export interface DatabaseStateExpectation { + readonly schemaControls: { + readonly constraints: RestoredDatabaseEvidence["constraints"]; + readonly triggers: RestoredDatabaseEvidence["triggers"]; + }; readonly catalog: { readonly snapshotCount: number; readonly revisionCount: number; @@ -110,13 +164,167 @@ const REQUIRED_CONSTRAINTS = [ "schema_migrations_pkey", ] as const; +const FUNCTION_BODY_SHA256 = { + reject_external_history_mutation: + "4adf876c6bd96600896c6d48b63a2900408722ed33f8603af7a413555788e83c", + protect_github_registration_identity: + "b7e09311ad9c92f1d002dabe2991ee0e987071de9dfcec937f17fe62fab8071e", + validate_external_classification_transition: + "7e29bd82153cfd0976b925d2dd6a18879f3c9e16a4c79faf1586fdddb9aad718", + validate_external_classification_transition_after_migration_010: + "511e1833ab29d867a5c5a7ad5036aaabac942dab4e4f55d3de3247a0322507cd", + validate_external_advisory_append: + "ec80f70ea91225d44339c1a6aef7da0569e6aee6630f57c495430546fb6178df", + guard_external_snapshot_finalization: + "37167092120146842dfe2976dfcc45034ea58ad6eb4a4a576b0e0fd0e3ada9c9", + require_external_snapshot_finalization: + "e26c0466292fb76c9f4cce6af78ea7f617617d7a5f117fd20e15f3bfbbfdde8c", + validate_external_revision_classification_transition: + "01461630956a69c1ead4bfd7bf1df621e217a352052123fcfe79e401dea840b8", +} as const; + const REQUIRED_TRIGGERS = [ - "external_advisory_append_valid", - "external_advisory_events_immutable", - "external_dependencies_immutable", - "external_resources_immutable", - "external_revisions_immutable", - "external_snapshots_immutable", + { + minimumMigration: 5, + triggerName: "external_content_immutable", + tableName: "external_content_objects", + functionName: "reject_external_history_mutation", + functionBodySha256: FUNCTION_BODY_SHA256.reject_external_history_mutation, + events: ["DELETE", "UPDATE"], + }, + { + minimumMigration: 5, + triggerName: "external_identities_immutable", + tableName: "external_skill_identities", + functionName: "reject_external_history_mutation", + functionBodySha256: FUNCTION_BODY_SHA256.reject_external_history_mutation, + events: ["DELETE", "UPDATE"], + }, + { + minimumMigration: 5, + triggerName: "external_revisions_immutable", + tableName: "external_skill_revisions", + functionName: "reject_external_history_mutation", + functionBodySha256: FUNCTION_BODY_SHA256.reject_external_history_mutation, + events: ["DELETE", "UPDATE"], + }, + { + minimumMigration: 5, + triggerName: "external_resources_immutable", + tableName: "external_revision_resources", + functionName: "reject_external_history_mutation", + functionBodySha256: FUNCTION_BODY_SHA256.reject_external_history_mutation, + events: ["DELETE", "UPDATE"], + }, + { + minimumMigration: 5, + triggerName: "external_dependencies_immutable", + tableName: "external_revision_dependencies", + functionName: "reject_external_history_mutation", + functionBodySha256: FUNCTION_BODY_SHA256.reject_external_history_mutation, + events: ["DELETE", "UPDATE"], + }, + { + minimumMigration: 5, + triggerName: "external_observations_immutable", + tableName: "external_snapshot_skill_observations", + functionName: "reject_external_history_mutation", + functionBodySha256: FUNCTION_BODY_SHA256.reject_external_history_mutation, + events: ["DELETE", "UPDATE"], + }, + { + minimumMigration: 6, + triggerName: "github_source_registration_identity_immutable", + tableName: "github_source_registrations", + functionName: "protect_github_registration_identity", + functionBodySha256: + FUNCTION_BODY_SHA256.protect_github_registration_identity, + events: ["DELETE", "UPDATE"], + }, + { + minimumMigration: 6, + triggerName: "external_classification_transition_valid", + tableName: "external_classification_events", + functionName: "validate_external_classification_transition", + functionBodySha256: + FUNCTION_BODY_SHA256.validate_external_classification_transition, + replacementFunctionBody: { + minimumMigration: 10, + functionBodySha256: + FUNCTION_BODY_SHA256.validate_external_classification_transition_after_migration_010, + }, + events: ["INSERT"], + }, + { + minimumMigration: 6, + triggerName: "external_advisory_append_valid", + tableName: "external_revision_advisory_events", + functionName: "validate_external_advisory_append", + functionBodySha256: FUNCTION_BODY_SHA256.validate_external_advisory_append, + events: ["INSERT"], + }, + ...[ + ["external_candidates_immutable", "external_import_candidates"], + ["external_reports_immutable", "external_verification_reports"], + ["external_findings_immutable", "external_validation_findings"], + [ + "external_classification_events_immutable", + "external_classification_events", + ], + ["external_curation_decisions_immutable", "external_curation_decisions"], + ["external_advisory_events_immutable", "external_revision_advisory_events"], + ].map(([triggerName, tableName]) => ({ + minimumMigration: 6, + triggerName: triggerName ?? "", + tableName: tableName ?? "", + functionName: "reject_external_history_mutation", + functionBodySha256: FUNCTION_BODY_SHA256.reject_external_history_mutation, + events: ["DELETE", "UPDATE"], + })), + { + minimumMigration: 7, + triggerName: "github_sync_candidate_results_immutable", + tableName: "github_sync_candidate_results", + functionName: "reject_external_history_mutation", + functionBodySha256: FUNCTION_BODY_SHA256.reject_external_history_mutation, + events: ["DELETE", "UPDATE"], + }, + { + minimumMigration: 8, + triggerName: "external_snapshots_immutable", + tableName: "external_source_snapshots", + functionName: "guard_external_snapshot_finalization", + functionBodySha256: + FUNCTION_BODY_SHA256.guard_external_snapshot_finalization, + events: ["DELETE", "UPDATE"], + }, + { + minimumMigration: 8, + triggerName: "external_snapshot_finalization_required", + tableName: "external_source_snapshots", + functionName: "require_external_snapshot_finalization", + functionBodySha256: + FUNCTION_BODY_SHA256.require_external_snapshot_finalization, + events: ["INSERT", "UPDATE"], + timing: "AFTER", + }, + { + minimumMigration: 10, + triggerName: "external_revision_classification_transition_valid", + tableName: "external_revision_classification_events", + functionName: "validate_external_revision_classification_transition", + functionBodySha256: + FUNCTION_BODY_SHA256.validate_external_revision_classification_transition, + events: ["INSERT"], + }, + { + minimumMigration: 10, + triggerName: "external_revision_classification_events_immutable", + tableName: "external_revision_classification_events", + functionName: "reject_external_history_mutation", + functionBodySha256: FUNCTION_BODY_SHA256.reject_external_history_mutation, + events: ["DELETE", "UPDATE"], + }, ] as const; export async function expectedMigrationInventory( @@ -210,12 +418,55 @@ export function assessRestoredDatabaseEvidence( JSON.stringify(evidence.migrations) !== JSON.stringify(expectedMigrations) ) throw new Error("Restored migration inventory or checksum is invalid"); - const constraints = new Set(evidence.constraints); - const triggers = new Set(evidence.triggers); - const constraintsValid = REQUIRED_CONSTRAINTS.every((name) => - constraints.has(name), + const latestMigration = Number(expectedMigrations.at(-1)?.version ?? "0"); + const constraintNames = new Set( + evidence.constraints.map(({ constraintName }) => constraintName), + ); + const uniqueConstraints = new Set( + evidence.constraints.map( + ({ schemaName, tableName, constraintName }) => + `${schemaName}\0${tableName}\0${constraintName}`, + ), ); - const triggersValid = REQUIRED_TRIGGERS.every((name) => triggers.has(name)); + const uniqueTriggers = new Set( + evidence.triggers.map( + ({ schemaName, tableName, triggerName }) => + `${schemaName}\0${tableName}\0${triggerName}`, + ), + ); + const constraintsValid = + uniqueConstraints.size === evidence.constraints.length && + evidence.constraints.every(({ validated }) => validated) && + REQUIRED_CONSTRAINTS.every((name) => constraintNames.has(name)) && + JSON.stringify(evidence.constraints) === + JSON.stringify(expectations.expectedState.schemaControls.constraints); + const triggersValid = + uniqueTriggers.size === evidence.triggers.length && + evidence.triggers.every(({ enabled }) => enabled === "origin") && + REQUIRED_TRIGGERS.filter( + ({ minimumMigration }) => minimumMigration <= latestMigration, + ).every((required) => + evidence.triggers.some( + (trigger) => + trigger.schemaName === "public" && + trigger.triggerName === required.triggerName && + trigger.tableName === required.tableName && + trigger.functionSchema === "public" && + trigger.functionName === required.functionName && + trigger.functionArguments === "" && + trigger.functionBodySha256 === + ("replacementFunctionBody" in required && + latestMigration >= required.replacementFunctionBody.minimumMigration + ? required.replacementFunctionBody.functionBodySha256 + : required.functionBodySha256) && + trigger.timing === + ("timing" in required ? required.timing : "BEFORE") && + trigger.level === "ROW" && + JSON.stringify(trigger.events) === JSON.stringify(required.events), + ), + ) && + JSON.stringify(evidence.triggers) === + JSON.stringify(expectations.expectedState.schemaControls.triggers); const catalogValid = evidence.catalog.invalidSnapshotCounts === 0 && evidence.catalog.invalidPublishedPointers === 0 && @@ -285,6 +536,10 @@ export function databaseStateExpectation( ): DatabaseStateExpectation { const evidence = EvidenceSchema.parse(input); return { + schemaControls: { + constraints: evidence.constraints, + triggers: evidence.triggers, + }, catalog: { snapshotCount: evidence.catalog.snapshotCount, revisionCount: evidence.catalog.revisionCount, @@ -308,8 +563,44 @@ function restoredDatabaseEvidenceQuery(installationAccountId: string): string { 'inRecovery', pg_is_in_recovery(), 'transactionReadOnly', current_setting('transaction_read_only'), 'migrations', (SELECT COALESCE(json_agg(json_build_object('version',version,'checksum',checksum) ORDER BY version),'[]'::json) FROM schema_migrations), - 'constraints', (SELECT COALESCE(json_agg(conname ORDER BY conname),'[]'::json) FROM pg_constraint JOIN pg_namespace ON pg_namespace.oid=pg_constraint.connamespace WHERE nspname='public'), - 'triggers', (SELECT COALESCE(json_agg(tgname ORDER BY tgname),'[]'::json) FROM pg_trigger JOIN pg_class ON pg_class.oid=pg_trigger.tgrelid JOIN pg_namespace ON pg_namespace.oid=pg_class.relnamespace WHERE nspname='public' AND NOT tgisinternal), + 'constraints', (SELECT COALESCE(json_agg(json_build_object( + 'schemaName', namespace.nspname, + 'tableName', relation.relname, + 'constraintName', constraint_entry.conname, + 'constraintType', CASE constraint_entry.contype WHEN 'c' THEN 'check' WHEN 'f' THEN 'foreign-key' WHEN 'p' THEN 'primary-key' WHEN 'u' THEN 'unique' WHEN 't' THEN 'constraint-trigger' WHEN 'n' THEN 'not-null' WHEN 'x' THEN 'exclusion' ELSE 'unsupported' END, + 'definition', pg_get_constraintdef(constraint_entry.oid, true), + 'validated', constraint_entry.convalidated + ) ORDER BY namespace.nspname, relation.relname, constraint_entry.conname),'[]'::json) + FROM pg_constraint constraint_entry + JOIN pg_namespace namespace ON namespace.oid=constraint_entry.connamespace + JOIN pg_class relation ON relation.oid=constraint_entry.conrelid + WHERE namespace.nspname='public' AND constraint_entry.conrelid<>0), + 'triggers', (SELECT COALESCE(json_agg(json_build_object( + 'schemaName', namespace.nspname, + 'tableName', relation.relname, + 'triggerName', trigger_entry.tgname, + 'functionSchema', function_namespace.nspname, + 'functionName', function_entry.proname, + 'functionArguments', pg_get_function_identity_arguments(function_entry.oid), + 'functionDefinition', pg_get_functiondef(function_entry.oid), + 'functionBodySha256', encode(sha256(convert_to(btrim(function_entry.prosrc), 'UTF8')), 'hex'), + 'timing', CASE WHEN (trigger_entry.tgtype & 64)<>0 THEN 'INSTEAD OF' WHEN (trigger_entry.tgtype & 2)<>0 THEN 'BEFORE' ELSE 'AFTER' END, + 'level', CASE WHEN (trigger_entry.tgtype & 1)<>0 THEN 'ROW' ELSE 'STATEMENT' END, + 'events', array_remove(ARRAY[ + CASE WHEN (trigger_entry.tgtype & 4)<>0 THEN 'INSERT' END, + CASE WHEN (trigger_entry.tgtype & 8)<>0 THEN 'DELETE' END, + CASE WHEN (trigger_entry.tgtype & 16)<>0 THEN 'UPDATE' END, + CASE WHEN (trigger_entry.tgtype & 32)<>0 THEN 'TRUNCATE' END + ], NULL), + 'enabled', CASE trigger_entry.tgenabled WHEN 'O' THEN 'origin' WHEN 'D' THEN 'disabled' WHEN 'R' THEN 'replica' WHEN 'A' THEN 'always' ELSE 'unknown' END, + 'definition', pg_get_triggerdef(trigger_entry.oid, true) + ) ORDER BY namespace.nspname, relation.relname, trigger_entry.tgname),'[]'::json) + FROM pg_trigger trigger_entry + JOIN pg_class relation ON relation.oid=trigger_entry.tgrelid + JOIN pg_namespace namespace ON namespace.oid=relation.relnamespace + JOIN pg_proc function_entry ON function_entry.oid=trigger_entry.tgfoid + JOIN pg_namespace function_namespace ON function_namespace.oid=function_entry.pronamespace + WHERE namespace.nspname='public' AND NOT trigger_entry.tgisinternal), 'catalog', json_build_object( 'snapshotCount', (SELECT count(*) FROM external_source_snapshots), 'revisionCount', (SELECT count(*) FROM external_skill_revisions), diff --git a/src/onboarding/application/production-continuation.ts b/src/onboarding/application/production-continuation.ts index 2173c2f..daf04a0 100644 --- a/src/onboarding/application/production-continuation.ts +++ b/src/onboarding/application/production-continuation.ts @@ -9,7 +9,11 @@ import { CodexClientAdapter } from "../adapters/clients/codex.js"; import { ClaudeClientAdapter } from "../adapters/clients/claude.js"; import { clientComponentIdentity } from "../adapters/clients/client-state.js"; import { DeploymentAdapter } from "../adapters/docker/deployment.js"; -import { dockerProcessEnvironment } from "../adapters/docker/environment.js"; +import { + assertLocalDockerContext, + dockerProcessEnvironment, + pinLocalDockerEndpoint, +} from "../adapters/docker/environment.js"; import { SecretToolCredentialStore } from "../adapters/credentials/secret-tool.js"; import { RestrictiveFileCredentialStore, @@ -187,6 +191,10 @@ export async function continueProductionSetup(options: { readonly environment: NodeJS.ProcessEnv; readonly signal: AbortSignal; readonly journal: OperationJournal; + readonly verifyPriorSelectedClients: ( + clients: readonly ClientName[], + dockerEnvironment: NodeJS.ProcessEnv, + ) => Promise; }): Promise { const { installation, stateRoot, dataRoot, environment, signal, journal } = options; @@ -224,7 +232,33 @@ export async function continueProductionSetup(options: { ) throw new Error("Retained setup installation identities differ"); - const dockerEnvironment = composeEnvironment(deployment, environment); + const localDockerEndpoint = await assertLocalDockerContext({ + dockerExecutable: "/usr/bin/docker", + environment, + signal, + }); + const operationEnvironment = pinLocalDockerEndpoint( + environment, + localDockerEndpoint, + ); + const dockerEnvironment = composeEnvironment( + deployment, + operationEnvironment, + ); + if (installation.status !== "data-retained") { + const priorSelectedClients = installation.selectedClients.filter( + (client) => { + const state = integrations.integrations.find( + (entry) => entry.client === client, + )?.state; + return state === "verified" || state === "external-verified"; + }, + ); + await options.verifyPriorSelectedClients( + priorSelectedClients, + operationEnvironment, + ); + } if (installation.status === "data-retained") { const adapter = new DeploymentAdapter({ dockerExecutable: "/usr/bin/docker", @@ -237,7 +271,7 @@ export async function continueProductionSetup(options: { applicationPepperFile: deployment.applicationPepperFile, runtimeSocketDirectory: deployment.runtimeSocketDirectory, socketPath: deployment.socketPath, - hostEnvironment: environment, + hostEnvironment: operationEnvironment, }); await journal.runEffect({ step: "retained-service-reactivation", @@ -290,9 +324,8 @@ export async function continueProductionSetup(options: { const mustReconcile = installation.status === "data-retained" || priorIntegration === undefined || - priorIntegration.state === "failed" || - priorIntegration.state === "removed" || - priorIntegration.state === "retained-external"; + (priorIntegration.state !== "verified" && + priorIntegration.state !== "external-verified"); if (!mustReconcile) { clientResults.push({ client, diff --git a/src/onboarding/application/production-lifecycle.ts b/src/onboarding/application/production-lifecycle.ts index 8032b86..58dd829 100644 --- a/src/onboarding/application/production-lifecycle.ts +++ b/src/onboarding/application/production-lifecycle.ts @@ -267,6 +267,18 @@ function deploymentEnvironment( }); } +async function resolveLocalLifecycleDockerEnvironment( + environment: NodeJS.ProcessEnv, + signal: AbortSignal, +): Promise { + const endpoint = await assertLocalDockerContext({ + dockerExecutable: "/usr/bin/docker", + environment, + signal, + }); + return pinLocalDockerEndpoint(environment, endpoint); +} + async function observeOwnedComposeService( deployment: z.infer, service: "skillwire" | "postgres", @@ -1307,7 +1319,7 @@ async function repairOperation( status: journal.hasUnprovenEffect() ? "recovery-required" : "failed", }) .catch(() => undefined); - throw error; + throw journal.failure(error); } finally { await lock.release(); } @@ -1629,10 +1641,12 @@ async function rotateClientKeyOperation( recovery: { rollbackBoundary: "none", backupId: null, instructions: [] }, }); } catch (error) { - await journal.cancel({ - status: journal.hasUnprovenEffect() ? "recovery-required" : "failed", - }); - throw error; + await journal + .cancel({ + status: journal.hasUnprovenEffect() ? "recovery-required" : "failed", + }) + .catch(() => undefined); + throw journal.failure(error); } finally { await lock.release(); } @@ -1918,7 +1932,7 @@ async function rotateServiceSecretOperation( status: journal.hasUnprovenEffect() ? "recovery-required" : "failed", }) .catch(() => undefined); - throw error; + throw journal.failure(error); } finally { await lock.release(); } @@ -2131,7 +2145,7 @@ async function backupOperation( status: journal.hasUnprovenEffect() ? "recovery-required" : "failed", }) .catch(() => undefined); - throw error; + throw journal.failure(error); } finally { await lock.release(); } @@ -2964,7 +2978,7 @@ async function upgradeOperation( }) .catch(() => undefined); if (recovery === undefined || !(error instanceof UpgradeRecoveryError)) - throw error; + throw journal.failure(error); return result({ command: "upgrade", status: @@ -3280,7 +3294,7 @@ async function defaultUninstallOperation( status: journal.hasUnprovenEffect() ? "recovery-required" : "failed", }) .catch(() => undefined); - throw error; + throw journal.failure(error); } finally { await lock.release(); } @@ -3643,7 +3657,7 @@ async function clientUninstallOperation( status: journal.hasUnprovenEffect() ? "recovery-required" : "failed", }) .catch(() => undefined); - throw error; + throw journal.failure(error); } finally { await lock.release(); } @@ -3932,7 +3946,7 @@ async function purgeOperation( status: journal.hasUnprovenEffect() ? "recovery-required" : "failed", }) .catch(() => undefined); - throw error; + throw journal.failure(error); } finally { await lock.release(); } @@ -3940,22 +3954,44 @@ async function purgeOperation( export function createProductionLifecycleOperations( environment: NodeJS.ProcessEnv = process.env, + dependencies: { + readonly resolveDockerEnvironment?: + | (( + environment: NodeJS.ProcessEnv, + signal: AbortSignal, + ) => Promise) + | undefined; + } = {}, ): AdministrativeOperations { + const resolveDockerEnvironment = + dependencies.resolveDockerEnvironment ?? + resolveLocalLifecycleDockerEnvironment; + const withLocalDocker = + ( + operation: ( + command: ParsedCommand, + signal: AbortSignal, + environment: NodeJS.ProcessEnv, + ) => Promise, + ) => + async (command: ParsedCommand, signal: AbortSignal) => + operation( + command, + signal, + await resolveDockerEnvironment(environment, signal), + ); return { status: (command, signal) => statusOperation(command, signal, environment), doctor: (command, signal) => doctorOperation(command, signal, environment), - repair: (command, signal) => repairOperation(command, signal, environment), - "clients:rotate-key": (command, signal) => - rotateClientKeyOperation(command, signal, environment), - "maintenance:rotate-service-secret": (command, signal) => - rotateServiceSecretOperation(command, signal, environment), - backup: (command, signal) => backupOperation(command, signal, environment), - upgrade: (command, signal) => - upgradeOperation(command, signal, environment), - "clients:uninstall": (command, signal) => - clientUninstallOperation(command, signal, environment), - uninstall: (command, signal) => - defaultUninstallOperation(command, signal, environment), - purge: (command, signal) => purgeOperation(command, signal, environment), + repair: withLocalDocker(repairOperation), + "clients:rotate-key": withLocalDocker(rotateClientKeyOperation), + "maintenance:rotate-service-secret": withLocalDocker( + rotateServiceSecretOperation, + ), + backup: withLocalDocker(backupOperation), + upgrade: withLocalDocker(upgradeOperation), + "clients:uninstall": withLocalDocker(clientUninstallOperation), + uninstall: withLocalDocker(defaultUninstallOperation), + purge: withLocalDocker(purgeOperation), }; } diff --git a/src/onboarding/application/production-setup.ts b/src/onboarding/application/production-setup.ts index 6695557..1729582 100644 --- a/src/onboarding/application/production-setup.ts +++ b/src/onboarding/application/production-setup.ts @@ -1946,6 +1946,24 @@ export async function runProductionSetup( environment, signal, journal, + verifyPriorSelectedClients: async ( + clientsToVerify, + dockerEnvironment, + ) => { + const integrations = ClientIntegrationsStateSchema.parse( + await readProtectedSetupJson( + resolve(setupRoots.stateRoot, "client-integrations.json"), + ), + ); + await verifyUnchangedProductionSetup({ + installation: current, + integrations, + clientsToVerify, + roots: setupRoots, + environment: dockerEnvironment, + signal, + }); + }, }); await journal.commit({ status: "success" }); return continued; diff --git a/src/onboarding/cli/command-router.ts b/src/onboarding/cli/command-router.ts index 71f2c04..684103b 100644 --- a/src/onboarding/cli/command-router.ts +++ b/src/onboarding/cli/command-router.ts @@ -13,6 +13,7 @@ import { runProductionSetup, } from "../application/production-setup.js"; import { inspectInstalledStatus } from "../application/status.js"; +import { JournaledOperationFailure } from "../domain/operation-journal.js"; import type { AdminResult, ExitClass } from "./output.js"; function emit( @@ -466,34 +467,47 @@ export async function routeAdministrativeCommand( try { return emit(await operation(command, signal), command, io); } catch (error) { + const mutated = error instanceof JournaledOperationFailure; + const cancelled = signalIsAborted(signal); return emit( AdminResultSchema.parse({ schemaVersion: "skillwire.admin-result/v1", command: command.route, operationId: randomUUID(), - status: signalIsAborted(signal) ? "cancelled" : "failure", - exitClass: signalIsAborted(signal) + status: cancelled + ? "cancelled" + : mutated + ? "recovery-required" + : "failure", + exitClass: cancelled ? "user-cancellation" - : failureClass(error), + : mutated + ? "rollback-required" + : failureClass(error), previewHash: null, - changed: false, - summary: `${command.route} stopped before successful completion`, + changed: mutated, + summary: mutated + ? `${command.route} stopped after an owned mutation began` + : `${command.route} stopped before successful completion`, components: [], findings: [ { - code: "LIFECYCLE_OPERATION_FAILED", - severity: "error", + code: mutated + ? "LIFECYCLE_RECOVERY_REQUIRED" + : "LIFECYCLE_OPERATION_FAILED", + severity: mutated ? "recovery-required" : "error", component: command.route, summary: error instanceof Error ? error.message.slice(0, 512) : "Lifecycle operation failed", - nextAction: - "Resolve the reported condition and generate a fresh preview", + nextAction: mutated + ? "Inspect and recover the owned operation journal before retrying" + : "Resolve the reported condition and generate a fresh preview", }, ], recovery: { - rollbackBoundary: "none", + rollbackBoundary: mutated ? error.rollbackBoundary : "none", backupId: null, instructions: [], }, diff --git a/src/onboarding/domain/operation-journal.ts b/src/onboarding/domain/operation-journal.ts index 3359aed..5408625 100644 --- a/src/onboarding/domain/operation-journal.ts +++ b/src/onboarding/domain/operation-journal.ts @@ -80,6 +80,25 @@ export class JournaledEffectError extends Error { } } +export class JournaledOperationFailure extends Error { + readonly changed = true; + readonly recoveryRequired = true; + + public constructor( + message: string, + readonly rollbackBoundary: + | "automatic" + | "client-only" + | "application-config" + | "database-restore-required" + | "none" = "application-config", + options?: ErrorOptions, + ) { + super(message, options); + this.name = "JournaledOperationFailure"; + } +} + export class OperationJournal { readonly entries: JournalEntry[] = []; @@ -265,6 +284,34 @@ export class OperationJournal { } return unproven.size > 0; } + + hasIncompleteMutation(): boolean { + const unresolved = new Set(); + for (const entry of this.entries) { + if (entry.phase === "effect") unresolved.add(entry.step); + if (entry.phase !== "compensate") continue; + if (entry.detail["completion"] === "unproven") unresolved.add(entry.step); + else unresolved.delete(entry.step); + } + return unresolved.size > 0; + } + + failure(error: unknown): unknown { + if (!this.hasIncompleteMutation()) return error; + if (error instanceof JournaledOperationFailure) return error; + return new JournaledOperationFailure( + error instanceof Error + ? error.message + : "Lifecycle operation stopped after an owned mutation began", + this.command === "clients-uninstall" || + this.command === "clients-rotate-key" + ? "client-only" + : this.command === "purge" + ? "none" + : "application-config", + { cause: error }, + ); + } } export interface ProcessIdentity { diff --git a/tests/contract/cli/lifecycle-operations.test.ts b/tests/contract/cli/lifecycle-operations.test.ts index 8478861..14de01f 100644 --- a/tests/contract/cli/lifecycle-operations.test.ts +++ b/tests/contract/cli/lifecycle-operations.test.ts @@ -12,6 +12,7 @@ import { import { createProductionLifecycleOperations } from "../../../src/onboarding/application/production-lifecycle.js"; import { ensureServiceSecrets } from "../../../src/onboarding/secrets/service-secrets.js"; import { createOwnershipLedger } from "../../../src/onboarding/domain/ownership.js"; +import { JournaledOperationFailure } from "../../../src/onboarding/domain/operation-journal.js"; import type { ParsedCommand } from "../../../src/onboarding/cli/main.js"; import { createOnboardingEnvironment, @@ -303,6 +304,90 @@ describe("administrative lifecycle routes", () => { expect(stderr).toBe(""); }, ); + + it.each([ + ["repair", { component: "service" }], + ["clients:rotate-key", { client: "codex" }], + [ + "maintenance:rotate-service-secret", + { serviceSecret: "database-password" }, + ], + ["backup", {}], + ["upgrade", { release: "/tmp/verified-release.tar.zst" }], + ["clients:uninstall", { client: "claude" }], + ["uninstall", {}], + ["purge", {}], + ] as const)( + "rejects a remote named Docker context before executing %s", + async (route, extra) => { + fixture = await createOnboardingEnvironment(); + const resolveDockerEnvironment = vi.fn(async () => { + throw new Error( + "A local Docker context is required; remote contexts are refused", + ); + }); + const operations = createProductionLifecycleOperations( + fixture.environment, + { resolveDockerEnvironment }, + ); + const operation = operations[route]; + expect(operation).toBeDefined(); + + await expect( + operation?.( + { + route, + output: "json", + previewOnly: false, + ...extra, + }, + new AbortController().signal, + ), + ).rejects.toThrow(/local Docker context|remote context/i); + expect(resolveDockerEnvironment).toHaveBeenCalledOnce(); + }, + ); + + it("reports a journaled partial lifecycle mutation as recovery-required", async () => { + let stdout = ""; + const code = await routeAdministrativeCommand( + { + route: "uninstall", + output: "json", + previewOnly: false, + confirmPreview: "a".repeat(64), + }, + { + stdout: (value) => (stdout += value), + stderr: vi.fn(), + }, + new AbortController().signal, + { + uninstall: async () => { + throw new JournaledOperationFailure( + "owned uninstall effect may have completed", + "application-config", + { cause: new Error("simulated post-effect failure") }, + ); + }, + }, + ); + + expect(code).toBe(10); + expect(JSON.parse(stdout)).toMatchObject({ + command: "uninstall", + status: "recovery-required", + exitClass: "rollback-required", + changed: true, + findings: [ + { + code: "LIFECYCLE_RECOVERY_REQUIRED", + severity: "recovery-required", + }, + ], + recovery: { rollbackBoundary: "application-config" }, + }); + }); }); async function fixtureSnapshot(root: string): Promise { diff --git a/tests/integration/onboarding/production-continuation.test.ts b/tests/integration/onboarding/production-continuation.test.ts new file mode 100644 index 0000000..c698529 --- /dev/null +++ b/tests/integration/onboarding/production-continuation.test.ts @@ -0,0 +1,448 @@ +import { randomUUID } from "node:crypto"; +import { chmod, lstat, mkdir, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { DeploymentOptions } from "../../../src/onboarding/adapters/docker/deployment.js"; +import type * as DockerEnvironment from "../../../src/onboarding/adapters/docker/environment.js"; + +const dockerBoundary = vi.hoisted(() => ({ + assertLocal: vi.fn<() => Promise>(), + deployments: [] as DeploymentOptions[], + databaseEnvironments: [] as NodeJS.ProcessEnv[], +})); + +vi.mock( + "../../../src/onboarding/adapters/docker/environment.js", + async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + assertLocalDockerContext: dockerBoundary.assertLocal, + }; + }, +); + +vi.mock("../../../src/onboarding/adapters/docker/deployment.js", () => ({ + DeploymentAdapter: class { + constructor(options: DeploymentOptions) { + dockerBoundary.deployments.push(options); + } + async probe(): Promise { + await Promise.resolve(); + } + async deploy(): Promise { + await Promise.resolve(); + } + }, +})); + +vi.mock( + "../../../src/onboarding/adapters/postgres/service-database.js", + () => ({ + ServiceDatabase: class { + constructor(options: { readonly environment: NodeJS.ProcessEnv }) { + dockerBoundary.databaseEnvironments.push(options.environment); + } + async verifyVolume(): Promise { + await Promise.resolve(); + } + async verifySchemaAndReadiness(): Promise { + await Promise.resolve(); + } + }, + }), +); + +import { continueProductionSetup } from "../../../src/onboarding/application/production-continuation.js"; +import { + InstallationSchema, + type Installation, +} from "../../../src/onboarding/domain/installation.js"; +import { OperationJournal } from "../../../src/onboarding/domain/operation-journal.js"; +import { createOwnershipLedger } from "../../../src/onboarding/domain/ownership.js"; +import { snapshotTree } from "../../helpers/filesystem-snapshot.js"; +import { + createOnboardingEnvironment, + type OnboardingEnvironment, +} from "../../helpers/onboarding-environment.js"; + +async function protectedJson(path: string, value: unknown): Promise { + await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + await writeFile(path, JSON.stringify(value), { mode: 0o600 }); + await chmod(path, 0o600); +} + +async function persistedContinuation(options: { + readonly fixture: OnboardingEnvironment; + readonly state: + "verified" | "external-verified" | "adapter-installed" | "none"; + readonly status?: "complete" | "data-retained"; +}): Promise<{ + readonly installation: Installation; + readonly stateRoot: string; + readonly dataRoot: string; + readonly launcherPath: string; + readonly operationId: string; +}> { + const installationId = randomUUID(); + const operationId = randomUUID(); + const stateRoot = resolve(options.fixture.xdgStateHome, "skillwire"); + const dataRoot = resolve(options.fixture.xdgDataHome, "skillwire"); + const launcherPath = resolve(options.fixture.root, "owned/bin/skillwire"); + const selected = options.state === "none" ? [] : (["codex"] as const); + const integrationId = options.state === "none" ? null : randomUUID(); + const credentialId = + options.state === "verified" || options.state === "adapter-installed" + ? randomUUID() + : null; + const now = new Date().toISOString(); + const installation = InstallationSchema.parse({ + schemaVersion: "skillwire.installation/v1", + installationId, + ownerUid: process.getuid?.() ?? 1000, + accountId: randomUUID(), + activeReleaseId: "1-amd64", + highestAcceptedReleaseSequence: 1, + activeTrustPolicySequence: 1, + endpoint: `unix://${resolve(options.fixture.runtimeRoot, "skillwire/mcp.sock")}`, + composeProject: options.fixture.composeProject, + postgresVolume: options.fixture.postgresVolume, + selectedClients: selected, + clientIntegrationIds: { codex: integrationId, claude: null }, + status: options.status ?? "complete", + createdAt: now, + updatedAt: now, + lastValidatedAt: now, + }); + await protectedJson(resolve(stateRoot, "deployment.json"), { + schemaVersion: "skillwire.deployment/v1", + installationId, + releaseRoot: options.fixture.root, + composePath: resolve(options.fixture.root, "compose.yaml"), + skillwireImage: `example.invalid/skillwire@sha256:${"a".repeat(64)}`, + postgresImage: `example.invalid/postgres@sha256:${"b".repeat(64)}`, + databasePasswordFile: resolve(options.fixture.root, "database-password"), + applicationPepperFile: resolve(options.fixture.root, "application-pepper"), + runtimeSocketDirectory: resolve(options.fixture.runtimeRoot, "skillwire"), + socketPath: resolve(options.fixture.runtimeRoot, "skillwire/mcp.sock"), + projectName: options.fixture.composeProject, + volumeName: options.fixture.postgresVolume, + }); + await protectedJson( + resolve(stateRoot, "installations", installationId, "bridge-state.json"), + { + schemaVersion: "skillwire.bridge-state/v1", + installationId, + transport: "unix-domain-socket", + endpoint: "http://localhost/mcp", + socketPath: resolve(options.fixture.runtimeRoot, "skillwire/mcp.sock"), + clients: + options.state === "verified" || options.state === "adapter-installed" + ? [ + { + client: "codex", + credentialReference: "restrictive-file:codex", + keyId: randomUUID(), + }, + ] + : [], + }, + ); + await protectedJson(resolve(stateRoot, "credential-references.json"), { + schemaVersion: "skillwire.credential-references/v1", + installationId, + credentials: + credentialId === null + ? [] + : [ + { + schemaVersion: "skillwire.credential-reference/v1", + credentialReferenceId: credentialId, + installationId, + client: "codex", + backend: "restrictive-file", + locator: "restrictive-file:codex", + keyPublicIdHash: "c".repeat(64), + createdByOperation: operationId, + state: "available", + fallbackRiskConfirmed: true, + }, + ], + }); + await protectedJson(resolve(stateRoot, "client-integrations.json"), { + schemaVersion: "skillwire.client-integrations/v1", + installationId, + integrations: + integrationId === null + ? [] + : [ + { + schemaVersion: "skillwire.client-integration/v1", + clientIntegrationId: integrationId, + installationId, + client: "codex", + clientVersion: "0.147.0", + profileScope: "normal-user", + state: options.state, + credentialReferenceId: credentialId, + keyPublicIdHash: + options.state === "verified" || + options.state === "adapter-installed" + ? "c".repeat(64) + : null, + mcpIdentitySha256: "d".repeat(64), + adapterIdentitySha256: "e".repeat(64), + }, + ], + }); + await protectedJson(resolve(stateRoot, "external-integrations.json"), { + schemaVersion: "skillwire.external-integrations/v1", + installationId, + dependencies: [], + }); + await protectedJson( + resolve(stateRoot, "ownership.json"), + createOwnershipLedger(installationId).record, + ); + return { installation, stateRoot, dataRoot, launcherPath, operationId }; +} + +describe("production setup continuation boundaries", () => { + let fixture: OnboardingEnvironment | undefined; + + beforeEach(() => { + dockerBoundary.assertLocal.mockReset(); + dockerBoundary.assertLocal.mockResolvedValue( + "unix:///tmp/disposable-docker.sock", + ); + dockerBoundary.deployments.length = 0; + dockerBoundary.databaseEnvironments.length = 0; + }); + + afterEach(async () => { + await fixture?.close(); + fixture = undefined; + }); + + it.each(["verified", "external-verified"] as const)( + "verifies a prior %s client before any newly requested client mutation", + async (state) => { + fixture = await createOnboardingEnvironment(); + const persisted = await persistedContinuation({ fixture, state }); + const verifyPriorSelectedClients = vi + .fn<(clients: readonly ("codex" | "claude")[]) => Promise>() + .mockRejectedValue(new Error("prior selected client drifted")); + const journal = await OperationJournal.create( + resolve(persisted.stateRoot, "operations"), + persisted.operationId, + "setup", + ); + const before = await snapshotTree(fixture.root); + + await expect( + continueProductionSetup({ + setup: { clients: "claude" }, + credentialBackend: "restrictive-file", + installation: persisted.installation, + home: fixture.home, + dataRoot: persisted.dataRoot, + stateRoot: persisted.stateRoot, + runtimeRoot: fixture.runtimeRoot, + launcherPath: persisted.launcherPath, + environment: fixture.environment, + signal: new AbortController().signal, + journal, + verifyPriorSelectedClients, + }), + ).rejects.toThrow(/prior selected client drifted/i); + + expect(verifyPriorSelectedClients).toHaveBeenCalledOnce(); + expect(verifyPriorSelectedClients).toHaveBeenCalledWith( + ["codex"], + expect.objectContaining({ + DOCKER_HOST: "unix:///tmp/disposable-docker.sock", + }), + ); + expect(journal.entries).toEqual([]); + expect(await snapshotTree(fixture.root)).toEqual(before); + }, + ); + + it("keeps an external prior integration credential-free during verification", async () => { + fixture = await createOnboardingEnvironment(); + const persisted = await persistedContinuation({ + fixture, + state: "external-verified", + }); + const verifyPriorSelectedClients = vi.fn(() => Promise.resolve()); + const journal = await OperationJournal.create( + resolve(persisted.stateRoot, "operations"), + persisted.operationId, + "setup", + ); + + const result = await continueProductionSetup({ + setup: { clients: "none" }, + credentialBackend: "not-selected", + installation: persisted.installation, + home: fixture.home, + dataRoot: persisted.dataRoot, + stateRoot: persisted.stateRoot, + runtimeRoot: fixture.runtimeRoot, + launcherPath: persisted.launcherPath, + environment: fixture.environment, + signal: new AbortController().signal, + journal, + verifyPriorSelectedClients, + }); + + expect(result).toMatchObject({ status: "success", serviceReady: true }); + expect(result.clients).toEqual([ + expect.objectContaining({ + client: "codex", + status: "external-verified", + owned: false, + }), + ]); + expect(verifyPriorSelectedClients).toHaveBeenCalledWith( + ["codex"], + expect.objectContaining({ + DOCKER_HOST: "unix:///tmp/disposable-docker.sock", + }), + ); + await expect( + lstat(resolve(persisted.dataRoot, "credentials")), + ).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("never promotes an unverified prior integration from persisted metadata", async () => { + fixture = await createOnboardingEnvironment(); + const persisted = await persistedContinuation({ + fixture, + state: "adapter-installed", + }); + const verifyPriorSelectedClients = vi.fn(() => Promise.resolve()); + const journal = await OperationJournal.create( + resolve(persisted.stateRoot, "operations"), + persisted.operationId, + "setup", + ); + + await expect( + continueProductionSetup({ + setup: { clients: "none" }, + credentialBackend: "restrictive-file", + installation: persisted.installation, + home: fixture.home, + dataRoot: persisted.dataRoot, + stateRoot: persisted.stateRoot, + runtimeRoot: fixture.runtimeRoot, + launcherPath: persisted.launcherPath, + environment: { ...fixture.environment, PATH: fixture.repository }, + signal: new AbortController().signal, + journal, + verifyPriorSelectedClients, + }), + ).rejects.toThrow(/codex.*unavailable/i); + + expect(verifyPriorSelectedClients).toHaveBeenCalledWith( + [], + expect.objectContaining({ + DOCKER_HOST: "unix:///tmp/disposable-docker.sock", + }), + ); + }); + + it("rejects a remote named Docker context before any continuation effect", async () => { + fixture = await createOnboardingEnvironment(); + const persisted = await persistedContinuation({ + fixture, + state: "verified", + }); + dockerBoundary.assertLocal.mockRejectedValueOnce( + new Error( + "A local Docker context is required; remote contexts are refused", + ), + ); + const journal = await OperationJournal.create( + resolve(persisted.stateRoot, "operations"), + persisted.operationId, + "setup", + ); + const before = await snapshotTree(fixture.root); + const verifyPriorSelectedClients = vi.fn(() => Promise.resolve()); + + await expect( + continueProductionSetup({ + setup: { clients: "none" }, + credentialBackend: "not-selected", + installation: persisted.installation, + home: fixture.home, + dataRoot: persisted.dataRoot, + stateRoot: persisted.stateRoot, + runtimeRoot: fixture.runtimeRoot, + launcherPath: persisted.launcherPath, + environment: { + ...fixture.environment, + DOCKER_CONTEXT: "remote-production", + }, + signal: new AbortController().signal, + journal, + verifyPriorSelectedClients, + }), + ).rejects.toThrow(/local Docker context|remote/i); + + expect(verifyPriorSelectedClients).not.toHaveBeenCalled(); + expect(journal.entries).toEqual([]); + expect(await snapshotTree(fixture.root)).toEqual(before); + }); + + it("pins an accepted named Docker context for retained-service Docker effects", async () => { + fixture = await createOnboardingEnvironment(); + const persisted = await persistedContinuation({ + fixture, + state: "none", + status: "data-retained", + }); + const journal = await OperationJournal.create( + resolve(persisted.stateRoot, "operations"), + persisted.operationId, + "setup", + ); + + await continueProductionSetup({ + setup: { clients: "none" }, + credentialBackend: "not-selected", + installation: persisted.installation, + home: fixture.home, + dataRoot: persisted.dataRoot, + stateRoot: persisted.stateRoot, + runtimeRoot: fixture.runtimeRoot, + launcherPath: persisted.launcherPath, + environment: { + ...fixture.environment, + DOCKER_CONTEXT: "rootless-local", + }, + signal: new AbortController().signal, + journal, + verifyPriorSelectedClients: vi.fn(() => Promise.resolve()), + }); + + expect(dockerBoundary.deployments).toHaveLength(1); + expect(dockerBoundary.deployments[0]?.hostEnvironment).toMatchObject({ + DOCKER_HOST: "unix:///tmp/disposable-docker.sock", + }); + expect(dockerBoundary.deployments[0]?.hostEnvironment).not.toHaveProperty( + "DOCKER_CONTEXT", + ); + expect(dockerBoundary.databaseEnvironments).toHaveLength(1); + expect(dockerBoundary.databaseEnvironments[0]).toMatchObject({ + DOCKER_HOST: "unix:///tmp/disposable-docker.sock", + }); + expect(dockerBoundary.databaseEnvironments[0]).not.toHaveProperty( + "DOCKER_CONTEXT", + ); + }); +}); diff --git a/tests/unit/onboarding/operation-journal.test.ts b/tests/unit/onboarding/operation-journal.test.ts index e93449c..bb75b95 100644 --- a/tests/unit/onboarding/operation-journal.test.ts +++ b/tests/unit/onboarding/operation-journal.test.ts @@ -11,6 +11,7 @@ import { import { currentProcessIdentity, InstallationLock, + JournaledOperationFailure, OperationJournal, } from "../../../src/onboarding/domain/operation-journal.js"; @@ -107,6 +108,33 @@ describe("operation journal and installation lock", () => { expect(journal.hasUnprovenEffect()).toBe(false); }); + it("wraps only unresolved journaled mutations for the CLI recovery envelope", async () => { + fixture = await createOnboardingEnvironment(); + const journal = await OperationJournal.create( + resolve(fixture.root, "journals"), + randomUUID(), + "uninstall", + ); + const failure = new Error("post-effect publication failed"); + await journal.intent("uninstall", { confirmed: true }); + expect(journal.failure(failure)).toBe(failure); + + await journal.runEffect({ + step: "uninstall-owned-client", + intent: { client: "codex" }, + signal: new AbortController().signal, + action: () => Promise.resolve(), + verification: () => ({ removed: true }), + }); + expect(journal.failure(failure)).toBeInstanceOf(JournaledOperationFailure); + + await journal.compensate("uninstall-owned-client", { + completion: "reverted", + recoveryRequired: false, + }); + expect(journal.failure(failure)).toBe(failure); + }); + it("persists the required setup effect inventory as intent/effect/verify triplets", async () => { fixture = await createOnboardingEnvironment(); const journal = await OperationJournal.create( diff --git a/tests/unit/onboarding/restored-database-validation.test.ts b/tests/unit/onboarding/restored-database-validation.test.ts index 8c83f3a..4d3d3a0 100644 --- a/tests/unit/onboarding/restored-database-validation.test.ts +++ b/tests/unit/onboarding/restored-database-validation.test.ts @@ -16,7 +16,7 @@ import { hashExternalAdvisoryEvent } from "../../../src/domain/external-catalog/ import type { CommandOptions } from "../../../src/onboarding/adapters/process/command-runner.js"; import { createOnboardingEnvironment } from "../../helpers/onboarding-environment.js"; -const REQUIRED_CONSTRAINTS = [ +const REQUIRED_CONSTRAINT_NAMES = [ "accounts_pkey", "accounts_status_check", "api_keys_account_id_fkey", @@ -31,18 +31,172 @@ const REQUIRED_CONSTRAINTS = [ "schema_migrations_pkey", ]; -const REQUIRED_TRIGGERS = [ - "external_advisory_append_valid", - "external_advisory_events_immutable", - "external_dependencies_immutable", - "external_resources_immutable", - "external_revisions_immutable", - "external_snapshots_immutable", +const REQUIRED_CONSTRAINTS: RestoredDatabaseEvidence["constraints"] = + REQUIRED_CONSTRAINT_NAMES.map((constraintName) => ({ + schemaName: "public", + tableName: constraintName.split("_").slice(0, -1).join("_") || "accounts", + constraintName, + constraintType: constraintName.endsWith("_pkey") + ? ("primary-key" as const) + : constraintName.endsWith("_fkey") + ? ("foreign-key" as const) + : ("check" as const), + definition: `fixture definition for ${constraintName}`, + validated: true, + })); + +function trigger( + triggerName: string, + tableName: string, + functionName: string, + events: RestoredDatabaseEvidence["triggers"][number]["events"], + timing: RestoredDatabaseEvidence["triggers"][number]["timing"] = "BEFORE", +): RestoredDatabaseEvidence["triggers"][number] { + const functionBodySha256 = { + reject_external_history_mutation: + "4adf876c6bd96600896c6d48b63a2900408722ed33f8603af7a413555788e83c", + protect_github_registration_identity: + "b7e09311ad9c92f1d002dabe2991ee0e987071de9dfcec937f17fe62fab8071e", + validate_external_classification_transition: + "511e1833ab29d867a5c5a7ad5036aaabac942dab4e4f55d3de3247a0322507cd", + validate_external_advisory_append: + "ec80f70ea91225d44339c1a6aef7da0569e6aee6630f57c495430546fb6178df", + guard_external_snapshot_finalization: + "37167092120146842dfe2976dfcc45034ea58ad6eb4a4a576b0e0fd0e3ada9c9", + require_external_snapshot_finalization: + "e26c0466292fb76c9f4cce6af78ea7f617617d7a5f117fd20e15f3bfbbfdde8c", + validate_external_revision_classification_transition: + "01461630956a69c1ead4bfd7bf1df621e217a352052123fcfe79e401dea840b8", + }[functionName]; + if (functionBodySha256 === undefined) + throw new Error("Test trigger function is not release-bound"); + return { + schemaName: "public", + tableName, + triggerName, + functionSchema: "public", + functionName, + functionArguments: "", + functionDefinition: `CREATE FUNCTION ${functionName}() RETURNS trigger LANGUAGE plpgsql AS 'fixture'`, + functionBodySha256, + timing, + level: "ROW", + events, + enabled: "origin", + definition: `CREATE TRIGGER ${triggerName} ${timing} ${events.join(" OR ")} ON ${tableName} FOR EACH ROW EXECUTE FUNCTION ${functionName}()`, + }; +} + +const REQUIRED_TRIGGERS: RestoredDatabaseEvidence["triggers"] = [ + trigger( + "external_content_immutable", + "external_content_objects", + "reject_external_history_mutation", + ["DELETE", "UPDATE"], + ), + trigger( + "external_identities_immutable", + "external_skill_identities", + "reject_external_history_mutation", + ["DELETE", "UPDATE"], + ), + trigger( + "external_revisions_immutable", + "external_skill_revisions", + "reject_external_history_mutation", + ["DELETE", "UPDATE"], + ), + trigger( + "external_resources_immutable", + "external_revision_resources", + "reject_external_history_mutation", + ["DELETE", "UPDATE"], + ), + trigger( + "external_dependencies_immutable", + "external_revision_dependencies", + "reject_external_history_mutation", + ["DELETE", "UPDATE"], + ), + trigger( + "external_observations_immutable", + "external_snapshot_skill_observations", + "reject_external_history_mutation", + ["DELETE", "UPDATE"], + ), + trigger( + "github_source_registration_identity_immutable", + "github_source_registrations", + "protect_github_registration_identity", + ["DELETE", "UPDATE"], + ), + trigger( + "external_classification_transition_valid", + "external_classification_events", + "validate_external_classification_transition", + ["INSERT"], + ), + trigger( + "external_advisory_append_valid", + "external_revision_advisory_events", + "validate_external_advisory_append", + ["INSERT"], + ), + ...[ + ["external_candidates_immutable", "external_import_candidates"], + ["external_reports_immutable", "external_verification_reports"], + ["external_findings_immutable", "external_validation_findings"], + [ + "external_classification_events_immutable", + "external_classification_events", + ], + ["external_curation_decisions_immutable", "external_curation_decisions"], + ["external_advisory_events_immutable", "external_revision_advisory_events"], + ].map(([triggerName, tableName]) => + trigger( + triggerName ?? "invalid", + tableName ?? "invalid", + "reject_external_history_mutation", + ["DELETE", "UPDATE"], + ), + ), + trigger( + "github_sync_candidate_results_immutable", + "github_sync_candidate_results", + "reject_external_history_mutation", + ["DELETE", "UPDATE"], + ), + trigger( + "external_snapshots_immutable", + "external_source_snapshots", + "guard_external_snapshot_finalization", + ["DELETE", "UPDATE"], + ), + trigger( + "external_snapshot_finalization_required", + "external_source_snapshots", + "require_external_snapshot_finalization", + ["INSERT", "UPDATE"], + "AFTER", + ), + trigger( + "external_revision_classification_transition_valid", + "external_revision_classification_events", + "validate_external_revision_classification_transition", + ["INSERT"], + ), + trigger( + "external_revision_classification_events_immutable", + "external_revision_classification_events", + "reject_external_history_mutation", + ["DELETE", "UPDATE"], + ), ]; function evidence(options: { readonly accountId: string; readonly checksums: readonly string[]; + readonly triggers?: RestoredDatabaseEvidence["triggers"]; }): RestoredDatabaseEvidence { const advisoryInput = { sequence: "1", @@ -63,7 +217,7 @@ function evidence(options: { checksum, })), constraints: REQUIRED_CONSTRAINTS, - triggers: REQUIRED_TRIGGERS, + triggers: options.triggers ?? [], catalog: { snapshotCount: 1, revisionCount: 2, @@ -274,6 +428,227 @@ describe("production restored-database validation", () => { }, ); + it("accepts the complete migration-010 schema-control inventory", () => { + const accountId = randomUUID(); + const checksums = Array.from({ length: 10 }, (_, index) => + (index + 1).toString(16).padStart(64, "0"), + ); + const valid = evidence({ + accountId, + checksums, + triggers: REQUIRED_TRIGGERS, + }); + + expect( + assessRestoredDatabaseEvidence(valid, { + expectedMigrations: valid.migrations, + installationAccountId: accountId, + expectedActiveApiKeys: 2, + expectedDatabase: "postgres", + expectedState: databaseStateExpectation(valid), + }), + ).toMatchObject({ + latestMigration: "010", + constraintsValid: true, + ready: true, + }); + }); + + it("rejects the superseded classification-trigger function body after migration 010", () => { + const accountId = randomUUID(); + const checksums = Array.from({ length: 10 }, (_, index) => + (index + 1).toString(16).padStart(64, "0"), + ); + const valid = evidence({ + accountId, + checksums, + triggers: REQUIRED_TRIGGERS, + }); + const superseded: RestoredDatabaseEvidence = { + ...valid, + triggers: valid.triggers.map((entry) => + entry.triggerName === "external_classification_transition_valid" + ? { + ...entry, + functionBodySha256: + "7e29bd82153cfd0976b925d2dd6a18879f3c9e16a4c79faf1586fdddb9aad718", + } + : entry, + ), + }; + + expect(() => + assessRestoredDatabaseEvidence(superseded, { + expectedMigrations: superseded.migrations, + installationAccountId: accountId, + expectedActiveApiKeys: 2, + expectedDatabase: "postgres", + expectedState: databaseStateExpectation(superseded), + }), + ).toThrow(/restore validation/i); + }); + + it.each([ + [ + "disabled trigger", + (value: RestoredDatabaseEvidence) => ({ + ...value, + triggers: value.triggers.map((entry) => + entry.triggerName === + "external_revision_classification_events_immutable" + ? { ...entry, enabled: "disabled" as const } + : entry, + ), + }), + ], + [ + "replacement trigger function", + (value: RestoredDatabaseEvidence) => ({ + ...value, + triggers: value.triggers.map((entry) => + entry.triggerName === + "external_revision_classification_events_immutable" + ? { ...entry, functionName: "attacker_owned_trigger" } + : entry, + ), + }), + ], + [ + "replacement trigger event", + (value: RestoredDatabaseEvidence) => ({ + ...value, + triggers: value.triggers.map((entry) => + entry.triggerName === + "external_revision_classification_events_immutable" + ? { ...entry, events: ["INSERT"] as const } + : entry, + ), + }), + ], + [ + "changed trigger function body", + (value: RestoredDatabaseEvidence) => ({ + ...value, + triggers: value.triggers.map((entry) => + entry.triggerName === + "external_revision_classification_events_immutable" + ? { ...entry, functionBodySha256: "f".repeat(64) } + : entry, + ), + }), + ], + [ + "missing migration-010 trigger", + (value: RestoredDatabaseEvidence) => ({ + ...value, + triggers: value.triggers.filter( + ({ triggerName }) => + triggerName !== "external_revision_classification_events_immutable", + ), + }), + ], + ] as const)("rejects a restored database with a %s", (_name, corrupt) => { + const accountId = randomUUID(); + const checksums = Array.from({ length: 10 }, (_, index) => + (index + 1).toString(16).padStart(64, "0"), + ); + const valid = evidence({ + accountId, + checksums, + triggers: REQUIRED_TRIGGERS, + }); + const corrupted = corrupt(valid); + + expect(() => + assessRestoredDatabaseEvidence(corrupted, { + expectedMigrations: valid.migrations, + installationAccountId: accountId, + expectedActiveApiKeys: 2, + expectedDatabase: "postgres", + expectedState: databaseStateExpectation(valid), + }), + ).toThrow(/restore validation/i); + }); + + it("rejects canonical schema-control drift even when the live expectation is already drifted", () => { + const accountId = randomUUID(); + const checksums = Array.from({ length: 10 }, (_, index) => + (index + 1).toString(16).padStart(64, "0"), + ); + const valid = evidence({ + accountId, + checksums, + triggers: REQUIRED_TRIGGERS, + }); + const drifted: RestoredDatabaseEvidence = { + ...valid, + triggers: valid.triggers.map((entry) => + entry.triggerName === + "external_revision_classification_events_immutable" + ? { + ...entry, + functionDefinition: `${entry.functionDefinition}\n-- replaced`, + functionBodySha256: "e".repeat(64), + } + : entry, + ), + }; + + expect(() => + assessRestoredDatabaseEvidence(drifted, { + expectedMigrations: drifted.migrations, + installationAccountId: accountId, + expectedActiveApiKeys: 2, + expectedDatabase: "postgres", + expectedState: databaseStateExpectation(drifted), + }), + ).toThrow(/restore validation/i); + }); + + it.each([ + [ + "constraint definition", + (entry: RestoredDatabaseEvidence["constraints"][number]) => ({ + ...entry, + definition: `${entry.definition} NOT VALID`, + }), + ], + [ + "constraint table identity", + (entry: RestoredDatabaseEvidence["constraints"][number]) => ({ + ...entry, + tableName: "attacker_shadow_table", + }), + ], + [ + "constraint validation state", + (entry: RestoredDatabaseEvidence["constraints"][number]) => ({ + ...entry, + validated: false, + }), + ], + ] as const)("rejects restored %s drift", (_name, corrupt) => { + const accountId = randomUUID(); + const checksum = "a".repeat(64); + const valid = evidence({ accountId, checksums: [checksum] }); + const corrupted = { + ...valid, + constraints: valid.constraints.map((entry, index) => + index === 0 ? corrupt(entry) : entry, + ), + }; + + expect(() => + assessRestoredDatabaseEvidence(corrupted, { + expectedMigrations: valid.migrations, + installationAccountId: accountId, + expectedActiveApiKeys: 2, + expectedDatabase: "postgres", + expectedState: databaseStateExpectation(valid), + }), + ).toThrow(/restore validation/i); + }); + it("queries raw production evidence and rejects corrupt callback output", async () => { const accountId = randomUUID(); const checksum = "b".repeat(64); @@ -318,9 +693,18 @@ describe("production restored-database validation", () => { const query = commands[0]?.args.at(-1) ?? ""; expect(query).toContain("schema_migrations"); expect(query).toContain("pg_constraint"); + expect(query).toContain("pg_get_constraintdef"); + expect(query).toContain("WHEN 't' THEN 'constraint-trigger'"); + expect(query).toContain("WHEN 'n' THEN 'not-null'"); + expect(query).toContain("constraint_entry.conrelid<>0"); expect(query).toContain("external_revision_advisory_events"); expect(query).toContain("external_content_objects"); expect(query).toContain("repository_skill_usage"); + expect(query).toContain("pg_get_triggerdef"); + expect(query).toContain("pg_get_functiondef"); + expect(query).toContain("functionBodySha256"); + expect(query).toContain("tgenabled"); + expect(query).toContain("proname"); expect(query).not.toMatch(/AS\s+(?:invariants|catalog|advisory)_valid/i); }); }); From a6f56e9879c85c12e320436ac1650a1e955b4b87 Mon Sep 17 00:00:00 2001 From: Lucenx9 Date: Fri, 14 Aug 2026 17:47:37 +0200 Subject: [PATCH 7/7] test: isolate lifecycle Docker context in containers --- tests/integration/onboarding/interruption-recovery.test.ts | 6 +++--- .../integration/onboarding/service-secret-rotation.test.ts | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/integration/onboarding/interruption-recovery.test.ts b/tests/integration/onboarding/interruption-recovery.test.ts index a6d283c..d16c9c9 100644 --- a/tests/integration/onboarding/interruption-recovery.test.ts +++ b/tests/integration/onboarding/interruption-recovery.test.ts @@ -384,9 +384,9 @@ describe("observation-based interruption recovery", () => { await interrupted.intent("upgrade-migration", { schema: 10 }); await interrupted.effect("upgrade-migration", { completion: "recorded" }); - const repair = createProductionLifecycleOperations( - fixture.environment, - ).repair; + const repair = createProductionLifecycleOperations(fixture.environment, { + resolveDockerEnvironment: async (environment) => environment, + }).repair; expect(repair).toBeDefined(); const preview = await repair?.( { diff --git a/tests/integration/onboarding/service-secret-rotation.test.ts b/tests/integration/onboarding/service-secret-rotation.test.ts index aec6143..763c61d 100644 --- a/tests/integration/onboarding/service-secret-rotation.test.ts +++ b/tests/integration/onboarding/service-secret-rotation.test.ts @@ -83,7 +83,12 @@ describe("explicit service-secret rotation", () => { it("blocks application-pepper rotation before mutation when the runtime has no safe overlap support", async () => { fixture = await createOnboardingEnvironment(); - const operations = createProductionLifecycleOperations(fixture.environment); + const operations = createProductionLifecycleOperations( + fixture.environment, + { + resolveDockerEnvironment: async (environment) => environment, + }, + ); const before = await snapshotTree(fixture.root); await expect(