feat(deployment): open hidden variables sealed under a team's data key

apps/api/docs/runbooks/deployment-secrets.md

@@ -117,7 +117,7 @@ DEPLOYMENT_ENV=staging NETWORK=sandbox doppler run -p console-api -c staging-san
    node --require ./dist/instrumentation.js ./dist/console.js rekey-user-data-key --user-id <uuid>
    ```
 
-   The run retires the user's active key and inserts its replacement in one transaction, then re-seals each of the user's tokens under the new key. Both keys stay openable throughout and the user keeps deploying. The report states `deploymentsResealed`, `secretsResealed`, `deploymentsAlreadyUnderActiveKey`, `deploymentsMovedByAnotherWriter`, `deploymentsUnderUnknownKey`, `retiredDataKeyDeleted` and `retiredDataKeyDeletableAfter`.
+   The run retires the user's active key and inserts its replacement in one transaction, then re-seals each of the user's tokens under the new key. Both keys stay openable throughout and the user keeps deploying. The report states `deploymentsResealed`, `secretsResealed`, `deploymentsAlreadyUnderActiveKey`, `deploymentsMovedByAnotherWriter`, `deploymentsSealedForOrganization`, `deploymentsUnderUnknownKey`, `retiredDataKeyDeleted` and `retiredDataKeyDeletableAfter`. `deploymentsSealedForOrganization` counts the user's team deployments whose token is sealed under the team's own data key; the user's key never sealed them, so the re-key leaves them where they are and they do not hold back the retired key's deletion.
 
 3. The first run always keeps the retired key and reports `retiredDataKeyDeleted: false`, because a request that read the old key just before it was retired may still be writing a value under it. Run the command again at or after `retiredDataKeyDeletableAfter`, a minute after the retirement. That run re-seals any straggler, checks that nothing is still sealed under the retired key, and deletes it. Only then is the leaked key useless.
 
@@ -165,7 +165,7 @@ There is no threshold to tune: a healthy system produces none. The event is writ
 
 What it means: a token's ciphertext or header was altered, or it names a data key the user does not hold. Both are wrong in a way a bug is not.
 
-What to look at first: the event carries `userId`, `dseq` and the token's header `claims`. Compare `claims.kid` with that user's `data_keys` rows. If a `DATA_KEY_REKEY_*` run for that user is in progress or just finished, and the kid is the key it retired, a request opened a token during the window; re-run the re-key. Otherwise treat it as tampering: snapshot the database, page security, and do not repair the row.
+What to look at first: the event carries `userId`, `dseq` and the token's header `claims`. A token sealed under a team's data key has an `org` claim instead of `sub`: compare its `claims.kid` with that organization's `data_keys` rows (`organization_id = claims.org`), not the user's. A re-key only ever moves a user's own tokens, so a team token whose kid the team does not hold is tampering. For a user's token, compare `claims.kid` with that user's `data_keys` rows. If a `DATA_KEY_REKEY_*` run for that user is in progress or just finished, and the kid is the key it retired, a request opened a token during the window; re-run the re-key. Otherwise treat it as tampering: snapshot the database, page security, and do not repair the row.
 
 ### 5.2 Many distinct users unwrapped in a short window
 
@@ -328,16 +328,16 @@ The same property is what makes section 2 step 5 load-bearing: destroying a KMS
 
 ## 8. Events and metrics
 
-| Name                                              | Kind         | Meaning                                                                                                                               |
-| ------------------------------------------------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------- |
-| `USER_DATA_KEY_UNWRAPPED`                         | event, info  | A request unwrapped a user's data key. Carries `userId`, `dataKeyId`, `kid`. The only per-user record of secret access.               |
-| `USER_DATA_KEY_WRAPPED_UNDER_UNKNOWN_KID`         | event, error | A data key names a KMS version the console cannot resolve or the service refuses (disabled, destroyed, unknown). The user gets a 503. |
-| `USER_DATA_KEY_UNWRAP_FAILED`                     | event, error | The key service was unreachable, answered with an error, or answered nonsense; `failure` says which. The user gets a 503.             |
-| `USER_DATA_KEY_UNREADABLE`                        | event, error | The wrapped key itself is corrupt. Permanent.                                                                                         |
-| `SECRET_DECRYPT_FAILED`                           | event, error | A stored token would not open. Section 5.1.                                                                                           |
-| `SDL_SECRETS_SEALED`, `SDL_SECRETS_STORED_OPENED` | event, info  | A token was written or read for a deployment.                                                                                         |
-| `DATA_KEY_REWRAP_*`                               | events       | The scheduled rotation's start, batches, refusals, failures and end report.                                                           |
-| `DATA_KEY_REKEY_*`                                | events       | The per-user re-key's start, retirement, batches, kept or deleted retired key, and end report.                                        |
-| `kms_key_service_calls_total{status,failure}`     | counter      | Key-service unwraps by outcome.                                                                                                       |
-| `kms_key_service_call_duration_ms`                | histogram    | Key-service latency.                                                                                                                  |
-| `user_data_key_unwraps_per_request`               | histogram    | Unwraps a request spent on one user's key. Expected 1; 2 during that user's re-key window.                                            |
+| Name                                              | Kind         | Meaning                                                                                                                                                         |
+| ------------------------------------------------- | ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `USER_DATA_KEY_UNWRAPPED`                         | event, info  | A request unwrapped a data key. Carries `userId`, or `organizationId` for a team's key, plus `dataKeyId` and `kid`. The only per-owner record of secret access. |
+| `USER_DATA_KEY_WRAPPED_UNDER_UNKNOWN_KID`         | event, error | A data key names a KMS version the console cannot resolve or the service refuses (disabled, destroyed, unknown). The user gets a 503.                           |
+| `USER_DATA_KEY_UNWRAP_FAILED`                     | event, error | The key service was unreachable, answered with an error, or answered nonsense; `failure` says which. The user gets a 503.                                       |
+| `USER_DATA_KEY_UNREADABLE`                        | event, error | The wrapped key itself is corrupt. Permanent.                                                                                                                   |
+| `SECRET_DECRYPT_FAILED`                           | event, error | A stored token would not open. Section 5.1.                                                                                                                     |
+| `SDL_SECRETS_SEALED`, `SDL_SECRETS_STORED_OPENED` | event, info  | A token was written or read for a deployment.                                                                                                                   |
+| `DATA_KEY_REWRAP_*`                               | events       | The scheduled rotation's start, batches, refusals, failures and end report.                                                                                     |
+| `DATA_KEY_REKEY_*`                                | events       | The per-user re-key's start, retirement, batches, kept or deleted retired key, and end report.                                                                  |
+| `kms_key_service_calls_total{status,failure}`     | counter      | Key-service unwraps by outcome.                                                                                                                                 |
+| `kms_key_service_call_duration_ms`                | histogram    | Key-service latency.                                                                                                                                            |
+| `user_data_key_unwraps_per_request`               | histogram    | Unwraps a request spent on one user's key. Expected 1; 2 during that user's re-key window.                                                                      |

apps/api/scripts/rehearseDeploymentSecrets.ts

@@ -251,7 +251,9 @@ async function inspectDeployment(userId: string, row: DeploymentStoredSecretsOfU
   const kid = kidOf(row.sealedSecrets);
 
   try {
-    const secrets = await container.resolve(SdlSecretsService).openStored({ userId, dseq: row.dseq, sealedSecrets: row.sealedSecrets });
+    const secrets = await container
+      .resolve(SdlSecretsService)
+      .openStored({ userId, organizationId: row.organizationId, dseq: row.dseq, sealedSecrets: row.sealedSecrets });
 
     return { dseq: row.dseq, kid, opens: true, secretCount: Object.keys(secrets).length };
   } catch (error) {

apps/api/src/deployment/repositories/deployment-setting/deployment-setting.repository.ts

@@ -145,7 +145,7 @@ export type DeploymentStoredSecrets = {
 };
 
 /** A user's stored secrets token with the deployment it is bound to, which re-sealing it under another key has to name. */
-export type DeploymentStoredSecretsOfUser = DeploymentStoredSecrets & { dseq: string };
+export type DeploymentStoredSecretsOfUser = DeploymentStoredSecrets & Pick<DeploymentSettingsDbOutput, "dseq" | "organizationId">;
 
 export type StoredLeaseGpus = { readings: LeaseGpuReading[]; offers: LeaseGpuOffer[] };
 
@@ -840,7 +840,13 @@ export class DeploymentSettingRepository extends OrgScopedRepository<Table, Depl
 
     while (true) {
       const batch = await this.pg
-        .select({ id: this.table.id, dseq: this.table.dseq, sealedSecrets: this.table.sealedSecrets, updatedAt: this.table.updatedAt })
+        .select({
+          id: this.table.id,
+          dseq: this.table.dseq,
+          organizationId: this.table.organizationId,
+          sealedSecrets: this.table.sealedSecrets,
+          updatedAt: this.table.updatedAt
+        })
         .from(this.table)
         .where(this.whereAccessibleBy(and(eq(this.table.userId, userId), isNotNull(this.table.sealedSecrets), ...(cursor ? [gt(this.table.id, cursor)] : []))))
         .orderBy(asc(this.table.id))

apps/api/src/deployment/services/deployment-writer/deployment-writer.service.spec.ts

@@ -3310,7 +3310,12 @@ describe(DeploymentWriterService.name, () => {
 
       await service.patchByUserIdAndDseq("user-1", "1234", { services: { web: { image: "nginx:1.27" } } }, ability);
 
-      expect(sdlSecretsService.openStored).toHaveBeenCalledWith({ userId: "user-1", dseq: "1234", sealedSecrets: STORED_TOKEN });
+      expect(sdlSecretsService.openStored).toHaveBeenCalledWith({
+        userId: "user-1",
+        organizationId: "personal-organization-1",
+        dseq: "1234",
+        sealedSecrets: STORED_TOKEN
+      });
     });
 
     it("opens the client's seal against the sdl the console stored, which is what it was sealed to", async () => {
@@ -4404,7 +4409,12 @@ describe(DeploymentWriterService.name, () => {
 
       await service.applySavedPatch(savedPatch, ability);
 
-      expect(sdlSecretsService.openStored).toHaveBeenCalledWith({ userId: "user-1", dseq: "100", sealedSecrets: "stored.token.aaa.bbb.ccc" });
+      expect(sdlSecretsService.openStored).toHaveBeenCalledWith({
+        userId: "user-1",
+        organizationId: "organization-of-the-row",
+        dseq: "100",
+        sealedSecrets: "stored.token.aaa.bbb.ccc"
+      });
     });
 
     it("rebuilds the manifest from the saved sdl and the values its token holds", async () => {
@@ -4603,7 +4613,12 @@ describe(DeploymentWriterService.name, () => {
 
         expect(signerService.filedUserIdOf).toHaveBeenCalledWith(organizationWallet, "user-1", "100");
         expect(scopedSettingRepository.findOneBy).toHaveBeenCalledWith({ userId: "filer-1", dseq: "100" });
-        expect(sdlSecretsService.openStored).toHaveBeenCalledWith({ userId: "filer-1", dseq: "100", sealedSecrets: "stored.token.aaa.bbb.ccc" });
+        expect(sdlSecretsService.openStored).toHaveBeenCalledWith({
+          userId: "filer-1",
+          organizationId: "organization-of-the-row",
+          dseq: "100",
+          sealedSecrets: "stored.token.aaa.bbb.ccc"
+        });
       });
 
       it("updates the chain and pushes the manifest with the organization's wallet, for the member who saved the patch", async () => {
@@ -4951,6 +4966,7 @@ describe(DeploymentWriterService.name, () => {
             sdl: input?.storedSdl ?? SDL_OF_A_REDEPLOY,
             runtimeLimitHours: input?.storedRuntimeLimitHours ?? null,
             sealedSecrets: "stored.token.aaa.bbb.ccc",
+            organizationId: "organization-of-the-row",
             manifestVersion: input?.storedManifestVersion ?? savedVersion
           })
     );

apps/api/src/deployment/services/deployment-writer/deployment-writer.service.ts

@@ -200,6 +200,15 @@ type PreparedPatch = {
   assertMayStillSign?: () => Promise<void>;
 };
 
+type StoredDefinition = {
+  sdl: string;
+  sealedSecrets: string | null;
+  manifestVersion?: string;
+  runtimeLimitHours?: number;
+  /** Absent for a deployment filed before organizations, whose row only its user reaches. */
+  organizationId?: string;
+};
+
 type PatchWrite = {
   definition: Parameters<DeploymentSettingRepository["replaceDefinitionIfVersionMatches"]>[0];
   counts: { patchedServiceCount: number; secretCount: number; guarded: boolean };
@@ -981,7 +990,9 @@ export class DeploymentWriterService {
 
     const [supplied, held] = await Promise.all([
       input.sealedSecrets ? this.sdlSecretsService.receiveForMerge({ rawSdl: stored.sdl, sealedSecrets: input.sealedSecrets }) : {},
-      stored.sealedSecrets ? this.sdlSecretsService.openStored({ userId: filedUserId, dseq, sealedSecrets: stored.sealedSecrets }) : {}
+      stored.sealedSecrets
+        ? this.sdlSecretsService.openStored({ userId: filedUserId, organizationId: stored.organizationId ?? null, dseq, sealedSecrets: stored.sealedSecrets })
+        : {}
     ]);
     const merged = this.#mergeAndPrune({ held, supplied, derived }, document);
 
@@ -1027,11 +1038,7 @@ export class DeploymentWriterService {
   }
 
   /** Read from the row even right after the save, so a patch a newer one replaced is never applied on chain, however late it runs. */
-  async #findDefinitionSavedAs(
-    saved: SavedPatch,
-    filedUserId: string,
-    ability: AnyAbility
-  ): Promise<{ sdl: string; sealedSecrets: string | null } | undefined> {
+  async #findDefinitionSavedAs(saved: SavedPatch, filedUserId: string, ability: AnyAbility): Promise<StoredDefinition | undefined> {
     const definition = await this.#findStoredDefinition({ userId: filedUserId, dseq: saved.dseq }, ability, NOT_PATCHABLE_MESSAGE);
 
     if (definition.manifestVersion === saved.manifestVersion) return definition;
@@ -1044,10 +1051,15 @@ export class DeploymentWriterService {
   /** Resolved without trial limits, which refuse a patch when it is saved and never change the manifest it hashes. */
   async #prepareSavedDefinition(
     { saved, wallet, filedUserId }: { saved: SavedPatch; wallet: WalletInitialized; filedUserId: string },
-    definition: { sdl: string; sealedSecrets: string | null }
+    definition: StoredDefinition
   ): Promise<PreparedPatch> {
     const secrets = definition.sealedSecrets
-      ? await this.sdlSecretsService.openStored({ userId: filedUserId, dseq: saved.dseq, sealedSecrets: definition.sealedSecrets })
+      ? await this.sdlSecretsService.openStored({
+          userId: filedUserId,
+          organizationId: definition.organizationId ?? null,
+          dseq: saved.dseq,
+          sealedSecrets: definition.sealedSecrets
+        })
       : {};
     const resolved = await this.#resolveSdl(definition.sdl, { secrets });
     const deployment = await this.deploymentReaderService.findByWalletAndDseqWithoutProviderStatus(wallet, saved.dseq);
@@ -1173,11 +1185,7 @@ export class DeploymentWriterService {
     return { ...deployment, name: persistedName };
   }
 
-  async #findStoredDefinition(
-    key: { userId: string; dseq: string },
-    ability: AnyAbility,
-    notFoundMessage: string
-  ): Promise<{ sdl: string; sealedSecrets: string | null; manifestVersion?: string; runtimeLimitHours?: number; organizationId?: string }> {
+  async #findStoredDefinition(key: { userId: string; dseq: string }, ability: AnyAbility, notFoundMessage: string): Promise<StoredDefinition> {
     const setting = await this.deploymentSettingRepository.accessibleBy(ability, "read").findOneBy(key);
 
     if (!setting?.sdl) {

apps/api/src/deployment/services/lease-manifest/lease-manifest.service.spec.ts

@@ -23,6 +23,7 @@ import { mockConfigService } from "@test/mocks/config-service.mock";
 
 const USER_ID = "3f1b6d4e-0000-4000-8000-000000000001";
 const DSEQ = "1420000000001";
+const ORGANIZATION_ID = "5c2a7e1f-0000-4000-8000-000000000002";
 
 function sdlWith(env: string[], options: { credentials?: Record<string, string>; gpuModel?: string } = {}) {
   const credentialsBlock = options.credentials
@@ -184,7 +185,7 @@ describe(LeaseManifestService.name, () => {
 
       await service.deriveFor({ dseq: DSEQ, userId: USER_ID });
 
-      expect(sdlSecretsService.openStored).toHaveBeenCalledWith({ userId: USER_ID, dseq: DSEQ, sealedSecrets: "sealed" });
+      expect(sdlSecretsService.openStored).toHaveBeenCalledWith({ userId: USER_ID, organizationId: ORGANIZATION_ID, dseq: DSEQ, sealedSecrets: "sealed" });
     });
 
     it("logs what it derived and no part of what it resolved", async () => {
@@ -400,7 +401,9 @@ describe(LeaseManifestService.name, () => {
 
     deploymentSettingRepository.accessibleBy.mockReturnValue(scopedRepository);
     scopedRepository.findOneBy.mockResolvedValue(
-      input.definition ? mock<DeploymentSettingsOutput>({ sdl: input.definition.sdl, sealedSecrets: input.definition.sealedSecrets }) : undefined
+      input.definition
+        ? mock<DeploymentSettingsOutput>({ sdl: input.definition.sdl, sealedSecrets: input.definition.sealedSecrets, organizationId: ORGANIZATION_ID })
+        : undefined
     );
 
     const createLogger: CreateLogger = () => logger;