feat(deployment): open hidden variables sealed under a team's data key

A stored token whose header names an organization instead of a member opens under
the data key and binding of the organization its row is filed in. Patching,
redeploying, inheriting and sending the manifest of a team deployment read it
that way, including after the member who filed the row deleted their account.
A user re-key leaves such a row to the team and counts it apart.

Nothing seals under a team's key yet, so every stored token still opens as before.

Part of CON-1213

Author: baktun14

apps/api/scripts/rehearseDeploymentSecrets.ts

@@ -251,7 +251,9 @@ async function inspectDeployment(userId: string, row: DeploymentStoredSecretsOfU
   const kid = kidOf(row.sealedSecrets);
 
   try {
-    const secrets = await container.resolve(SdlSecretsService).openStored({ userId, dseq: row.dseq, sealedSecrets: row.sealedSecrets });
+    const secrets = await container
+      .resolve(SdlSecretsService)
+      .openStored({ userId, organizationId: row.organizationId, dseq: row.dseq, sealedSecrets: row.sealedSecrets });
 
     return { dseq: row.dseq, kid, opens: true, secretCount: Object.keys(secrets).length };
   } catch (error) {

apps/api/src/deployment/repositories/deployment-setting/deployment-setting.repository.ts

@@ -145,7 +145,7 @@ export type DeploymentStoredSecrets = {
 };
 
 /** A user's stored secrets token with the deployment it is bound to, which re-sealing it under another key has to name. */
-export type DeploymentStoredSecretsOfUser = DeploymentStoredSecrets & { dseq: string };
+export type DeploymentStoredSecretsOfUser = DeploymentStoredSecrets & Pick<DeploymentSettingsDbOutput, "dseq" | "organizationId">;
 
 export type StoredLeaseGpus = { readings: LeaseGpuReading[]; offers: LeaseGpuOffer[] };
 
@@ -840,7 +840,13 @@ export class DeploymentSettingRepository extends OrgScopedRepository<Table, Depl
 
     while (true) {
       const batch = await this.pg
-        .select({ id: this.table.id, dseq: this.table.dseq, sealedSecrets: this.table.sealedSecrets, updatedAt: this.table.updatedAt })
+        .select({
+          id: this.table.id,
+          dseq: this.table.dseq,
+          organizationId: this.table.organizationId,
+          sealedSecrets: this.table.sealedSecrets,
+          updatedAt: this.table.updatedAt
+        })
         .from(this.table)
         .where(this.whereAccessibleBy(and(eq(this.table.userId, userId), isNotNull(this.table.sealedSecrets), ...(cursor ? [gt(this.table.id, cursor)] : []))))
         .orderBy(asc(this.table.id))

apps/api/src/deployment/services/deployment-writer/deployment-writer.service.spec.ts

@@ -3310,7 +3310,12 @@ describe(DeploymentWriterService.name, () => {
 
       await service.patchByUserIdAndDseq("user-1", "1234", { services: { web: { image: "nginx:1.27" } } }, ability);
 
-      expect(sdlSecretsService.openStored).toHaveBeenCalledWith({ userId: "user-1", dseq: "1234", sealedSecrets: STORED_TOKEN });
+      expect(sdlSecretsService.openStored).toHaveBeenCalledWith({
+        userId: "user-1",
+        organizationId: "personal-organization-1",
+        dseq: "1234",
+        sealedSecrets: STORED_TOKEN
+      });
     });
 
     it("opens the client's seal against the sdl the console stored, which is what it was sealed to", async () => {
@@ -4404,7 +4409,12 @@ describe(DeploymentWriterService.name, () => {
 
       await service.applySavedPatch(savedPatch, ability);
 
-      expect(sdlSecretsService.openStored).toHaveBeenCalledWith({ userId: "user-1", dseq: "100", sealedSecrets: "stored.token.aaa.bbb.ccc" });
+      expect(sdlSecretsService.openStored).toHaveBeenCalledWith({
+        userId: "user-1",
+        organizationId: "organization-of-the-row",
+        dseq: "100",
+        sealedSecrets: "stored.token.aaa.bbb.ccc"
+      });
     });
 
     it("rebuilds the manifest from the saved sdl and the values its token holds", async () => {
@@ -4603,7 +4613,12 @@ describe(DeploymentWriterService.name, () => {
 
         expect(signerService.filedUserIdOf).toHaveBeenCalledWith(organizationWallet, "user-1", "100");
         expect(scopedSettingRepository.findOneBy).toHaveBeenCalledWith({ userId: "filer-1", dseq: "100" });
-        expect(sdlSecretsService.openStored).toHaveBeenCalledWith({ userId: "filer-1", dseq: "100", sealedSecrets: "stored.token.aaa.bbb.ccc" });
+        expect(sdlSecretsService.openStored).toHaveBeenCalledWith({
+          userId: "filer-1",
+          organizationId: "organization-of-the-row",
+          dseq: "100",
+          sealedSecrets: "stored.token.aaa.bbb.ccc"
+        });
       });
 
       it("updates the chain and pushes the manifest with the organization's wallet, for the member who saved the patch", async () => {
@@ -4951,6 +4966,7 @@ describe(DeploymentWriterService.name, () => {
             sdl: input?.storedSdl ?? SDL_OF_A_REDEPLOY,
             runtimeLimitHours: input?.storedRuntimeLimitHours ?? null,
             sealedSecrets: "stored.token.aaa.bbb.ccc",
+            organizationId: "organization-of-the-row",
             manifestVersion: input?.storedManifestVersion ?? savedVersion
           })
     );

apps/api/src/deployment/services/deployment-writer/deployment-writer.service.ts

@@ -200,6 +200,15 @@ type PreparedPatch = {
   assertMayStillSign?: () => Promise<void>;
 };
 
+type StoredDefinition = {
+  sdl: string;
+  sealedSecrets: string | null;
+  manifestVersion?: string;
+  runtimeLimitHours?: number;
+  /** Absent for a deployment filed before organizations, whose row only its user reaches. */
+  organizationId?: string;
+};
+
 type PatchWrite = {
   definition: Parameters<DeploymentSettingRepository["replaceDefinitionIfVersionMatches"]>[0];
   counts: { patchedServiceCount: number; secretCount: number; guarded: boolean };
@@ -981,7 +990,9 @@ export class DeploymentWriterService {
 
     const [supplied, held] = await Promise.all([
       input.sealedSecrets ? this.sdlSecretsService.receiveForMerge({ rawSdl: stored.sdl, sealedSecrets: input.sealedSecrets }) : {},
-      stored.sealedSecrets ? this.sdlSecretsService.openStored({ userId: filedUserId, dseq, sealedSecrets: stored.sealedSecrets }) : {}
+      stored.sealedSecrets
+        ? this.sdlSecretsService.openStored({ userId: filedUserId, organizationId: stored.organizationId ?? null, dseq, sealedSecrets: stored.sealedSecrets })
+        : {}
     ]);
     const merged = this.#mergeAndPrune({ held, supplied, derived }, document);
 
@@ -1027,11 +1038,7 @@ export class DeploymentWriterService {
   }
 
   /** Read from the row even right after the save, so a patch a newer one replaced is never applied on chain, however late it runs. */
-  async #findDefinitionSavedAs(
-    saved: SavedPatch,
-    filedUserId: string,
-    ability: AnyAbility
-  ): Promise<{ sdl: string; sealedSecrets: string | null } | undefined> {
+  async #findDefinitionSavedAs(saved: SavedPatch, filedUserId: string, ability: AnyAbility): Promise<StoredDefinition | undefined> {
     const definition = await this.#findStoredDefinition({ userId: filedUserId, dseq: saved.dseq }, ability, NOT_PATCHABLE_MESSAGE);
 
     if (definition.manifestVersion === saved.manifestVersion) return definition;
@@ -1044,10 +1051,15 @@ export class DeploymentWriterService {
   /** Resolved without trial limits, which refuse a patch when it is saved and never change the manifest it hashes. */
   async #prepareSavedDefinition(
     { saved, wallet, filedUserId }: { saved: SavedPatch; wallet: WalletInitialized; filedUserId: string },
-    definition: { sdl: string; sealedSecrets: string | null }
+    definition: StoredDefinition
   ): Promise<PreparedPatch> {
     const secrets = definition.sealedSecrets
-      ? await this.sdlSecretsService.openStored({ userId: filedUserId, dseq: saved.dseq, sealedSecrets: definition.sealedSecrets })
+      ? await this.sdlSecretsService.openStored({
+          userId: filedUserId,
+          organizationId: definition.organizationId ?? null,
+          dseq: saved.dseq,
+          sealedSecrets: definition.sealedSecrets
+        })
       : {};
     const resolved = await this.#resolveSdl(definition.sdl, { secrets });
     const deployment = await this.deploymentReaderService.findByWalletAndDseqWithoutProviderStatus(wallet, saved.dseq);
@@ -1173,11 +1185,7 @@ export class DeploymentWriterService {
     return { ...deployment, name: persistedName };
   }
 
-  async #findStoredDefinition(
-    key: { userId: string; dseq: string },
-    ability: AnyAbility,
-    notFoundMessage: string
-  ): Promise<{ sdl: string; sealedSecrets: string | null; manifestVersion?: string; runtimeLimitHours?: number; organizationId?: string }> {
+  async #findStoredDefinition(key: { userId: string; dseq: string }, ability: AnyAbility, notFoundMessage: string): Promise<StoredDefinition> {
     const setting = await this.deploymentSettingRepository.accessibleBy(ability, "read").findOneBy(key);
 
     if (!setting?.sdl) {

apps/api/src/deployment/services/lease-manifest/lease-manifest.service.spec.ts

@@ -23,6 +23,7 @@ import { mockConfigService } from "@test/mocks/config-service.mock";
 
 const USER_ID = "3f1b6d4e-0000-4000-8000-000000000001";
 const DSEQ = "1420000000001";
+const ORGANIZATION_ID = "5c2a7e1f-0000-4000-8000-000000000002";
 
 function sdlWith(env: string[], options: { credentials?: Record<string, string>; gpuModel?: string } = {}) {
   const credentialsBlock = options.credentials
@@ -184,7 +185,7 @@ describe(LeaseManifestService.name, () => {
 
       await service.deriveFor({ dseq: DSEQ, userId: USER_ID });
 
-      expect(sdlSecretsService.openStored).toHaveBeenCalledWith({ userId: USER_ID, dseq: DSEQ, sealedSecrets: "sealed" });
+      expect(sdlSecretsService.openStored).toHaveBeenCalledWith({ userId: USER_ID, organizationId: ORGANIZATION_ID, dseq: DSEQ, sealedSecrets: "sealed" });
     });
 
     it("logs what it derived and no part of what it resolved", async () => {
@@ -400,7 +401,9 @@ describe(LeaseManifestService.name, () => {
 
     deploymentSettingRepository.accessibleBy.mockReturnValue(scopedRepository);
     scopedRepository.findOneBy.mockResolvedValue(
-      input.definition ? mock<DeploymentSettingsOutput>({ sdl: input.definition.sdl, sealedSecrets: input.definition.sealedSecrets }) : undefined
+      input.definition
+        ? mock<DeploymentSettingsOutput>({ sdl: input.definition.sdl, sealedSecrets: input.definition.sealedSecrets, organizationId: ORGANIZATION_ID })
+        : undefined
     );
 
     const createLogger: CreateLogger = () => logger;

apps/api/src/deployment/services/lease-manifest/lease-manifest.service.ts

@@ -10,7 +10,7 @@ import { SdlService } from "@src/deployment/services/sdl/sdl.service";
 import { SdlReferenceService } from "@src/deployment/services/sdl-reference/sdl-reference.service";
 import { SdlSecretsService } from "@src/deployment/services/sdl-secrets/sdl-secrets.service";
 
-type StoredDefinition = { sdl: string; sealedSecrets: string | null };
+type StoredDefinition = { sdl: string; sealedSecrets: string | null; organizationId: string | null };
 
 /** Why a deployment has no manifest of its own, which is what the residual volume of client-supplied ones is measured by. */
 type StoredDefinitionMiss = "nothing-recorded" | "unresolvable";
@@ -48,7 +48,8 @@ export class LeaseManifestService {
       return this.#fallBack({ userId, dseq, reason: "nothing-recorded" });
     }
 
-    const secrets = definition.sealedSecrets ? await this.#openStored(definition.sealedSecrets, { userId, dseq }) : {};
+    const { sealedSecrets, organizationId } = definition;
+    const secrets = sealedSecrets ? await this.#openStored({ sealedSecrets, organizationId }, { userId, dseq }) : {};
 
     const resolved = await this.sdlService.generateResolvedManifest({ sdl: definition.sdl, secrets });
 
@@ -65,7 +66,7 @@ export class LeaseManifestService {
   async #findDefinition(key: { userId: string; dseq: string }): Promise<StoredDefinition | null> {
     const setting = await this.deploymentSettingRepository.accessibleBy(this.authService.ability, "read").findOneBy(key);
 
-    if (setting?.sdl) return { sdl: setting.sdl, sealedSecrets: setting.sealedSecrets };
+    if (setting?.sdl) return { sdl: setting.sdl, sealedSecrets: setting.sealedSecrets, organizationId: setting.organizationId };
 
     if (setting?.sealedSecrets) throw this.#refuse(key);
 
@@ -95,9 +96,9 @@ export class LeaseManifestService {
   }
 
   /** Keeps the cause's own error, so a key service that is merely unreachable still answers 503 rather than reading as a definition that cannot be derived. */
-  async #openStored(sealedSecrets: string, key: { userId: string; dseq: string }) {
+  async #openStored(stored: { sealedSecrets: string; organizationId: string | null }, key: { userId: string; dseq: string }) {
     try {
-      return await this.sdlSecretsService.openStored({ ...key, sealedSecrets });
+      return await this.sdlSecretsService.openStored({ ...key, ...stored });
     } catch (error) {
       throw isRetryable(error) ? this.#reportUnreachable(key, error) : this.#refuse(key, error);
     }

apps/api/src/deployment/services/sdl-secrets-inheritance/sdl-secrets-inheritance.service.spec.ts

@@ -30,7 +30,7 @@ describe(SdlSecretsInheritanceService.name, () => {
 
     await service.open(REQUEST);
 
-    expect(sdlSecretsService.openStored).toHaveBeenCalledWith({ ...SOURCE, sealedSecrets: TOKEN });
+    expect(sdlSecretsService.openStored).toHaveBeenCalledWith({ ...SOURCE, organizationId: null, sealedSecrets: TOKEN });
   });
 
   it("finds the source through the wallet that owns it, only where the caller may change it", async () => {
@@ -43,11 +43,13 @@ describe(SdlSecretsInheritanceService.name, () => {
   });
 
   it("opens an organization's source under the member it was filed by", async () => {
-    const { service, sdlSecretsService } = setup({ setting: mock<DeploymentSettingsOutput>({ sealedSecrets: TOKEN, userId: "filer-1" }) });
+    const { service, sdlSecretsService } = setup({
+      setting: mock<DeploymentSettingsOutput>({ sealedSecrets: TOKEN, userId: "filer-1", organizationId: "organization-1" })
+    });
 
     await service.open({ wallet: { userId: null, organizationId: "organization-1" }, dseq: SOURCE.dseq });
 
-    expect(sdlSecretsService.openStored).toHaveBeenCalledWith({ userId: "filer-1", dseq: SOURCE.dseq, sealedSecrets: TOKEN });
+    expect(sdlSecretsService.openStored).toHaveBeenCalledWith({ userId: "filer-1", organizationId: "organization-1", dseq: SOURCE.dseq, sealedSecrets: TOKEN });
   });
 
   it("inherits from a closed deployment, because closing a deployment keeps its secrets", async () => {
@@ -92,6 +94,26 @@ describe(SdlSecretsInheritanceService.name, () => {
       expect(sdlSecretsService.openStored).not.toHaveBeenCalled();
     });
 
+    it("answers not found for a team's source whose filer has deleted their account and that holds no token", async () => {
+      const { service } = setup({ setting: mock<DeploymentSettingsOutput>({ sealedSecrets: null, userId: null }) });
+
+      await expect(service.open({ wallet: { userId: null, organizationId: "organization-1" }, dseq: SOURCE.dseq })).rejects.toMatchObject({ status: 404 });
+    });
+  });
+
+  describe("a team's source whose token is sealed under the team's key", () => {
+    it("opens it after its filer has deleted their account, since the team's key outlives them", async () => {
+      const carried = { TOKEN: faker.string.alphanumeric(32) };
+      const sealedSecrets = tokenSealedForTeam("organization-1");
+      const { service, sdlSecretsService } = setup({
+        setting: mock<DeploymentSettingsOutput>({ sealedSecrets, userId: null, organizationId: "organization-1" }),
+        opened: carried
+      });
+
+      await expect(service.open({ wallet: { userId: null, organizationId: "organization-1" }, dseq: SOURCE.dseq })).resolves.toEqual(carried);
+      expect(sdlSecretsService.openStored).toHaveBeenCalledWith({ userId: null, organizationId: "organization-1", dseq: SOURCE.dseq, sealedSecrets });
+    });
+
     it("says nothing about whether the deployment exists", async () => {
       const { service } = setup({ setting: undefined });
 
@@ -241,12 +263,19 @@ describe(SdlSecretsInheritanceService.name, () => {
     expect(JSON.stringify(vi.mocked(logger.info).mock.calls)).not.toContain(value);
   });
 
+  function tokenSealedForTeam(organizationId: string) {
+    const header = Buffer.from(JSON.stringify({ org: organizationId, dseq: SOURCE.dseq, alg: "dir", enc: "A256GCM", kid: "team-key" })).toString("base64url");
+
+    return `${header}..iv.ciphertext.tag`;
+  }
+
   function unreadableTokenError() {
     return createError(500, SECRET_UNREADABLE_ERROR_MESSAGE, { errorCode: SECRET_UNREADABLE_ERROR_CODE });
   }
 
   function setup(input?: { setting?: DeploymentSettingsOutput; opened?: SdlSecrets; openStoredError?: Error }) {
-    const setting = "setting" in (input ?? {}) ? input!.setting : mock<DeploymentSettingsOutput>({ sealedSecrets: TOKEN, userId: SOURCE.userId });
+    const setting =
+      "setting" in (input ?? {}) ? input!.setting : mock<DeploymentSettingsOutput>({ sealedSecrets: TOKEN, userId: SOURCE.userId, organizationId: null });
 
     const scoped = mock<DeploymentSettingRepository>();
     scoped.findOneOfWallet.mockResolvedValue(setting);