feat(deployment): move the hidden variables members sealed for teams onto team keys

apps/api/src/app/console.ts

@@ -17,6 +17,7 @@ import { LeaseGpuController } from "@src/deployment/controllers/lease-gpu.contro
 import { ProviderController } from "@src/provider/controllers/provider/provider.controller";
 import { DataKeyRekeyController } from "@src/secret/controllers/data-key-rekey/data-key-rekey.controller";
 import { DataKeyRewrapController } from "@src/secret/controllers/data-key-rewrap/data-key-rewrap.controller";
+import { TeamSecretsResealController } from "@src/secret/controllers/team-secrets-reseal/team-secrets-reseal.controller";
 import { WorkloadAbuseController } from "@src/workload-abuse/controllers/workload-abuse.controller";
 import { APP_INITIALIZER, ON_APP_START } from "../core/providers/app-initializer";
 
@@ -188,6 +189,17 @@ program
     });
   });
 
+program
+  .command("reseal-team-deployment-secrets")
+  .description("Move the hidden variables members sealed for their teams onto each team's own data key, once team_data_keys is on for everyone")
+  .option("-b, --batch-size <number>", "How many team deployments are read per page", value => z.number({ coerce: true }).int().positive().parse(value))
+  .option("-d, --dry-run", "Open and count what would be re-sealed, without writing anything", false)
+  .action(async (options, command) => {
+    await executeCliHandler(command.name(), async () => {
+      return container.resolve(TeamSecretsResealController).resealTeamDeployments(options);
+    });
+  });
+
 program
   .command("link-auth0-accounts")
   .description("Interactively link a secondary Auth0 account into a primary one (Auth0 only, no DB writes)")

apps/api/src/deployment/repositories/deployment-setting/deployment-setting.repository.integration.ts

@@ -2920,6 +2920,46 @@ describe(DeploymentSettingRepository.name, () => {
     }
   });
 
+  describe("re-sealing the stored secrets filed in teams", () => {
+    it("pages only the live teams' deployments that hold a secret, across batches smaller than the set", async () => {
+      const { deploymentSettingRepository, teamSealedIds, passedOverIds } = await setupTeamSecrets();
+      const batchSizes: number[] = [];
+      const pagedIds: string[] = [];
+
+      for await (const batch of deploymentSettingRepository.findStoredSecretsOfTeamsIteratively({ batchSize: 1 })) {
+        batchSizes.push(batch.length);
+        pagedIds.push(...batch.map(row => row.id));
+      }
+
+      expect(batchSizes.every(size => size === 1)).toBe(true);
+      expect(new Set(pagedIds).size).toBe(pagedIds.length);
+      expect(pagedIds).toEqual(expect.arrayContaining(teamSealedIds));
+      expect(pagedIds.filter(id => passedOverIds.includes(id))).toEqual([]);
+    });
+
+    async function setupTeamSecrets() {
+      const deploymentSettingRepository = container.resolve(DeploymentSettingRepository);
+      const { user, organization: team } = await seedOrganizationWithOwner();
+      const { organization: personal } = await seedOrganizationWithOwner({ type: "personal" });
+      const { organization: deletedTeam } = await seedOrganizationWithOwner({ deletedAt: new Date() });
+      const seedFiledIn = async (organizationId: string, sealedSecrets: string | null) =>
+        (await seedDeploymentSetting({ userId: user.id, organizationId, sealedSecrets })).id;
+
+      const teamSealedIds = [
+        await seedFiledIn(team.id, newSealedToken()),
+        await seedFiledIn(team.id, newSealedToken()),
+        await seedFiledIn(team.id, newSealedToken())
+      ];
+      const passedOverIds = [
+        await seedFiledIn(team.id, null),
+        await seedFiledIn(personal.id, newSealedToken()),
+        await seedFiledIn(deletedTeam.id, newSealedToken())
+      ];
+
+      return { deploymentSettingRepository, teamSealedIds, passedOverIds };
+    }
+  });
+
   describe("when a deployment is filed in a team organization", () => {
     it("funds the team deployment from the organization's wallet, never from the member's own", async () => {
       const { deploymentSettingRepository, teamWallet, personalWallet, teamDeployment } = await setupTeamDeployment();

apps/api/src/deployment/repositories/deployment-setting/deployment-setting.repository.ts

@@ -147,6 +147,9 @@ export type DeploymentStoredSecrets = {
 /** A user's stored secrets token with the deployment it is bound to, which re-sealing it under another key has to name. */
 export type DeploymentStoredSecretsOfUser = DeploymentStoredSecrets & Pick<DeploymentSettingsDbOutput, "dseq" | "organizationId">;
 
+/** A team deployment's stored secrets token with the member the row is filed under, null once they deleted their account. */
+export type DeploymentStoredSecretsOfTeam = DeploymentStoredSecrets & Pick<DeploymentSettingsDbOutput, "dseq" | "userId"> & { organizationId: string };
+
 export type StoredLeaseGpus = { readings: LeaseGpuReading[]; offers: LeaseGpuOffer[] };
 
 export type LiveManagedDeployment = {
@@ -866,6 +869,42 @@ export class DeploymentSettingRepository extends OrgScopedRepository<Table, Depl
     }
   }
 
+  /** Every deployment filed in a team that still exists and holds a secret, sealed for a member or for the team, since telling the two apart needs the token's header. */
+  async *findStoredSecretsOfTeamsIteratively({ batchSize }: { batchSize: number }): AsyncGenerator<DeploymentStoredSecretsOfTeam[]> {
+    assertBatchSize(batchSize);
+
+    let cursor: string | undefined;
+
+    while (true) {
+      const batch = await this.pg
+        .select({
+          id: this.table.id,
+          dseq: this.table.dseq,
+          userId: this.table.userId,
+          organizationId: this.table.organizationId,
+          sealedSecrets: this.table.sealedSecrets,
+          updatedAt: this.table.updatedAt
+        })
+        .from(this.table)
+        .innerJoin(Organizations, and(eq(Organizations.id, this.table.organizationId), eq(Organizations.type, "team"), isNull(Organizations.deletedAt)))
+        .where(this.whereAccessibleBy(and(isNotNull(this.table.sealedSecrets), ...(cursor ? [gt(this.table.id, cursor)] : []))))
+        .orderBy(asc(this.table.id))
+        .limit(batchSize);
+
+      if (!batch.length) {
+        return;
+      }
+
+      yield batch as DeploymentStoredSecretsOfTeam[];
+
+      if (batch.length < batchSize) {
+        return;
+      }
+
+      cursor = batch[batch.length - 1].id;
+    }
+  }
+
   /** Guards on the token as it was read, so a value the user replaced in between is left as the user wrote it; the timestamp moves because this is a legitimate secrets write. */
   async resealIfUnchanged(id: string, sealedSecrets: string, resealed: string | null): Promise<boolean> {
     const [row] = await this.cursor

apps/api/src/deployment/services/sdl-secrets/sdl-secrets.service.spec.ts

@@ -517,6 +517,23 @@ describe(SdlSecretsService.name, () => {
       });
     });
 
+    describe("sealForTeam", () => {
+      it("seals under the team's key even while team data keys are off", async () => {
+        const { service, secretCipherService } = setup({ teamDataKeys: false });
+        const secrets = { TOKEN: "one" };
+
+        await expect(service.sealForTeam({ teamId: "team-1", dseq: "1420000", secrets })).resolves.toBe("encrypted");
+        expect(secretCipherService.encrypt).toHaveBeenCalledWith({ organizationId: "team-1" }, JSON.stringify(secrets), { org: "team-1", dseq: "1420000" });
+      });
+
+      it("writes no token for a deployment that holds nothing", async () => {
+        const { service, secretCipherService } = setup();
+
+        await expect(service.sealForTeam({ teamId: "team-1", dseq: "1420000", secrets: {} })).resolves.toBeNull();
+        expect(secretCipherService.encrypt).not.toHaveBeenCalled();
+      });
+    });
+
     it("never seals a personal deployment for a team", async () => {
       const { service, secretCipherService } = setup({ teamDataKeys: true });
 

apps/api/src/deployment/services/sdl-secrets/sdl-secrets.service.ts

@@ -163,11 +163,19 @@ export class SdlSecretsService {
 
   /** Returns null when nothing was supplied, so a create always has a value to write and a retry cannot inherit an abandoned attempt's token. */
   async sealForStorage(input: SecretsToSeal): Promise<string | null> {
+    return await this.#seal(input, this.#sealerFor(input));
+  }
+
+  /** Seals for the team whatever `team_data_keys` says, which is how a value a member sealed moves onto the team's key. */
+  async sealForTeam(input: { teamId: string; dseq: string; secrets: SdlSecrets }): Promise<string | null> {
+    return await this.#seal(input, sealerOfOrganization(input.teamId, input.dseq));
+  }
+
+  async #seal(input: { userId?: string; dseq: string; secrets: SdlSecrets }, { owner, binding }: Sealer): Promise<string | null> {
     const names = Object.keys(input.secrets);
 
     if (names.length === 0) return null;
 
-    const { owner, binding } = this.#sealerFor(input);
     const sealed = await this.secretCipherService.encrypt(owner, JSON.stringify(input.secrets), binding);
 
     this.#loggerService.info({

apps/api/src/secret/controllers/team-secrets-reseal/team-secrets-reseal.controller.ts

@@ -0,0 +1,13 @@
+import { singleton } from "tsyringe";
+
+import type { TeamSecretsResealOptions } from "@src/secret/services/team-secrets-reseal/team-secrets-reseal.service";
+import { TeamSecretsResealService } from "@src/secret/services/team-secrets-reseal/team-secrets-reseal.service";
+
+@singleton()
+export class TeamSecretsResealController {
+  constructor(private readonly teamSecretsResealService: TeamSecretsResealService) {}
+
+  async resealTeamDeployments(options: TeamSecretsResealOptions) {
+    return await this.teamSecretsResealService.resealTeamDeployments(options);
+  }
+}

apps/api/src/secret/services/team-secrets-reseal/team-secrets-reseal.service.integration.ts

@@ -0,0 +1,91 @@
+import { faker } from "@faker-js/faker";
+import { eq } from "drizzle-orm";
+import { container } from "tsyringe";
+import { describe, expect, it } from "vitest";
+
+import { type ApiPgDatabase, POSTGRES_DB, resolveTable } from "@src/core";
+import { ExecutionContextService } from "@src/core/services/execution-context/execution-context.service";
+import type { StoredSecretsOfDeployment } from "@src/deployment/services/sdl-secrets/sdl-secrets.service";
+import { isSealedForOrganization, SdlSecretsService } from "@src/deployment/services/sdl-secrets/sdl-secrets.service";
+import { TeamSecretsResealService } from "./team-secrets-reseal.service";
+
+import { registerFakeSdlSecretsKms, warmSealingKeyAsBootWould } from "@test/mocks/sdl-secrets-kms.mock";
+import { createDseq, seedDeploymentSetting } from "@test/seeders/db/deployment-setting.seeder";
+import { seedOrganization, seedOrganizationWithOwner } from "@test/seeders/db/organization.seeder";
+import { seedUser } from "@test/seeders/db/user-with-wallet.seeder";
+
+registerFakeSdlSecretsKms();
+
+describe(TeamSecretsResealService.name, () => {
+  it("moves what members sealed for live teams onto each team's key, and leaves every other value as it was", async () => {
+    const { reseal, seedDeployment, orphan, findToken, open } = await setup();
+    const team = await seedOrganizationWithOwner();
+    const deletedTeam = await seedOrganization({ deletedAt: new Date() });
+    const moved = await seedDeployment({ filerId: team.user.id, organizationId: team.organization.id });
+    const orphaned = await orphan(await seedDeployment({ filerId: team.user.id, organizationId: team.organization.id }));
+    const inDeletedTeam = await seedDeployment({ filerId: (await seedUser()).id, organizationId: deletedTeam.id });
+    const personal = await seedDeployment({ filerId: (await seedUser()).id, organizationId: null });
+
+    const dryRun = (await reseal({ dryRun: true })).unwrap();
+
+    expect(dryRun).toMatchObject({
+      deploymentsScanned: 2,
+      deploymentsResealed: 1,
+      deploymentsWhoseMemberKeyIsGone: [`${team.organization.id}/${orphaned.dseq}`]
+    });
+    expect(await findToken(moved)).toBe(moved.sealedSecrets);
+
+    const report = (await reseal({ dryRun: false })).unwrap();
+
+    expect(report).toMatchObject({ deploymentsResealed: 1, secretsResealed: 1, teamsResealed: 1, deploymentsMovedByAnotherWriter: 0 });
+    const resealed = await findToken(moved);
+    expect(isSealedForOrganization(resealed)).toBe(true);
+    await expect(open({ userId: null, organizationId: team.organization.id, dseq: moved.dseq, sealedSecrets: resealed })).resolves.toEqual(moved.secrets);
+    expect(await findToken(orphaned)).toBe(orphaned.sealedSecrets);
+    expect(await findToken(inDeletedTeam)).toBe(inDeletedTeam.sealedSecrets);
+    expect(await findToken(personal)).toBe(personal.sealedSecrets);
+
+    expect((await reseal({ dryRun: false })).unwrap()).toMatchObject({ deploymentsResealed: 0, deploymentsAlreadyUnderTeamKey: 1 });
+  });
+
+  async function setup() {
+    await warmSealingKeyAsBootWould();
+    const db = container.resolve<ApiPgDatabase>(POSTGRES_DB);
+    const deploymentSettings = resolveTable("DeploymentSettings");
+    const executionContextService = container.resolve(ExecutionContextService);
+    const sdlSecretsService = container.resolve(SdlSecretsService);
+
+    async function seedDeployment({ filerId, organizationId }: { filerId: string; organizationId: string | null }) {
+      const dseq = createDseq();
+      const secrets = { API_TOKEN: faker.string.alphanumeric(24) };
+      const sealedSecrets = (await executionContextService.runWithContext(
+        async () => await sdlSecretsService.sealForStorage({ userId: filerId, dseq, secrets })
+      ))!;
+      const { id } = await seedDeploymentSetting({ userId: filerId, organizationId, dseq, sealedSecrets, autoTopUpEnabled: false });
+
+      return { id, dseq, secrets, sealedSecrets };
+    }
+
+    async function orphan<T extends { id: string }>(deployment: T) {
+      await db.update(deploymentSettings).set({ userId: null }).where(eq(deploymentSettings.id, deployment.id));
+
+      return deployment;
+    }
+
+    async function findToken({ id }: { id: string }) {
+      const [row] = await db.select({ sealedSecrets: deploymentSettings.sealedSecrets }).from(deploymentSettings).where(eq(deploymentSettings.id, id));
+
+      return row.sealedSecrets!;
+    }
+
+    async function open(stored: StoredSecretsOfDeployment) {
+      return await executionContextService.runWithContext(async () => await sdlSecretsService.openStored(stored));
+    }
+
+    async function reseal(options: { dryRun: boolean }) {
+      return await container.resolve(TeamSecretsResealService).resealTeamDeployments(options);
+    }
+
+    return { reseal, seedDeployment, orphan, findToken, open };
+  }
+});