From 1981d4263a2b6386533b62ae701f66a74cb419bc Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Sat, 17 Oct 2026 19:38:04 -0700 Subject: [PATCH 1/2] feat(deployment): move the hidden variables members sealed for teams onto team keys The reseal-team-deployment-secrets command reads every deployment filed in a live team that holds a secret, opens a value its member sealed through the path a deployment reads it by, seals it again under the team's key and writes it back only if the row still holds what was read. A value already sealed for the team is counted and left alone, so a re-run picks up where an earlier one stopped. A value whose member key went with the member's account cannot be moved and is reported. A dry run opens and counts without writing anything. Part of CON-1213 --- apps/api/src/app/console.ts | 12 ++ .../deployment-setting.repository.ts | 39 ++++ .../sdl-secrets/sdl-secrets.service.spec.ts | 17 ++ .../sdl-secrets/sdl-secrets.service.ts | 10 +- .../team-secrets-reseal.controller.ts | 13 ++ ...team-secrets-reseal.service.integration.ts | 91 +++++++++ .../team-secrets-reseal.service.spec.ts | 184 ++++++++++++++++++ .../team-secrets-reseal.service.ts | 129 ++++++++++++ 8 files changed, 494 insertions(+), 1 deletion(-) create mode 100644 apps/api/src/secret/controllers/team-secrets-reseal/team-secrets-reseal.controller.ts create mode 100644 apps/api/src/secret/services/team-secrets-reseal/team-secrets-reseal.service.integration.ts create mode 100644 apps/api/src/secret/services/team-secrets-reseal/team-secrets-reseal.service.spec.ts create mode 100644 apps/api/src/secret/services/team-secrets-reseal/team-secrets-reseal.service.ts diff --git a/apps/api/src/app/console.ts b/apps/api/src/app/console.ts index 210e747766..883560113b 100644 --- a/apps/api/src/app/console.ts +++ b/apps/api/src/app/console.ts @@ -17,6 +17,7 @@ import { LeaseGpuController } from "@src/deployment/controllers/lease-gpu.contro import { ProviderController } from "@src/provider/controllers/provider/provider.controller"; import { DataKeyRekeyController } from "@src/secret/controllers/data-key-rekey/data-key-rekey.controller"; import { DataKeyRewrapController } from "@src/secret/controllers/data-key-rewrap/data-key-rewrap.controller"; +import { TeamSecretsResealController } from "@src/secret/controllers/team-secrets-reseal/team-secrets-reseal.controller"; import { WorkloadAbuseController } from "@src/workload-abuse/controllers/workload-abuse.controller"; import { APP_INITIALIZER, ON_APP_START } from "../core/providers/app-initializer"; @@ -188,6 +189,17 @@ program }); }); +program + .command("reseal-team-deployment-secrets") + .description("Move the hidden variables members sealed for their teams onto each team's own data key, once team_data_keys is on for everyone") + .option("-b, --batch-size ", "How many team deployments are read per page", value => z.number({ coerce: true }).int().positive().parse(value)) + .option("-d, --dry-run", "Open and count what would be re-sealed, without writing anything", false) + .action(async (options, command) => { + await executeCliHandler(command.name(), async () => { + return container.resolve(TeamSecretsResealController).resealTeamDeployments(options); + }); + }); + program .command("link-auth0-accounts") .description("Interactively link a secondary Auth0 account into a primary one (Auth0 only, no DB writes)") diff --git a/apps/api/src/deployment/repositories/deployment-setting/deployment-setting.repository.ts b/apps/api/src/deployment/repositories/deployment-setting/deployment-setting.repository.ts index 7bb154db26..3669e29b64 100644 --- a/apps/api/src/deployment/repositories/deployment-setting/deployment-setting.repository.ts +++ b/apps/api/src/deployment/repositories/deployment-setting/deployment-setting.repository.ts @@ -147,6 +147,9 @@ export type DeploymentStoredSecrets = { /** A user's stored secrets token with the deployment it is bound to, which re-sealing it under another key has to name. */ export type DeploymentStoredSecretsOfUser = DeploymentStoredSecrets & Pick; +/** A team deployment's stored secrets token with the member the row is filed under, null once they deleted their account. */ +export type DeploymentStoredSecretsOfTeam = DeploymentStoredSecrets & Pick & { organizationId: string }; + export type StoredLeaseGpus = { readings: LeaseGpuReading[]; offers: LeaseGpuOffer[] }; export type LiveManagedDeployment = { @@ -866,6 +869,42 @@ export class DeploymentSettingRepository extends OrgScopedRepository { + assertBatchSize(batchSize); + + let cursor: string | undefined; + + while (true) { + const batch = await this.pg + .select({ + id: this.table.id, + dseq: this.table.dseq, + userId: this.table.userId, + organizationId: this.table.organizationId, + sealedSecrets: this.table.sealedSecrets, + updatedAt: this.table.updatedAt + }) + .from(this.table) + .innerJoin(Organizations, and(eq(Organizations.id, this.table.organizationId), eq(Organizations.type, "team"), isNull(Organizations.deletedAt))) + .where(this.whereAccessibleBy(and(isNotNull(this.table.sealedSecrets), ...(cursor ? [gt(this.table.id, cursor)] : [])))) + .orderBy(asc(this.table.id)) + .limit(batchSize); + + if (!batch.length) { + return; + } + + yield batch as DeploymentStoredSecretsOfTeam[]; + + if (batch.length < batchSize) { + return; + } + + cursor = batch[batch.length - 1].id; + } + } + /** Guards on the token as it was read, so a value the user replaced in between is left as the user wrote it; the timestamp moves because this is a legitimate secrets write. */ async resealIfUnchanged(id: string, sealedSecrets: string, resealed: string | null): Promise { const [row] = await this.cursor diff --git a/apps/api/src/deployment/services/sdl-secrets/sdl-secrets.service.spec.ts b/apps/api/src/deployment/services/sdl-secrets/sdl-secrets.service.spec.ts index 54ef4f59e6..5bf25e28ea 100644 --- a/apps/api/src/deployment/services/sdl-secrets/sdl-secrets.service.spec.ts +++ b/apps/api/src/deployment/services/sdl-secrets/sdl-secrets.service.spec.ts @@ -517,6 +517,23 @@ describe(SdlSecretsService.name, () => { }); }); + describe("sealForTeam", () => { + it("seals under the team's key even while team data keys are off", async () => { + const { service, secretCipherService } = setup({ teamDataKeys: false }); + const secrets = { TOKEN: "one" }; + + await expect(service.sealForTeam({ teamId: "team-1", dseq: "1420000", secrets })).resolves.toBe("encrypted"); + expect(secretCipherService.encrypt).toHaveBeenCalledWith({ organizationId: "team-1" }, JSON.stringify(secrets), { org: "team-1", dseq: "1420000" }); + }); + + it("writes no token for a deployment that holds nothing", async () => { + const { service, secretCipherService } = setup(); + + await expect(service.sealForTeam({ teamId: "team-1", dseq: "1420000", secrets: {} })).resolves.toBeNull(); + expect(secretCipherService.encrypt).not.toHaveBeenCalled(); + }); + }); + it("never seals a personal deployment for a team", async () => { const { service, secretCipherService } = setup({ teamDataKeys: true }); diff --git a/apps/api/src/deployment/services/sdl-secrets/sdl-secrets.service.ts b/apps/api/src/deployment/services/sdl-secrets/sdl-secrets.service.ts index 7faa6a27c7..16e2458ee7 100644 --- a/apps/api/src/deployment/services/sdl-secrets/sdl-secrets.service.ts +++ b/apps/api/src/deployment/services/sdl-secrets/sdl-secrets.service.ts @@ -163,11 +163,19 @@ export class SdlSecretsService { /** Returns null when nothing was supplied, so a create always has a value to write and a retry cannot inherit an abandoned attempt's token. */ async sealForStorage(input: SecretsToSeal): Promise { + return await this.#seal(input, this.#sealerFor(input)); + } + + /** Seals for the team whatever `team_data_keys` says, which is how a value a member sealed moves onto the team's key. */ + async sealForTeam(input: { teamId: string; dseq: string; secrets: SdlSecrets }): Promise { + return await this.#seal(input, sealerOfOrganization(input.teamId, input.dseq)); + } + + async #seal(input: { userId?: string; dseq: string; secrets: SdlSecrets }, { owner, binding }: Sealer): Promise { const names = Object.keys(input.secrets); if (names.length === 0) return null; - const { owner, binding } = this.#sealerFor(input); const sealed = await this.secretCipherService.encrypt(owner, JSON.stringify(input.secrets), binding); this.#loggerService.info({ diff --git a/apps/api/src/secret/controllers/team-secrets-reseal/team-secrets-reseal.controller.ts b/apps/api/src/secret/controllers/team-secrets-reseal/team-secrets-reseal.controller.ts new file mode 100644 index 0000000000..7ff7de0e46 --- /dev/null +++ b/apps/api/src/secret/controllers/team-secrets-reseal/team-secrets-reseal.controller.ts @@ -0,0 +1,13 @@ +import { singleton } from "tsyringe"; + +import type { TeamSecretsResealOptions } from "@src/secret/services/team-secrets-reseal/team-secrets-reseal.service"; +import { TeamSecretsResealService } from "@src/secret/services/team-secrets-reseal/team-secrets-reseal.service"; + +@singleton() +export class TeamSecretsResealController { + constructor(private readonly teamSecretsResealService: TeamSecretsResealService) {} + + async resealTeamDeployments(options: TeamSecretsResealOptions) { + return await this.teamSecretsResealService.resealTeamDeployments(options); + } +} diff --git a/apps/api/src/secret/services/team-secrets-reseal/team-secrets-reseal.service.integration.ts b/apps/api/src/secret/services/team-secrets-reseal/team-secrets-reseal.service.integration.ts new file mode 100644 index 0000000000..257ff3c3d6 --- /dev/null +++ b/apps/api/src/secret/services/team-secrets-reseal/team-secrets-reseal.service.integration.ts @@ -0,0 +1,91 @@ +import { faker } from "@faker-js/faker"; +import { eq } from "drizzle-orm"; +import { container } from "tsyringe"; +import { describe, expect, it } from "vitest"; + +import { type ApiPgDatabase, POSTGRES_DB, resolveTable } from "@src/core"; +import { ExecutionContextService } from "@src/core/services/execution-context/execution-context.service"; +import type { StoredSecretsOfDeployment } from "@src/deployment/services/sdl-secrets/sdl-secrets.service"; +import { isSealedForOrganization, SdlSecretsService } from "@src/deployment/services/sdl-secrets/sdl-secrets.service"; +import { TeamSecretsResealService } from "./team-secrets-reseal.service"; + +import { registerFakeSdlSecretsKms, warmSealingKeyAsBootWould } from "@test/mocks/sdl-secrets-kms.mock"; +import { createDseq, seedDeploymentSetting } from "@test/seeders/db/deployment-setting.seeder"; +import { seedOrganization, seedOrganizationWithOwner } from "@test/seeders/db/organization.seeder"; +import { seedUser } from "@test/seeders/db/user-with-wallet.seeder"; + +registerFakeSdlSecretsKms(); + +describe(TeamSecretsResealService.name, () => { + it("moves what members sealed for live teams onto each team's key, and leaves every other value as it was", async () => { + const { reseal, seedDeployment, orphan, findToken, open } = await setup(); + const team = await seedOrganizationWithOwner(); + const deletedTeam = await seedOrganization({ deletedAt: new Date() }); + const moved = await seedDeployment({ filerId: team.user.id, organizationId: team.organization.id }); + const orphaned = await orphan(await seedDeployment({ filerId: team.user.id, organizationId: team.organization.id })); + const inDeletedTeam = await seedDeployment({ filerId: (await seedUser()).id, organizationId: deletedTeam.id }); + const personal = await seedDeployment({ filerId: (await seedUser()).id, organizationId: null }); + + const dryRun = (await reseal({ dryRun: true })).unwrap(); + + expect(dryRun).toMatchObject({ + deploymentsScanned: 2, + deploymentsResealed: 1, + deploymentsWhoseMemberKeyIsGone: [`${team.organization.id}/${orphaned.dseq}`] + }); + expect(await findToken(moved)).toBe(moved.sealedSecrets); + + const report = (await reseal({ dryRun: false })).unwrap(); + + expect(report).toMatchObject({ deploymentsResealed: 1, secretsResealed: 1, teamsResealed: 1, deploymentsMovedByAnotherWriter: 0 }); + const resealed = await findToken(moved); + expect(isSealedForOrganization(resealed)).toBe(true); + await expect(open({ userId: null, organizationId: team.organization.id, dseq: moved.dseq, sealedSecrets: resealed })).resolves.toEqual(moved.secrets); + expect(await findToken(orphaned)).toBe(orphaned.sealedSecrets); + expect(await findToken(inDeletedTeam)).toBe(inDeletedTeam.sealedSecrets); + expect(await findToken(personal)).toBe(personal.sealedSecrets); + + expect((await reseal({ dryRun: false })).unwrap()).toMatchObject({ deploymentsResealed: 0, deploymentsAlreadyUnderTeamKey: 1 }); + }); + + async function setup() { + await warmSealingKeyAsBootWould(); + const db = container.resolve(POSTGRES_DB); + const deploymentSettings = resolveTable("DeploymentSettings"); + const executionContextService = container.resolve(ExecutionContextService); + const sdlSecretsService = container.resolve(SdlSecretsService); + + async function seedDeployment({ filerId, organizationId }: { filerId: string; organizationId: string | null }) { + const dseq = createDseq(); + const secrets = { API_TOKEN: faker.string.alphanumeric(24) }; + const sealedSecrets = (await executionContextService.runWithContext( + async () => await sdlSecretsService.sealForStorage({ userId: filerId, dseq, secrets }) + ))!; + const { id } = await seedDeploymentSetting({ userId: filerId, organizationId, dseq, sealedSecrets, autoTopUpEnabled: false }); + + return { id, dseq, secrets, sealedSecrets }; + } + + async function orphan(deployment: T) { + await db.update(deploymentSettings).set({ userId: null }).where(eq(deploymentSettings.id, deployment.id)); + + return deployment; + } + + async function findToken({ id }: { id: string }) { + const [row] = await db.select({ sealedSecrets: deploymentSettings.sealedSecrets }).from(deploymentSettings).where(eq(deploymentSettings.id, id)); + + return row.sealedSecrets!; + } + + async function open(stored: StoredSecretsOfDeployment) { + return await executionContextService.runWithContext(async () => await sdlSecretsService.openStored(stored)); + } + + async function reseal(options: { dryRun: boolean }) { + return await container.resolve(TeamSecretsResealService).resealTeamDeployments(options); + } + + return { reseal, seedDeployment, orphan, findToken, open }; + } +}); diff --git a/apps/api/src/secret/services/team-secrets-reseal/team-secrets-reseal.service.spec.ts b/apps/api/src/secret/services/team-secrets-reseal/team-secrets-reseal.service.spec.ts new file mode 100644 index 0000000000..198174a9aa --- /dev/null +++ b/apps/api/src/secret/services/team-secrets-reseal/team-secrets-reseal.service.spec.ts @@ -0,0 +1,184 @@ +import createError from "http-errors"; +import { randomUUID } from "node:crypto"; +import { describe, expect, it, vi } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import type { CreateLogger } from "@src/core/providers/logging.provider"; +import { ExecutionContextService } from "@src/core/services/execution-context/execution-context.service"; +import type { DeploymentSettingRepository, DeploymentStoredSecretsOfTeam } from "@src/deployment/repositories/deployment-setting/deployment-setting.repository"; +import type { SdlSecretsService } from "@src/deployment/services/sdl-secrets/sdl-secrets.service"; +import { STORED_SECRETS_UNAVAILABLE_ERROR_CODE } from "@src/deployment/services/sdl-secrets/sdl-secrets.service"; +import { TeamSecretsResealService } from "./team-secrets-reseal.service"; + +const TWO_SECRETS = { DB_URL: "postgres://db", API_TOKEN: "token" }; + +function tokenWithClaims(claims: Record) { + const header = Buffer.from(JSON.stringify({ ...claims, alg: "dir", enc: "A256GCM", kid: randomUUID() })).toString("base64url"); + + return `${header}..iv.ciphertext.tag`; +} + +function memberSealed(input: { organizationId?: string; dseq?: string; userId?: string | null } = {}): DeploymentStoredSecretsOfTeam { + const dseq = input.dseq ?? String(1000 + Math.floor(Math.random() * 9000)); + const userId = input.userId === undefined ? randomUUID() : input.userId; + + return { + id: randomUUID(), + dseq, + userId, + organizationId: input.organizationId ?? "team-1", + sealedSecrets: tokenWithClaims({ sub: userId ?? randomUUID(), dseq }), + updatedAt: new Date() + }; +} + +function teamSealed(organizationId = "team-1"): DeploymentStoredSecretsOfTeam { + const row = memberSealed({ organizationId }); + + return { ...row, sealedSecrets: tokenWithClaims({ org: organizationId, dseq: row.dseq }) }; +} + +describe(TeamSecretsResealService.name, () => { + it("moves a value a member sealed for the team onto the team's key", async () => { + const row = memberSealed({ organizationId: "team-1", dseq: "100" }); + const { service, sdlSecretsService, deploymentSettingRepository } = setup({ rows: [row] }); + + const report = (await service.resealTeamDeployments({ dryRun: false })).unwrap(); + + expect(sdlSecretsService.openStored).toHaveBeenCalledWith({ userId: row.userId, organizationId: "team-1", dseq: "100", sealedSecrets: row.sealedSecrets }); + expect(sdlSecretsService.sealForTeam).toHaveBeenCalledWith({ teamId: "team-1", dseq: "100", secrets: TWO_SECRETS }); + expect(deploymentSettingRepository.resealIfUnchanged).toHaveBeenCalledWith(row.id, row.sealedSecrets, "team-1/100.sealed-for-team"); + expect(report).toMatchObject({ dryRun: false, deploymentsScanned: 1, deploymentsResealed: 1, secretsResealed: 2, teamsResealed: 1 }); + }); + + it("leaves a value already sealed for its team as it is", async () => { + const { service, sdlSecretsService } = setup({ rows: [teamSealed()] }); + + const report = (await service.resealTeamDeployments({ dryRun: false })).unwrap(); + + expect(sdlSecretsService.openStored).not.toHaveBeenCalled(); + expect(report).toMatchObject({ deploymentsScanned: 1, deploymentsAlreadyUnderTeamKey: 1, deploymentsResealed: 0, teamsResealed: 0 }); + }); + + it("opens and counts on a dry run without sealing or writing anything", async () => { + const { service, sdlSecretsService, deploymentSettingRepository } = setup({ rows: [memberSealed(), memberSealed()] }); + + const report = (await service.resealTeamDeployments({ dryRun: true })).unwrap(); + + expect(sdlSecretsService.openStored).toHaveBeenCalledTimes(2); + expect(sdlSecretsService.sealForTeam).not.toHaveBeenCalled(); + expect(deploymentSettingRepository.resealIfUnchanged).not.toHaveBeenCalled(); + expect(report).toMatchObject({ dryRun: true, deploymentsResealed: 2, secretsResealed: 4, teamsResealed: 1 }); + }); + + it("reports a value whose member key went with the member's account, without failing the run", async () => { + const gone = memberSealed({ organizationId: "team-2", dseq: "200", userId: null }); + const { service, deploymentSettingRepository } = setup({ rows: [gone, memberSealed()], departedDseqs: ["200"] }); + + const result = await service.resealTeamDeployments({ dryRun: false }); + + expect(result.unwrap()).toMatchObject({ deploymentsWhoseMemberKeyIsGone: ["team-2/200"], deploymentsResealed: 1 }); + expect(deploymentSettingRepository.resealIfUnchanged).toHaveBeenCalledTimes(1); + }); + + it("re-seals the rest and fails the run at the end when a value will not open", async () => { + const broken = memberSealed({ dseq: "300" }); + const readable = memberSealed({ dseq: "400" }); + const { service, deploymentSettingRepository } = setup({ rows: [broken, readable], unopenableDseqs: ["300"] }); + + const result = await service.resealTeamDeployments({ dryRun: false }); + + expect(result.err).toBe(true); + expect(result.val).toEqual([expect.objectContaining({ message: "deployment 300 would not open" })]); + expect(deploymentSettingRepository.resealIfUnchanged).toHaveBeenCalledWith(readable.id, readable.sealedSecrets, expect.any(String)); + }); + + it("fails the run for a value the key service would not open, rather than reporting its key gone", async () => { + const { service, sdlSecretsService } = setup({ rows: [memberSealed({ dseq: "500" })] }); + const unavailable = createError(503, "key service unavailable"); + sdlSecretsService.openStored.mockRejectedValue(unavailable); + + const result = await service.resealTeamDeployments({ dryRun: false }); + + expect(result.err).toBe(true); + expect(result.val).toEqual([unavailable]); + }); + + it("counts a value another writer replaced in between rather than overwriting it", async () => { + const row = memberSealed(); + const { service } = setup({ rows: [row], movedIds: [row.id] }); + + const report = (await service.resealTeamDeployments({ dryRun: false })).unwrap(); + + expect(report).toMatchObject({ deploymentsMovedByAnotherWriter: 1, deploymentsResealed: 0, secretsResealed: 0, teamsResealed: 0 }); + }); + + it("counts every team it moved values for once", async () => { + const { service } = setup({ + rows: [memberSealed({ organizationId: "team-1" }), memberSealed({ organizationId: "team-1" }), memberSealed({ organizationId: "team-2" })] + }); + + const report = (await service.resealTeamDeployments({ dryRun: false })).unwrap(); + + expect(report).toMatchObject({ deploymentsResealed: 3, teamsResealed: 2 }); + }); + + it("opens every page inside a request context of its own, as holding a data key requires", async () => { + const { service, sdlSecretsService, executionContextService } = setup({ pages: [[memberSealed()], [memberSealed()]] }); + const contexts: unknown[] = []; + sdlSecretsService.openStored.mockImplementation(async () => { + executionContextService.set("HELD_DATA_KEYS", executionContextService.get("HELD_DATA_KEYS") ?? new Map()); + contexts.push(executionContextService.get("HELD_DATA_KEYS")); + + return TWO_SECRETS; + }); + + await service.resealTeamDeployments({ dryRun: true }); + + expect(contexts).toHaveLength(2); + expect(contexts[0]).not.toBe(contexts[1]); + }); + + it("reads the team deployments a page at a time, a hundred unless told otherwise", async () => { + const { service, deploymentSettingRepository } = setup({ rows: [] }); + + await service.resealTeamDeployments({ dryRun: true }); + await service.resealTeamDeployments({ dryRun: true, batchSize: 25 }); + + expect(deploymentSettingRepository.findStoredSecretsOfTeamsIteratively).toHaveBeenNthCalledWith(1, { batchSize: 100 }); + expect(deploymentSettingRepository.findStoredSecretsOfTeamsIteratively).toHaveBeenNthCalledWith(2, { batchSize: 25 }); + }); + + function setup(input: { + rows?: DeploymentStoredSecretsOfTeam[]; + pages?: DeploymentStoredSecretsOfTeam[][]; + departedDseqs?: string[]; + unopenableDseqs?: string[]; + movedIds?: string[]; + }) { + const pages = input.pages ?? (input.rows?.length ? [input.rows] : []); + const deploymentSettingRepository = mock(); + deploymentSettingRepository.findStoredSecretsOfTeamsIteratively.mockImplementation(async function* () { + yield* pages; + }); + deploymentSettingRepository.resealIfUnchanged.mockImplementation(async id => !input.movedIds?.includes(id)); + + const sdlSecretsService = mock(); + sdlSecretsService.openStored.mockImplementation(async ({ dseq }) => { + if (input.departedDseqs?.includes(dseq)) { + throw createError(409, "sealed for a departed member", { errorCode: STORED_SECRETS_UNAVAILABLE_ERROR_CODE }); + } + + if (input.unopenableDseqs?.includes(dseq)) throw new Error(`deployment ${dseq} would not open`); + + return TWO_SECRETS; + }); + sdlSecretsService.sealForTeam.mockImplementation(async ({ teamId, dseq }) => `${teamId}/${dseq}.sealed-for-team`); + + const createLogger = vi.fn(() => mock>()); + const executionContextService = new ExecutionContextService(createLogger); + const service = new TeamSecretsResealService(deploymentSettingRepository, sdlSecretsService, executionContextService, createLogger); + + return { service, deploymentSettingRepository, sdlSecretsService, executionContextService }; + } +}); diff --git a/apps/api/src/secret/services/team-secrets-reseal/team-secrets-reseal.service.ts b/apps/api/src/secret/services/team-secrets-reseal/team-secrets-reseal.service.ts new file mode 100644 index 0000000000..8f57422f00 --- /dev/null +++ b/apps/api/src/secret/services/team-secrets-reseal/team-secrets-reseal.service.ts @@ -0,0 +1,129 @@ +import { isHttpError } from "http-errors"; +import { Err, Ok, type Result } from "ts-results"; +import { inject, singleton } from "tsyringe"; + +import { assertBatchSize } from "@src/core/lib/batch-size/batch-size"; +import { type CreateLogger, LOGGER_FACTORY } from "@src/core/providers/logging.provider"; +import { ExecutionContextService } from "@src/core/services/execution-context/execution-context.service"; +import type { DeploymentStoredSecretsOfTeam } from "@src/deployment/repositories/deployment-setting/deployment-setting.repository"; +import { DeploymentSettingRepository } from "@src/deployment/repositories/deployment-setting/deployment-setting.repository"; +import { isSealedForOrganization, SdlSecretsService, STORED_SECRETS_UNAVAILABLE_ERROR_CODE } from "@src/deployment/services/sdl-secrets/sdl-secrets.service"; + +const DEFAULT_BATCH_SIZE = 100; + +export interface TeamSecretsResealOptions { + dryRun: boolean; + batchSize?: number; +} + +export interface TeamSecretsResealReport { + dryRun: boolean; + deploymentsScanned: number; + deploymentsResealed: number; + secretsResealed: number; + teamsResealed: number; + deploymentsAlreadyUnderTeamKey: number; + deploymentsMovedByAnotherWriter: number; + deploymentsWhoseMemberKeyIsGone: string[]; + elapsedMs: number; +} + +type ResealPass = Omit & { teams: Set; errors: unknown[] }; + +function isSealedForADepartedMember(error: unknown): boolean { + return isHttpError(error) && error.errorCode === STORED_SECRETS_UNAVAILABLE_ERROR_CODE; +} + +/** Moves the values members sealed for their teams onto each team's own key, so a member deleting their account no longer takes them along; a value whose member key is already gone cannot be moved and is only reported. */ +@singleton() +export class TeamSecretsResealService { + private readonly logger: ReturnType; + + constructor( + private readonly deploymentSettingRepository: DeploymentSettingRepository, + private readonly sdlSecretsService: SdlSecretsService, + private readonly executionContextService: ExecutionContextService, + @inject(LOGGER_FACTORY) createLogger: CreateLogger + ) { + this.logger = createLogger({ context: TeamSecretsResealService.name }); + } + + async resealTeamDeployments({ dryRun, batchSize }: TeamSecretsResealOptions): Promise> { + const startedAt = Date.now(); + const pageSize = assertBatchSize(batchSize ?? DEFAULT_BATCH_SIZE); + const pass: ResealPass = { + deploymentsScanned: 0, + deploymentsResealed: 0, + secretsResealed: 0, + deploymentsAlreadyUnderTeamKey: 0, + deploymentsMovedByAnotherWriter: 0, + deploymentsWhoseMemberKeyIsGone: [], + teams: new Set(), + errors: [] + }; + + this.logger.info({ event: "TEAM_SECRETS_RESEAL_START", dryRun }); + + for await (const batch of this.deploymentSettingRepository.findStoredSecretsOfTeamsIteratively({ batchSize: pageSize })) { + await this.executionContextService.runWithContext(async () => { + for (const row of batch) { + await this.#resealDeployment(row, pass, dryRun); + } + }); + + this.logger.info({ event: "TEAM_SECRETS_RESEAL_BATCH", deploymentsScanned: pass.deploymentsScanned, deploymentsResealed: pass.deploymentsResealed }); + } + + const { teams, errors, ...counts } = pass; + const report: TeamSecretsResealReport = { dryRun, ...counts, teamsResealed: teams.size, elapsedMs: Date.now() - startedAt }; + + this.logger.info({ event: "TEAM_SECRETS_RESEAL_END", report }); + + return errors.length > 0 ? Err(errors) : Ok(report); + } + + async #resealDeployment(row: DeploymentStoredSecretsOfTeam, pass: ResealPass, dryRun: boolean): Promise { + pass.deploymentsScanned += 1; + + if (isSealedForOrganization(row.sealedSecrets)) { + pass.deploymentsAlreadyUnderTeamKey += 1; + + return; + } + + try { + await this.#reseal(row, pass, dryRun); + } catch (error) { + if (isSealedForADepartedMember(error)) { + pass.deploymentsWhoseMemberKeyIsGone.push(`${row.organizationId}/${row.dseq}`); + this.logger.warn({ event: "TEAM_SECRETS_RESEAL_MEMBER_KEY_GONE", organizationId: row.organizationId, dseq: row.dseq }); + + return; + } + + pass.errors.push(error); + this.logger.error({ event: "TEAM_SECRETS_RESEAL_DEPLOYMENT_FAILED", organizationId: row.organizationId, dseq: row.dseq, error }); + } + } + + /** Opened through the path a deployment reads it by, so a value bound to a member other than the row's filer is refused rather than moved. */ + async #reseal(row: DeploymentStoredSecretsOfTeam, pass: ResealPass, dryRun: boolean): Promise { + const { organizationId, dseq } = row; + const secrets = await this.sdlSecretsService.openStored({ userId: row.userId, organizationId, dseq, sealedSecrets: row.sealedSecrets }); + + if (!dryRun) { + const resealed = await this.sdlSecretsService.sealForTeam({ teamId: organizationId, dseq, secrets }); + + if (!(await this.deploymentSettingRepository.resealIfUnchanged(row.id, row.sealedSecrets, resealed))) { + pass.deploymentsMovedByAnotherWriter += 1; + this.logger.warn({ event: "TEAM_SECRETS_RESEAL_MOVED_BY_ANOTHER_WRITER", organizationId, dseq }); + + return; + } + } + + pass.deploymentsResealed += 1; + pass.secretsResealed += Object.keys(secrets).length; + pass.teams.add(organizationId); + } +} From f72ceddc2587337a870a4e138f391e279b006cb9 Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Sat, 17 Oct 2026 23:24:57 -0700 Subject: [PATCH 2/2] test(deployment): page the team stored secrets across batches against a real database Part of CON-1213 --- ...ployment-setting.repository.integration.ts | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/apps/api/src/deployment/repositories/deployment-setting/deployment-setting.repository.integration.ts b/apps/api/src/deployment/repositories/deployment-setting/deployment-setting.repository.integration.ts index 9a39047649..61a5b7bb69 100644 --- a/apps/api/src/deployment/repositories/deployment-setting/deployment-setting.repository.integration.ts +++ b/apps/api/src/deployment/repositories/deployment-setting/deployment-setting.repository.integration.ts @@ -2920,6 +2920,46 @@ describe(DeploymentSettingRepository.name, () => { } }); + describe("re-sealing the stored secrets filed in teams", () => { + it("pages only the live teams' deployments that hold a secret, across batches smaller than the set", async () => { + const { deploymentSettingRepository, teamSealedIds, passedOverIds } = await setupTeamSecrets(); + const batchSizes: number[] = []; + const pagedIds: string[] = []; + + for await (const batch of deploymentSettingRepository.findStoredSecretsOfTeamsIteratively({ batchSize: 1 })) { + batchSizes.push(batch.length); + pagedIds.push(...batch.map(row => row.id)); + } + + expect(batchSizes.every(size => size === 1)).toBe(true); + expect(new Set(pagedIds).size).toBe(pagedIds.length); + expect(pagedIds).toEqual(expect.arrayContaining(teamSealedIds)); + expect(pagedIds.filter(id => passedOverIds.includes(id))).toEqual([]); + }); + + async function setupTeamSecrets() { + const deploymentSettingRepository = container.resolve(DeploymentSettingRepository); + const { user, organization: team } = await seedOrganizationWithOwner(); + const { organization: personal } = await seedOrganizationWithOwner({ type: "personal" }); + const { organization: deletedTeam } = await seedOrganizationWithOwner({ deletedAt: new Date() }); + const seedFiledIn = async (organizationId: string, sealedSecrets: string | null) => + (await seedDeploymentSetting({ userId: user.id, organizationId, sealedSecrets })).id; + + const teamSealedIds = [ + await seedFiledIn(team.id, newSealedToken()), + await seedFiledIn(team.id, newSealedToken()), + await seedFiledIn(team.id, newSealedToken()) + ]; + const passedOverIds = [ + await seedFiledIn(team.id, null), + await seedFiledIn(personal.id, newSealedToken()), + await seedFiledIn(deletedTeam.id, newSealedToken()) + ]; + + return { deploymentSettingRepository, teamSealedIds, passedOverIds }; + } + }); + describe("when a deployment is filed in a team organization", () => { it("funds the team deployment from the organization's wallet, never from the member's own", async () => { const { deploymentSettingRepository, teamWallet, personalWallet, teamDeployment } = await setupTeamDeployment();