diff --git a/apps/api/src/app/providers/jobs.provider.ts b/apps/api/src/app/providers/jobs.provider.ts index 59e226d410..20f8c7211e 100644 --- a/apps/api/src/app/providers/jobs.provider.ts +++ b/apps/api/src/app/providers/jobs.provider.ts @@ -10,7 +10,9 @@ import { DeleteUnbackedDeploymentSettingHandler } from "@src/deployment/services import { ReconcileManagedTxHandler } from "@src/deployment/services/reconcile-managed-tx/reconcile-managed-tx.handler"; import { RecordDeploymentSettingHandler } from "@src/deployment/services/record-deployment-setting/record-deployment-setting.handler"; import { NotificationHandler } from "@src/notifications/services/notification-handler/notification.handler"; +import { BlockEmailDomainOfWalletHandler } from "@src/workload-abuse/services/block-email-domain-of-wallet/block-email-domain-of-wallet.handler"; import { EnforceTrialAbuseHandler } from "@src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler"; +import { LockBlockedDomainWalletHandler } from "@src/workload-abuse/services/lock-blocked-domain-wallet/lock-blocked-domain-wallet.handler"; import { ProbeTrialDeploymentHandler } from "@src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler"; import { AutoRechargeSucceededHandler } from "../services/auto-recharge-succeeded/auto-recharge-succeeded.handler"; import { CloseExpiredDeploymentHandler } from "../services/close-expired-deployment/close-expired-deployment.handler"; @@ -46,7 +48,9 @@ export async function startJobQueues(): Promise { container.resolve(RecordDeploymentSettingHandler), container.resolve(ReconcileManagedTxHandler), container.resolve(ProbeTrialDeploymentHandler), - container.resolve(EnforceTrialAbuseHandler) + container.resolve(EnforceTrialAbuseHandler), + container.resolve(LockBlockedDomainWalletHandler), + container.resolve(BlockEmailDomainOfWalletHandler) ]); } diff --git a/apps/api/src/billing/repositories/stripe-transaction/stripe-transaction.repository.integration.ts b/apps/api/src/billing/repositories/stripe-transaction/stripe-transaction.repository.integration.ts index 53fda4d79c..94ac61a741 100644 --- a/apps/api/src/billing/repositories/stripe-transaction/stripe-transaction.repository.integration.ts +++ b/apps/api/src/billing/repositories/stripe-transaction/stripe-transaction.repository.integration.ts @@ -270,6 +270,64 @@ describe(StripeTransactionRepository.name, () => { }; } + describe("hasPaidUserWithEmailDomain", () => { + it.each([ + { status: "succeeded" as const, expected: true }, + { status: "refunded" as const, expected: true }, + { status: "created" as const, expected: false }, + { status: "failed" as const, expected: false } + ])("answers $expected for a payment_intent in $status", async ({ status, expected }) => { + const { stripeTransactionRepository, createUserOnDomain } = setup(); + const domain = uniqueDomain(); + const user = await createUserOnDomain(domain); + await stripeTransactionRepository.create({ userId: user.id, type: "payment_intent", status, amount: 1000, currency: "usd" }); + + await expect(stripeTransactionRepository.hasPaidUserWithEmailDomain(domain)).resolves.toBe(expected); + }); + + it.each([["manual_credit" as const], ["coupon_claim" as const]])("does not count a %s, so a granted credit cannot shield a domain", async type => { + const { stripeTransactionRepository, createUserOnDomain } = setup(); + const domain = uniqueDomain(); + const user = await createUserOnDomain(domain); + await stripeTransactionRepository.create({ userId: user.id, type, status: "succeeded", amount: 1000, currency: "usd" }); + + await expect(stripeTransactionRepository.hasPaidUserWithEmailDomain(domain)).resolves.toBe(false); + }); + + it("matches the domain part only, never a domain that merely contains it", async () => { + const { stripeTransactionRepository, createUserOnDomain } = setup(); + const domain = uniqueDomain(); + const lookalikes = [`x${domain}`, `${domain}.attacker.net`, `mail.${domain}`]; + for (const lookalike of lookalikes) { + const user = await createUserOnDomain(lookalike); + await stripeTransactionRepository.create({ userId: user.id, type: "payment_intent", status: "succeeded", amount: 1000, currency: "usd" }); + } + + await expect(stripeTransactionRepository.hasPaidUserWithEmailDomain(domain)).resolves.toBe(false); + }); + + it("matches a stored address whatever its case", async () => { + const { stripeTransactionRepository, createUserOnDomain } = setup(); + const domain = uniqueDomain(); + const user = await createUserOnDomain(domain.toUpperCase()); + await stripeTransactionRepository.create({ userId: user.id, type: "payment_intent", status: "succeeded", amount: 1000, currency: "usd" }); + + await expect(stripeTransactionRepository.hasPaidUserWithEmailDomain(domain)).resolves.toBe(true); + }); + + it("answers false for a domain nobody has paid from", async () => { + const { stripeTransactionRepository, createUserOnDomain } = setup(); + const domain = uniqueDomain(); + await createUserOnDomain(domain); + + await expect(stripeTransactionRepository.hasPaidUserWithEmailDomain(domain)).resolves.toBe(false); + }); + }); + + function uniqueDomain() { + return `${faker.string.alphanumeric(16).toLowerCase()}.com`; + } + function setup() { const stripeTransactionRepository = container.resolve(StripeTransactionRepository); const userRepository = container.resolve(UserRepository); @@ -292,6 +350,12 @@ describe(StripeTransactionRepository.name, () => { return testUserId; } + async function createUserOnDomain(domain: string) { + const user = await userRepository.create({ userId: faker.string.uuid(), email: `${faker.string.alphanumeric(10)}@${domain}` }); + createdUserIds.push(user.id); + return user; + } + async function createTestTransaction(overrides: Partial = {}) { return stripeTransactionRepository.create({ userId: await getTestUserId(), @@ -309,6 +373,6 @@ describe(StripeTransactionRepository.name, () => { return user; } - return { stripeTransactionRepository, userRepository, createTestTransaction, createTestUser }; + return { stripeTransactionRepository, userRepository, createTestTransaction, createTestUser, createUserOnDomain }; } }); diff --git a/apps/api/src/billing/repositories/stripe-transaction/stripe-transaction.repository.ts b/apps/api/src/billing/repositories/stripe-transaction/stripe-transaction.repository.ts index 76f2ad72b7..1ea1ecc0ec 100644 --- a/apps/api/src/billing/repositories/stripe-transaction/stripe-transaction.repository.ts +++ b/apps/api/src/billing/repositories/stripe-transaction/stripe-transaction.repository.ts @@ -1,9 +1,10 @@ -import { and, count, desc, eq, gte, inArray, lte, notInArray, SQL, sql } from "drizzle-orm"; +import { and, count, desc, eq, exists, gte, inArray, lte, notInArray, SQL, sql } from "drizzle-orm"; import { singleton } from "tsyringe"; import { type ApiPgDatabase, type ApiPgTables, InjectPg, InjectPgTable } from "@src/core/providers"; import { type AbilityParams, BaseRepository } from "@src/core/repositories/base.repository"; import { TxService } from "@src/core/services"; +import { Users } from "@src/user/model-schemas"; type Table = ApiPgTables["StripeTransactions"]; export type StripeTransactionInput = Table["$inferInsert"]; @@ -176,6 +177,31 @@ export class StripeTransactionRepository extends BaseRepository { + const [match] = await this.cursor + .select({ id: Users.id }) + .from(Users) + .where( + and( + sql`lower(${Users.email}) LIKE ${"%@"} || ${domain}`, + exists( + this.cursor + .select({ id: this.table.id }) + .from(this.table) + .where(and(eq(this.table.userId, Users.id), eq(this.table.type, "payment_intent"), inArray(this.table.status, ["succeeded", "refunded"]))) + ) + ) + ) + .limit(1); + + return !!match; + } + async countByUserId(userId: string, options?: { startDate?: Date; endDate?: Date }): Promise { const conditions: SQL[] = [eq(this.table.userId, userId)]; diff --git a/apps/api/src/billing/repositories/user-wallet/user-wallet.repository.integration.ts b/apps/api/src/billing/repositories/user-wallet/user-wallet.repository.integration.ts index bdc96b8e6c..f1e2e5ddd8 100644 --- a/apps/api/src/billing/repositories/user-wallet/user-wallet.repository.integration.ts +++ b/apps/api/src/billing/repositories/user-wallet/user-wallet.repository.integration.ts @@ -4,6 +4,7 @@ import subMinutes from "date-fns/subMinutes"; import { container } from "tsyringe"; import { describe, expect, it } from "vitest"; +import { StripeTransactionRepository } from "@src/billing/repositories/stripe-transaction/stripe-transaction.repository"; import { UserRepository } from "@src/user/repositories"; import { UserWalletRepository } from "./user-wallet.repository"; @@ -233,6 +234,101 @@ describe(UserWalletRepository.name, () => { }); }); + describe("findLockableTrialWalletsByEmailDomain", () => { + it("returns the trialing, unlocked, never-paid wallets on the domain", async () => { + const { domain, createWalletOnDomain, userWalletRepository } = await setupDomain(); + const target = await createWalletOnDomain({}); + const trigger = await createWalletOnDomain({}); + + const lockable = await userWalletRepository.findLockableTrialWalletsByEmailDomain(domain, { excludeWalletId: trigger.id, limit: 10 }); + + expect(lockable).toEqual([{ walletId: target.id, userId: target.userId }]); + }); + + it.each([ + { label: "already locked", overrides: { abuseLockedAt: new Date(), abuseLockedReason: "workload_abuse" } }, + { label: "no longer trialing", overrides: { isTrialing: false } } + ])("excludes a wallet that is $label", async ({ overrides }) => { + const { domain, createWalletOnDomain, userWalletRepository } = await setupDomain(); + await createWalletOnDomain(overrides); + const trigger = await createWalletOnDomain({}); + + const lockable = await userWalletRepository.findLockableTrialWalletsByEmailDomain(domain, { excludeWalletId: trigger.id, limit: 10 }); + + expect(lockable).toEqual([]); + }); + + it("excludes a wallet whose owner has ever paid", async () => { + const { domain, createWalletOnDomain, userWalletRepository, stripeTransactionRepository } = await setupDomain(); + const paid = await createWalletOnDomain({}); + await stripeTransactionRepository.create({ userId: paid.userId, type: "payment_intent", status: "succeeded", amount: 1000, currency: "usd" }); + const trigger = await createWalletOnDomain({}); + + const lockable = await userWalletRepository.findLockableTrialWalletsByEmailDomain(domain, { excludeWalletId: trigger.id, limit: 10 }); + + expect(lockable).toEqual([]); + }); + + it("still returns a wallet whose owner only ever received a manual credit", async () => { + const { domain, createWalletOnDomain, userWalletRepository, stripeTransactionRepository } = await setupDomain(); + const comped = await createWalletOnDomain({}); + await stripeTransactionRepository.create({ userId: comped.userId, type: "manual_credit", status: "succeeded", amount: 1000, currency: "usd" }); + const trigger = await createWalletOnDomain({}); + + const lockable = await userWalletRepository.findLockableTrialWalletsByEmailDomain(domain, { excludeWalletId: trigger.id, limit: 10 }); + + expect(lockable).toEqual([{ walletId: comped.id, userId: comped.userId }]); + }); + + it("excludes the wallet that triggered the block", async () => { + const { domain, createWalletOnDomain, userWalletRepository } = await setupDomain(); + const trigger = await createWalletOnDomain({}); + + const lockable = await userWalletRepository.findLockableTrialWalletsByEmailDomain(domain, { excludeWalletId: trigger.id, limit: 10 }); + + expect(lockable).toEqual([]); + }); + + it("matches the domain part only, never a domain that merely contains it", async () => { + const { domain, createWalletOnDomain, userWalletRepository } = await setupDomain(); + await createWalletOnDomain({}, `x${domain}`); + await createWalletOnDomain({}, `${domain}.attacker.net`); + await createWalletOnDomain({}, `mail.${domain}`); + const trigger = await createWalletOnDomain({}); + + const lockable = await userWalletRepository.findLockableTrialWalletsByEmailDomain(domain, { excludeWalletId: trigger.id, limit: 10 }); + + expect(lockable).toEqual([]); + }); + + it("returns no more wallets than the limit allows", async () => { + const { domain, createWalletOnDomain, userWalletRepository } = await setupDomain(); + await createWalletOnDomain({}); + await createWalletOnDomain({}); + await createWalletOnDomain({}); + const trigger = await createWalletOnDomain({}); + + const lockable = await userWalletRepository.findLockableTrialWalletsByEmailDomain(domain, { excludeWalletId: trigger.id, limit: 2 }); + + expect(lockable).toHaveLength(2); + }); + }); + + async function setupDomain() { + const userRepository = container.resolve(UserRepository); + const userWalletRepository = container.resolve(UserWalletRepository); + const stripeTransactionRepository = container.resolve(StripeTransactionRepository); + const domain = `${faker.string.alphanumeric(16).toLowerCase()}.com`; + + async function createWalletOnDomain(overrides: Parameters[1], onDomain = domain) { + const user = await userRepository.create({ userId: faker.string.uuid(), email: `${faker.string.alphanumeric(10)}@${onDomain}` }); + const created = await userWalletRepository.create({ userId: user.id, address: createAkashAddress() }); + return await userWalletRepository.updateById(created.id, { isTrialing: true, abuseLockedAt: null, ...overrides }, { returning: true }); + } + + return { domain, createWalletOnDomain, userRepository, userWalletRepository, stripeTransactionRepository }; + } + async function setup(input: { creditsLowNotifiedAt?: Date; creditsSufficientSince?: Date; creditsLowSince?: Date } = {}) { const userRepository = container.resolve(UserRepository); const userWalletRepository = container.resolve(UserWalletRepository); diff --git a/apps/api/src/billing/repositories/user-wallet/user-wallet.repository.ts b/apps/api/src/billing/repositories/user-wallet/user-wallet.repository.ts index c59265ef2e..879954c3b8 100644 --- a/apps/api/src/billing/repositories/user-wallet/user-wallet.repository.ts +++ b/apps/api/src/billing/repositories/user-wallet/user-wallet.repository.ts @@ -1,11 +1,13 @@ import { Trace } from "@akashnetwork/instrumentation"; import subDays from "date-fns/subDays"; -import { and, count, eq, gt, inArray, isNotNull, isNull, lte, or, sql } from "drizzle-orm"; +import { and, count, eq, gt, inArray, isNotNull, isNull, lte, ne, notExists, or, sql } from "drizzle-orm"; import { singleton } from "tsyringe"; +import { StripeTransactions } from "@src/billing/model-schemas"; import { type ApiPgDatabase, type ApiPgTables, InjectPg, InjectPgTable } from "@src/core/providers"; import { type AbilityParams, BaseRepository } from "@src/core/repositories/base.repository"; import { TxService } from "@src/core/services"; +import { Users } from "@src/user/model-schemas"; export type DbCreateUserWalletInput = ApiPgTables["UserWallets"]["$inferInsert"]; export type DbUserWalletInput = Partial; @@ -166,6 +168,42 @@ export class UserWalletRepository extends BaseRepository> { + return await this.cursor + .select({ walletId: this.table.id, userId: this.table.userId }) + .from(this.table) + .innerJoin(Users, eq(Users.id, this.table.userId)) + .where( + and( + sql`lower(${Users.email}) LIKE ${"%@"} || ${domain}`, + eq(this.table.isTrialing, true), + isNull(this.table.abuseLockedAt), + ne(this.table.id, options.excludeWalletId), + notExists( + this.cursor + .select({ id: StripeTransactions.id }) + .from(StripeTransactions) + .where( + and( + eq(StripeTransactions.userId, Users.id), + eq(StripeTransactions.type, "payment_intent"), + inArray(StripeTransactions.status, ["succeeded", "refunded"]) + ) + ) + ) + ) + ) + .limit(options.limit); + } + /** One write, so a wallet is never left with zeroed allowances but no lock or the other way round. */ async lockForAbuse(id: UserWalletOutput["id"], reason: string): Promise { await this.updateById(id, { deploymentAllowance: 0, feeAllowance: 0, isTrialing: false, abuseLockedAt: new Date(), abuseLockedReason: reason }); diff --git a/apps/api/src/user/repositories/user/user.repository.integration.ts b/apps/api/src/user/repositories/user/user.repository.integration.ts index 57e59444e2..3b5acacd43 100644 --- a/apps/api/src/user/repositories/user/user.repository.integration.ts +++ b/apps/api/src/user/repositories/user/user.repository.integration.ts @@ -1,4 +1,5 @@ import { faker } from "@faker-js/faker"; +import subDays from "date-fns/subDays"; import { container } from "tsyringe"; import { afterEach, describe, expect, it } from "vitest"; @@ -142,6 +143,68 @@ describe(UserRepository.name, () => { }; } + describe("hasEstablishedUserWithEmailDomain", () => { + it("answers true for a domain with an account older than the window", async () => { + const { userRepository, domain, createUserOnDomain, someoneElse } = setupDomain(); + await createUserOnDomain({ createdAt: subDays(new Date(), 45) }); + + await expect(userRepository.hasEstablishedUserWithEmailDomain(domain, 30, someoneElse)).resolves.toBe(true); + }); + + it("answers false when every account on the domain is newer than the window", async () => { + const { userRepository, domain, createUserOnDomain, someoneElse } = setupDomain(); + await createUserOnDomain({ createdAt: subDays(new Date(), 5) }); + + await expect(userRepository.hasEstablishedUserWithEmailDomain(domain, 30, someoneElse)).resolves.toBe(false); + }); + + it("ignores the excluded account, so aging one signup cannot vouch for its own domain", async () => { + const { userRepository, domain, createUserOnDomain } = setupDomain(); + const attacker = await createUserOnDomain({ createdAt: subDays(new Date(), 45) }); + + await expect(userRepository.hasEstablishedUserWithEmailDomain(domain, 30, attacker.id)).resolves.toBe(false); + }); + + it("still answers true when another account on the domain predates the window", async () => { + const { userRepository, domain, createUserOnDomain } = setupDomain(); + const attacker = await createUserOnDomain({ createdAt: subDays(new Date(), 45) }); + await createUserOnDomain({ createdAt: subDays(new Date(), 45) }); + + await expect(userRepository.hasEstablishedUserWithEmailDomain(domain, 30, attacker.id)).resolves.toBe(true); + }); + + it("matches the domain part only, never a domain that merely contains it", async () => { + const { userRepository, domain, createUserOnDomain, someoneElse } = setupDomain(); + await createUserOnDomain({ createdAt: subDays(new Date(), 45) }, `x${domain}`); + await createUserOnDomain({ createdAt: subDays(new Date(), 45) }, `${domain}.attacker.net`); + await createUserOnDomain({ createdAt: subDays(new Date(), 45) }, `mail.${domain}`); + + await expect(userRepository.hasEstablishedUserWithEmailDomain(domain, 30, someoneElse)).resolves.toBe(false); + }); + + it("matches a stored address whatever its case", async () => { + const { userRepository, domain, createUserOnDomain, someoneElse } = setupDomain(); + await createUserOnDomain({ createdAt: subDays(new Date(), 45) }, domain.toUpperCase()); + + await expect(userRepository.hasEstablishedUserWithEmailDomain(domain, 30, someoneElse)).resolves.toBe(true); + }); + }); + + function setupDomain() { + const userRepository = container.resolve(UserRepository); + const domain = `${faker.string.alphanumeric(16).toLowerCase()}.com`; + + async function createUserOnDomain(overrides: { createdAt: Date }, onDomain = domain) { + return await userRepository.create({ + userId: faker.string.uuid(), + email: `${faker.string.alphanumeric(10)}@${onDomain}`, + ...overrides + }); + } + + return { userRepository, domain, createUserOnDomain, someoneElse: faker.string.uuid() }; + } + function setup() { const userRepository = container.resolve(UserRepository); const createdUserIds: string[] = []; diff --git a/apps/api/src/user/repositories/user/user.repository.ts b/apps/api/src/user/repositories/user/user.repository.ts index 24ba81e32b..74e6228458 100644 --- a/apps/api/src/user/repositories/user/user.repository.ts +++ b/apps/api/src/user/repositories/user/user.repository.ts @@ -103,6 +103,23 @@ export class UserRepository extends BaseRepository { + const [match] = await this.pg + .select({ id: this.table.id }) + .from(this.table) + .where( + and( + sql`lower(${this.table.email}) LIKE ${"%@"} || ${domain}`, + lt(this.table.createdAt, sql`now() - make_interval(days => ${minAgeDays})`), + ne(this.table.id, excludeUserId) + ) + ) + .limit(1); + + return !!match; + } + private async findUserWithWallet(whereClause: SQL) { const result = await this.cursor.query.Users.findFirst({ where: this.whereAccessibleBy(whereClause), diff --git a/apps/api/src/workload-abuse/services/block-email-domain-of-wallet/block-email-domain-of-wallet.handler.spec.ts b/apps/api/src/workload-abuse/services/block-email-domain-of-wallet/block-email-domain-of-wallet.handler.spec.ts new file mode 100644 index 0000000000..d2a886589d --- /dev/null +++ b/apps/api/src/workload-abuse/services/block-email-domain-of-wallet/block-email-domain-of-wallet.handler.spec.ts @@ -0,0 +1,65 @@ +import { describe, expect, it, vi } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import type { UserWalletRepository } from "@src/billing/repositories"; +import type { CreateLogger } from "@src/core"; +import type { EmailDomainBlockService } from "@src/workload-abuse/services/email-domain-block/email-domain-block.service"; +import type { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service"; +import { BlockEmailDomainOfWalletHandler, blockEmailDomainOfWalletKeyFor } from "./block-email-domain-of-wallet.handler"; + +import { createUserWallet } from "@test/seeders/user-wallet.seeder"; + +const PAYLOAD = { walletId: 42, version: 1 as const }; + +describe(BlockEmailDomainOfWalletHandler.name, () => { + it("keys the block by wallet, so a second detection of the same wallet does not queue a second evaluation", () => { + expect(blockEmailDomainOfWalletKeyFor(7)).toBe("blockEmailDomainOfWallet.7"); + }); + + it("blocks the domain of the wallet it was given", async () => { + const { handler, wallet, emailDomainBlockService } = setup({ wallet: createUserWallet({ isTrialing: false, abuseLockedAt: new Date() }) }); + + await handler.handle(PAYLOAD); + + expect(emailDomainBlockService.blockDomainOf).toHaveBeenCalledWith(wallet); + }); + + it("skips a wallet it cannot find", async () => { + const { handler, emailDomainBlockService, instrumentation, logger } = setup({ wallet: null }); + + await handler.handle(PAYLOAD); + + expect(emailDomainBlockService.blockDomainOf).not.toHaveBeenCalled(); + expect(instrumentation.recordDomainBlock).toHaveBeenCalledWith("skipped", "wallet_not_found"); + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "EMAIL_DOMAIN_AUTO_BLOCK_SKIPPED", reason: "wallet_not_found" })); + }); + + it("records a failure and rethrows it, so the queue retries the block", async () => { + const { handler, emailDomainBlockService, instrumentation, logger } = setup({ wallet: createUserWallet({ isTrialing: false }) }); + emailDomainBlockService.blockDomainOf.mockRejectedValue(new Error("connection terminated")); + + await expect(handler.handle(PAYLOAD)).rejects.toThrow("connection terminated"); + + expect(instrumentation.recordDomainBlock).toHaveBeenCalledWith("failed"); + expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ event: "EMAIL_DOMAIN_AUTO_BLOCK_FAILED", walletId: PAYLOAD.walletId })); + }); + + it("declares no permissions for its execution", () => { + const { handler } = setup({ wallet: null }); + + expect(handler.requiresPermission()).toEqual([]); + }); + + function setup(input: { wallet: ReturnType | null }) { + const userWalletRepository = mock(); + userWalletRepository.findById.mockResolvedValue(input.wallet ?? undefined); + const emailDomainBlockService = mock(); + const instrumentation = mock(); + const logger = mock>(); + const createLogger = vi.fn(() => logger); + + const handler = new BlockEmailDomainOfWalletHandler(userWalletRepository, emailDomainBlockService, instrumentation, createLogger); + + return { handler, wallet: input.wallet!, userWalletRepository, emailDomainBlockService, instrumentation, logger }; + } +}); diff --git a/apps/api/src/workload-abuse/services/block-email-domain-of-wallet/block-email-domain-of-wallet.handler.ts b/apps/api/src/workload-abuse/services/block-email-domain-of-wallet/block-email-domain-of-wallet.handler.ts new file mode 100644 index 0000000000..8a25e94ef5 --- /dev/null +++ b/apps/api/src/workload-abuse/services/block-email-domain-of-wallet/block-email-domain-of-wallet.handler.ts @@ -0,0 +1,70 @@ +import { inject, singleton } from "tsyringe"; + +import { isWalletInitialized, UserWalletRepository } from "@src/billing/repositories"; +import { type CreateLogger, type Job, JOB_NAME, type JobHandler, type JobPayload, type JobPermissions, LOGGER_FACTORY } from "@src/core"; +import { EmailDomainBlockService } from "@src/workload-abuse/services/email-domain-block/email-domain-block.service"; +import { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service"; + +export class BlockEmailDomainOfWallet implements Job { + static readonly [JOB_NAME] = "BlockEmailDomainOfWallet"; + readonly name = BlockEmailDomainOfWallet[JOB_NAME]; + readonly version = 1; + + constructor( + public readonly data: { + walletId: number; + } + ) {} +} + +/** Keyed by wallet: the domain is re-read on every run, so one pending evaluation per wiped wallet is all the queue needs to hold. */ +export function blockEmailDomainOfWalletKeyFor(walletId: number): string { + return `blockEmailDomainOfWallet.${walletId}`; +} + +/** + * Blocks the domain of a wallet the wipe has already locked. On the queue rather than inline, so a database + * blip while the guardrails are being read costs a retry instead of the block for the whole incident. + */ +@singleton() +export class BlockEmailDomainOfWalletHandler implements JobHandler { + public readonly accepts = BlockEmailDomainOfWallet; + + public readonly concurrency = 1; + + public readonly policy = "stately"; + + private readonly logger: ReturnType; + + constructor( + private readonly userWalletRepository: UserWalletRepository, + private readonly emailDomainBlockService: EmailDomainBlockService, + private readonly instrumentation: WorkloadAbuseInstrumentationService, + @inject(LOGGER_FACTORY) createLogger: CreateLogger + ) { + this.logger = createLogger({ context: BlockEmailDomainOfWalletHandler.name }); + } + + requiresPermission(): JobPermissions { + return []; + } + + async handle(payload: JobPayload): Promise { + const { walletId } = payload; + const wallet = await this.userWalletRepository.findById(walletId); + + if (!wallet || !isWalletInitialized(wallet)) { + this.instrumentation.recordDomainBlock("skipped", "wallet_not_found"); + this.logger.warn({ event: "EMAIL_DOMAIN_AUTO_BLOCK_SKIPPED", reason: "wallet_not_found", walletId }); + return; + } + + try { + await this.emailDomainBlockService.blockDomainOf(wallet); + } catch (error) { + this.instrumentation.recordDomainBlock("failed"); + this.logger.error({ event: "EMAIL_DOMAIN_AUTO_BLOCK_FAILED", walletId, userId: wallet.userId, error }); + throw error; + } + } +} diff --git a/apps/api/src/workload-abuse/services/email-domain-block/email-domain-block.service.spec.ts b/apps/api/src/workload-abuse/services/email-domain-block/email-domain-block.service.spec.ts new file mode 100644 index 0000000000..2ad1adcba3 --- /dev/null +++ b/apps/api/src/workload-abuse/services/email-domain-block/email-domain-block.service.spec.ts @@ -0,0 +1,368 @@ +import { describe, expect, it, vi } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import type { UserWalletRepository, WalletInitialized } from "@src/billing/repositories"; +import type { StripeTransactionRepository } from "@src/billing/repositories/stripe-transaction/stripe-transaction.repository"; +import type { CreateLogger } from "@src/core"; +import type { JobQueueService } from "@src/core"; +import type { UserRepository } from "@src/user/repositories"; +import type { BlockedEmailDomainRepository } from "@src/workload-abuse/repositories/blocked-email-domain/blocked-email-domain.repository"; +import type { BlockedEmailDomainService } from "@src/workload-abuse/services/blocked-email-domain/blocked-email-domain.service"; +import { LockBlockedDomainWallet } from "@src/workload-abuse/services/lock-blocked-domain-wallet/lock-blocked-domain-wallet.handler"; +import type { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; +import type { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service"; +import { DOMAIN_BLOCK_REASON, EmailDomainBlockService } from "./email-domain-block.service"; + +import { mockConfigService } from "@test/mocks/config-service.mock"; +import { createBlockedEmailDomain } from "@test/seeders/blocked-email-domain.seeder"; +import { createUser } from "@test/seeders/user.seeder"; +import { createUserWallet } from "@test/seeders/user-wallet.seeder"; + +describe(EmailDomainBlockService.name, () => { + describe("blockDomainOf", () => { + it("blocks the domain of the wallet it was given", async () => { + const { service, wallet, blockedEmailDomainRepository, instrumentation } = setup({ email: "miner@attacker.com" }); + + await service.blockDomainOf(wallet); + + expect(blockedEmailDomainRepository.blockIfAbsent).toHaveBeenCalledWith({ + domain: "attacker.com", + reason: DOMAIN_BLOCK_REASON, + triggeredByUserId: wallet.userId + }); + expect(instrumentation.recordDomainBlock).toHaveBeenCalledWith("blocked"); + }); + + it("primes the lookup cache so the sweep does not wait out a stale negative", async () => { + const { service, wallet, blockedEmailDomainService } = setup({ email: "miner@attacker.com" }); + + await service.blockDomainOf(wallet); + + expect(blockedEmailDomainService.rememberBlocked).toHaveBeenCalledWith("attacker.com"); + }); + + it("blocks the normalized domain of a mixed-case address", async () => { + const { service, wallet, blockedEmailDomainRepository } = setup({ email: "Miner@Attacker.COM" }); + + await service.blockDomainOf(wallet); + + expect(blockedEmailDomainRepository.blockIfAbsent).toHaveBeenCalledWith(expect.objectContaining({ domain: "attacker.com" })); + }); + + describe("guardrails", () => { + it.each([ + { reason: "no_domain", input: { email: null } }, + { reason: "public_provider", input: { email: "miner@gmail.com" } }, + { reason: "allowlisted", input: { existing: createBlockedEmailDomain({ domain: "attacker.com", status: "allowed" }) } }, + { reason: "domain_has_paid_user", input: { hasPaidUser: true } }, + { reason: "domain_predates_attack", input: { hasEstablishedUser: true } } + ])("skips with $reason and writes nothing", async ({ reason, input }) => { + const { service, wallet, blockedEmailDomainRepository, instrumentation } = setup({ email: "miner@attacker.com", ...input }); + + await service.blockDomainOf(wallet); + + expect(blockedEmailDomainRepository.blockIfAbsent).not.toHaveBeenCalled(); + expect(instrumentation.recordDomainBlock).toHaveBeenCalledWith("skipped", reason); + }); + + it("does not sweep siblings of a domain an operator has allowed", async () => { + const { service, wallet, jobQueueService } = setup({ + email: "miner@customer.com", + existing: createBlockedEmailDomain({ domain: "customer.com", status: "allowed" }), + siblings: [{ walletId: 7, userId: "user-7" }] + }); + + await service.blockDomainOf(wallet); + + expect(jobQueueService.enqueue).not.toHaveBeenCalled(); + }); + + it("sweeps siblings of a domain that is already blocked, because the first sweep can have missed one", async () => { + const { service, wallet, blockedEmailDomainRepository, jobQueueService, instrumentation } = setup({ + email: "miner@attacker.com", + existing: createBlockedEmailDomain({ domain: "attacker.com", status: "blocked" }), + siblings: [{ walletId: 7, userId: "user-7" }] + }); + + await service.blockDomainOf(wallet); + + expect(blockedEmailDomainRepository.blockIfAbsent).not.toHaveBeenCalled(); + expect(instrumentation.recordDomainBlock).toHaveBeenCalledWith("skipped", "already_blocked"); + expect(jobQueueService.enqueue).toHaveBeenCalledWith(new LockBlockedDomainWallet({ walletId: 7, domain: "attacker.com" }), { + singletonKey: "lockBlockedDomainWallet.7" + }); + }); + + it("checks the paid guardrail before the age guardrail", async () => { + const { service, wallet, userRepository } = setup({ email: "miner@attacker.com", hasPaidUser: true }); + + await service.blockDomainOf(wallet); + + expect(userRepository.hasEstablishedUserWithEmailDomain).not.toHaveBeenCalled(); + }); + + it("asks for established accounts using the configured window", async () => { + const { service, wallet, userRepository } = setup({ email: "miner@attacker.com", minAccountAgeDays: 45 }); + + await service.blockDomainOf(wallet); + + expect(userRepository.hasEstablishedUserWithEmailDomain).toHaveBeenCalledWith("attacker.com", 45, wallet.userId); + }); + + it("does not let the caught account vouch for its own domain", async () => { + const { service, wallet, userRepository, blockedEmailDomainRepository } = setup({ email: "miner@attacker.com", hasEstablishedUser: false }); + + await service.blockDomainOf(wallet); + + expect(userRepository.hasEstablishedUserWithEmailDomain).toHaveBeenCalledWith("attacker.com", expect.any(Number), wallet.userId); + expect(blockedEmailDomainRepository.blockIfAbsent).toHaveBeenCalled(); + }); + }); + + describe("in detect mode", () => { + it("evaluates the guardrails but writes nothing and sweeps nothing", async () => { + const { service, wallet, blockedEmailDomainRepository, jobQueueService, instrumentation, userRepository } = setup({ + email: "miner@attacker.com", + mode: "detect", + siblings: [{ walletId: 7, userId: "user-7" }] + }); + + await service.blockDomainOf(wallet); + + expect(userRepository.hasEstablishedUserWithEmailDomain).toHaveBeenCalled(); + expect(blockedEmailDomainRepository.blockIfAbsent).not.toHaveBeenCalled(); + expect(jobQueueService.enqueue).not.toHaveBeenCalled(); + expect(instrumentation.recordDomainBlock).toHaveBeenCalledWith("dry_run"); + }); + }); + + describe("sibling sweep", () => { + it("enqueues one wipe per sibling, keyed by wallet", async () => { + const { service, wallet, jobQueueService } = setup({ + email: "miner@attacker.com", + siblings: [ + { walletId: 7, userId: "user-7" }, + { walletId: 9, userId: "user-9" } + ] + }); + + await service.blockDomainOf(wallet); + + expect(jobQueueService.enqueue).toHaveBeenCalledWith(new LockBlockedDomainWallet({ walletId: 7, domain: "attacker.com" }), { + singletonKey: "lockBlockedDomainWallet.7" + }); + expect(jobQueueService.enqueue).toHaveBeenCalledWith(new LockBlockedDomainWallet({ walletId: 9, domain: "attacker.com" }), { + singletonKey: "lockBlockedDomainWallet.9" + }); + }); + + it("excludes the wallet that triggered the block and bounds the query by the configured limit", async () => { + const { service, wallet, userWalletRepository } = setup({ email: "miner@attacker.com", maxSiblings: 50 }); + + await service.blockDomainOf(wallet); + + expect(userWalletRepository.findLockableTrialWalletsByEmailDomain).toHaveBeenCalledWith("attacker.com", { + excludeWalletId: wallet.id, + limit: 51 + }); + }); + + it("skips a sibling whose wipe is already queued", async () => { + const { service, wallet, jobQueueService } = setup({ + email: "miner@attacker.com", + siblings: [{ walletId: 7, userId: "user-7" }], + pendingKeys: ["lockBlockedDomainWallet.7"] + }); + + await service.blockDomainOf(wallet); + + expect(jobQueueService.enqueue).not.toHaveBeenCalled(); + }); + + it("keeps enqueuing the rest when one sibling fails, then rethrows so the sweep is retried", async () => { + const { service, wallet, jobQueueService, logger } = setup({ + email: "miner@attacker.com", + siblings: [ + { walletId: 7, userId: "user-7" }, + { walletId: 9, userId: "user-9" } + ] + }); + jobQueueService.enqueue.mockRejectedValueOnce(new Error("queue unavailable")); + + await expect(service.blockDomainOf(wallet)).rejects.toThrow("queue unavailable"); + + expect(jobQueueService.enqueue).toHaveBeenCalledTimes(2); + expect(jobQueueService.enqueue).toHaveBeenCalledWith(new LockBlockedDomainWallet({ walletId: 9, domain: "attacker.com" }), { + singletonKey: "lockBlockedDomainWallet.9" + }); + expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ event: "BLOCKED_DOMAIN_SIBLING_ENQUEUE_FAILED", walletId: 7 })); + expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ event: "BLOCKED_DOMAIN_SIBLINGS_SWEPT", enqueued: 1, failed: 1 })); + }); + + it("records reaching the sibling limit, because a match that broad needs a human", async () => { + const { service, wallet, instrumentation, jobQueueService } = setup({ + email: "miner@attacker.com", + maxSiblings: 2, + siblings: [ + { walletId: 7, userId: "user-7" }, + { walletId: 9, userId: "user-9" }, + { walletId: 11, userId: "user-11" } + ] + }); + + await service.blockDomainOf(wallet); + + expect(instrumentation.recordDomainBlock).toHaveBeenCalledWith("sibling_limit_reached"); + expect(jobQueueService.enqueue).toHaveBeenCalledTimes(2); + expect(jobQueueService.enqueue).not.toHaveBeenCalledWith(new LockBlockedDomainWallet({ walletId: 11, domain: "attacker.com" }), expect.anything()); + }); + + it("leaves the limit unrecorded for a domain holding exactly as many siblings as the limit allows", async () => { + const { service, wallet, instrumentation, jobQueueService } = setup({ + email: "miner@attacker.com", + maxSiblings: 2, + siblings: [ + { walletId: 7, userId: "user-7" }, + { walletId: 9, userId: "user-9" } + ] + }); + + await service.blockDomainOf(wallet); + + expect(instrumentation.recordDomainBlock).not.toHaveBeenCalledWith("sibling_limit_reached"); + expect(jobQueueService.enqueue).toHaveBeenCalledTimes(2); + }); + + it("does not record the limit when fewer siblings came back than the limit allows", async () => { + const { service, wallet, instrumentation } = setup({ email: "miner@attacker.com", maxSiblings: 2, siblings: [{ walletId: 7, userId: "user-7" }] }); + + await service.blockDomainOf(wallet); + + expect(instrumentation.recordDomainBlock).not.toHaveBeenCalledWith("sibling_limit_reached"); + }); + }); + + describe("when another pod wrote the row first", () => { + it("still sweeps the siblings", async () => { + const { service, wallet, jobQueueService } = setup({ + email: "miner@attacker.com", + raceWinner: "blocked", + siblings: [{ walletId: 7, userId: "user-7" }] + }); + + await service.blockDomainOf(wallet); + + expect(jobQueueService.enqueue).toHaveBeenCalledWith(new LockBlockedDomainWallet({ walletId: 7, domain: "attacker.com" }), { + singletonKey: "lockBlockedDomainWallet.7" + }); + }); + + it("primes the lookup cache with the block that won, so signups on this pod stop waiting out a stale negative", async () => { + const { service, wallet, blockedEmailDomainService, instrumentation, logger } = setup({ email: "miner@attacker.com", raceWinner: "blocked" }); + + await service.blockDomainOf(wallet); + + expect(blockedEmailDomainService.rememberBlocked).toHaveBeenCalledWith("attacker.com"); + expect(instrumentation.recordDomainBlock).toHaveBeenCalledWith("raced"); + expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ event: "EMAIL_DOMAIN_AUTO_BLOCK_RACED", domain: "attacker.com" })); + }); + + it.each([{ raceWinner: "allowed" as const }, { raceWinner: "gone" as const }])( + "leaves the domain alone when the row that won is $raceWinner", + async ({ raceWinner }) => { + const { service, wallet, blockedEmailDomainService, jobQueueService, instrumentation } = setup({ + email: "miner@attacker.com", + raceWinner, + siblings: [{ walletId: 7, userId: "user-7" }] + }); + + await service.blockDomainOf(wallet); + + expect(blockedEmailDomainService.rememberBlocked).not.toHaveBeenCalled(); + expect(jobQueueService.enqueue).not.toHaveBeenCalled(); + expect(instrumentation.recordDomainBlock).toHaveBeenCalledWith("skipped", "allowlisted"); + } + ); + }); + + it("lets a failure through, so the queue retries the block rather than losing it", async () => { + const { service, wallet } = setup({ email: "miner@attacker.com", lookupError: new Error("connection terminated") }); + + await expect(service.blockDomainOf(wallet)).rejects.toThrow("connection terminated"); + }); + }); + + function setup(input?: { + email?: string | null; + existing?: ReturnType; + hasPaidUser?: boolean; + hasEstablishedUser?: boolean; + siblings?: Array<{ walletId: number; userId: string }>; + pendingKeys?: string[]; + mode?: "detect" | "enforce"; + maxSiblings?: number; + minAccountAgeDays?: number; + raceWinner?: "blocked" | "allowed" | "gone"; + lookupError?: Error; + }) { + const raceWinner = + input?.raceWinner && input.raceWinner !== "gone" ? createBlockedEmailDomain({ domain: "attacker.com", status: input.raceWinner }) : undefined; + const wallet = createUserWallet({ isTrialing: true }) as WalletInitialized; + const user = createUser({ id: wallet.userId, email: input?.email === undefined ? "miner@attacker.com" : (input.email as string) }); + + const userRepository = mock({ + findById: input?.lookupError ? vi.fn().mockRejectedValue(input.lookupError) : vi.fn().mockResolvedValue(user), + hasEstablishedUserWithEmailDomain: vi.fn().mockResolvedValue(input?.hasEstablishedUser ?? false) + }); + const userWalletRepository = mock({ + findLockableTrialWalletsByEmailDomain: vi.fn().mockResolvedValue(input?.siblings ?? []) + }); + const stripeTransactionRepository = mock({ + hasPaidUserWithEmailDomain: vi.fn().mockResolvedValue(input?.hasPaidUser ?? false) + }); + const blockedEmailDomainRepository = mock({ + findByDomain: vi.fn().mockResolvedValueOnce(input?.existing).mockResolvedValue(raceWinner), + blockIfAbsent: vi.fn().mockResolvedValue(input?.raceWinner ? undefined : createBlockedEmailDomain()) + }); + const blockedEmailDomainService = mock(); + const jobQueueService = mock({ + findPendingSingletonKeys: vi.fn().mockResolvedValue(new Set(input?.pendingKeys ?? [])), + enqueue: vi.fn().mockResolvedValue("job-id") + }); + const config = mockConfigService({ + WORKLOAD_ABUSE_DOMAIN_BLOCK_MODE: input?.mode ?? "enforce", + WORKLOAD_ABUSE_DOMAIN_BLOCK_MAX_SIBLINGS: input?.maxSiblings ?? 200, + WORKLOAD_ABUSE_DOMAIN_BLOCK_MIN_ACCOUNT_AGE_DAYS: input?.minAccountAgeDays ?? 30 + }); + const instrumentation = mock(); + const logger = mock>(); + const createLogger = vi.fn(() => logger); + + const service = new EmailDomainBlockService( + userRepository, + userWalletRepository, + stripeTransactionRepository, + blockedEmailDomainRepository, + blockedEmailDomainService, + jobQueueService, + config, + instrumentation, + createLogger + ); + + return { + service, + wallet, + user, + userRepository, + userWalletRepository, + stripeTransactionRepository, + blockedEmailDomainRepository, + blockedEmailDomainService, + jobQueueService, + config, + instrumentation, + logger + }; + } +}); diff --git a/apps/api/src/workload-abuse/services/email-domain-block/email-domain-block.service.ts b/apps/api/src/workload-abuse/services/email-domain-block/email-domain-block.service.ts new file mode 100644 index 0000000000..094d96a3f8 --- /dev/null +++ b/apps/api/src/workload-abuse/services/email-domain-block/email-domain-block.service.ts @@ -0,0 +1,167 @@ +import { inject, singleton } from "tsyringe"; + +import { UserWalletRepository, type WalletInitialized } from "@src/billing/repositories"; +import { StripeTransactionRepository } from "@src/billing/repositories/stripe-transaction/stripe-transaction.repository"; +import { type CreateLogger, JOB_NAME, JobQueueService, LOGGER_FACTORY } from "@src/core"; +import { UserRepository } from "@src/user/repositories"; +import { extractEmailDomain } from "@src/workload-abuse/lib/email-domain/email-domain"; +import { isPublicEmailProvider } from "@src/workload-abuse/lib/email-domain/public-email-providers"; +import { BlockedEmailDomainRepository } from "@src/workload-abuse/repositories/blocked-email-domain/blocked-email-domain.repository"; +import { BlockedEmailDomainService } from "@src/workload-abuse/services/blocked-email-domain/blocked-email-domain.service"; +import { + LockBlockedDomainWallet, + lockBlockedDomainWalletKeyFor +} from "@src/workload-abuse/services/lock-blocked-domain-wallet/lock-blocked-domain-wallet.handler"; +import { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; +import { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service"; + +export const DOMAIN_BLOCK_REASON = "workload_abuse"; + +type SkipReason = "no_domain" | "public_provider" | "allowlisted" | "already_blocked" | "domain_has_paid_user" | "domain_predates_attack"; + +/** + * Turns one wallet caught mining into a block on the whole email domain it signed up from, so the next + * account on that domain never gets a trial. Blunt by design, so three guardrails stand in front of it: + * a public provider, a domain anybody has ever paid from, and a domain older than the attack are all + * left alone and merely recorded. + */ +@singleton() +export class EmailDomainBlockService { + private readonly logger: ReturnType; + + constructor( + private readonly userRepository: UserRepository, + private readonly userWalletRepository: UserWalletRepository, + private readonly stripeTransactionRepository: StripeTransactionRepository, + private readonly blockedEmailDomainRepository: BlockedEmailDomainRepository, + private readonly blockedEmailDomainService: BlockedEmailDomainService, + private readonly jobQueueService: JobQueueService, + private readonly config: WorkloadAbuseConfigService, + private readonly instrumentation: WorkloadAbuseInstrumentationService, + @inject(LOGGER_FACTORY) createLogger: CreateLogger + ) { + this.logger = createLogger({ context: EmailDomainBlockService.name }); + } + + /** Throws: it runs on the queue, where a transient failure is worth a retry rather than a lost block. */ + async blockDomainOf(wallet: WalletInitialized): Promise { + const user = await this.userRepository.findById(wallet.userId); + const domain = extractEmailDomain(user?.email); + + if (!domain) return this.#skip("no_domain", { walletId: wallet.id, userId: wallet.userId }); + + const context = { walletId: wallet.id, userId: wallet.userId, domain }; + + if (isPublicEmailProvider(domain)) return this.#skip("public_provider", context); + + const existing = await this.blockedEmailDomainRepository.findByDomain(domain); + + if (existing?.status === "allowed") return this.#skip("allowlisted", context); + + if (existing?.status === "blocked") { + this.#skip("already_blocked", context); + await this.#sweepSiblings(domain, wallet); + return; + } + + if (await this.stripeTransactionRepository.hasPaidUserWithEmailDomain(domain)) return this.#skip("domain_has_paid_user", context); + + const minAccountAgeDays = this.config.get("WORKLOAD_ABUSE_DOMAIN_BLOCK_MIN_ACCOUNT_AGE_DAYS"); + if (await this.userRepository.hasEstablishedUserWithEmailDomain(domain, minAccountAgeDays, wallet.userId)) + return this.#skip("domain_predates_attack", context); + + if (!this.#isEnforcing) { + this.instrumentation.recordDomainBlock("dry_run"); + this.logger.info({ event: "EMAIL_DOMAIN_AUTO_BLOCK_DRY_RUN", ...context }); + return; + } + + const blocked = await this.blockedEmailDomainRepository.blockIfAbsent({ + domain, + reason: DOMAIN_BLOCK_REASON, + triggeredByUserId: wallet.userId + }); + + if (blocked) { + this.blockedEmailDomainService.rememberBlocked(domain); + this.instrumentation.recordDomainBlock("blocked"); + this.logger.warn({ event: "EMAIL_DOMAIN_AUTO_BLOCKED", ...context }); + } else if (!(await this.#adoptRacedBlock(domain, context))) { + return; + } + + await this.#sweepSiblings(domain, wallet); + } + + /** The insert lost to a row written since the read above, so the row that won says whether this is another pod's block to adopt or an operator's allow that outranks it. */ + async #adoptRacedBlock(domain: string, context: Record): Promise { + const winner = await this.blockedEmailDomainRepository.findByDomain(domain); + + if (winner?.status !== "blocked") { + this.#skip("allowlisted", context); + return false; + } + + this.blockedEmailDomainService.rememberBlocked(domain); + this.instrumentation.recordDomainBlock("raced"); + this.logger.info({ event: "EMAIL_DOMAIN_AUTO_BLOCK_RACED", ...context }); + + return true; + } + + /** Nothing else requeues a sibling this sweep drops, so every sibling is attempted and the first enqueue failure is rethrown onto the job's retry budget. */ + async #sweepSiblings(domain: string, wallet: WalletInitialized): Promise { + if (!this.#isEnforcing) return; + + const limit = this.config.get("WORKLOAD_ABUSE_DOMAIN_BLOCK_MAX_SIBLINGS"); + const overfetched = await this.userWalletRepository.findLockableTrialWalletsByEmailDomain(domain, { excludeWalletId: wallet.id, limit: limit + 1 }); + + if (overfetched.length === 0) return; + + const siblings = overfetched.slice(0, limit); + const moreThanTheLimitExist = overfetched.length > limit; + + if (moreThanTheLimitExist) { + this.instrumentation.recordDomainBlock("sibling_limit_reached"); + this.logger.warn({ event: "BLOCKED_DOMAIN_SIBLING_LIMIT_REACHED", domain, limit }); + } + + const pendingKeys = await this.jobQueueService.findPendingSingletonKeys(LockBlockedDomainWallet[JOB_NAME]); + let enqueued = 0; + let alreadyQueued = 0; + let failed = 0; + let firstError: unknown; + + for (const { walletId } of siblings) { + const singletonKey = lockBlockedDomainWalletKeyFor(walletId); + + if (pendingKeys.has(singletonKey)) { + alreadyQueued++; + continue; + } + + try { + const jobId = await this.jobQueueService.enqueue(new LockBlockedDomainWallet({ walletId, domain }), { singletonKey }); + if (jobId) enqueued++; + else alreadyQueued++; + } catch (error) { + this.logger.error({ event: "BLOCKED_DOMAIN_SIBLING_ENQUEUE_FAILED", domain, walletId, error }); + failed++; + firstError ??= error; + } + } + + this.logger.info({ event: "BLOCKED_DOMAIN_SIBLINGS_SWEPT", domain, found: siblings.length, enqueued, alreadyQueued, failed }); + + if (firstError) throw firstError; + } + + get #isEnforcing(): boolean { + return this.config.get("WORKLOAD_ABUSE_DOMAIN_BLOCK_MODE") === "enforce"; + } + + #skip(reason: SkipReason, context: Record): void { + this.instrumentation.recordDomainBlock("skipped", reason); + this.logger.info({ event: "EMAIL_DOMAIN_AUTO_BLOCK_SKIPPED", reason, ...context }); + } +} diff --git a/apps/api/src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler.integration.ts b/apps/api/src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler.integration.ts index 86a53bf713..7ed4710f75 100644 --- a/apps/api/src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler.integration.ts +++ b/apps/api/src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler.integration.ts @@ -11,6 +11,11 @@ import { TxManagerService } from "@src/billing/services/tx-manager/tx-manager.se import { type ApiPgDatabase, JOB_NAME, POSTGRES_DB, resolveTable } from "@src/core"; import { DeploymentWriterService } from "@src/deployment/services/deployment-writer/deployment-writer.service"; import { WorkloadAbuseDetectionRepository } from "@src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository"; +import { + BlockEmailDomainOfWallet, + BlockEmailDomainOfWalletHandler, + blockEmailDomainOfWalletKeyFor +} from "@src/workload-abuse/services/block-email-domain-of-wallet/block-email-domain-of-wallet.handler"; import { ProbeTrialDeploymentHandler } from "@src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler"; import { ABUSE_LOCK_REASON } from "@src/workload-abuse/services/trial-abuse-enforcement/trial-abuse-enforcement.service"; import { ProbeTrialDeployment, probeTrialDeploymentKeyFor } from "@src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service"; @@ -20,7 +25,11 @@ import { createAkashAddress } from "@test/seeders/akash-address.seeder"; import { seedUserWithWallet } from "@test/seeders/db/user-with-wallet.seeder"; import { expectJobCompleted, findJobRows, useJobWorkers } from "@test/services/job-queue-harness"; -const jobWorkers = useJobWorkers(() => [container.resolve(EnforceTrialAbuseHandler), container.resolve(ProbeTrialDeploymentHandler)]); +const jobWorkers = useJobWorkers(() => [ + container.resolve(EnforceTrialAbuseHandler), + container.resolve(ProbeTrialDeploymentHandler), + container.resolve(BlockEmailDomainOfWalletHandler) +]); describe(EnforceTrialAbuseHandler.name, () => { afterEach(() => { @@ -65,6 +74,14 @@ describe(EnforceTrialAbuseHandler.name, () => { expect(executeFundingTx).not.toHaveBeenCalled(); }); + it("queues the domain block of a wallet already locked, so a run interrupted before it resumes on the retry", async () => { + const { handler, wallet, detection, findDomainBlockJob } = await setup({ abuseLockedAt: new Date() }); + + await handler.handle({ walletId: wallet.id, detectionId: detection.id, version: 1 }); + + expect(await findDomainBlockJob()).toMatchObject({ data: { walletId: wallet.id, version: 1 } }); + }); + it("leaves a wallet that has since paid alone", async () => { const { handler, wallet, detection, close, executeFundingTx, findWallet, findDetection } = await setup({ isTrialing: false }); @@ -187,6 +204,11 @@ describe(EnforceTrialAbuseHandler.name, () => { enqueue(new EnforceTrialAbuse({ walletId: wallet.id, detectionId }), { singletonKey: enforceTrialAbuseKeyFor(wallet.id) }), findWallet: () => userWalletRepository.findById(wallet.id), findDetection: (id: string) => detectionRepository.findById(id), + findDomainBlockJob: async () => { + const [row] = await findJobRows(BlockEmailDomainOfWallet[JOB_NAME], { singletonKey: blockEmailDomainOfWalletKeyFor(wallet.id) }); + + return row; + }, findProbeJob: async (dseq: string) => { const [row] = await findJobRows(ProbeTrialDeployment[JOB_NAME], { singletonKey: probeTrialDeploymentKeyFor({ walletId: wallet.id, dseq }) }); diff --git a/apps/api/src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler.spec.ts b/apps/api/src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler.spec.ts index a19bbe5bbf..ce75414de2 100644 --- a/apps/api/src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler.spec.ts +++ b/apps/api/src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler.spec.ts @@ -2,15 +2,17 @@ import { describe, expect, it, vi } from "vitest"; import { mock } from "vitest-mock-extended"; import type { UserWalletRepository } from "@src/billing/repositories"; -import type { CreateLogger } from "@src/core"; +import type { CreateLogger, JobQueueService } from "@src/core"; import type { WorkloadAbuseDetectionRepository } from "@src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository"; -import type { TrialAbuseEnforcementService } from "@src/workload-abuse/services/trial-abuse-enforcement/trial-abuse-enforcement.service"; +import { BlockEmailDomainOfWallet } from "@src/workload-abuse/services/block-email-domain-of-wallet/block-email-domain-of-wallet.handler"; +import type { EnforcementOutcome, TrialAbuseEnforcementService } from "@src/workload-abuse/services/trial-abuse-enforcement/trial-abuse-enforcement.service"; import type { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service"; import { EnforceTrialAbuseHandler, enforceTrialAbuseKeyFor } from "./enforce-trial-abuse.handler"; import { createUserWallet } from "@test/seeders/user-wallet.seeder"; const PAYLOAD = { walletId: 42, detectionId: "detection-1", version: 1 as const }; +const WIPED: EnforcementOutcome = { depositGrantRevoked: true, feeGrantRevoked: true, closedDseqs: ["123"] }; describe(EnforceTrialAbuseHandler.name, () => { it("keys the wipe by wallet", () => { @@ -25,6 +27,31 @@ describe(EnforceTrialAbuseHandler.name, () => { expect(enforcementService.enforce).toHaveBeenCalledWith({ wallet, detectionId: PAYLOAD.detectionId }); }); + it("queues the email domain block of a wallet it wiped rather than running it inline", async () => { + const { handler, jobQueueService } = setup({ wallet: createUserWallet({ isTrialing: true }) }); + + await handler.handle(PAYLOAD); + + expect(jobQueueService.enqueue).toHaveBeenCalledWith(new BlockEmailDomainOfWallet({ walletId: PAYLOAD.walletId }), { + singletonKey: "blockEmailDomainOfWallet.42" + }); + }); + + it("leaves the email domain alone when the wipe was skipped because the wallet paid", async () => { + const { handler, jobQueueService } = setup({ wallet: createUserWallet({ isTrialing: true }), enforcementOutcome: null }); + + await handler.handle(PAYLOAD); + + expect(jobQueueService.enqueue).not.toHaveBeenCalled(); + }); + + it("fails the job when the domain block cannot be queued, so the retry picks it up", async () => { + const { handler, jobQueueService } = setup({ wallet: createUserWallet({ isTrialing: true }) }); + jobQueueService.enqueue.mockRejectedValue(new Error("queue unavailable")); + + await expect(handler.handle(PAYLOAD)).rejects.toThrow("queue unavailable"); + }); + it("settles the wallet's detections without acting again when the wallet is already locked", async () => { const { handler, enforcementService, detectionRepository, instrumentation } = setup({ wallet: createUserWallet({ isTrialing: false, abuseLockedAt: new Date() }) @@ -37,21 +64,33 @@ describe(EnforceTrialAbuseHandler.name, () => { expect(instrumentation.recordEnforcement).toHaveBeenCalledWith("skipped"); }); + it("resumes the domain block of a wallet an interrupted run had already locked", async () => { + const { handler, jobQueueService } = setup({ wallet: createUserWallet({ isTrialing: false, abuseLockedAt: new Date() }) }); + + await handler.handle(PAYLOAD); + + expect(jobQueueService.enqueue).toHaveBeenCalledWith(new BlockEmailDomainOfWallet({ walletId: PAYLOAD.walletId }), { + singletonKey: "blockEmailDomainOfWallet.42" + }); + }); + it("leaves a wallet that has since paid alone", async () => { - const { handler, enforcementService, logger } = setup({ wallet: createUserWallet({ isTrialing: false }) }); + const { handler, enforcementService, jobQueueService, logger } = setup({ wallet: createUserWallet({ isTrialing: false }) }); await handler.handle(PAYLOAD); expect(enforcementService.enforce).not.toHaveBeenCalled(); + expect(jobQueueService.enqueue).not.toHaveBeenCalled(); expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ event: "TRIAL_WORKLOAD_ABUSE_ENFORCEMENT_SKIPPED", reason: "NOT_TRIALING" })); }); it("skips a wallet it cannot find", async () => { - const { handler, enforcementService, logger } = setup({ wallet: null }); + const { handler, enforcementService, jobQueueService, logger } = setup({ wallet: null }); await handler.handle(PAYLOAD); expect(enforcementService.enforce).not.toHaveBeenCalled(); + expect(jobQueueService.enqueue).not.toHaveBeenCalled(); expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ reason: "WALLET_NOT_FOUND" })); }); @@ -61,17 +100,29 @@ describe(EnforceTrialAbuseHandler.name, () => { expect(handler.requiresPermission()).toEqual([]); }); - function setup(input: { wallet: ReturnType | null }) { + function setup(input: { wallet: ReturnType | null; enforcementOutcome?: EnforcementOutcome | null }) { const userWalletRepository = mock(); userWalletRepository.findById.mockResolvedValue(input.wallet ?? undefined); const detectionRepository = mock(); - const enforcementService = mock(); + const enforcementService = mock({ + enforce: vi.fn().mockResolvedValue(input.enforcementOutcome === undefined ? WIPED : input.enforcementOutcome) + }); + const jobQueueService = mock({ enqueue: vi.fn().mockResolvedValue("job-id") }); const instrumentation = mock(); const logger = mock>(); const createLogger = vi.fn(() => logger); - const handler = new EnforceTrialAbuseHandler(userWalletRepository, detectionRepository, enforcementService, instrumentation, createLogger); - - return { handler, wallet: input.wallet!, userWalletRepository, detectionRepository, enforcementService, instrumentation, logger }; + const handler = new EnforceTrialAbuseHandler(userWalletRepository, detectionRepository, enforcementService, jobQueueService, instrumentation, createLogger); + + return { + handler, + wallet: input.wallet!, + userWalletRepository, + detectionRepository, + enforcementService, + jobQueueService, + instrumentation, + logger + }; } }); diff --git a/apps/api/src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler.ts b/apps/api/src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler.ts index e420bebdf1..bcd549194f 100644 --- a/apps/api/src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler.ts +++ b/apps/api/src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler.ts @@ -1,8 +1,12 @@ import { inject, singleton } from "tsyringe"; import { isWalletInitialized, UserWalletRepository } from "@src/billing/repositories"; -import { type CreateLogger, type Job, JOB_NAME, type JobHandler, type JobPayload, type JobPermissions, LOGGER_FACTORY } from "@src/core"; +import { type CreateLogger, type Job, JOB_NAME, type JobHandler, type JobPayload, type JobPermissions, JobQueueService, LOGGER_FACTORY } from "@src/core"; import { WorkloadAbuseDetectionRepository } from "@src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository"; +import { + BlockEmailDomainOfWallet, + blockEmailDomainOfWalletKeyFor +} from "@src/workload-abuse/services/block-email-domain-of-wallet/block-email-domain-of-wallet.handler"; import { TrialAbuseEnforcementService } from "@src/workload-abuse/services/trial-abuse-enforcement/trial-abuse-enforcement.service"; import { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service"; @@ -39,6 +43,7 @@ export class EnforceTrialAbuseHandler implements JobHandler { private readonly userWalletRepository: UserWalletRepository, private readonly detectionRepository: WorkloadAbuseDetectionRepository, private readonly enforcementService: TrialAbuseEnforcementService, + private readonly jobQueueService: JobQueueService, private readonly instrumentation: WorkloadAbuseInstrumentationService, @inject(LOGGER_FACTORY) createLogger: CreateLogger ) { @@ -64,6 +69,7 @@ export class EnforceTrialAbuseHandler implements JobHandler { this.logger.info({ event: "TRIAL_WORKLOAD_ABUSE_ENFORCEMENT_SKIPPED", reason: "ALREADY_LOCKED", ...context, userId: wallet.userId }); await this.detectionRepository.markWalletEnforced(walletId); this.instrumentation.recordEnforcement("skipped"); + await this.#queueDomainBlock(walletId); return; } @@ -73,6 +79,19 @@ export class EnforceTrialAbuseHandler implements JobHandler { return; } - await this.enforcementService.enforce({ wallet, detectionId }); + const outcome = await this.enforcementService.enforce({ wallet, detectionId }); + + if (outcome) { + await this.#queueDomainBlock(walletId); + } + } + + /** + * Queued rather than run inline, so a blip while the guardrails are read costs a retry instead of the block. + * Also queued from the already-locked branch, which is where a run interrupted between the wipe and this + * enqueue lands when the queue retries it. + */ + async #queueDomainBlock(walletId: number): Promise { + await this.jobQueueService.enqueue(new BlockEmailDomainOfWallet({ walletId }), { singletonKey: blockEmailDomainOfWalletKeyFor(walletId) }); } } diff --git a/apps/api/src/workload-abuse/services/lock-blocked-domain-wallet/lock-blocked-domain-wallet.handler.spec.ts b/apps/api/src/workload-abuse/services/lock-blocked-domain-wallet/lock-blocked-domain-wallet.handler.spec.ts new file mode 100644 index 0000000000..e6eedb4933 --- /dev/null +++ b/apps/api/src/workload-abuse/services/lock-blocked-domain-wallet/lock-blocked-domain-wallet.handler.spec.ts @@ -0,0 +1,141 @@ +import { describe, expect, it, vi } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import type { UserWalletRepository } from "@src/billing/repositories"; +import type { CreateLogger } from "@src/core"; +import type { UserRepository } from "@src/user/repositories"; +import type { BlockedEmailDomainRepository } from "@src/workload-abuse/repositories/blocked-email-domain/blocked-email-domain.repository"; +import type { EnforcementOutcome, TrialAbuseEnforcementService } from "@src/workload-abuse/services/trial-abuse-enforcement/trial-abuse-enforcement.service"; +import type { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service"; +import { LockBlockedDomainWalletHandler, lockBlockedDomainWalletKeyFor } from "./lock-blocked-domain-wallet.handler"; + +import { createBlockedEmailDomain } from "@test/seeders/blocked-email-domain.seeder"; +import { createUser } from "@test/seeders/user.seeder"; +import { createUserWallet } from "@test/seeders/user-wallet.seeder"; + +const PAYLOAD = { walletId: 42, domain: "attacker.com", version: 1 as const }; +const WIPED: EnforcementOutcome = { depositGrantRevoked: true, feeGrantRevoked: true, closedDseqs: ["123"] }; + +describe(LockBlockedDomainWalletHandler.name, () => { + it("keys the wipe by wallet, so two sweeps of one domain do not suppress each other", () => { + expect(lockBlockedDomainWalletKeyFor(7)).toBe("lockBlockedDomainWallet.7"); + }); + + it("wipes a trialing wallet with the blocked-domain reason", async () => { + const { handler, wallet, enforcementService } = setup({ wallet: createUserWallet({ isTrialing: true }) }); + + await handler.handle(PAYLOAD); + + expect(enforcementService.wipeTrialWallet).toHaveBeenCalledWith(wallet, "blocked_domain"); + }); + + it("records the wipe it performed", async () => { + const { handler, instrumentation, logger } = setup({ wallet: createUserWallet({ isTrialing: true }) }); + + await handler.handle(PAYLOAD); + + expect(instrumentation.recordEnforcement).toHaveBeenCalledWith("enforced"); + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "BLOCKED_DOMAIN_WALLET_LOCKED", walletId: PAYLOAD.walletId })); + }); + + it.each([ + { reason: "WALLET_NOT_FOUND", input: { wallet: null } }, + { reason: "ALREADY_LOCKED", input: { wallet: createUserWallet({ isTrialing: true, abuseLockedAt: new Date() }) } }, + { reason: "NOT_TRIALING", input: { wallet: createUserWallet({ isTrialing: false }) } }, + { reason: "DOMAIN_NOT_BLOCKED", input: { wallet: createUserWallet({ isTrialing: true }), blockedDomain: null } }, + { reason: "DOMAIN_CHANGED", input: { wallet: createUserWallet({ isTrialing: true }), ownerEmail: "someone@elsewhere.com" } }, + { reason: "DOMAIN_CHANGED", input: { wallet: createUserWallet({ isTrialing: true }), ownerEmail: null } } + ])("skips with $reason without wiping", async ({ reason, input }) => { + const { handler, enforcementService, instrumentation, logger } = setup(input); + + await handler.handle(PAYLOAD); + + expect(enforcementService.wipeTrialWallet).not.toHaveBeenCalled(); + expect(instrumentation.recordEnforcement).toHaveBeenCalledWith("skipped"); + expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ event: "BLOCKED_DOMAIN_WALLET_LOCK_SKIPPED", reason })); + }); + + it("leaves a domain an operator has allowed since the sweep started", async () => { + const { handler, enforcementService, logger } = setup({ + wallet: createUserWallet({ isTrialing: true }), + blockedDomain: createBlockedEmailDomain({ domain: "attacker.com", status: "allowed" }) + }); + + await handler.handle(PAYLOAD); + + expect(enforcementService.wipeTrialWallet).not.toHaveBeenCalled(); + expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ event: "BLOCKED_DOMAIN_WALLET_LOCK_SKIPPED", reason: "DOMAIN_NOT_BLOCKED" })); + }); + + it("reads the domain row rather than a cached verdict", async () => { + const { handler, blockedEmailDomainRepository } = setup({ wallet: createUserWallet({ isTrialing: true }) }); + + await handler.handle(PAYLOAD); + + expect(blockedEmailDomainRepository.findByDomain).toHaveBeenCalledWith("attacker.com"); + }); + + it("records a failure and rethrows it when the wipe fails on the chain", async () => { + const { handler, enforcementService, instrumentation, logger } = setup({ wallet: createUserWallet({ isTrialing: true }) }); + enforcementService.wipeTrialWallet.mockRejectedValue(new Error("escrow not settled")); + + await expect(handler.handle(PAYLOAD)).rejects.toThrow("escrow not settled"); + + expect(instrumentation.recordEnforcement).toHaveBeenCalledWith("failed"); + expect(logger.error).toHaveBeenCalledWith( + expect.objectContaining({ event: "BLOCKED_DOMAIN_WALLET_LOCK_FAILED", walletId: PAYLOAD.walletId, domain: PAYLOAD.domain }) + ); + }); + + it("records a skip when the wallet paid between the sweep and the wipe", async () => { + const { handler, logger, instrumentation } = setup({ wallet: createUserWallet({ isTrialing: true }), enforcementOutcome: null }); + + await handler.handle(PAYLOAD); + + expect(instrumentation.recordEnforcement).toHaveBeenCalledWith("skipped"); + expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ reason: "PAID_DURING_ENFORCEMENT" })); + }); + + it("declares no permissions for its execution", () => { + const { handler } = setup({ wallet: null }); + + expect(handler.requiresPermission()).toEqual([]); + }); + + function setup(input: { + wallet: ReturnType | null; + blockedDomain?: ReturnType | null; + enforcementOutcome?: EnforcementOutcome | null; + ownerEmail?: string | null; + }) { + const userWalletRepository = mock(); + userWalletRepository.findById.mockResolvedValue(input.wallet ?? undefined); + const userRepository = mock({ + findById: vi.fn().mockResolvedValue(createUser({ email: input.ownerEmail === undefined ? `owner@${PAYLOAD.domain}` : input.ownerEmail })) + }); + const blockedEmailDomainRepository = mock({ + findByDomain: vi + .fn() + .mockResolvedValue( + input.blockedDomain === undefined ? createBlockedEmailDomain({ domain: PAYLOAD.domain, status: "blocked" }) : input.blockedDomain ?? undefined + ) + }); + const enforcementService = mock({ + wipeTrialWallet: vi.fn().mockResolvedValue(input.enforcementOutcome === undefined ? WIPED : input.enforcementOutcome) + }); + const instrumentation = mock(); + const logger = mock>(); + const createLogger = vi.fn(() => logger); + + const handler = new LockBlockedDomainWalletHandler( + userWalletRepository, + userRepository, + blockedEmailDomainRepository, + enforcementService, + instrumentation, + createLogger + ); + + return { handler, wallet: input.wallet!, userWalletRepository, userRepository, blockedEmailDomainRepository, enforcementService, instrumentation, logger }; + } +}); diff --git a/apps/api/src/workload-abuse/services/lock-blocked-domain-wallet/lock-blocked-domain-wallet.handler.ts b/apps/api/src/workload-abuse/services/lock-blocked-domain-wallet/lock-blocked-domain-wallet.handler.ts new file mode 100644 index 0000000000..e1a4fc5385 --- /dev/null +++ b/apps/api/src/workload-abuse/services/lock-blocked-domain-wallet/lock-blocked-domain-wallet.handler.ts @@ -0,0 +1,127 @@ +import { inject, singleton } from "tsyringe"; + +import { isWalletInitialized, UserWalletRepository, type WalletInitialized } from "@src/billing/repositories"; +import { type CreateLogger, type Job, JOB_NAME, type JobHandler, type JobPayload, type JobPermissions, LOGGER_FACTORY } from "@src/core"; +import { UserRepository } from "@src/user/repositories"; +import { extractEmailDomain } from "@src/workload-abuse/lib/email-domain/email-domain"; +import { BlockedEmailDomainRepository } from "@src/workload-abuse/repositories/blocked-email-domain/blocked-email-domain.repository"; +import { + BLOCKED_DOMAIN_LOCK_REASON, + type EnforcementOutcome, + TrialAbuseEnforcementService +} from "@src/workload-abuse/services/trial-abuse-enforcement/trial-abuse-enforcement.service"; +import { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service"; + +export class LockBlockedDomainWallet implements Job { + static readonly [JOB_NAME] = "LockBlockedDomainWallet"; + readonly name = LockBlockedDomainWallet[JOB_NAME]; + readonly version = 1; + + constructor( + public readonly data: { + walletId: number; + domain: string; + } + ) {} +} + +/** Keyed by wallet, not by domain: two sweeps of the same domain must not suppress each other's wallets. */ +export function lockBlockedDomainWalletKeyFor(walletId: number): string { + return `lockBlockedDomainWallet.${walletId}`; +} + +/** Wipes one trial wallet caught by its email domain, re-reading every precondition so a wallet that paid or was already locked is left alone. */ +@singleton() +export class LockBlockedDomainWalletHandler implements JobHandler { + public readonly accepts = LockBlockedDomainWallet; + + public readonly concurrency = 1; + + public readonly policy = "stately"; + + private readonly logger: ReturnType; + + constructor( + private readonly userWalletRepository: UserWalletRepository, + private readonly userRepository: UserRepository, + private readonly blockedEmailDomainRepository: BlockedEmailDomainRepository, + private readonly enforcementService: TrialAbuseEnforcementService, + private readonly instrumentation: WorkloadAbuseInstrumentationService, + @inject(LOGGER_FACTORY) createLogger: CreateLogger + ) { + this.logger = createLogger({ context: LockBlockedDomainWalletHandler.name }); + } + + requiresPermission(): JobPermissions { + return []; + } + + async handle(payload: JobPayload): Promise { + const { walletId, domain } = payload; + const context = { job: LockBlockedDomainWallet[JOB_NAME], walletId, domain }; + const wallet = await this.userWalletRepository.findById(walletId); + + if (!wallet || !isWalletInitialized(wallet)) { + this.#skip("WALLET_NOT_FOUND", context); + return; + } + + if (wallet.abuseLockedAt) { + this.#skip("ALREADY_LOCKED", { ...context, userId: wallet.userId }); + return; + } + + if (!wallet.isTrialing) { + this.#skip("NOT_TRIALING", { ...context, userId: wallet.userId }); + return; + } + + if (!(await this.#isDomainStillBlocked(domain))) { + this.#skip("DOMAIN_NOT_BLOCKED", { ...context, userId: wallet.userId }); + return; + } + + if (!(await this.#isWalletStillOnDomain(wallet.userId, domain))) { + this.#skip("DOMAIN_CHANGED", { ...context, userId: wallet.userId }); + return; + } + + const outcome = await this.#wipe(wallet, { ...context, userId: wallet.userId }); + + if (!outcome) { + this.#skip("PAID_DURING_ENFORCEMENT", { ...context, userId: wallet.userId }); + return; + } + + this.instrumentation.recordEnforcement("enforced"); + this.logger.warn({ event: "BLOCKED_DOMAIN_WALLET_LOCKED", ...context, userId: wallet.userId, owner: wallet.address, ...outcome }); + } + + /** Counted and logged the way the detection path counts its own failures, so a sweep failing on the chain is as visible as a wipe the probe triggered. */ + async #wipe(wallet: WalletInitialized, context: Record): Promise { + try { + return await this.enforcementService.wipeTrialWallet(wallet, BLOCKED_DOMAIN_LOCK_REASON); + } catch (error) { + this.instrumentation.recordEnforcement("failed"); + this.logger.error({ event: "BLOCKED_DOMAIN_WALLET_LOCK_FAILED", ...context, owner: wallet.address, error }); + throw error; + } + } + + /** Reads the row rather than the cached verdict, so an operator un-blocking the domain mid-sweep stops the wipes still queued behind it. */ + async #isDomainStillBlocked(domain: string): Promise { + return (await this.blockedEmailDomainRepository.findByDomain(domain))?.status === "blocked"; + } + + /** The sweep picked this wallet off its owner's email, which every login rewrites, so the domain is re-read rather than trusted from the payload. */ + async #isWalletStillOnDomain(userId: string, domain: string): Promise { + const user = await this.userRepository.findById(userId); + + return extractEmailDomain(user?.email) === domain; + } + + #skip(reason: string, context: Record): void { + this.logger.info({ event: "BLOCKED_DOMAIN_WALLET_LOCK_SKIPPED", reason, ...context }); + this.instrumentation.recordEnforcement("skipped"); + } +} diff --git a/apps/api/src/workload-abuse/services/trial-abuse-enforcement/trial-abuse-enforcement.service.spec.ts b/apps/api/src/workload-abuse/services/trial-abuse-enforcement/trial-abuse-enforcement.service.spec.ts index 052b6b2b3c..1cdef900be 100644 --- a/apps/api/src/workload-abuse/services/trial-abuse-enforcement/trial-abuse-enforcement.service.spec.ts +++ b/apps/api/src/workload-abuse/services/trial-abuse-enforcement/trial-abuse-enforcement.service.spec.ts @@ -12,7 +12,7 @@ import type { DeploymentWriterService } from "@src/deployment/services/deploymen import type { WorkloadAbuseDetectionRepository } from "@src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository"; import type { TrialWorkloadProbeJobService } from "@src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service"; import type { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service"; -import { ABUSE_LOCK_REASON, TrialAbuseEnforcementService } from "./trial-abuse-enforcement.service"; +import { ABUSE_LOCK_REASON, BLOCKED_DOMAIN_LOCK_REASON, TrialAbuseEnforcementService } from "./trial-abuse-enforcement.service"; import { createInitializedUserWallet } from "@test/seeders/user-wallet.seeder"; @@ -151,6 +151,28 @@ describe(TrialAbuseEnforcementService.name, () => { expect(probeJobService.cancelForWallet).not.toHaveBeenCalled(); }); + describe("wipeTrialWallet", () => { + it("locks the wallet with the reason it was given, without touching the detection ledger", async () => { + const { service, wallet, userWalletRepository, detectionRepository, instrumentation } = setup({ liveDseqs: ["11"] }); + + const outcome = await service.wipeTrialWallet(wallet, BLOCKED_DOMAIN_LOCK_REASON); + + expect(userWalletRepository.lockForAbuse).toHaveBeenCalledWith(wallet.id, BLOCKED_DOMAIN_LOCK_REASON); + expect(outcome).toEqual({ depositGrantRevoked: true, feeGrantRevoked: true, closedDseqs: ["11"] }); + expect(detectionRepository.updateById).not.toHaveBeenCalled(); + expect(detectionRepository.markWalletEnforced).not.toHaveBeenCalled(); + expect(instrumentation.recordEnforcement).not.toHaveBeenCalled(); + }); + + it("leaves a wallet that paid under the row lock alone", async () => { + const { service, wallet, userWalletRepository } = setup({ liveDseqs: [], paidUnderLock: true }); + + await expect(service.wipeTrialWallet(wallet, BLOCKED_DOMAIN_LOCK_REASON)).resolves.toBeNull(); + + expect(userWalletRepository.lockForAbuse).not.toHaveBeenCalled(); + }); + }); + function setup(input: { liveDseqs: string[]; hasDepositGrant?: boolean; hasFeeGrant?: boolean; paidUnderLock?: boolean }) { const wallet = createInitializedUserWallet({ isTrialing: true }); const calls: string[] = []; diff --git a/apps/api/src/workload-abuse/services/trial-abuse-enforcement/trial-abuse-enforcement.service.ts b/apps/api/src/workload-abuse/services/trial-abuse-enforcement/trial-abuse-enforcement.service.ts index 952d54babb..f5b0e0474e 100644 --- a/apps/api/src/workload-abuse/services/trial-abuse-enforcement/trial-abuse-enforcement.service.ts +++ b/apps/api/src/workload-abuse/services/trial-abuse-enforcement/trial-abuse-enforcement.service.ts @@ -16,6 +16,10 @@ import { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/service export const ABUSE_LOCK_REASON = "workload_abuse"; +export const BLOCKED_DOMAIN_LOCK_REASON = "blocked_domain"; + +export type AbuseLockReason = typeof ABUSE_LOCK_REASON | typeof BLOCKED_DOMAIN_LOCK_REASON; + export type EnforcementOutcome = { depositGrantRevoked: boolean; feeGrantRevoked: boolean; @@ -59,7 +63,7 @@ export class TrialAbuseEnforcementService { let outcome: EnforcementOutcome | null; try { - outcome = await this.txService.transaction(() => this.#wipeUnlessPaid(wallet)); + outcome = await this.wipeTrialWallet(wallet, ABUSE_LOCK_REASON); } catch (error) { this.instrumentation.recordEnforcement("failed"); this.logger.error({ @@ -95,6 +99,11 @@ export class TrialAbuseEnforcementService { return outcome; } + /** The wipe without the detection bookkeeping, for a wallet caught by its email domain rather than by its own workload. */ + async wipeTrialWallet(wallet: WalletInitialized, reason: AbuseLockReason): Promise { + return await this.txService.transaction(() => this.#wipeUnlessPaid(wallet, reason)); + } + /** findStalledEnforcements re-queues a detection left in enforcing, so a record that cannot be written is worth a log rather than the failure it was recording. */ async #recordEnforcementFailure(detectionId: string, error: unknown): Promise { try { @@ -109,21 +118,21 @@ export class TrialAbuseEnforcementService { } /** Holds the wallet row for the whole wipe, so a payment settling at the same time waits for it and then clears the lock instead of re-granting between the revokes. */ - async #wipeUnlessPaid(wallet: WalletInitialized): Promise { + async #wipeUnlessPaid(wallet: WalletInitialized, reason: AbuseLockReason): Promise { const lockedWallet = await this.userWalletRepository.findOneByAndLock({ id: wallet.id }); if (!lockedWallet?.isTrialing) return null; - return await this.#wipe(wallet); + return await this.#wipe(wallet, reason); } /** The lock and the probe cancellations ride the row-lock transaction, so a wipe that fails partway rolls them back and leaves the wallet unlocked and still monitored for the retry. */ - async #wipe(wallet: WalletInitialized): Promise { + async #wipe(wallet: WalletInitialized, reason: AbuseLockReason): Promise { const granter = await this.txManagerService.getFundingWalletAddress(); const depositGrantRevoked = await this.#revokeDepositGrant(granter, wallet.address); const closedDseqs = await this.#closeLiveDeployments(wallet); const feeGrantRevoked = await this.#revokeFeeGrant(granter, wallet.address); - await this.userWalletRepository.lockForAbuse(wallet.id, ABUSE_LOCK_REASON); + await this.userWalletRepository.lockForAbuse(wallet.id, reason); await this.probeJobService.cancelForWallet(wallet.id); return { depositGrantRevoked, feeGrantRevoked, closedDseqs }; diff --git a/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.spec.ts b/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.spec.ts new file mode 100644 index 0000000000..2459418db9 --- /dev/null +++ b/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.spec.ts @@ -0,0 +1,97 @@ +import type { Counter } from "@opentelemetry/api"; +import { describe, expect, it } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import type { MetricsService } from "@src/core"; +import { WorkloadAbuseInstrumentationService } from "./workload-abuse-instrumentation.service"; + +describe(WorkloadAbuseInstrumentationService.name, () => { + it("creates a counter per outcome it reports", () => { + const { metricsService } = setup(); + + expect(metricsService.createCounter).toHaveBeenCalledWith(expect.anything(), "workload_abuse_probes_total", expect.anything()); + expect(metricsService.createCounter).toHaveBeenCalledWith(expect.anything(), "workload_abuse_detections_total", expect.anything()); + expect(metricsService.createCounter).toHaveBeenCalledWith(expect.anything(), "workload_abuse_enforcements_total", expect.anything()); + expect(metricsService.createCounter).toHaveBeenCalledWith(expect.anything(), "workload_abuse_blocked_domain_lookup_failures_total", expect.anything()); + expect(metricsService.createCounter).toHaveBeenCalledWith(expect.anything(), "workload_abuse_domain_blocks_total", expect.anything()); + }); + + describe("recordProbe", () => { + it("tags the probe with its verdict and status", () => { + const { service, probes } = setup(); + + service.recordProbe({ verdict: "hard", probeStatus: "completed" }); + + expect(probes.add).toHaveBeenCalledWith(1, { verdict: "hard", probe_status: "completed" }); + }); + }); + + describe("recordDetection", () => { + it("tags the detection with its verdict", () => { + const { service, detections } = setup(); + + service.recordDetection("soft"); + + expect(detections.add).toHaveBeenCalledWith(1, { verdict: "soft" }); + }); + }); + + describe("recordEnforcement", () => { + it("tags the enforcement with its result", () => { + const { service, enforcements } = setup(); + + service.recordEnforcement("enforced"); + + expect(enforcements.add).toHaveBeenCalledWith(1, { result: "enforced" }); + }); + }); + + describe("recordBlockedDomainLookupFailure", () => { + it("counts a lookup that failed, because enforcement is off while it climbs", () => { + const { service, blockedDomainLookupFailures } = setup(); + + service.recordBlockedDomainLookupFailure(); + + expect(blockedDomainLookupFailures.add).toHaveBeenCalledWith(1); + }); + }); + + describe("recordDomainBlock", () => { + it("tags a block with its result", () => { + const { service, domainBlocks } = setup(); + + service.recordDomainBlock("blocked"); + + expect(domainBlocks.add).toHaveBeenCalledWith(1, { result: "blocked" }); + }); + + it("adds the reason to a skip, so the guardrail that fired is visible on the dashboard", () => { + const { service, domainBlocks } = setup(); + + service.recordDomainBlock("skipped", "public_provider"); + + expect(domainBlocks.add).toHaveBeenCalledWith(1, { result: "skipped", reason: "public_provider" }); + }); + }); + + function setup() { + const probes = mock(); + const detections = mock(); + const enforcements = mock(); + const blockedDomainLookupFailures = mock(); + const domainBlocks = mock(); + + const metricsService = mock(); + metricsService.getMeter.mockReturnValue(mock()); + metricsService.createCounter + .mockReturnValueOnce(probes) + .mockReturnValueOnce(detections) + .mockReturnValueOnce(enforcements) + .mockReturnValueOnce(blockedDomainLookupFailures) + .mockReturnValueOnce(domainBlocks); + + const service = new WorkloadAbuseInstrumentationService(metricsService); + + return { service, metricsService, probes, detections, enforcements, blockedDomainLookupFailures, domainBlocks }; + } +}); diff --git a/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts b/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts index 61233dd7a5..6307f13c87 100644 --- a/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts +++ b/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts @@ -4,6 +4,8 @@ import { singleton } from "tsyringe"; import { MetricsService } from "@src/core/services/metrics/metrics.service"; import type { WorkloadVerdict } from "@src/workload-abuse/lib/evidence-scanner/evidence-scanner"; +export type DomainBlockResult = "blocked" | "raced" | "skipped" | "dry_run" | "failed" | "sibling_limit_reached"; + @singleton() export class WorkloadAbuseInstrumentationService { private readonly meter: Meter; @@ -11,6 +13,7 @@ export class WorkloadAbuseInstrumentationService { private readonly detections: Counter; private readonly enforcements: Counter; private readonly blockedDomainLookupFailures: Counter; + private readonly domainBlocks: Counter; constructor(metricsService: MetricsService) { this.meter = metricsService.getMeter("workload-abuse", "1.0.0"); @@ -26,6 +29,9 @@ export class WorkloadAbuseInstrumentationService { this.blockedDomainLookupFailures = metricsService.createCounter(this.meter, "workload_abuse_blocked_domain_lookup_failures_total", { description: "Blocked email domain lookups that failed and were answered as not blocked, so enforcement is off for as long as this climbs" }); + this.domainBlocks = metricsService.createCounter(this.meter, "workload_abuse_domain_blocks_total", { + description: "Email domain auto-block outcomes, by result and (on a skip) reason" + }); } recordProbe(input: { verdict: WorkloadVerdict; probeStatus: string }): void { @@ -43,4 +49,8 @@ export class WorkloadAbuseInstrumentationService { recordBlockedDomainLookupFailure(): void { this.blockedDomainLookupFailures.add(1); } + + recordDomainBlock(result: DomainBlockResult, reason?: string): void { + this.domainBlocks.add(1, reason ? { result, reason } : { result }); + } }