From 4b92a5d53b458ad3ec572e7fb0846bfcbb41d85d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N=C3=ADcolas=20Pastorello?= Date: Sat, 11 Apr 2026 00:26:04 -0300 Subject: [PATCH 1/5] feat: add link group correlation for multi-link monitoring Monitors can be assigned to a named group (e.g. branch-sp-01). When checks run, the system evaluates all monitors in the group: - Some links down: severity high - All links down: severity critical Backend: - Add findByGroupAndTeamId to monitors repository (Mongo + Timescale) - Add GroupCorrelation interface and groupCorrelation field to MonitorActionDecision - Evaluate group status in SuperSimpleQueueHelper after each check (Step 5b) - Persist severity field on Incident model and type - Enrich notification title/summary with group context and severity override Frontend: - Add group field to monitor create/edit form (all monitor types) - Add i18n keys for group label, placeholder, helper text --- CLAUDE.md | 46 +++++++++++++++--- client/src/Hooks/useMonitorForm.ts | 1 + client/src/Pages/CreateMonitor/index.tsx | 22 +++++++++ client/src/Validation/monitor.ts | 5 ++ client/src/locales/en.json | 5 ++ server/src/db/models/Incident.ts | 5 ++ .../incidents/MongoIncidentsRepository.ts | 1 + .../monitors/IMonitorsRepository.ts | 1 + .../monitors/MongoMonitorsRepository.ts | 9 ++++ .../monitors/TimescaleMonitorsRepository.ts | 5 ++ .../src/service/business/incidentService.ts | 1 + .../SuperSimpleQueueHelper.ts | 31 ++++++++++++ .../notificationMessageBuilder.ts | 47 +++++++++++++++---- server/src/types/incident.ts | 1 + server/src/types/notificationMessage.ts | 6 +++ 15 files changed, 172 insertions(+), 14 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1535e84cfb..ac03d7951d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,9 +45,10 @@ docker run -d -p 27017:27017 -v uptime_mongo_data:/data/db --name uptime_databas CLIENT_HOST="http://localhost:5173" JWT_SECRET="my_secret_key_change_this" DB_CONNECTION_STRING="mongodb://localhost:27017/uptime_db" +DB_TYPE="mongodb" # mongodb (default) or timescaledb TOKEN_TTL="99d" ORIGIN="localhost" -LOG_LEVEL="debug" +LOG_LEVEL="debug" # error | warn | info | debug ``` ### Client `.env` @@ -88,11 +89,13 @@ client/src/ ├── Pages/ # Page components (Auth, Uptime, Infrastructure, Incidents, etc.) ├── Features/ # Redux slices (Auth, UI) ├── Hooks/ # Custom React hooks -├── Utils/ # Utilities (NetworkService.js is main API client) +├── Utils/ # Utilities (ApiClient.ts is the main Axios client) ├── Validation/ # Input validation └── locales/ # i18n translations ``` +`ApiClient.ts` injects Bearer tokens from Redux auth state on every request and redirects to `/login` on 401. It also detects `ERR_NETWORK` to trigger the offline banner via `setServerUnreachableCallback()`. + ### API - Base URL: `/api/v1` - Documentation: `http://localhost:52345/api-docs` (Swagger UI) @@ -101,8 +104,8 @@ client/src/ ### Key Technologies - **State Management**: Redux Toolkit + Redux-Persist - **Data Fetching**: SWR + Axios -- **Database**: MongoDB with Mongoose ODM -- **Queue/Cache**: Redis + BullMQ + Pulse (cron scheduling) +- **Database**: MongoDB (default) or TimescaleDB/PostgreSQL — selected via `DB_TYPE` env var +- **Job Scheduler**: `super-simple-scheduler` (in-memory, NOT Redis/BullMQ/Pulse despite those being listed as dependencies) - **i18n**: i18next + react-i18next (translations via PoEditor) ## Code Conventions @@ -126,8 +129,9 @@ t('your.key') // Never hardcode UI strings ### Testing Server tests use Mocha + Chai + Sinon: ```bash -npm test # Run all tests with coverage -npm test -- --grep "pattern" # Run specific tests +npm test # Run all tests with coverage +npm run test:services # Run only service/provider tests +npm test -- --grep "pattern" # Run tests matching a string pattern ``` Test files: `server/tests/**/*.test.js` @@ -143,3 +147,33 @@ Key Mongoose models in `/server/src/db/models/`: - **Notification** - Alert configuration (email, Discord, Slack, webhooks) - **MaintenanceWindow** - Scheduled maintenance periods - **AppSettings** - Global application settings + +## Monitoring Loop Architecture + +On startup, `initializeServices()` in `server/src/config/services.ts` wires up a dependency-injection graph: +1. Connects DB (MongoDB or TimescaleDB based on `DB_TYPE`) +2. Instantiates the matching repository implementations (`Mongo*Repository` or `Timescale*Repository`) +3. Creates all network check providers (HTTP, Ping, Port, Docker, Hardware, PageSpeed, GameDig, GRPC, WebSocket) +4. Creates all notification providers (email, Slack, Discord, Teams, Telegram, PagerDuty, Matrix, webhook) +5. Creates `SuperSimpleQueue` with a `SuperSimpleQueueJobHelper` that ties it all together + +**Job templates registered at startup:** +- `monitor-job` — executes each monitor's check on its configured interval +- `geo-check-job` — geo-distributed check for supported HTTP monitors +- `cleanup-orphaned` / `cleanup-retention-job` — database cleanup (every 24h) + +**Per-check execution order** (in `SuperSimpleQueueJobHelper`): +1. Skip if monitor is in an active maintenance window +2. Run the appropriate network provider check +3. Buffer result via `BufferService` +4. Update monitor status via `StatusService` +5. Call `evaluateMonitorAction()` → produces 4 decision flags: + - `shouldCreateIncident` / `shouldResolveIncident` (status down/breached/recovered) + - `shouldSendNotification` / reason (only fires on status *changes*, not every check) +6. Dispatch notifications and incident mutations fire-and-forget (non-blocking) + +## Repository Pattern + +Every entity has an interface (e.g., `IMonitorsRepository`) with concrete implementations for each supported database. The correct implementation is selected at startup and injected into all services — services never import a concrete repository class directly. When adding a new DB operation, add the method to the interface and implement it in all concrete classes. + +Repositories live in `server/src/repositories/`; service constructors accept the interface type. diff --git a/client/src/Hooks/useMonitorForm.ts b/client/src/Hooks/useMonitorForm.ts index 963409fc8a..7b6084f365 100644 --- a/client/src/Hooks/useMonitorForm.ts +++ b/client/src/Hooks/useMonitorForm.ts @@ -17,6 +17,7 @@ const getBaseDefaults = (data?: Monitor | null) => ({ geoCheckEnabled: data?.geoCheckEnabled ?? false, geoCheckLocations: data?.geoCheckLocations || [], geoCheckInterval: data?.geoCheckInterval || 300000, + group: data?.group ?? null, }); export const useMonitorForm = ({ diff --git a/client/src/Pages/CreateMonitor/index.tsx b/client/src/Pages/CreateMonitor/index.tsx index 15b76eab36..54c95cb9c4 100644 --- a/client/src/Pages/CreateMonitor/index.tsx +++ b/client/src/Pages/CreateMonitor/index.tsx @@ -498,6 +498,28 @@ const CreateMonitorPage = () => { /> )} /> + ( + field.onChange(e.target.value || null)} + type="text" + fieldLabel={t("pages.createMonitor.form.general.option.group.label")} + placeholder={t( + "pages.createMonitor.form.general.option.group.placeholder" + )} + fullWidth + error={!!fieldState.error} + helperText={ + fieldState.error?.message ?? + t("pages.createMonitor.form.general.option.group.helper") + } + /> + )} + /> } /> diff --git a/client/src/Validation/monitor.ts b/client/src/Validation/monitor.ts index 9acffe6fed..f737331cbc 100644 --- a/client/src/Validation/monitor.ts +++ b/client/src/Validation/monitor.ts @@ -27,6 +27,11 @@ const baseSchema = z.object({ .number() .min(300000, "Interval must be at least 5 minutes") .optional(), + group: z + .string() + .max(50, "Group name must be at most 50 characters") + .optional() + .nullable(), }); // HTTP monitor schema diff --git a/client/src/locales/en.json b/client/src/locales/en.json index 83ca4fa332..65946885ce 100644 --- a/client/src/locales/en.json +++ b/client/src/locales/en.json @@ -503,6 +503,11 @@ "wsUrl": { "label": "WebSocket URL", "placeholder": "wss://example.com/socket" + }, + "group": { + "label": "Link Group", + "placeholder": "e.g. branch-sp-01", + "helper": "Assign to a group to enable multi-link correlation alerts" } }, "title": "General settings", diff --git a/server/src/db/models/Incident.ts b/server/src/db/models/Incident.ts index 82e2b5eb2b..7f558c1b23 100644 --- a/server/src/db/models/Incident.ts +++ b/server/src/db/models/Incident.ts @@ -72,6 +72,11 @@ const IncidentSchema = new Schema( type: String, default: null, }, + severity: { + type: String, + enum: ["none", "high", "critical", null], + default: "none", + }, }, { timestamps: true } ); diff --git a/server/src/repositories/incidents/MongoIncidentsRepository.ts b/server/src/repositories/incidents/MongoIncidentsRepository.ts index e24551462f..60f1411f46 100644 --- a/server/src/repositories/incidents/MongoIncidentsRepository.ts +++ b/server/src/repositories/incidents/MongoIncidentsRepository.ts @@ -60,6 +60,7 @@ class MongoIncidentsRepository implements IIncidentsRepository { resolvedBy: doc.resolvedBy ? this.toStringId(doc.resolvedBy) : null, resolvedByEmail: doc.resolvedByEmail ?? null, comment: doc.comment ?? null, + severity: doc.severity ?? "none", createdAt: this.toDateString(doc.createdAt), updatedAt: this.toDateString(doc.updatedAt), }; diff --git a/server/src/repositories/monitors/IMonitorsRepository.ts b/server/src/repositories/monitors/IMonitorsRepository.ts index 8c5f63e7de..9cb8bfce21 100644 --- a/server/src/repositories/monitors/IMonitorsRepository.ts +++ b/server/src/repositories/monitors/IMonitorsRepository.ts @@ -40,6 +40,7 @@ export interface IMonitorsRepository { // other findMonitorsSummaryByTeamId(teamId: string, config?: SummaryConfig): Promise; findGroupsByTeamId(teamId: string): Promise; + findByGroupAndTeamId(group: string, teamId: string): Promise; removeNotificationFromMonitors(notificationId: string): Promise; updateNotifications(teamId: string, monitorIds: string[], notificationIds: string[], action: "add" | "remove" | "set"): Promise; deleteByTeamIdsNotIn(teamIds: string[]): Promise; diff --git a/server/src/repositories/monitors/MongoMonitorsRepository.ts b/server/src/repositories/monitors/MongoMonitorsRepository.ts index b2d7594483..74c8accea5 100644 --- a/server/src/repositories/monitors/MongoMonitorsRepository.ts +++ b/server/src/repositories/monitors/MongoMonitorsRepository.ts @@ -292,6 +292,15 @@ class MongoMonitorsRepository implements IMonitorsRepository { return groups.sort(); }; + findByGroupAndTeamId = async (group: string, teamId: string): Promise => { + const docs = await MonitorModel.find({ + teamId: new mongoose.Types.ObjectId(teamId), + group: group, + isActive: true, + }); + return this.mapDocuments(docs); + }; + removeNotificationFromMonitors = async (notificationId: string): Promise => { await MonitorModel.updateMany({ notifications: notificationId }, { $pull: { notifications: notificationId } }); }; diff --git a/server/src/repositories/monitors/TimescaleMonitorsRepository.ts b/server/src/repositories/monitors/TimescaleMonitorsRepository.ts index a94d385042..cb2856276a 100644 --- a/server/src/repositories/monitors/TimescaleMonitorsRepository.ts +++ b/server/src/repositories/monitors/TimescaleMonitorsRepository.ts @@ -742,6 +742,11 @@ export class TimescaleMonitorsRepository implements IMonitorsRepository { return result.rows.map((row) => row.monitor_group); }; + findByGroupAndTeamId = async (group: string, teamId: string): Promise => { + const result = await this.pool.query(`SELECT * FROM monitors WHERE team_id = $1 AND monitor_group = $2 AND is_active = true`, [teamId, group]); + return result.rows.map(this.toEntity); + }; + removeNotificationFromMonitors = async (notificationId: string): Promise => { await this.pool.query(`DELETE FROM monitor_notifications WHERE notification_id = $1`, [notificationId]); }; diff --git a/server/src/service/business/incidentService.ts b/server/src/service/business/incidentService.ts index 2070a89370..7e2df4d9c7 100644 --- a/server/src/service/business/incidentService.ts +++ b/server/src/service/business/incidentService.ts @@ -90,6 +90,7 @@ export class IncidentService implements IIncidentService { status: true, statusCode, message, + severity: decision.groupCorrelation?.severity ?? "none", }; return await this.incidentsRepository.create(incident); } diff --git a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts index 956d19c984..e035e4f6fd 100644 --- a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts +++ b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts @@ -33,6 +33,13 @@ export interface ISuperSimpleQueueHelper { isInMaintenanceWindow(monitorId: string, teamId: string): Promise; } +export interface GroupCorrelation { + groupName: string; + downCount: number; + totalCount: number; + severity: "high" | "critical"; +} + export interface MonitorActionDecision { shouldCreateIncident: boolean; shouldResolveIncident: boolean; @@ -45,6 +52,7 @@ export interface MonitorActionDecision { disk?: boolean; temp?: boolean; }; + groupCorrelation?: GroupCorrelation; } export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { @@ -156,6 +164,29 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { // Step 5. Get decisions const decision = this.evaluateMonitorAction(statusChangeResult); + // Step 5b. Evaluate group correlation if monitor belongs to a group + if (monitor.group && (decision.shouldCreateIncident || decision.shouldResolveIncident)) { + try { + const groupMonitors = await this.monitorsRepository.findByGroupAndTeamId(monitor.group, teamId); + const totalCount = groupMonitors.length; + const downCount = groupMonitors.filter((m) => m.status === "down").length; + if (downCount > 0 && totalCount > 1) { + decision.groupCorrelation = { + groupName: monitor.group, + downCount, + totalCount, + severity: downCount === totalCount ? "critical" : "high", + }; + } + } catch (error: unknown) { + this.logger.warn({ + message: `Could not evaluate group correlation for monitor ${monitorId}: ${error instanceof Error ? error.message : "Unknown error"}`, + service: SERVICE_NAME, + method: "getMonitorJob", + }); + } + } + // Step 6. Handle notifications (best effort, continue even in event of failure, don't wait) if (decision.shouldSendNotification) { this.notificationsService.handleNotifications(statusChangeResult.monitor, status, decision).catch((error: unknown) => { diff --git a/server/src/service/infrastructure/notificationMessageBuilder.ts b/server/src/service/infrastructure/notificationMessageBuilder.ts index 934163b2a9..34abd61bb3 100644 --- a/server/src/service/infrastructure/notificationMessageBuilder.ts +++ b/server/src/service/infrastructure/notificationMessageBuilder.ts @@ -1,5 +1,5 @@ import type { HardwareStatusPayload, Monitor, MonitorStatusResponse } from "@/types/index.js"; -import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; +import type { MonitorActionDecision, GroupCorrelation } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import type { NotificationMessage, NotificationType, @@ -30,8 +30,13 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { clientHost: string ): NotificationMessage { const type = this.determineNotificationType(decision, monitor); - const severity = this.determineSeverity(type); - const content = this.buildContent(type, monitor, monitorStatusResponse); + let severity = this.determineSeverity(type); + + if (decision.groupCorrelation && monitor.status === "down") { + severity = decision.groupCorrelation.severity === "critical" ? "critical" : "warning"; + } + + const content = this.buildContent(type, monitor, monitorStatusResponse, decision.groupCorrelation); return { type, @@ -48,6 +53,14 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { metadata: { teamId: monitor.teamId, notificationReason: decision.notificationReason || "status_change", + groupCorrelation: decision.groupCorrelation + ? { + groupName: decision.groupCorrelation.groupName, + downCount: decision.groupCorrelation.downCount, + totalCount: decision.groupCorrelation.totalCount, + severity: decision.groupCorrelation.severity, + } + : undefined, }, }; } @@ -93,10 +106,15 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { } } - private buildContent(type: NotificationType, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent { + private buildContent( + type: NotificationType, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + groupCorrelation?: GroupCorrelation + ): NotificationContent { switch (type) { case "monitor_down": - return this.buildMonitorDownContent(monitor, monitorStatusResponse); + return this.buildMonitorDownContent(monitor, monitorStatusResponse, groupCorrelation); case "monitor_up": return this.buildMonitorUpContent(monitor); case "threshold_breach": @@ -108,9 +126,22 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { } } - private buildMonitorDownContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent { - const title = `Monitor Down: ${monitor.name}`; - const summary = `Monitor "${monitor.name}" is currently down and unreachable.`; + private buildMonitorDownContent( + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + groupCorrelation?: GroupCorrelation + ): NotificationContent { + const title = + groupCorrelation?.severity === "critical" + ? `[CRITICAL] All Links Down: ${groupCorrelation.groupName}` + : groupCorrelation + ? `[HIGH] Link Down: ${monitor.name}` + : `Monitor Down: ${monitor.name}`; + + const summary = groupCorrelation + ? `Monitor "${monitor.name}" is down. Group "${groupCorrelation.groupName}": ${groupCorrelation.downCount}/${groupCorrelation.totalCount} link(s) down.${groupCorrelation.severity === "critical" ? " ALL links are down — critical outage." : ""}` + : `Monitor "${monitor.name}" is currently down and unreachable.`; + const details = [`URL: ${monitor.url}`, `Status: Down`, `Type: ${monitor.type}`]; // Add response code if available diff --git a/server/src/types/incident.ts b/server/src/types/incident.ts index 6b076ff835..44850e96fa 100644 --- a/server/src/types/incident.ts +++ b/server/src/types/incident.ts @@ -16,6 +16,7 @@ export interface Incident { resolvedBy?: string | null; resolvedByEmail?: string | null; comment?: string | null; + severity?: "none" | "high" | "critical" | null; createdAt: string; updatedAt: string; } diff --git a/server/src/types/notificationMessage.ts b/server/src/types/notificationMessage.ts index f06ff1bd9a..631f336fae 100644 --- a/server/src/types/notificationMessage.ts +++ b/server/src/types/notificationMessage.ts @@ -49,5 +49,11 @@ export interface NotificationMessage { metadata: { teamId: string; notificationReason: string; + groupCorrelation?: { + groupName: string; + downCount: number; + totalCount: number; + severity: "high" | "critical"; + }; }; } From 222cc58717b1cd2dbfeb48743f14a2557aebc3bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N=C3=ADcolas=20Pastorello?= Date: Sat, 11 Apr 2026 00:56:59 -0300 Subject: [PATCH 2/5] fix: persist incident severity in TimescaleDB backend --- .../timescaledb/0022_add_incident_severity.ts | 13 +++++++++++++ server/src/db/migration/timescaledb/index.ts | 2 ++ .../incidents/TimescaleIncidentsRepository.ts | 10 +++++++--- 3 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 server/src/db/migration/timescaledb/0022_add_incident_severity.ts diff --git a/server/src/db/migration/timescaledb/0022_add_incident_severity.ts b/server/src/db/migration/timescaledb/0022_add_incident_severity.ts new file mode 100644 index 0000000000..4a9d962b7d --- /dev/null +++ b/server/src/db/migration/timescaledb/0022_add_incident_severity.ts @@ -0,0 +1,13 @@ +import type { Pool } from "pg"; + +export const addIncidentSeverity = async (pool: Pool) => { + await pool.query(` + ALTER TABLE incidents + ADD COLUMN IF NOT EXISTS severity TEXT NOT NULL DEFAULT 'none' + CHECK (severity IN ('none', 'high', 'critical')); + `); +}; + +export const dropIncidentSeverity = async (pool: Pool) => { + await pool.query(`ALTER TABLE incidents DROP COLUMN IF EXISTS severity;`); +}; diff --git a/server/src/db/migration/timescaledb/index.ts b/server/src/db/migration/timescaledb/index.ts index 14084ebf3d..15a7aaee08 100644 --- a/server/src/db/migration/timescaledb/index.ts +++ b/server/src/db/migration/timescaledb/index.ts @@ -21,6 +21,7 @@ import { createStatusPages, dropStatusPages } from "./0018_create_status_pages.j import { createAppSettings, dropAppSettings } from "./0019_create_app_settings.js"; import { createContinuousAggregates, dropContinuousAggregates } from "./0020_create_continuous_aggregates.js"; import { createRetentionCompression, dropRetentionCompression } from "./0021_create_retention_compression.js"; +import { addIncidentSeverity, dropIncidentSeverity } from "./0022_add_incident_severity.js"; const SERVICE_NAME = "TimescaleDB Migrations"; @@ -52,6 +53,7 @@ const migrations: MigrationEntry[] = [ { name: "0019_create_app_settings", up: createAppSettings, down: dropAppSettings }, { name: "0020_create_continuous_aggregates", up: createContinuousAggregates, down: dropContinuousAggregates }, { name: "0021_create_retention_compression", up: createRetentionCompression, down: dropRetentionCompression }, + { name: "0022_add_incident_severity", up: addIncidentSeverity, down: dropIncidentSeverity }, ]; const ensureMigrationsTable = async (pool: Pool) => { diff --git a/server/src/repositories/incidents/TimescaleIncidentsRepository.ts b/server/src/repositories/incidents/TimescaleIncidentsRepository.ts index 8a6ad7d22e..b5e2385364 100644 --- a/server/src/repositories/incidents/TimescaleIncidentsRepository.ts +++ b/server/src/repositories/incidents/TimescaleIncidentsRepository.ts @@ -16,20 +16,21 @@ interface IncidentRow { resolved_by: string | null; resolved_by_email: string | null; comment: string | null; + severity: "none" | "high" | "critical"; created_at: Date; updated_at: Date; } const COLUMNS = `id, monitor_id, team_id, start_time, end_time, status, message, status_code, - resolution_type, resolved_by, resolved_by_email, comment, created_at, updated_at`; + resolution_type, resolved_by, resolved_by_email, comment, severity, created_at, updated_at`; export class TimescaleIncidentsRepository implements IIncidentsRepository { constructor(private pool: Pool) {} create = async (incident: Partial): Promise => { const result = await this.pool.query( - `INSERT INTO incidents (monitor_id, team_id, start_time, end_time, status, message, status_code, resolution_type, resolved_by, resolved_by_email, comment) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + `INSERT INTO incidents (monitor_id, team_id, start_time, end_time, status, message, status_code, resolution_type, resolved_by, resolved_by_email, comment, severity) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING ${COLUMNS}`, [ incident.monitorId, @@ -43,6 +44,7 @@ export class TimescaleIncidentsRepository implements IIncidentsRepository { incident.resolvedBy ?? null, incident.resolvedByEmail ?? null, incident.comment ?? null, + incident.severity ?? "none", ] ); const row = result.rows[0]; @@ -207,6 +209,7 @@ export class TimescaleIncidentsRepository implements IIncidentsRepository { ["resolvedBy", "resolved_by"], ["resolvedByEmail", "resolved_by_email"], ["comment", "comment"], + ["severity", "severity"], ]; for (const [key, column] of fieldMap) { @@ -288,6 +291,7 @@ export class TimescaleIncidentsRepository implements IIncidentsRepository { resolvedBy: row.resolved_by ?? null, resolvedByEmail: row.resolved_by_email ?? null, comment: row.comment ?? null, + severity: row.severity ?? "none", createdAt: row.created_at.toISOString(), updatedAt: row.updated_at.toISOString(), }); From 0506d81b6eb2b07454fc12c6fcd53c80aa6a1ad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N=C3=ADcolas=20Pastorello?= Date: Sat, 11 Apr 2026 19:07:59 -0300 Subject: [PATCH 3/5] fix: cast severity type to satisfy TypeScript in incidentService --- server/src/service/business/incidentService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/service/business/incidentService.ts b/server/src/service/business/incidentService.ts index 7e2df4d9c7..a365a00eb8 100644 --- a/server/src/service/business/incidentService.ts +++ b/server/src/service/business/incidentService.ts @@ -90,7 +90,7 @@ export class IncidentService implements IIncidentService { status: true, statusCode, message, - severity: decision.groupCorrelation?.severity ?? "none", + severity: (decision.groupCorrelation?.severity ?? "none") as "none" | "high" | "critical", }; return await this.incidentsRepository.create(incident); } From 345a47a500124535de8f0b5c5aac95bc39ce802f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N=C3=ADcolas=20Pastorello?= Date: Sat, 11 Apr 2026 19:57:11 -0300 Subject: [PATCH 4/5] refactor: address review feedback on group correlation types and architecture - Add IncidentSeverity type following as-const pattern (like MonitorType) - Extract IncidentContext interface; remove groupCorrelation from MonitorActionDecision - Pass IncidentContext as separate param to handleIncident and handleNotifications - Step 5b: evaluate group correlation only on shouldCreateIncident, not resolve - Remove null from Incident.severity type and Mongoose enum - Annotate incident object as Partial to avoid severity type widening --- server/src/db/models/Incident.ts | 2 +- .../src/service/business/incidentService.ts | 12 ++++---- .../SuperSimpleQueueHelper.ts | 29 ++++++++++++------- .../notificationMessageBuilder.ts | 24 ++++++++------- .../infrastructure/notificationsService.ts | 27 +++++++++++++---- server/src/types/incident.ts | 5 +++- 6 files changed, 64 insertions(+), 35 deletions(-) diff --git a/server/src/db/models/Incident.ts b/server/src/db/models/Incident.ts index 7f558c1b23..b0c9ddd508 100644 --- a/server/src/db/models/Incident.ts +++ b/server/src/db/models/Incident.ts @@ -74,7 +74,7 @@ const IncidentSchema = new Schema( }, severity: { type: String, - enum: ["none", "high", "critical", null], + enum: ["none", "high", "critical"], default: "none", }, }, diff --git a/server/src/service/business/incidentService.ts b/server/src/service/business/incidentService.ts index a365a00eb8..636314f905 100644 --- a/server/src/service/business/incidentService.ts +++ b/server/src/service/business/incidentService.ts @@ -5,7 +5,7 @@ import { AppError } from "@/utils/AppError.js"; import { getDateForRange } from "@/utils/dataUtils.js"; import type { IIncidentsRepository, IMonitorsRepository, IUsersRepository } from "@/repositories/index.js"; import type { Incident, IncidentSummary, User } from "@/types/index.js"; -import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; +import type { MonitorActionDecision, IncidentContext } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import type { INotificationMessageBuilder } from "@/service/infrastructure/notificationMessageBuilder.js"; import type { ILogger } from "@/utils/logger.js"; @@ -14,7 +14,8 @@ export interface IIncidentService { monitor: Monitor, code: number, decision: MonitorActionDecision, - monitorStatusResponse?: MonitorStatusResponse + monitorStatusResponse?: MonitorStatusResponse, + context?: IncidentContext ): Promise; resolveIncident(incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string): Promise; getIncidentsByTeam( @@ -62,7 +63,8 @@ export class IncidentService implements IIncidentService { monitor: Monitor, code: number, decision: MonitorActionDecision, - monitorStatusResponse?: MonitorStatusResponse + monitorStatusResponse?: MonitorStatusResponse, + context?: IncidentContext ): Promise => { if (!decision.shouldCreateIncident && !decision.shouldResolveIncident) { return null; @@ -83,14 +85,14 @@ export class IncidentService implements IIncidentService { message = this.buildThresholdBreachMessage(monitor, monitorStatusResponse); } - const incident = { + const incident: Partial = { monitorId: monitor.id, teamId: monitor.teamId, startTime: Date.now().toString(), status: true, statusCode, message, - severity: (decision.groupCorrelation?.severity ?? "none") as "none" | "high" | "critical", + severity: context?.groupCorrelation?.severity ?? "none", }; return await this.incidentsRepository.create(incident); } diff --git a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts index e035e4f6fd..e2062624ce 100644 --- a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts +++ b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts @@ -23,6 +23,7 @@ import { } from "@/repositories/index.js"; import { ILogger } from "@/utils/logger.js"; import { IBufferService } from "@/service/index.js"; +import type { IncidentSeverity } from "@/types/incident.js"; export interface ISuperSimpleQueueHelper { readonly serviceName: string; @@ -37,7 +38,11 @@ export interface GroupCorrelation { groupName: string; downCount: number; totalCount: number; - severity: "high" | "critical"; + severity: Exclude; +} + +export interface IncidentContext { + groupCorrelation?: GroupCorrelation; } export interface MonitorActionDecision { @@ -52,7 +57,6 @@ export interface MonitorActionDecision { disk?: boolean; temp?: boolean; }; - groupCorrelation?: GroupCorrelation; } export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { @@ -164,18 +168,21 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { // Step 5. Get decisions const decision = this.evaluateMonitorAction(statusChangeResult); - // Step 5b. Evaluate group correlation if monitor belongs to a group - if (monitor.group && (decision.shouldCreateIncident || decision.shouldResolveIncident)) { + // Step 5b. Evaluate group correlation if monitor belongs to a group and an incident will be created + let incidentContext: IncidentContext | undefined; + if (monitor.group && decision.shouldCreateIncident) { try { const groupMonitors = await this.monitorsRepository.findByGroupAndTeamId(monitor.group, teamId); const totalCount = groupMonitors.length; const downCount = groupMonitors.filter((m) => m.status === "down").length; if (downCount > 0 && totalCount > 1) { - decision.groupCorrelation = { - groupName: monitor.group, - downCount, - totalCount, - severity: downCount === totalCount ? "critical" : "high", + incidentContext = { + groupCorrelation: { + groupName: monitor.group, + downCount, + totalCount, + severity: downCount === totalCount ? "critical" : "high", + }, }; } } catch (error: unknown) { @@ -189,7 +196,7 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { // Step 6. Handle notifications (best effort, continue even in event of failure, don't wait) if (decision.shouldSendNotification) { - this.notificationsService.handleNotifications(statusChangeResult.monitor, status, decision).catch((error: unknown) => { + this.notificationsService.handleNotifications(statusChangeResult.monitor, status, decision, incidentContext).catch((error: unknown) => { this.logger.error({ message: `Error sending notifications for job ${statusChangeResult.monitor.id}: ${error instanceof Error ? error.message : "Unknown error"}`, service: SERVICE_NAME, @@ -201,7 +208,7 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { // Step 7. Handle incidents try { - await this.incidentService.handleIncident(statusChangeResult.monitor, statusChangeResult.code, decision, status); + await this.incidentService.handleIncident(statusChangeResult.monitor, statusChangeResult.code, decision, status, incidentContext); } catch (error: unknown) { this.logger.warn({ message: `Error handling incident for job ${monitor.id}: ${error instanceof Error ? error.message : "Unknown error"}`, diff --git a/server/src/service/infrastructure/notificationMessageBuilder.ts b/server/src/service/infrastructure/notificationMessageBuilder.ts index 34abd61bb3..4ada377298 100644 --- a/server/src/service/infrastructure/notificationMessageBuilder.ts +++ b/server/src/service/infrastructure/notificationMessageBuilder.ts @@ -1,5 +1,5 @@ import type { HardwareStatusPayload, Monitor, MonitorStatusResponse } from "@/types/index.js"; -import type { MonitorActionDecision, GroupCorrelation } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; +import type { MonitorActionDecision, GroupCorrelation, IncidentContext } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import type { NotificationMessage, NotificationType, @@ -13,7 +13,8 @@ export interface INotificationMessageBuilder { monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision, - clientHost: string + clientHost: string, + context?: IncidentContext ): NotificationMessage; extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[]; } @@ -27,16 +28,17 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision, - clientHost: string + clientHost: string, + context?: IncidentContext ): NotificationMessage { const type = this.determineNotificationType(decision, monitor); let severity = this.determineSeverity(type); - if (decision.groupCorrelation && monitor.status === "down") { - severity = decision.groupCorrelation.severity === "critical" ? "critical" : "warning"; + if (context?.groupCorrelation && monitor.status === "down") { + severity = context.groupCorrelation.severity === "critical" ? "critical" : "warning"; } - const content = this.buildContent(type, monitor, monitorStatusResponse, decision.groupCorrelation); + const content = this.buildContent(type, monitor, monitorStatusResponse, context?.groupCorrelation); return { type, @@ -53,12 +55,12 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { metadata: { teamId: monitor.teamId, notificationReason: decision.notificationReason || "status_change", - groupCorrelation: decision.groupCorrelation + groupCorrelation: context?.groupCorrelation ? { - groupName: decision.groupCorrelation.groupName, - downCount: decision.groupCorrelation.downCount, - totalCount: decision.groupCorrelation.totalCount, - severity: decision.groupCorrelation.severity, + groupName: context.groupCorrelation.groupName, + downCount: context.groupCorrelation.downCount, + totalCount: context.groupCorrelation.totalCount, + severity: context.groupCorrelation.severity, } : undefined, }, diff --git a/server/src/service/infrastructure/notificationsService.ts b/server/src/service/infrastructure/notificationsService.ts index b131c4ff9c..36623dfb48 100644 --- a/server/src/service/infrastructure/notificationsService.ts +++ b/server/src/service/infrastructure/notificationsService.ts @@ -2,7 +2,7 @@ import type { Monitor, MonitorStatusResponse, Notification } from "@/types/index import type { NotificationMessage } from "@/types/notificationMessage.js"; import { IMonitorsRepository, INotificationsRepository } from "@/repositories/index.js"; import { INotificationProvider } from "./notificationProviders/INotificationProvider.js"; -import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; +import type { MonitorActionDecision, IncidentContext } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import type { ISettingsService } from "@/service/system/settingsService.js"; import { ILogger } from "@/utils/logger.js"; import type { INotificationMessageBuilder } from "@/service/infrastructure/notificationMessageBuilder.js"; @@ -13,7 +13,12 @@ export interface INotificationsService { findNotificationsByTeamId: (teamId: string) => Promise; updateById(id: string, teamId: string, updateData: Partial): Promise; deleteById: (id: string, teamId: string) => Promise; - handleNotifications: (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => Promise; + handleNotifications: ( + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision, + context?: IncidentContext + ) => Promise; sendTestNotification: (notification: Partial) => Promise; testAllNotifications: (notificationIds: string[]) => Promise; @@ -112,14 +117,19 @@ export class NotificationsService implements INotificationsService { } }; - private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { + private sendNotifications = async ( + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision, + context?: IncidentContext + ) => { const notificationIds = monitor.notifications ?? []; const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds); // Build notification message once for all notifications const settings = this.settingsService.getSettings(); const clientHost = settings.clientHost || "Host not defined"; - const notificationMessage = this.notificationMessageBuilder.buildMessage(monitor, monitorStatusResponse, decision, clientHost); + const notificationMessage = this.notificationMessageBuilder.buildMessage(monitor, monitorStatusResponse, decision, clientHost, context); const tasks = notifications.map((notification) => this.send(notification, monitor, monitorStatusResponse, decision, notificationMessage)); @@ -137,13 +147,18 @@ export class NotificationsService implements INotificationsService { return succeeded === notifications.length; }; - handleNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { + handleNotifications = async ( + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision, + context?: IncidentContext + ) => { if (!decision.shouldSendNotification) { return false; } // Send notifications based on decision - return await this.sendNotifications(monitor, monitorStatusResponse, decision); + return await this.sendNotifications(monitor, monitorStatusResponse, decision, context); }; sendTestNotification = async (notification: Partial) => { diff --git a/server/src/types/incident.ts b/server/src/types/incident.ts index 44850e96fa..255da3f719 100644 --- a/server/src/types/incident.ts +++ b/server/src/types/incident.ts @@ -3,6 +3,9 @@ export const IncidentResolutionTypes = ["automatic", "manual", null] as const; export type IncidentResolutionType = (typeof IncidentResolutionTypes)[number]; +export const IncidentSeverities = ["none", "high", "critical"] as const; +export type IncidentSeverity = (typeof IncidentSeverities)[number]; + export interface Incident { id: string; monitorId: string; @@ -16,7 +19,7 @@ export interface Incident { resolvedBy?: string | null; resolvedByEmail?: string | null; comment?: string | null; - severity?: "none" | "high" | "critical" | null; + severity?: IncidentSeverity; createdAt: string; updatedAt: string; } From a0e31d68b9683b5fd81acc9dace6fc948d0f0321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N=C3=ADcolas=20Pastorello?= Date: Sun, 19 Apr 2026 01:09:52 -0300 Subject: [PATCH 5/5] test: update mock expectations for new IncidentContext arg --- .../services/notificationsService.test.ts | 3 ++- .../services/superSimpleQueueHelper.test.ts | 21 ++++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/server/test/unit/services/notificationsService.test.ts b/server/test/unit/services/notificationsService.test.ts index 00dd644f83..a2b6b05f98 100644 --- a/server/test/unit/services/notificationsService.test.ts +++ b/server/test/unit/services/notificationsService.test.ts @@ -225,7 +225,8 @@ describe("NotificationsService", () => { expect.anything(), expect.anything(), expect.anything(), - "Host not defined" + "Host not defined", + undefined ); }); }); diff --git a/server/test/unit/services/superSimpleQueueHelper.test.ts b/server/test/unit/services/superSimpleQueueHelper.test.ts index 4d8ecfbc01..5e44ca3f9f 100644 --- a/server/test/unit/services/superSimpleQueueHelper.test.ts +++ b/server/test/unit/services/superSimpleQueueHelper.test.ts @@ -187,7 +187,8 @@ describe("SuperSimpleQueueHelper", () => { expect(defaults.notificationsService.handleNotifications).toHaveBeenCalledWith( statusResult.monitor, networkResponse, - expect.objectContaining({ shouldSendNotification: true, shouldCreateIncident: true }) + expect.objectContaining({ shouldSendNotification: true, shouldCreateIncident: true }), + undefined ); }); @@ -627,7 +628,8 @@ describe("SuperSimpleQueueHelper", () => { expect.anything(), expect.anything(), expect.objectContaining({ shouldCreateIncident: false, shouldResolveIncident: false, shouldSendNotification: false }), - expect.anything() + expect.anything(), + undefined ); }); @@ -647,7 +649,8 @@ describe("SuperSimpleQueueHelper", () => { incidentReason: "status_down", notificationReason: "status_change", }), - expect.anything() + expect.anything(), + undefined ); }); @@ -666,7 +669,8 @@ describe("SuperSimpleQueueHelper", () => { incidentReason: "threshold_breach", notificationReason: "threshold_breach", }), - expect.anything() + expect.anything(), + undefined ); }); @@ -681,7 +685,8 @@ describe("SuperSimpleQueueHelper", () => { expect.anything(), expect.anything(), expect.objectContaining({ shouldResolveIncident: true, shouldSendNotification: true }), - expect.anything() + expect.anything(), + undefined ); }); @@ -696,7 +701,8 @@ describe("SuperSimpleQueueHelper", () => { expect.anything(), expect.anything(), expect.objectContaining({ shouldResolveIncident: true }), - expect.anything() + expect.anything(), + undefined ); }); @@ -711,7 +717,8 @@ describe("SuperSimpleQueueHelper", () => { expect.anything(), expect.anything(), expect.objectContaining({ shouldCreateIncident: false, shouldResolveIncident: false, shouldSendNotification: false }), - expect.anything() + expect.anything(), + undefined ); }); });