From b48087f0ef66723785dd8e3259dc2f80fd071963 Mon Sep 17 00:00:00 2001 From: Matthias Hartmann Date: Thu, 23 Apr 2026 13:36:38 +0200 Subject: [PATCH 1/2] feat(metrics): add service monitors for all backup-restore-sidecars --- control-plane/roles/monitoring/tasks/main.yaml | 3 +++ .../servicemonitors/auditing-timescaledb.yaml | 16 ++++++++++++++++ .../templates/servicemonitors/headscale-db.yaml | 16 ++++++++++++++++ .../templates/servicemonitors/zitadel-db.yaml | 16 ++++++++++++++++ 4 files changed, 51 insertions(+) create mode 100644 control-plane/roles/monitoring/templates/servicemonitors/auditing-timescaledb.yaml create mode 100644 control-plane/roles/monitoring/templates/servicemonitors/headscale-db.yaml create mode 100644 control-plane/roles/monitoring/templates/servicemonitors/zitadel-db.yaml diff --git a/control-plane/roles/monitoring/tasks/main.yaml b/control-plane/roles/monitoring/tasks/main.yaml index 6de2e7a32..4f8df9409 100644 --- a/control-plane/roles/monitoring/tasks/main.yaml +++ b/control-plane/roles/monitoring/tasks/main.yaml @@ -97,12 +97,15 @@ definition: "{{ lookup('template', 'servicemonitors/' + item) }}" namespace: "{{ monitoring_namespace }}" loop: + - auditing-timescaledb.yaml + - headscale-db.yaml - ipam-db.yaml - masterdata-api.yaml - masterdata-db.yaml - metal-api.yaml - metal-db.yaml - metal-metrics-exporter.yaml + - zitadel-db.yaml - name: Create service monitors for Gardener k8s: diff --git a/control-plane/roles/monitoring/templates/servicemonitors/auditing-timescaledb.yaml b/control-plane/roles/monitoring/templates/servicemonitors/auditing-timescaledb.yaml new file mode 100644 index 000000000..750c116dc --- /dev/null +++ b/control-plane/roles/monitoring/templates/servicemonitors/auditing-timescaledb.yaml @@ -0,0 +1,16 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + release: kube-prometheus-stack + name: auditing-timescaledb +spec: + selector: + matchLabels: + app: auditing-timescaledb + namespaceSelector: + matchNames: + - {{ metal_control_plane_namespace }} + endpoints: + - port: "metrics" + interval: 60s diff --git a/control-plane/roles/monitoring/templates/servicemonitors/headscale-db.yaml b/control-plane/roles/monitoring/templates/servicemonitors/headscale-db.yaml new file mode 100644 index 000000000..55d3a1901 --- /dev/null +++ b/control-plane/roles/monitoring/templates/servicemonitors/headscale-db.yaml @@ -0,0 +1,16 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + release: kube-prometheus-stack + name: headscale-db +spec: + selector: + matchLabels: + app: headscale-db + namespaceSelector: + matchNames: + - {{ metal_control_plane_namespace }} + endpoints: + - port: "metrics" + interval: 60s diff --git a/control-plane/roles/monitoring/templates/servicemonitors/zitadel-db.yaml b/control-plane/roles/monitoring/templates/servicemonitors/zitadel-db.yaml new file mode 100644 index 000000000..030208549 --- /dev/null +++ b/control-plane/roles/monitoring/templates/servicemonitors/zitadel-db.yaml @@ -0,0 +1,16 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + release: kube-prometheus-stack + name: zitadel-db +spec: + selector: + matchLabels: + app: zitadel-db + namespaceSelector: + matchNames: + - {{ metal_control_plane_namespace }} + endpoints: + - port: "metrics" + interval: 60s From a9915b6602337770690c89c41092c65d61ecd925 Mon Sep 17 00:00:00 2001 From: Matthias Hartmann Date: Thu, 23 Apr 2026 13:37:38 +0200 Subject: [PATCH 2/2] feat(alerts): introduce alerts for backup-restore-sidecar --- .../templates/prometheus-stack-values.yaml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/control-plane/roles/monitoring/templates/prometheus-stack-values.yaml b/control-plane/roles/monitoring/templates/prometheus-stack-values.yaml index 0814830f6..e5413a00b 100644 --- a/control-plane/roles/monitoring/templates/prometheus-stack-values.yaml +++ b/control-plane/roles/monitoring/templates/prometheus-stack-values.yaml @@ -478,6 +478,50 @@ additionalPrometheusRulesMap: annotations: summary: "Switch sync is slow on {{ $labels.switchname }}" description: "Average sync duration of {{ $labels.switchname }} in partition {{ $labels.partition }} is exceeding two seconds for more than 5 minutes." + "backup-restore-sidecar.rules": + groups: + - name: backup-restore-sidecar.rules + rules: + - alert: BackupFailed + expr: backup_success == 0 and backup_total_backups > 0 + for: 10m + labels: + severity: critical + annotations: + summary: "Backup failed on {{ $labels.pod }}" + description: "backup-restore-sidecar on {{ $labels.pod }} in namespace {{ $labels.namespace }} had successful backups before, but the last one failed (backup_success == 0) for more than 10 minutes." + - alert: BackupDatabaseNotInitialized + expr: backup_database_initialized == 0 + for: 10m + labels: + severity: critical + annotations: + summary: "Database not initialized for backup on {{ $labels.pod }}" + description: "backup-restore-sidecar on {{ $labels.pod }} in namespace {{ $labels.namespace }} has not yet become initialized for more than 10 minutes. This covers two cases: (1) the sidecar is stuck in its startup probe loop (wrong credentials, wrong port, database not accepting connections), or (2) an ongoing restore from the backup provider is taking unexpectedly long. The metric only transitions from 0 to 1 once per container lifetime, so runtime database outages are instead covered by BackupFailed." + - alert: BackupErrorsIncreasing + expr: increase(backup_errors_total[10m]) > 0 + for: 10m + labels: + severity: warning + annotations: + summary: "Backup errors on {{ $labels.pod }}" + description: "backup-restore-sidecar on {{ $labels.pod }} in namespace {{ $labels.namespace }} is reporting errors for operation {{ $labels.operation }} over the last 10 minutes." + - alert: BackupNoSuccessfulBackup + expr: backup_total_backups == 0 + for: 25h # TODO because the longest backup schedule is currently 24h for headscale-db + labels: + severity: warning + annotations: + summary: "No successful backup on {{ $labels.pod }}" + description: "backup-restore-sidecar on {{ $labels.pod }} in namespace {{ $labels.namespace }} has not completed any successful backup since startup (more than 25 hours). This likely indicates a misconfiguration. The `for:` duration is chosen to tolerate the slowest backup cron schedule in use (currently 24h for headscale-db)." + - alert: BackupSizeZero + expr: backup_size == 0 and backup_total_backups > 0 + for: 10m + labels: + severity: warning + annotations: + summary: "Backup size is zero on {{ $labels.pod }}" + description: "backup-restore-sidecar on {{ $labels.pod }} in namespace {{ $labels.namespace }} reports a successful backup of zero bytes. This may indicate an empty database or a corrupt backup file." {% endraw %} {% if monitoring_gardener_enabled %} {% raw %}