diff --git a/dashboards/dashboards/AGENTS.md b/dashboards/dashboards/AGENTS.md index 201725de8e..ff9128a11f 100644 --- a/dashboards/dashboards/AGENTS.md +++ b/dashboards/dashboards/AGENTS.md @@ -30,7 +30,7 @@ dashboards/dashboards/*.json | `Valkey/` | Valkey/Redis clients, cluster, memory, replication, slowlog | | `Insight/` | Home Dashboard, Advanced Data Exploration, VictoriaMetrics, Exporters | | `Experimental/` | Databases Overview, DB Cluster Summary | -| `PMM Health/` | Environments Overview, PMM Health Overview, HA Health Overview | +| `PMM Health/` | Environments Overview, PMM Health Overview, PMM HA Health Overview | | `Query Analytics/` | QAN panel wrapper (`pmm-qan.json`) | | `Kubernetes (experimental)/` | Kubernetes operator monitoring | diff --git a/dashboards/dashboards/PMM Health/PMM_HA_Health_Overview.json b/dashboards/dashboards/PMM Health/PMM_HA_Health_Overview.json index eb2ad26b96..ab1515288d 100644 --- a/dashboards/dashboards/PMM Health/PMM_HA_Health_Overview.json +++ b/dashboards/dashboards/PMM Health/PMM_HA_Health_Overview.json @@ -15,7 +15,7 @@ } ] }, - "description": "High-level health overview of PMM-HA components: PostgreSQL/Patroni, ClickHouse, VictoriaMetrics, HAProxy", + "description": "GA overview of cluster-level and instance-level health for PMM HA, PostgreSQL/Patroni, ClickHouse, VictoriaMetrics, and HAProxy.", "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -32,59 +32,98 @@ }, "id": 200, "panels": [], - "title": "📊 Overview", + "title": "\ud83d\udcca Overview", "type": "row" }, { "datasource": "Metrics", - "description": "Click row below to see PMM pod details", + "description": "PMM instances alive versus expected. Green means more than 2/3 of expected instances are alive, yellow means between 1/3 and 2/3 are alive (degraded), red means 1/3 or fewer are alive or none is discovered \u2014 matching the health shown on the PMM HA sidebar badge.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "fixedColor": "text", + "mode": "fixed" }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Not Healthy" - } - }, - "type": "value" - }, - { - "options": { - "1": { - "color": "green", - "index": 0, - "text": "Healthy" - } - }, - "type": "value" - } - ], + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "red" - }, - { - "color": "green", - "value": 1 + "color": "green" } ] }, "unit": "none" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Expected" + }, + "properties": [ + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Alive" + }, + "properties": [ + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Health" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 33.34 + }, + { + "color": "green", + "value": 66.67 + } + ] + } + }, + { + "id": "unit", + "value": "percent" + } + ] + } + ] }, "gridPos": { "h": 4, - "w": 4, - "x": 0, + "w": 8, + "x": 8, "y": 1 }, "id": 5, @@ -92,7 +131,7 @@ "colorMode": "background", "graphMode": "none", "justifyMode": "center", - "orientation": "auto", + "orientation": "vertical", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ @@ -102,73 +141,134 @@ "values": false }, "showPercentChange": false, - "textMode": "value", - "wideLayout": true + "textMode": "value_and_name", + "wideLayout": false, + "text": { + "valueSize": 24 + } }, "pluginVersion": "11.6.4", "targets": [ { "datasource": "Metrics", "editorMode": "code", - "expr": "(\n count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${release}-${pmm_suffix_regex}\"}) == 1)\n ==\n count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-${pmm_suffix_regex}\"}))\n and\n count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-${pmm_suffix_regex}\"})) > 0\n) * 0 + 1 or vector(0)", - "legendFormat": "PMM", - "range": true, + "expr": "count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${release}-${pmm_suffix_regex}\"}) == 1)", + "instant": true, + "legendFormat": "Alive", + "range": false, "refId": "A" + }, + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-${pmm_suffix_regex}\"}))", + "instant": true, + "legendFormat": "Expected", + "range": false, + "refId": "B" + }, + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "((count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${release}-${pmm_suffix_regex}\"}) == 1) or vector(0)) / clamp_min((count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-${pmm_suffix_regex}\"})) or vector(0)), 1)) * 100", + "instant": true, + "legendFormat": "Health", + "range": false, + "refId": "C" } ], - "title": "🧭 PMM", + "title": "\ud83e\udded PMM", "type": "stat" }, { "datasource": "Metrics", - "description": "Click row below to see PostgreSQL pod details", + "description": "PostgreSQL instances alive versus expected. Green means more than 2/3 of expected instances are alive, yellow means between 1/3 and 2/3 are alive (degraded), and red means 1/3 or fewer are alive or none is discovered.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "fixedColor": "text", + "mode": "fixed" }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Not Healthy" - } - }, - "type": "value" - }, - { - "options": { - "1": { - "color": "green", - "index": 0, - "text": "Healthy" - } - }, - "type": "value" - } - ], + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "red" - }, - { - "color": "green", - "value": 1 + "color": "green" } ] }, "unit": "none" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Expected" + }, + "properties": [ + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Alive" + }, + "properties": [ + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Health" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 33.34 + }, + { + "color": "green", + "value": 66.67 + } + ] + } + }, + { + "id": "unit", + "value": "percent" + } + ] + } + ] }, "gridPos": { "h": 4, - "w": 4, - "x": 4, + "w": 8, + "x": 16, "y": 1 }, "id": 1, @@ -176,7 +276,7 @@ "colorMode": "background", "graphMode": "none", "justifyMode": "center", - "orientation": "auto", + "orientation": "vertical", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ @@ -186,165 +286,142 @@ "values": false }, "showPercentChange": false, - "textMode": "value", - "wideLayout": true + "textMode": "value_and_name", + "wideLayout": false, + "text": { + "valueSize": 24 + } }, "pluginVersion": "11.6.4", "targets": [ { "datasource": "Metrics", "editorMode": "code", - "expr": "(\n count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${release}-pg-db-.*\", exported_pod!~\".*(${pg_exclude_regex}).*\"}) == 1)\n ==\n count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-pg-db-.*\", exported_pod!~\".*(${pg_exclude_regex}).*\"}))\n and\n count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-pg-db-.*\", exported_pod!~\".*(${pg_exclude_regex}).*\"})) > 0\n) * 0 + 1 or vector(0)", - "legendFormat": "PostgreSQL", - "range": true, + "expr": "count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${release}-pg-db-.*\", exported_pod!~\".*(${pg_exclude_regex}).*\"}) == 1)", + "instant": true, + "legendFormat": "Alive", + "range": false, "refId": "A" + }, + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-pg-db-.*\", exported_pod!~\".*(${pg_exclude_regex}).*\"}))", + "instant": true, + "legendFormat": "Expected", + "range": false, + "refId": "B" + }, + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "((count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${release}-pg-db-.*\", exported_pod!~\".*(${pg_exclude_regex}).*\"}) == 1) or vector(0)) / clamp_min((count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-pg-db-.*\", exported_pod!~\".*(${pg_exclude_regex}).*\"})) or vector(0)), 1)) * 100", + "instant": true, + "legendFormat": "Health", + "range": false, + "refId": "C" } ], - "title": "🐘 PostgreSQL", + "title": "\ud83d\udc18 PostgreSQL", "type": "stat" }, { "datasource": "Metrics", - "description": "Click row below to see ClickHouse pod details", + "description": "ClickHouse instances alive versus expected. Green means more than 2/3 of expected instances are alive, yellow means between 1/3 and 2/3 are alive (degraded), and red means 1/3 or fewer are alive or none is discovered.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "fixedColor": "text", + "mode": "fixed" }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Not Healthy" - } - }, - "type": "value" - }, - { - "options": { - "1": { - "color": "green", - "index": 0, - "text": "Healthy" - } - }, - "type": "value" - } - ], + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "red" - }, - { - "color": "green", - "value": 1 + "color": "green" } ] }, "unit": "none" }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.6.4", - "targets": [ - { - "datasource": "Metrics", - "editorMode": "code", - "expr": "(\n count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${release}-${clickhouse_suffix_regex}\"}) == 1)\n ==\n count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-${clickhouse_suffix_regex}\"}))\n and\n count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-${clickhouse_suffix_regex}\"})) > 0\n) * 0 + 1 or vector(0)", - "legendFormat": "ClickHouse", - "range": true, - "refId": "A" - } - ], - "title": "🖱️ ClickHouse", - "type": "stat" - }, - { - "datasource": "Metrics", - "description": "Click row below to see VictoriaMetrics pod details", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Expected" + }, + "properties": [ + { + "id": "decimals", + "value": 0 + } + ] }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Not Healthy" - } - }, - "type": "value" + { + "matcher": { + "id": "byName", + "options": "Alive" }, - { - "options": { - "1": { - "color": "green", - "index": 0, - "text": "Healthy" + "properties": [ + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Health" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "thresholds" } }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ { - "color": "red" + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 33.34 + }, + { + "color": "green", + "value": 66.67 + } + ] + } }, { - "color": "green", - "value": 1 + "id": "unit", + "value": "percent" } ] - }, - "unit": "none" - }, - "overrides": [] + } + ] }, "gridPos": { "h": 4, - "w": 4, - "x": 12, - "y": 1 + "w": 8, + "x": 0, + "y": 5 }, - "id": 3, + "id": 2, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", - "orientation": "auto", + "orientation": "vertical", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ @@ -354,81 +431,142 @@ "values": false }, "showPercentChange": false, - "textMode": "value", - "wideLayout": true + "textMode": "value_and_name", + "wideLayout": false, + "text": { + "valueSize": 24 + } }, "pluginVersion": "11.6.4", "targets": [ { "datasource": "Metrics", "editorMode": "code", - "expr": "(\n count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${vm_suffix_regex}.*${release}.*\"}) == 1)\n ==\n count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${vm_suffix_regex}.*${release}.*\"}))\n and\n count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${vm_suffix_regex}.*${release}.*\"})) > 0\n) * 0 + 1 or vector(0)", - "legendFormat": "VictoriaMetrics", - "range": true, + "expr": "count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${release}-${clickhouse_suffix_regex}\"}) == 1)", + "instant": true, + "legendFormat": "Alive", + "range": false, "refId": "A" + }, + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-${clickhouse_suffix_regex}\"}))", + "instant": true, + "legendFormat": "Expected", + "range": false, + "refId": "B" + }, + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "((count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${release}-${clickhouse_suffix_regex}\"}) == 1) or vector(0)) / clamp_min((count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-${clickhouse_suffix_regex}\"})) or vector(0)), 1)) * 100", + "instant": true, + "legendFormat": "Health", + "range": false, + "refId": "C" } ], - "title": "📊 VictoriaMetrics", + "title": "\ud83d\uddb1\ufe0f ClickHouse", "type": "stat" }, { "datasource": "Metrics", - "description": "Click row below to see HAProxy pod details", + "description": "VictoriaMetrics instances alive versus expected. Green means more than 2/3 of expected instances are alive, yellow means between 1/3 and 2/3 are alive (degraded), and red means 1/3 or fewer are alive or none is discovered.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "fixedColor": "text", + "mode": "fixed" }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Not Healthy" - } - }, - "type": "value" - }, - { - "options": { - "1": { - "color": "green", - "index": 0, - "text": "Healthy" - } - }, - "type": "value" - } - ], + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "red" - }, - { - "color": "green", - "value": 1 + "color": "green" } ] }, "unit": "none" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Expected" + }, + "properties": [ + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Alive" + }, + "properties": [ + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Health" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 33.34 + }, + { + "color": "green", + "value": 66.67 + } + ] + } + }, + { + "id": "unit", + "value": "percent" + } + ] + } + ] }, "gridPos": { "h": 4, - "w": 4, - "x": 16, - "y": 1 + "w": 8, + "x": 8, + "y": 5 }, - "id": 4, + "id": 3, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", - "orientation": "auto", + "orientation": "vertical", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ @@ -438,97 +576,53 @@ "values": false }, "showPercentChange": false, - "textMode": "value", - "wideLayout": true + "textMode": "value_and_name", + "wideLayout": false, + "text": { + "valueSize": 24 + } }, "pluginVersion": "11.6.4", "targets": [ { "datasource": "Metrics", "editorMode": "code", - "expr": "(\n count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"$haproxy_pod_regex\"}) == 1)\n ==\n count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"$haproxy_pod_regex\"}))\n and\n count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"$haproxy_pod_regex\"})) > 0\n) * 0 + 1 or vector(0)", - "legendFormat": "HAProxy", - "range": true, + "expr": "count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${vm_suffix_regex}.*${release}.*\"}) == 1)", + "instant": true, + "legendFormat": "Alive", + "range": false, "refId": "A" - } - ], - "title": "⚖️ HAProxy", - "type": "stat" - }, - { - "datasource": "Metrics", - "description": "Overall health percentage of all PMM HA pods", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "yellow", - "value": 80 - }, - { - "color": "green", - "value": 95 - } - ] - }, - "unit": "percent" }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 5 - }, - "id": 201, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${vm_suffix_regex}.*${release}.*\"}))", + "instant": true, + "legendFormat": "Expected", + "range": false, + "refId": "B" }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.6.4", - "targets": [ { "datasource": "Metrics", "editorMode": "code", - "expr": "(\n count(kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod!~\".*($health_exclude_regex).*\"} == 1)\n /\n count(kube_pod_info{exported_namespace=\"$namespace\", exported_pod!~\".*($health_exclude_regex).*\"})\n) * 100", + "expr": "((count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${vm_suffix_regex}.*${release}.*\"}) == 1) or vector(0)) / clamp_min((count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${vm_suffix_regex}.*${release}.*\"})) or vector(0)), 1)) * 100", + "instant": true, "legendFormat": "Health", - "range": true, - "refId": "A" + "range": false, + "refId": "C" } ], - "title": "🎯 Overall System Health", - "type": "gauge" + "title": "\ud83d\udcca VictoriaMetrics", + "type": "stat" }, { "datasource": "Metrics", - "description": "Number of running pods vs total pods", + "description": "HAProxy instances alive versus expected. Green means more than 2/3 of expected instances are alive, yellow means between 1/3 and 2/3 are alive (degraded), and red means 1/3 or fewer are alive or none is discovered.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "fixedColor": "text", + "mode": "fixed" }, "mappings": [], "thresholds": { @@ -541,20 +635,231 @@ }, "unit": "none" }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Expected" + }, + "properties": [ + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Alive" + }, + "properties": [ + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Health" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 33.34 + }, + { + "color": "green", + "value": 66.67 + } + ] + } + }, + { + "id": "unit", + "value": "percent" + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, "y": 5 }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "vertical", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": false, + "text": { + "valueSize": 24 + } + }, + "pluginVersion": "11.6.4", + "targets": [ + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"$haproxy_pod_regex\"}) == 1)", + "instant": true, + "legendFormat": "Alive", + "range": false, + "refId": "A" + }, + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"$haproxy_pod_regex\"}))", + "instant": true, + "legendFormat": "Expected", + "range": false, + "refId": "B" + }, + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "((count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"$haproxy_pod_regex\"}) == 1) or vector(0)) / clamp_min((count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"$haproxy_pod_regex\"})) or vector(0)), 1)) * 100", + "instant": true, + "legendFormat": "Health", + "range": false, + "refId": "C" + } + ], + "title": "\u2696\ufe0f HAProxy", + "type": "stat" + }, + { + "datasource": "Metrics", + "description": "Overall health percentage of all PMM HA pods", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "green", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 201, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.6.4", + "targets": [ + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "(\n count(kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod!~\".*($health_exclude_regex).*\"} == 1)\n /\n count(kube_pod_info{exported_namespace=\"$namespace\", exported_pod!~\".*($health_exclude_regex).*\"})\n) * 100", + "legendFormat": "Health", + "range": true, + "refId": "A" + } + ], + "title": "\ud83c\udfaf Overall System Health", + "type": "gauge" + }, + { + "datasource": "Metrics", + "description": "Number of running pods vs total pods", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, "id": 202, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", - "orientation": "auto", + "orientation": "vertical", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ @@ -564,8 +869,12 @@ "values": false }, "showPercentChange": false, - "textMode": "value_and_name", - "wideLayout": true + "textMode": "auto", + "wideLayout": true, + "text": { + "titleSize": 11, + "valueSize": 30 + } }, "pluginVersion": "11.6.4", "targets": [ @@ -586,7 +895,7 @@ "refId": "B" } ], - "title": "🔢 Pod Count", + "title": "\ud83d\udd22 Pod Count", "type": "stat" }, { @@ -619,17 +928,17 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 5, "w": 6, - "x": 12, - "y": 5 + "x": 0, + "y": 9 }, "id": 203, "options": { "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", + "graphMode": "none", + "justifyMode": "center", + "orientation": "vertical", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ @@ -640,20 +949,23 @@ }, "showPercentChange": false, "textMode": "value_and_name", - "wideLayout": true + "wideLayout": false, + "text": { + "valueSize": 24 + } }, "pluginVersion": "11.6.4", "targets": [ { "datasource": "Metrics", "editorMode": "code", - "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=\"$namespace\", pod!~\".*($health_exclude_regex).*\"}[24h]))", - "legendFormat": "Restarts (24h)", + "expr": "sum(increase(kube_pod_container_status_restarts_total{exported_namespace=\"$namespace\", exported_pod!~\".*($health_exclude_regex).*\"}[24h]))", + "legendFormat": "Restarts", "range": true, "refId": "A" } ], - "title": "🔄 Pod Restarts", + "title": "\ud83d\udd04 Pod Restarts", "type": "stat" }, { @@ -711,10 +1023,10 @@ ] }, "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 5 + "h": 10, + "w": 24, + "x": 0, + "y": 14 }, "id": 204, "options": { @@ -735,7 +1047,7 @@ "datasource": "Metrics", "editorMode": "code", "exemplar": false, - "expr": "sum by(pod) (increase(kube_pod_container_status_restarts_total{namespace=\"$namespace\", pod!~\".*($health_exclude_regex).*\"}[24h])) > 0", + "expr": "sum by(exported_pod) (increase(kube_pod_container_status_restarts_total{exported_namespace=\"$namespace\", exported_pod!~\".*($health_exclude_regex).*\"}[24h])) > 0", "format": "table", "instant": true, "legendFormat": "__auto", @@ -743,7 +1055,7 @@ "refId": "A" } ], - "title": "⚠️ Pods with Restarts", + "title": "\u26a0\ufe0f Pods with Restarts", "transformations": [ { "id": "organize", @@ -753,77 +1065,318 @@ }, "indexByName": { "Value": 1, - "pod": 0 - }, - "renameByName": { - "Value": "Restarts (24h)", - "pod": "Pod" + "exported_pod": 0 } } } ], "type": "table" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 205, - "panels": [], - "title": "💻 Resource Usage", - "type": "row" - }, { "datasource": "Metrics", - "description": "CPU usage by pod for selected namespace", + "description": "Current PMM Raft leader. Shows the active instance name, or No active instance when no leader is visible.", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "cpu", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 213, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true, + "text": { + "valueSize": 24 + } + }, + "pluginVersion": "11.6.4", + "targets": [ + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "(\n max by(node_id) (\n (pmm_ha_leader_status == 1)\n and on(node_id)\n label_replace(\n kube_pod_info{\n exported_namespace=\"$namespace\",\n exported_pod=~\"${release}-${pmm_suffix_regex}\"\n },\n \"node_id\", \"$1\", \"exported_pod\", \"(.*)\"\n )\n )\n)\nor (\n label_replace(vector(0), \"node_id\", \"No active instance\", \"\", \"\")\n unless on() (\n max(\n (pmm_ha_leader_status == 1)\n and on(node_id)\n label_replace(\n kube_pod_info{\n exported_namespace=\"$namespace\",\n exported_pod=~\"${release}-${pmm_suffix_regex}\"\n },\n \"node_id\", \"$1\", \"exported_pod\", \"(.*)\"\n )\n ) == 1\n )\n)", + "instant": true, + "legendFormat": "{{node_id}}", + "range": false, + "refId": "A" + } + ], + "title": "Active PMM Instance", + "type": "stat" + }, + { + "datasource": "Metrics", + "description": "Current Patroni primary instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "No active instance", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 9 + }, + "id": 214, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true, + "text": { + "valueSize": 24 + } + }, + "pluginVersion": "11.6.4", + "targets": [ + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "(\n max by(name) (\n (patroni_primary == 1)\n and on(name)\n label_replace(\n kube_pod_info{\n exported_namespace=\"$namespace\",\n exported_pod=~\"${release}-pg-db-.*\",\n exported_pod!~\".*(${pg_exclude_regex}).*\"\n },\n \"name\", \"$1\", \"exported_pod\", \"(.*)\"\n )\n )\n)\nor (\n label_replace(vector(0), \"name\", \"No active instance\", \"\", \"\")\n unless on() (\n max(\n (patroni_primary == 1)\n and on(name)\n label_replace(\n kube_pod_info{\n exported_namespace=\"$namespace\",\n exported_pod=~\"${release}-pg-db-.*\",\n exported_pod!~\".*(${pg_exclude_regex}).*\"\n },\n \"name\", \"$1\", \"exported_pod\", \"(.*)\"\n )\n ) == 1\n )\n)", + "instant": true, + "legendFormat": "{{name}}", + "range": false, + "refId": "A" + } + ], + "title": "PostgreSQL Primary", + "type": "stat" + }, + { + "datasource": "Metrics", + "description": "Health percentage for vmauth, vminsert, vmselect, vmstorage, and vmagent.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 100 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 215, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true, + "text": { + "valueSize": 24 + } + }, + "pluginVersion": "11.6.4", + "targets": [ + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "((count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"vmauth.*${release}.*\"}) == 1)) / clamp_min((count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"vmauth.*${release}.*\"}))), 1)) * 100", + "instant": true, + "legendFormat": "vmauth", + "range": false, + "refId": "A" + }, + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "((count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"vminsert.*${release}.*\"}) == 1)) / clamp_min((count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"vminsert.*${release}.*\"}))), 1)) * 100", + "instant": true, + "legendFormat": "vminsert", + "range": false, + "refId": "B" + }, + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "((count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"vmselect.*${release}.*\"}) == 1)) / clamp_min((count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"vmselect.*${release}.*\"}))), 1)) * 100", + "instant": true, + "legendFormat": "vmselect", + "range": false, + "refId": "C" + }, + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "((count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"vmstorage.*${release}.*\"}) == 1)) / clamp_min((count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"vmstorage.*${release}.*\"}))), 1)) * 100", + "instant": true, + "legendFormat": "vmstorage", + "range": false, + "refId": "D" + }, + { + "datasource": "Metrics", + "editorMode": "code", + "expr": "((count(max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"vmagent.*${release}.*\"}) == 1)) / clamp_min((count(max by(exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"vmagent.*${release}.*\"}))), 1)) * 100", + "instant": true, + "legendFormat": "vmagent", + "range": false, + "refId": "E" + } + ], + "title": "VictoriaMetrics Components", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 205, + "panels": [], + "title": "\ud83d\udcbb Resource Usage", + "type": "row" + }, + { + "datasource": "Metrics", + "description": "CPU usage by pod for selected namespace", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "cpu", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, { "color": "red", @@ -839,7 +1392,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 33 }, "id": 6, "options": { @@ -870,7 +1423,7 @@ "refId": "A" } ], - "title": "🧮 CPU Usage", + "title": "\ud83e\uddee CPU Usage", "type": "timeseries" }, { @@ -935,7 +1488,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 33 }, "id": 206, "options": { @@ -966,7 +1519,7 @@ "refId": "A" } ], - "title": "💾 Memory Usage", + "title": "\ud83d\udcbe Memory Usage", "type": "timeseries" }, { @@ -975,11 +1528,11 @@ "h": 1, "w": 24, "x": 0, - "y": 20 + "y": 41 }, "id": 207, "panels": [], - "title": "💿 Storage Usage", + "title": "\ud83d\udcbf Storage Usage", "type": "row" }, { @@ -1017,7 +1570,7 @@ "h": 6, "w": 8, "x": 0, - "y": 21 + "y": 42 }, "id": 208, "options": { @@ -1046,7 +1599,7 @@ "refId": "A" } ], - "title": "🐘 PostgreSQL Storage", + "title": "\ud83d\udc18 PostgreSQL Storage", "type": "gauge" }, { @@ -1084,7 +1637,7 @@ "h": 6, "w": 8, "x": 8, - "y": 21 + "y": 42 }, "id": 209, "options": { @@ -1113,7 +1666,7 @@ "refId": "A" } ], - "title": "🖱️ ClickHouse Storage", + "title": "\ud83d\uddb1\ufe0f ClickHouse Storage", "type": "gauge" }, { @@ -1151,7 +1704,7 @@ "h": 6, "w": 8, "x": 16, - "y": 21 + "y": 42 }, "id": 210, "options": { @@ -1180,7 +1733,7 @@ "refId": "A" } ], - "title": "📊 VictoriaMetrics Storage", + "title": "\ud83d\udcca VictoriaMetrics Storage", "type": "gauge" }, { @@ -1189,11 +1742,11 @@ "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 48 }, "id": 211, "panels": [], - "title": "📈 Service Availability", + "title": "\ud83d\udcc8 Service Availability", "type": "row" }, { @@ -1202,19 +1755,40 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", "axisPlacement": "auto", - "fillOpacity": 70, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, - "lineWidth": 0, - "spanNulls": false + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "max": 100, @@ -1240,7 +1814,8 @@ } ] }, - "unit": "percent" + "unit": "percent", + "noValue": "No data" }, "overrides": [] }, @@ -1248,22 +1823,22 @@ "h": 8, "w": 24, "x": 0, - "y": 28 + "y": 49 }, "id": 212, "options": { - "alignValue": "left", "legend": { - "displayMode": "list", + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "auto", "tooltip": { "hideZeros": false, - "mode": "single", + "mode": "multi", "sort": "none" } }, @@ -1273,185 +1848,50 @@ "datasource": "Metrics", "editorMode": "code", "expr": "(\n count(kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${release}-${pmm_suffix_regex}\", exported_pod!~\".*($health_exclude_regex).*\"} == 1)\n /\n count(kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-${pmm_suffix_regex}\", exported_pod!~\".*($health_exclude_regex).*\"})\n) * 100", - "legendFormat": "🧭 PMM", + "legendFormat": "\ud83e\udded PMM", "range": true, - "refId": "PMM" + "refId": "PMM", + "instant": false }, { "datasource": "Metrics", "editorMode": "code", "expr": "(\n count(kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${release}-pg-db-.*\", exported_pod!~\".*(${pg_exclude_regex}|$health_exclude_regex).*\"} == 1)\n /\n count(kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-pg-db-.*\", exported_pod!~\".*(${pg_exclude_regex}|$health_exclude_regex).*\"})\n) * 100", - "legendFormat": "🐘 PostgreSQL", + "legendFormat": "\ud83d\udc18 PostgreSQL", "range": true, - "refId": "PostgreSQL" + "refId": "PostgreSQL", + "instant": false }, { "datasource": "Metrics", "editorMode": "code", "expr": "(\n count(kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${release}-${clickhouse_suffix_regex}\", exported_pod!~\".*($health_exclude_regex).*\"} == 1)\n /\n count(kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-${clickhouse_suffix_regex}\", exported_pod!~\".*($health_exclude_regex).*\"})\n) * 100", - "legendFormat": "🖱️ ClickHouse", + "legendFormat": "\ud83d\uddb1\ufe0f ClickHouse", "range": true, - "refId": "ClickHouse" + "refId": "ClickHouse", + "instant": false }, { "datasource": "Metrics", "editorMode": "code", "expr": "(\n count(kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${vm_suffix_regex}.*${release}.*\", exported_pod!~\".*($health_exclude_regex).*\"} == 1)\n /\n count(kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${vm_suffix_regex}.*${release}.*\", exported_pod!~\".*($health_exclude_regex).*\"})\n) * 100", - "legendFormat": "📊 VictoriaMetrics", + "legendFormat": "\ud83d\udcca VictoriaMetrics", "range": true, - "refId": "VictoriaMetrics" + "refId": "VictoriaMetrics", + "instant": false }, { "datasource": "Metrics", "editorMode": "code", "expr": "(\n count(kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"$haproxy_pod_regex\", exported_pod!~\".*($health_exclude_regex).*\"} == 1)\n /\n count(kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"$haproxy_pod_regex\", exported_pod!~\".*($health_exclude_regex).*\"})\n) * 100", - "legendFormat": "⚖️ HAProxy", + "legendFormat": "\u2696\ufe0f HAProxy", "range": true, - "refId": "HAProxy" - } - ], - "title": "⏱️ Service Availability by Component", - "type": "state-timeline" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 105, - "panels": [ - { - "datasource": "Metrics", - "description": "Status of each PMM server pod (Running = UP).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "color-background" - }, - "inspect": false - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "DOWN" - } - }, - "type": "value" - }, - { - "options": { - "1": { - "color": "green", - "index": 0, - "text": "UP" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Pod" - }, - "properties": [ - { - "id": "custom.width", - "value": 400 - }, - { - "id": "custom.cellOptions", - "value": { - "type": "auto" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 37 - }, - "id": 50, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "11.6.4", - "targets": [ - { - "datasource": "Metrics", - "editorMode": "code", - "exemplar": false, - "expr": "max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_pod=~\"${release}-${pmm_suffix_regex}\", exported_namespace=\"$namespace\"})", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "Status" - } - ], - "title": "PMM Pods", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "service_name": true - }, - "indexByName": { - "Value": 1, - "exported_pod": 0 - }, - "renameByName": { - "Value": "Status", - "exported_pod": "Pod" - } - } - } - ], - "type": "table" + "refId": "HAProxy", + "instant": false } ], - "title": "🧭 PMM Pod's Status", - "type": "row" + "title": "\u23f1\ufe0f Service Availability by Component", + "type": "timeseries" }, { "collapsed": false, @@ -1459,16 +1899,16 @@ "h": 1, "w": 24, "x": 0, - "y": 37 + "y": 57 }, - "id": 101, + "id": 105, "panels": [], - "title": "🐘 PostgreSQL Pod's Status", + "title": "\ud83e\udded PMM Pod's Status", "type": "row" }, { "datasource": "Metrics", - "description": "Status of each PostgreSQL pod based on Kubernetes phase (Running = UP).", + "description": "Status and Raft role of every PMM server instance.", "fieldConfig": { "defaults": { "color": { @@ -1520,13 +1960,21 @@ { "matcher": { "id": "byName", - "options": "Pod" + "options": "Status" }, "properties": [ { "id": "custom.width", - "value": 400 - }, + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Instance" + }, + "properties": [ { "id": "custom.cellOptions", "value": { @@ -1543,13 +1991,7 @@ "properties": [ { "id": "custom.width", - "value": 120 - }, - { - "id": "custom.cellOptions", - "value": { - "type": "auto" - } + "value": 150 }, { "id": "mappings", @@ -1557,17 +1999,14 @@ { "options": { "0": { + "color": "yellow", "index": 1, - "text": "Replica" - } - }, - "type": "value" - }, - { - "options": { + "text": "Follower" + }, "1": { + "color": "green", "index": 0, - "text": "Primary" + "text": "Active" } }, "type": "value" @@ -1579,12 +2018,12 @@ ] }, "gridPos": { - "h": 6, + "h": 10, "w": 24, "x": 0, - "y": 38 + "y": 58 }, - "id": 10, + "id": 50, "options": { "cellHeight": "sm", "footer": { @@ -1603,26 +2042,26 @@ "datasource": "Metrics", "editorMode": "code", "exemplar": false, - "expr": "max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_pod=~\"${release}-pg-db-.*\", exported_pod!~\".*(${pg_exclude_regex}).*\", exported_namespace=\"$namespace\"})", + "expr": "max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_pod=~\"${release}-${pmm_suffix_regex}\", exported_namespace=\"$namespace\"})", "format": "table", - "instant": false, + "instant": true, "legendFormat": "__auto", - "range": true, + "range": false, "refId": "Status" }, { "datasource": "Metrics", "editorMode": "code", "exemplar": false, - "expr": "label_replace(max by(name) (patroni_primary), \"exported_pod\", \"$1\", \"name\", \"(.*)\")", + "expr": "label_replace(\n max by(node_id) (\n pmm_ha_leader_status\n and on(node_id)\n label_replace(\n kube_pod_info{\n exported_namespace=\"$namespace\",\n exported_pod=~\"${release}-${pmm_suffix_regex}\"\n },\n \"node_id\", \"$1\", \"exported_pod\", \"(.*)\"\n )\n ),\n \"exported_pod\", \"$1\", \"node_id\", \"(.*)\"\n)", "format": "table", - "instant": false, + "instant": true, "legendFormat": "__auto", - "range": true, - "refId": "Primary" + "range": false, + "refId": "Leader" } ], - "title": "PostgreSQL Pods", + "title": "PMM Pods", "transformations": [ { "id": "seriesToColumns", @@ -1635,19 +2074,18 @@ "options": { "excludeByName": { "Time": true, - "name": true, + "node_id": true, "service_name": true }, - "includeByName": {}, "indexByName": { - "Value #Primary": 2, + "Value #Leader": 2, "Value #Status": 1, "exported_pod": 0 }, "renameByName": { - "Value #Primary": "Role", + "Value #Leader": "Role", "Value #Status": "Status", - "exported_pod": "Pod" + "exported_pod": "Instance" } } } @@ -1655,663 +2093,1007 @@ "type": "table" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 44 + "y": 68 }, - "id": 102, - "panels": [ - { - "datasource": "Metrics", - "description": "Status of each ClickHouse pod with leader/follower role.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "id": 101, + "panels": [], + "title": "\ud83d\udc18 PostgreSQL Pod's Status", + "type": "row" + }, + { + "datasource": "Metrics", + "description": "Status, primary/replica role, and replication streaming health for every PostgreSQL instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "color-background" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "DOWN" + } }, - "custom": { - "align": "center", - "cellOptions": { - "type": "color-background" - }, - "inspect": false + "type": "value" + }, + { + "options": { + "1": { + "color": "green", + "index": 0, + "text": "UP" + } }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "DOWN" - } - }, - "type": "value" - }, - { - "options": { - "1": { - "color": "green", - "index": 0, - "text": "UP" - } - }, - "type": "value" + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pod" + }, + "properties": [ + { + "id": "custom.width", + "value": 400 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "auto" } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Role" }, - "overrides": [ + "properties": [ { - "matcher": { - "id": "byName", - "options": "Pod" - }, - "properties": [ - { - "id": "custom.width", - "value": 400 - }, - { - "id": "custom.cellOptions", - "value": { - "type": "auto" - } - } - ] + "id": "custom.width", + "value": 150 }, { - "matcher": { - "id": "byName", - "options": "Role" - }, - "properties": [ - { - "id": "custom.width", - "value": 120 - }, + "id": "custom.cellOptions", + "value": { + "type": "color-background", + "mode": "basic" + } + }, + { + "id": "mappings", + "value": [ { - "id": "custom.cellOptions", - "value": { - "type": "auto" + "type": "value", + "options": { + "0": { + "color": "yellow", + "index": 1, + "text": "Replica" + }, + "1": { + "color": "green", + "index": 0, + "text": "Primary" + } } }, { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 1, - "text": "Follower" - } - }, - "type": "value" - }, - { - "options": { - "1": { - "index": 0, - "text": "Leader" - } - }, - "type": "value" + "type": "value", + "options": { + "-1": { + "text": "No data", + "color": "text", + "index": 99 } - ] + } } ] } ] }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 84 - }, - "id": 20, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": "Metrics", - "editorMode": "code", - "exemplar": false, - "expr": "max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${release}-${clickhouse_suffix_regex}\"})", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "Status" - }, - { - "datasource": "Metrics", - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(max by(instance) (ClickHouseAsyncMetrics_KeeperIsLeader), \"pod_ip\", \"$1\", \"instance\", \"([0-9.]+):.*\") * on(pod_ip) group_left(exported_pod) max by(pod_ip, exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-${clickhouse_suffix_regex}\"})", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "Leader" - } - ], - "title": "ClickHouse Pods", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "exported_pod" - } + { + "matcher": { + "id": "byName", + "options": "Replication" }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "instance": true, - "pod_ip": true - }, - "indexByName": { - "Value #Leader": 2, - "Value #Status": 1, - "exported_pod": 0 - }, - "renameByName": { - "Value #Leader": "Role", - "Value #Status": "Status", - "exported_pod": "Pod" - } + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "yellow", + "index": 1, + "text": "Not streaming" + }, + "1": { + "color": "green", + "index": 0, + "text": "Healthy" + } + }, + "type": "value" + }, + { + "type": "value", + "options": { + "-1": { + "text": "No data", + "color": "text", + "index": 99 + } + } + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 10, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.6.4", + "targets": [ + { + "datasource": "Metrics", + "editorMode": "code", + "exemplar": false, + "expr": "max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_pod=~\"${release}-pg-db-.*\", exported_pod!~\".*(${pg_exclude_regex}).*\", exported_namespace=\"$namespace\"})", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "Status" + }, + { + "datasource": "Metrics", + "editorMode": "code", + "exemplar": false, + "expr": "max by(exported_pod) (\n (\n (\n label_replace(max by(name) (patroni_primary), \"exported_pod\", \"$1\", \"name\", \"(.*)\")\n or\n label_replace(\n max by(node_name) (1 - clamp_max(pg_replication_is_replica, 1)),\n \"exported_pod\", \"$1\", \"node_name\", \".*(${release}-pg-db-[a-zA-Z0-9-]+)\"\n )\n )\n and on(exported_pod)\n max by(exported_pod) (\n kube_pod_info{\n exported_namespace=\"$namespace\",\n exported_pod=~\"${release}-pg-db-.*\",\n exported_pod!~\".*(${pg_exclude_regex}).*\"\n }\n )\n )\n or\n (max by(exported_pod) (\n kube_pod_info{\n exported_namespace=\"$namespace\",\n exported_pod=~\"${release}-pg-db-.*\",\n exported_pod!~\".*(${pg_exclude_regex}).*\"\n }\n ) * 0 - 1)\n)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "Primary" + }, + { + "datasource": "Metrics", + "editorMode": "code", + "exemplar": false, + "expr": "max by(exported_pod) (\n (\n (\n label_replace(\n max by(name) (patroni_postgres_streaming) + on(name) max by(name) (patroni_primary),\n \"exported_pod\", \"$1\", \"name\", \"(.*)\"\n )\n or\n label_replace(\n max by(node_name) (\n (1 - clamp_max(pg_replication_is_replica, 1))\n +\n (clamp_max(pg_replication_is_replica, 1) * (pg_replication_lag_seconds < bool 60))\n ),\n \"exported_pod\", \"$1\", \"node_name\", \".*(${release}-pg-db-[a-zA-Z0-9-]+)\"\n )\n )\n and on(exported_pod)\n max by(exported_pod) (\n kube_pod_info{\n exported_namespace=\"$namespace\",\n exported_pod=~\"${release}-pg-db-.*\",\n exported_pod!~\".*(${pg_exclude_regex}).*\"\n }\n )\n )\n or\n (max by(exported_pod) (\n kube_pod_info{\n exported_namespace=\"$namespace\",\n exported_pod=~\"${release}-pg-db-.*\",\n exported_pod!~\".*(${pg_exclude_regex}).*\"\n }\n ) * 0 - 1)\n)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "Replication" + } + ], + "title": "PostgreSQL Pods", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "exported_pod" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "name": true, + "service_name": true + }, + "includeByName": {}, + "indexByName": { + "Value #Primary": 2, + "Value #Replication": 3, + "Value #Status": 1, + "exported_pod": 0 + }, + "renameByName": { + "Value #Primary": "Role", + "Value #Replication": "Replication", + "Value #Status": "Status", + "exported_pod": "Pod" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 102, + "panels": [], + "title": "\ud83d\uddb1\ufe0f ClickHouse Pod's Status", + "type": "row" + }, + { + "datasource": "Metrics", + "description": "Health of all ClickHouse database and Keeper nodes. Keeper nodes additionally show leader/follower role.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "color-background" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "DOWN" + } + }, + "type": "value" + }, + { + "options": { + "1": { + "color": "green", + "index": 0, + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pod" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Role" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background", + "mode": "basic" + } + }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "0": { + "color": "yellow", + "index": 1, + "text": "Follower" + }, + "1": { + "color": "green", + "index": 0, + "text": "Leader" + }, + "2": { + "color": "yellow", + "index": 2, + "text": "Replica" + } + } + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Component" + }, + "properties": [ + { + "id": "custom.width", + "value": 160 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 80 + }, + "id": 20, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "Metrics", + "editorMode": "code", + "exemplar": false, + "expr": "max by(exported_pod, component) (label_replace(kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${release}-${clickhouse_suffix_regex}\"}, \"component\", \"$1\", \"exported_pod\", \".*(pmmdb|keeper).*\"))", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "Status" + }, + { + "datasource": "Metrics", + "editorMode": "code", + "exemplar": false, + "expr": "max by(exported_pod) (\n (\n label_replace(max by(instance) (ClickHouseAsyncMetrics_KeeperIsLeader), \"pod_ip\", \"$1\", \"instance\", \"([0-9.]+):.*\")\n * on(pod_ip) group_left(exported_pod)\n max by(pod_ip, exported_pod) (kube_pod_info{exported_namespace=\"$namespace\", exported_pod=~\"${release}-${clickhouse_suffix_regex}\"})\n )\n or\n (\n max by(exported_pod) (\n (kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${release}-pmmdb.*\"} == 1) * 2\n )\n )\n)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "Leader" + } + ], + "title": "ClickHouse Pods", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "exported_pod" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "instance": true, + "pod_ip": true + }, + "indexByName": { + "Value #Leader": 3, + "Value #Status": 2, + "component": 1, + "exported_pod": 0 + }, + "renameByName": { + "Value #Leader": "Role", + "Value #Status": "Status", + "component": "Component", + "exported_pod": "Pod" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 90 + }, + "id": 103, + "panels": [], + "title": "\ud83d\udcca VictoriaMetrics Pod's Status", + "type": "row" + }, + { + "datasource": "Metrics", + "description": "Status of each VictoriaMetrics pod (Kubernetes phase Running).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "color-background" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "DOWN" + } + }, + "type": "value" + }, + { + "options": { + "1": { + "color": "green", + "index": 0, + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Pod" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Component" + }, + "properties": [ + { + "id": "custom.width", + "value": 500 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } + }, + { + "id": "mappings", + "value": [] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 91 + }, + "id": 30, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.6.4", + "targets": [ + { + "datasource": "Metrics", + "editorMode": "code", + "exemplar": false, + "expr": "max by(exported_pod, component) (label_replace(kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${vm_suffix_regex}.*${release}.*\"}, \"component\", \"$1\", \"exported_pod\", \".*(vmauth|vminsert|vmselect|vmstorage|vmagent).*\") )", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "Status" + } + ], + "title": "VictoriaMetrics Pods", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { + "Value": 2, + "component": 1, + "exported_pod": 0 + }, + "renameByName": { + "Value": "Status", + "component": "Component", + "exported_pod": "Pod" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Component" } + ] + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 101 + }, + "id": 104, + "panels": [], + "title": "\u2696\ufe0f HAProxy Pod's Status", + "type": "row" + }, + { + "datasource": "Metrics", + "description": "Status of each HAProxy pod (Running = UP).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "color-background" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "DOWN" + } + }, + "type": "value" + }, + { + "options": { + "1": { + "color": "green", + "index": 0, + "text": "UP" + } + }, + "type": "value" } ], - "type": "table" + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Pod" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 102 + }, + "id": 40, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.6.4", + "targets": [ + { + "datasource": "Metrics", + "editorMode": "code", + "exemplar": false, + "expr": "max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_pod=~\"$haproxy_pod_regex\", exported_namespace=\"$namespace\"})", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "Status" + } + ], + "title": "HAProxy Instances", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { + "Value": 1, + "exported_pod": 0 + }, + "renameByName": { + "Value": "Status", + "exported_pod": "Pod" + } + } } ], - "title": "🖱️ ClickHouse Pod's Status", - "type": "row" + "type": "table" }, { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 45 - }, - "id": 103, - "panels": [ - { - "datasource": "Metrics", - "description": "Status of each VictoriaMetrics pod (Kubernetes phase Running).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "color-background" - }, - "inspect": false - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "DOWN" - } - }, - "type": "value" - }, - { - "options": { - "1": { - "color": "green", - "index": 0, - "text": "UP" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Pod" - }, - "properties": [ - { - "id": "custom.width", - "value": 400 - }, - { - "id": "custom.cellOptions", - "value": { - "type": "auto" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Component" - }, - "properties": [ - { - "id": "custom.width", - "value": 100 - }, - { - "id": "custom.cellOptions", - "value": { - "type": "auto" - } - }, - { - "id": "mappings", - "value": [] - } - ] - } - ] + "datasource": "Metrics", + "description": "UP/DOWN status of every HAProxy backend server.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 50 + "custom": { + "align": "center", + "cellOptions": { + "type": "color-background" + }, + "inspect": false }, - "id": 30, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "11.6.4", - "targets": [ - { - "datasource": "Metrics", - "editorMode": "code", - "exemplar": false, - "expr": "max by(exported_pod, component) (label_replace(kube_pod_status_phase{phase=\"Running\", exported_namespace=\"$namespace\", exported_pod=~\"${vm_suffix_regex}.*${release}.*\"}, \"component\", \"$1\", \"exported_pod\", \".*(${vm_suffix_regex}).*\") )", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "Status" - } - ], - "title": "VictoriaMetrics Pods", - "transformations": [ + "mappings": [ { - "id": "organize", "options": { - "excludeByName": { - "Time": true - }, - "indexByName": { - "Value": 2, - "component": 1, - "exported_pod": 0 - }, - "renameByName": { - "Value": "Status", - "component": "Component", - "exported_pod": "Pod" + "0": { + "color": "red", + "index": 1, + "text": "DOWN" } - } + }, + "type": "value" }, { - "id": "sortBy", "options": { - "fields": {}, - "sort": [ - { - "field": "Component" - } - ] - } + "1": { + "color": "green", + "index": 0, + "text": "UP" + } + }, + "type": "value" } ], - "type": "table" - } - ], - "title": "📊 VictoriaMetrics Pod's Status", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 46 - }, - "id": 104, - "panels": [ - { - "datasource": "Metrics", - "description": "Status of each HAProxy pod (Running = UP).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "color-background" - }, - "inspect": false + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "DOWN" - } - }, - "type": "value" - }, - { - "options": { - "1": { - "color": "green", - "index": 0, - "text": "UP" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] + { + "color": "green", + "value": 1 } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Service" }, - "overrides": [ + "properties": [ { - "matcher": { - "id": "byName", - "options": "Pod" - }, - "properties": [ - { - "id": "custom.width", - "value": 400 - }, - { - "id": "custom.cellOptions", - "value": { - "type": "auto" - } - } - ] + "id": "custom.width", + "value": 180 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } } ] }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 40, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "11.6.4", - "targets": [ - { - "datasource": "Metrics", - "editorMode": "code", - "exemplar": false, - "expr": "max by(exported_pod) (kube_pod_status_phase{phase=\"Running\", exported_pod=~\"$haproxy_pod_regex\", exported_namespace=\"$namespace\"})", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "Status" - } - ], - "title": "HAProxy Instances", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true - }, - "indexByName": { - "Value": 1, - "exported_pod": 0 - }, - "renameByName": { - "Value": "Status", - "exported_pod": "Pod" - } - } - } - ], - "type": "table" - }, - { - "datasource": "Metrics", - "description": "Status of HAProxy backends", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "color-background" - }, - "inspect": false - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "DOWN" - } - }, - "type": "value" - }, - { - "options": { - "1": { - "color": "green", - "index": 0, - "text": "UP" - } - }, - "type": "value" + { + "matcher": { + "id": "byName", + "options": "Backend" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "auto" } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Server" }, - "overrides": [ + "properties": [ { - "matcher": { - "id": "byName", - "options": "Service" - }, - "properties": [ - { - "id": "custom.width", - "value": 300 - }, - { - "id": "custom.cellOptions", - "value": { - "type": "auto" - } - } - ] + "id": "custom.width", + "value": 200 }, { - "matcher": { - "id": "byName", - "options": "Backend" - }, - "properties": [ - { - "id": "custom.width", - "value": 150 - }, - { - "id": "custom.cellOptions", - "value": { - "type": "auto" - } - } - ] + "id": "custom.cellOptions", + "value": { + "type": "auto" + } } ] }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 33 - }, - "id": 41, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "11.6.4", - "targets": [ - { - "datasource": "Metrics", - "editorMode": "code", - "exemplar": false, - "expr": "max by(service_name, proxy) (haproxy_backend_status)", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "Backend" - } - ], - "title": "HAProxy Backends", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true - }, - "indexByName": { - "Value": 2, - "proxy": 1, - "service_name": 0 - }, - "renameByName": { - "Value": "Status", - "proxy": "Backend", - "service_name": "Service" - } + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 } - } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 112 + }, + "id": 41, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" ], - "type": "table" + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.6.4", + "targets": [ + { + "datasource": "Metrics", + "editorMode": "code", + "exemplar": false, + "expr": "max by(service, proxy, server) (\n haproxy_server_status{\n namespace=\"$namespace\",\n service=~\"${release}-.*\"\n }\n)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "Backend" } ], - "title": "⚖️ HAProxy Pod's Status", - "type": "row" + "title": "HAProxy Backend Servers", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { + "Value": 3, + "proxy": 1, + "server": 2, + "service": 0 + }, + "renameByName": { + "Value": "Status", + "proxy": "Backend", + "server": "Server", + "service": "Service" + } + } + } + ], + "type": "table" } ], "preload": false, @@ -2352,7 +3134,8 @@ "query": "label_values(grafana_live_node_num_nodes, instance)", "refresh": 1, "regex": "/^(.*)-\\d+$/", - "type": "query" + "type": "query", + "hide": 2 }, { "current": { @@ -2489,6 +3272,5 @@ "timepicker": {}, "timezone": "", "title": "PMM HA Health Overview", - "uid": "pmm-ha-health-overview", - "version": 3 + "uid": "pmm-ha-health-overview" } diff --git a/documentation/docs/install-pmm/install-HA-clustered.md b/documentation/docs/install-pmm/install-HA-clustered.md index 704fe512ed..26c13178cb 100644 --- a/documentation/docs/install-pmm/install-HA-clustered.md +++ b/documentation/docs/install-pmm/install-HA-clustered.md @@ -773,17 +773,17 @@ To create additional service tokens manually, see the [PMM documentation on serv #### Monitor cluster health -Use the **PMM HA Health Overview** dashboard to monitor your entire HA deployment from a single view. +Use the **PMM HA Health Overview** dashboard to monitor your entire HA deployment from a single view. -This dashboard shows real-time health status for all critical components including PMM server replicas, PostgreSQL, ClickHouse, VictoriaMetrics, and HAProxy. +This dashboard shows alive versus expected instance counts, active or primary nodes, replication health, and real-time status for PMM, PostgreSQL, ClickHouse, VictoriaMetrics, and HAProxy. -Access the dashboard from **All Dashboards > Browse all dashboards > Experimental > PMM HA Health Overview**. +Access the dashboard from **PMM HA > Overview** in the main menu, or from **All Dashboards > Browse all dashboards > PMM Health > PMM HA Health Overview**. The dashboard helps you quickly identify component failures, resource constraints, and stability issues across your high-availability infrastructure. For detailed information about each panel and what to check, see the [PMM HA Health Overview dashboard reference](../reference/dashboards/dashboard-ha-health-overview.md). -#### Identify the leader node +#### Identify the health status PMM displays a visual badge on the side menu and displays the name of the active PMM instance that's currently handling all monitoring operations. For example, `pmm-ha-0`, `pmm-ha-1`, or `pmm-ha-2`. diff --git a/documentation/docs/reference/dashboards/dashboard-ha-health-overview.md b/documentation/docs/reference/dashboards/dashboard-ha-health-overview.md index 4a3c9e64be..c5ff6bf9bd 100644 --- a/documentation/docs/reference/dashboards/dashboard-ha-health-overview.md +++ b/documentation/docs/reference/dashboards/dashboard-ha-health-overview.md @@ -2,25 +2,25 @@ ![PMM HA Health Overview Dashboard](../../images/PMM_HA_Health_Overview.png) -The PMM HA Health Overview dashboard provides at-a-glance monitoring of your PMM High Availability Cluster deployment health. +The PMM HA Health Overview dashboard provides at-a-glance monitoring of cluster-level and instance-level health for your PMM High Availability deployment. -Use this dashboard to quickly identify component failures, resource constraints, and stability issues across your high-availability infrastructure. +Use the **Alive**, **Expected**, and **Health** values to identify component failures. Green means all expected instances are alive, yellow means the service is degraded, and red means no instances are alive or no expected instances were discovered. -This dashboard monitors all critical components: PMM server replicas, PostgreSQL database cluster, ClickHouse query analytics storage, VictoriaMetrics time series storage, and HAProxy load balancers. +The dashboard monitors PMM server replicas, PostgreSQL, ClickHouse, VictoriaMetrics, and HAProxy. It also identifies the active PMM instance, PostgreSQL primary, ClickHouse Keeper leader, VictoriaMetrics component health, and HAProxy backend server status. ## Overview ### PMM -Shows the overall health status of PMM server pods with green **Healthy** when all pods are running normally or red **Not Healthy** when one or more pods are down. +Shows PMM instances alive versus expected and the resulting health percentage. -In HA Cluster mode, you have three PMM server replicas providing full redundancy. If one replica fails, the remaining two continue serving requests with no user-visible impact, but you should investigate quickly to restore full redundancy. +In HA Cluster mode, multiple PMM server replicas (three by default) provide full redundancy through Raft consensus, which requires a quorum — a majority of replicas, `floor(N/2)+1` — to elect a leader and serve requests. If one replica fails, failover stays transparent to users as long as quorum holds, but you should investigate quickly to restore full redundancy. -Check the **PMM Pods** table in the Pod Status Details section below to see which specific pods are affected. +The **Active PMM Instance** panel identifies the current Raft leader. Check the **PMM Pods** table to see every instance's status and Active or Follower role. ### PostgreSQL -Shows the overall health status of PostgreSQL database pods. Green **Healthy** indicates the cluster is fully operational with primary and replicas running. Red **Not Healthy** signals the cluster is degraded, putting metadata and configuration storage at risk. +Shows PostgreSQL instances alive versus expected. The **PostgreSQL Primary** panel identifies the current primary, and the instance table reports replication streaming health. PostgreSQL stores PMM's critical data including user accounts, dashboard configurations, alerting rules, and inventory information. @@ -30,20 +30,21 @@ Check the [**PostgreSQL Pods**](#postgresql-pods) panel in the **PostgreSQL Pod' ### ClickHouse -Shows the overall health status of ClickHouse pods. Green **Healthy** indicates the cluster is operating normally with all keeper and database nodes running. Red **Not Healthy** signals Query Analytics data storage is degraded. +Shows ClickHouse database and Keeper nodes alive versus expected. ClickHouse stores all Query Analytics (QAN) data. When pods fail, QAN dashboards will show incomplete query performance metrics, you won't be able to analyze slow queries during the outage, and you'll have permanent gaps in your historical query analysis data. -Check the **ClickHouse Pods** panel in the **ClickHouse Pod's Status** section below to see which pods are down. +Check the **ClickHouse Pods** panel to distinguish database (`pmmdb`) and Keeper nodes and to identify the Keeper leader. The dashboard reports Kubernetes node role and Keeper election state; it does not infer ClickHouse logical shard topology. Loss of keeper nodes (which handle coordination) is more critical than loss of a single database node (which handles storage). ### VictoriaMetrics -Shows the overall health status of VictoriaMetrics components. Green **Healthy** indicates all components are operational and metrics collection and querying are working properly. Red **Not Healthy** signals the time series infrastructure is degraded. +Shows VictoriaMetrics instances alive versus expected. The **VictoriaMetrics Components** panel breaks health down by `vmauth`, `vminsert`, `vmselect`, `vmstorage`, and `vmagent`. -VictoriaMetrics uses multiple components working together: -- `vminsert` receives metrics from monitored services, - `vmselect` processes dashboard queries +VictoriaMetrics uses multiple components working together: +- `vminsert` receives metrics from monitored services +- `vmselect` processes dashboard queries - `vmstorage` stores time series data - `vmagent` scrapes metrics from targets - `vmauth` handles authentication @@ -54,7 +55,7 @@ Failures in `vminsert` or `vmstorage` are most critical as they prevent metrics ### HAProxy -Shows the overall health status of HAProxy load balancer pods. Green **Healthy** means load balancing is working and traffic is properly distributed across your PMM replicas. Red **Not Healthy** means the load balancer has problems and you may not be able to access PMM. +Shows HAProxy instances alive versus expected. The backend server table reports the UP or DOWN state of each server in each backend. HAProxy is how you get into your PMM cluster. It routes web traffic to healthy PMM server replicas and handles automatic failover when the leader changes. @@ -196,7 +197,7 @@ A healthy PMM HA deployment should be mostly green with only short gaps during p ### PostgreSQL Pods -Shows each of your PostgreSQL pods with their status (UP or DOWN) and role (Primary or Replica). In a healthy cluster, you should see one Primary and the rest as Replicas, all showing UP in green. +Shows each PostgreSQL instance with its status (UP or DOWN), role (Primary or Replica), and replication health. In a healthy cluster, you should see one Primary, the remaining instances as Replicas, and all rows showing UP and Healthy. If the Primary shows DOWN, a failover is either in progress or just completed. Check that a Replica was promoted to become the new Primary. @@ -212,18 +213,18 @@ After a failover, verify that the new Primary is handling writes correctly. ### PMM Pods -Shows each of your PMM server pods with their current status. Green UP means the pod is running normally. Red DOWN means the pod has failed or isn't running. +Shows each PMM server instance with its current status and Raft role. Green UP means the pod is running normally; **Active** identifies the Raft leader and **Follower** identifies the remaining instances. -If one pod shows DOWN, identify which replica is affected and investigate the cause. Two or more DOWN pods means your deployment is at serious risk—investigate immediately. +If one pod shows DOWN, identify which replica is affected and investigate the cause. If enough pods show DOWN that fewer than a quorum (`floor(N/2)+1` of your configured replicas) remain alive, your deployment is at serious risk—investigate immediately. -If all three show DOWN, your entire PMM system is unavailable. +If all configured replicas show DOWN, your entire PMM system is unavailable. -Use this table to identify which specific PMM server pods need attention when the [**PMM**](#pmm) health indicator shows **Not Healthy**. +Use this table to identify which PMM server instances need attention when the [**PMM**](#pmm) health percentage is below 100%. ### ClickHouse Pods -Shows each of your ClickHouse pods with their status (UP or DOWN) and role (Leader or Follower). +Shows every ClickHouse database and Keeper node with its UP or DOWN status. Keeper nodes also show their Leader or Follower role. ClickHouse uses two types of pods: keeper nodes that handle coordination and metadata, and database nodes that handle storage. @@ -243,7 +244,7 @@ Check for failures in `vmstorage` pods first—each stores a subset of your metr Failures in `vmselect` pods reduce query capacity but don't cause data loss. Down `vmagent` or `vmauth` pods affect metric scraping and authentication respectively. -Use this table to identify which specific components need attention when the [**VictoriaMetrics**](#victoriametrics) health indicator shows **Not Healthy**. +Use this table to identify which components need attention when the [**VictoriaMetrics**](#victoriametrics) health percentage is below 100%. ## HAProxy Pod's Status @@ -257,11 +258,11 @@ If one pod shows DOWN, your load balancing is still working but with reduced cap Investigate DOWN pods quickly to restore full redundancy and prevent a single point of failure. Check pod logs to identify whether the issue is configuration, resource limits, or connectivity problems. -Use this table to identify which specific HAProxy pods need attention when the [**HAProxy**](#haproxy) health indicator shows **Not Healthy**. +Use this table to identify which HAProxy pods need attention when the [**HAProxy**](#haproxy) health percentage is below 100%. -### HAProxy Backends +### HAProxy Backend Servers -Shows the health status of each backend service that HAProxy routes traffic to. +Shows the health status of every server in each backend that HAProxy routes traffic to. Green UP means HAProxy can reach the backend and will route traffic to it. Red DOWN means HAProxy detected the backend is unavailable and won't route traffic there. @@ -278,10 +279,10 @@ Check pod status in the tables above to identify why backends remain unavailable ## Dashboard Usage Tips ### Refresh rate -This dashboard auto-refreshes every 30 seconds to provide near real-time monitoring. +Use Grafana's refresh control to update the dashboard while investigating failovers or degraded components. ### Filters -Use the namespace and Helm release variables at the top to focus on your specific PMM HA deployment if you have multiple installations. +Use the namespace variable at the top to focus on your specific PMM HA deployment if you have multiple installations. ### How to find what's wrong @@ -304,4 +305,4 @@ When you notice problems, follow this approach to quickly figure out what's wron - **Storage full**: Red storage gauges + pods crashing = expand storage immediately - **Resource exhaustion**: High CPU/memory + pod restarts = increase resource limits - **Network issues**: Multiple components partially down + high restart counts = investigate cluster networking -- **Single pod failure**: One component shows "Not Healthy" but no restarts = stuck pod requiring manual intervention \ No newline at end of file +- **Single pod failure**: One component's health percentage is below 100% but no restarts = stuck pod requiring manual intervention \ No newline at end of file diff --git a/ui/apps/pmm/src/contexts/navigation/navigation.constants.ts b/ui/apps/pmm/src/contexts/navigation/navigation.constants.ts index 8e055172d2..9b0023f82b 100644 --- a/ui/apps/pmm/src/contexts/navigation/navigation.constants.ts +++ b/ui/apps/pmm/src/contexts/navigation/navigation.constants.ts @@ -790,13 +790,14 @@ export const NAV_HIGH_AVAILABILITY: NavItem = { id: 'high-availability', icon: 'cluster', text: 'PMM HA', - url: `${PMM_NEW_NAV_GRAFANA_PATH}/high-availability`, + url: `${PMM_NEW_NAV_GRAFANA_PATH}/d/pmm-ha-health-overview/pmm-ha-health-overview`, }; -export const NAV_HIGH_AVAILABILITY_LEADER: NavItem = { - id: 'high-availability-leader', - text: 'Leader:', - type: 'menu-text', +export const NAV_HIGH_AVAILABILITY_OVERVIEW: NavItem = { + id: 'high-availability-overview', + icon: 'overview', + text: 'Overview', + url: `${PMM_NEW_NAV_GRAFANA_PATH}/d/pmm-ha-health-overview/pmm-ha-health-overview`, }; export const NAV_HIGH_AVAILABILITY_NODES: NavItem = { diff --git a/ui/apps/pmm/src/contexts/navigation/navigation.utils.tsx b/ui/apps/pmm/src/contexts/navigation/navigation.utils.tsx index 62e54718f2..4ab078ea67 100644 --- a/ui/apps/pmm/src/contexts/navigation/navigation.utils.tsx +++ b/ui/apps/pmm/src/contexts/navigation/navigation.utils.tsx @@ -43,8 +43,8 @@ import { NAV_HIGH_AVAILABILITY, NAV_USERS_AND_ACCESS, NAV_ACCESS_CONTROL, - NAV_HIGH_AVAILABILITY_LEADER, NAV_HIGH_AVAILABILITY_NODES, + NAV_HIGH_AVAILABILITY_OVERVIEW, NAV_HOME_PAGE, } from './navigation.constants'; import { CombinedSettings } from 'contexts/settings'; @@ -249,20 +249,14 @@ export const addConfiguration = ( return NAV_CONFIGURATION; }; -export const addHighAvailability = ({ health, leader }: HAInfo): NavItem => { +export const addHighAvailability = ({ health }: HAInfo): NavItem => { const item = { ...NAV_HIGH_AVAILABILITY }; item.badge = ; item.icon = ; item.badgeAlwaysVisible = true; - item.children = [ - { - ...NAV_HIGH_AVAILABILITY_LEADER, - secondaryText: leader?.nodeName || 'Unknown', - }, - NAV_HIGH_AVAILABILITY_NODES, - ]; + item.children = [NAV_HIGH_AVAILABILITY_OVERVIEW, NAV_HIGH_AVAILABILITY_NODES]; return item; };