From 9bbd9a5fda631b52fecb2e4793d9113fe87ec9c7 Mon Sep 17 00:00:00 2001 From: Ante Gulin Date: Tue, 4 Aug 2026 11:04:05 +0200 Subject: [PATCH 1/4] PMM-14956 Add pmm_ha_expected_nodes metric --- managed/services/ha/ha.go | 3 +- managed/services/ha/ha_metrics.go | 46 ++++--- managed/services/ha/ha_metrics_test.go | 170 +++++++++++++++++++++++++ managed/services/ha/haservice.go | 32 +++-- 4 files changed, 221 insertions(+), 30 deletions(-) create mode 100644 managed/services/ha/ha_metrics_test.go diff --git a/managed/services/ha/ha.go b/managed/services/ha/ha.go index 0a0bef8147..135d90d5f6 100644 --- a/managed/services/ha/ha.go +++ b/managed/services/ha/ha.go @@ -52,8 +52,7 @@ func (s *HAServer) ListNodes(_ context.Context, _ *hav1beta1.ListNodesRequest) ( return &hav1beta1.ListNodesResponse{Nodes: []*hav1beta1.HANode{}}, nil } - // Default to 1 for single-node deployment where no peers are configured. - expectedNodes := max(len(s.service.params.Nodes), 1) + expectedNodes := s.service.expectedNodes() s.service.rw.RLock() memberlist := s.service.memberlist diff --git a/managed/services/ha/ha_metrics.go b/managed/services/ha/ha_metrics.go index 9228f91eba..62ddd940d4 100644 --- a/managed/services/ha/ha_metrics.go +++ b/managed/services/ha/ha_metrics.go @@ -26,23 +26,27 @@ const ( // // The following metrics are exposed (only when HA mode is enabled): // -// - pmm_ha_leader_status – 1 if this node is the current Raft leader, 0 -// otherwise. Summing this across all nodes in the cluster enables the -// PMMHALeaderMissing (sum == 0) and PMMHASplitBrain (sum > 1) alerts. +// - pmm_ha_leader_status - 1 if this node is the current Raft leader, 0 +// otherwise. Consumed by the pmm_ha_no_leader and pmm_ha_split_brain +// alert templates, which sum it across the cluster. // -// - pmm_ha_raft_term – The current Raft consensus term. Rapid growth -// (changes(pmm_ha_raft_term[10m]) > 5) triggers the PMMHALeaderFlapping -// alert that indicates an unstable network or crashing leader. +// - pmm_ha_raft_term - The current Raft consensus term. Rapid growth +// indicates an unstable network or a crashing leader. Consumed by the +// pmm_ha_leader_flapping alert template. // -// - pmm_ha_up{role="voter|nonvoter"} – Always 1 for a live node, labelled -// with the node's Raft suffrage role. count(pmm_ha_up{role="voter"}) < 3 -// triggers the PMMHAQuorumAtRisk alert for a three-node cluster. +// - pmm_ha_up{role="voter|nonvoter"} - Always 1 for a live node. The role +// label reports whether the node is in the current Raft configuration. A +// node that goes down stops emitting the series rather than reporting 0. +// +// - pmm_ha_expected_nodes - The number of nodes configured for this cluster, +// used as the denominator for node-down and quorum alerting. type HAMetricsCollector struct { //nolint:revive haService *Service - mLeaderStatus *prom.Desc - mRaftTerm *prom.Desc - mUp *prom.Desc + mLeaderStatus *prom.Desc + mRaftTerm *prom.Desc + mUp *prom.Desc + mExpectedNodes *prom.Desc } // NewHAMetricsCollector creates a new HAMetricsCollector backed by the @@ -53,28 +57,31 @@ func NewHAMetricsCollector(haService *Service) *HAMetricsCollector { mLeaderStatus: prom.NewDesc( prom.BuildFQName(haPrometheusNamespace, haPrometheusSubsystem, "leader_status"), "Reports whether this PMM node currently holds the Raft leader lease. "+ - "Value is 1 for the leader and 0 for followers. "+ - "Use sum(pmm_ha_leader_status) to detect split-brain (>1) or a missing leader (==0).", + "Value is 1 for the leader and 0 for followers.", []string{"node_id"}, nil, ), mRaftTerm: prom.NewDesc( prom.BuildFQName(haPrometheusNamespace, haPrometheusSubsystem, "raft_term"), "The current Raft consensus term number as seen by this node. "+ - "Rapid increases indicate leader instability or frequent elections (leader flapping). "+ - "Use changes(pmm_ha_raft_term[10m]) > 5 to fire the PMMHALeaderFlapping alert.", + "Rapid increases indicate leader instability or frequent elections (leader flapping).", []string{"node_id"}, nil, ), mUp: prom.NewDesc( prom.BuildFQName(haPrometheusNamespace, haPrometheusSubsystem, "up"), "Reports that this PMM node is up and participating in the cluster. "+ - "The 'role' label indicates the node's Raft suffrage: 'voter' nodes participate "+ - "in elections, 'nonvoter' nodes only replicate logs. "+ - "Use count(pmm_ha_up{role=\"voter\"}) to evaluate quorum health.", + "The 'role' label is 'voter' when the node is in the current Raft configuration "+ + "and votes in elections, and 'nonvoter' when it is absent from that configuration.", []string{"node_id", "role"}, nil, ), + mExpectedNodes: prom.NewDesc( + prom.BuildFQName(haPrometheusNamespace, haPrometheusSubsystem, "expected_nodes"), + "The number of PMM nodes configured for this HA cluster.", + []string{"node_id"}, + nil, + ), } } @@ -107,6 +114,7 @@ func (c *HAMetricsCollector) Collect(ch chan<- prom.Metric) { role = "voter" } ch <- prom.MustNewConstMetric(c.mUp, prom.GaugeValue, 1, nodeID, role) + ch <- prom.MustNewConstMetric(c.mExpectedNodes, prom.GaugeValue, float64(m.ExpectedNodes), nodeID) } var _ prom.Collector = (*HAMetricsCollector)(nil) diff --git a/managed/services/ha/ha_metrics_test.go b/managed/services/ha/ha_metrics_test.go new file mode 100644 index 0000000000..67d4b4df5b --- /dev/null +++ b/managed/services/ha/ha_metrics_test.go @@ -0,0 +1,170 @@ +// Copyright (C) 2023 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ha + +import ( + "fmt" + "sort" + "strings" + "testing" + + prom "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/percona/pmm/managed/models" +) + +// collectSamples gathers the collector's metrics and returns them keyed by +// `name{label="value",...}`, so assertions do not depend on the exact wording of +// the metric HELP strings. +func collectSamples(t *testing.T, c *HAMetricsCollector) map[string]float64 { + t.Helper() + + reg := prom.NewPedanticRegistry() + require.NoError(t, reg.Register(c)) + + families, err := reg.Gather() + require.NoError(t, err) + + samples := make(map[string]float64) + for _, mf := range families { + for _, m := range mf.GetMetric() { + labels := make([]string, 0, len(m.GetLabel())) + for _, l := range m.GetLabel() { + labels = append(labels, fmt.Sprintf("%s=%q", l.GetName(), l.GetValue())) + } + sort.Strings(labels) + + key := mf.GetName() + if len(labels) != 0 { + key = fmt.Sprintf("%s{%s}", key, strings.Join(labels, ",")) + } + samples[key] = m.GetGauge().GetValue() + } + } + + return samples +} + +func TestHAMetricsCollector_Disabled(t *testing.T) { + t.Parallel() + + c := NewHAMetricsCollector(&Service{ + params: &models.HAParams{ + Enabled: false, + NodeID: "node-1", + Nodes: []string{"node-1", "node-2", "node-3"}, + }, + }) + + // Nothing at all must be emitted when HA is disabled: the built-in HA alert + // templates rely on the absence of these series to stay silent on standalone PMM. + assert.Empty(t, collectSamples(t, c)) +} + +func TestHAMetricsCollector_EnabledBeforeRaftInit(t *testing.T) { + t.Parallel() + + // raftNode is nil, which is the state during early startup: HA is enabled but + // Raft has not been initialised yet. + c := NewHAMetricsCollector(&Service{ + params: &models.HAParams{ + Enabled: true, + NodeID: "node-1", + Nodes: []string{"node-1", "node-2", "node-3"}, + }, + }) + + assert.Equal(t, map[string]float64{ + `pmm_ha_leader_status{node_id="node-1"}`: 0, + `pmm_ha_raft_term{node_id="node-1"}`: 0, + `pmm_ha_up{node_id="node-1",role="nonvoter"}`: 1, + `pmm_ha_expected_nodes{node_id="node-1"}`: 3, + }, collectSamples(t, c)) +} + +func TestHAMetricsCollector_ExpectedNodes(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + nodes []string + want float64 + }{ + { + name: "no peers defaults to one", + nodes: nil, + want: 1, + }, + { + name: "single node cluster", + nodes: []string{"node-1"}, + want: 1, + }, + { + name: "three node cluster", + nodes: []string{"node-1", "node-2", "node-3"}, + want: 3, + }, + { + name: "five node cluster", + nodes: []string{"node-1", "node-2", "node-3", "node-4", "node-5"}, + want: 5, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + c := NewHAMetricsCollector(&Service{ + params: &models.HAParams{ + Enabled: true, + NodeID: "node-1", + Nodes: tt.nodes, + }, + }) + + samples := collectSamples(t, c) + assert.InDelta(t, tt.want, samples[`pmm_ha_expected_nodes{node_id="node-1"}`], 0.0001) + }) + } +} + +func TestHAMetricsCollector_Describe(t *testing.T) { + t.Parallel() + + c := NewHAMetricsCollector(&Service{ + params: &models.HAParams{ + Enabled: true, + NodeID: "node-1", + Nodes: []string{"node-1"}, + }, + }) + + ch := make(chan *prom.Desc, 10) + c.Describe(ch) + close(ch) + + descs := make([]string, 0, len(ch)) + for d := range ch { + descs = append(descs, d.String()) + } + + // Guards against emitting a metric in Collect without a matching descriptor. + assert.Len(t, descs, 4) +} diff --git a/managed/services/ha/haservice.go b/managed/services/ha/haservice.go index 19de9cd018..ca7ff1cb8f 100644 --- a/managed/services/ha/haservice.go +++ b/managed/services/ha/haservice.go @@ -605,6 +605,12 @@ func (s *Service) Params() *models.HAParams { return s.params } +// expectedNodes returns the number of PMM Server nodes declared for this cluster. +// Defaults to 1 for a single-node deployment where no peers are configured. +func (s *Service) expectedNodes() int { + return max(len(s.params.Nodes), 1) +} + // Metrics holds HA-related Prometheus metric values for this node. type Metrics struct { // Enabled indicates whether HA mode is active. @@ -614,15 +620,20 @@ type Metrics struct { // RaftTerm is the current Raft consensus term. Rapid increases indicate // an unstable leader or frequent elections (leader flapping). RaftTerm uint64 - // IsVoter is true when this node participates in Raft leader elections. - // Nonvoter nodes replicate logs but never vote. + // IsVoter is true when this node is in the current Raft configuration and votes + // in leader elections. It is false when the node is absent from that configuration, + // and also when the configuration could not be read. PMM never adds true Raft + // non-voting members, so a false value always means "not in the configuration". IsVoter bool + // ExpectedNodes is the number of nodes configured for this cluster. + ExpectedNodes int } // GetMetrics returns current HA Raft metrics for this node. The returned // values are intended to be exposed as Prometheus gauges so that VictoriaMetrics -// can evaluate cluster-health alerting rules such as PMMHALeaderMissing, -// PMMHASplitBrain, PMMHALeaderFlapping and PMMHAQuorumAtRisk. +// can evaluate the built-in cluster-health alert templates: pmm_ha_no_leader, +// pmm_ha_split_brain, pmm_ha_leader_flapping, pmm_ha_node_unreachable and +// pmm_ha_quorum_at_risk. // // When HA is disabled, Enabled is false and all other fields are zero values. func (s *Service) GetMetrics() Metrics { @@ -630,13 +641,15 @@ func (s *Service) GetMetrics() Metrics { return Metrics{Enabled: false} } + expectedNodes := s.expectedNodes() + s.rw.RLock() raftNode := s.raftNode s.rw.RUnlock() if raftNode == nil { // HA enabled but Raft not yet initialised (early startup). - return Metrics{Enabled: true} + return Metrics{Enabled: true, ExpectedNodes: expectedNodes} } isLeader := raftNode.State() == raft.Leader @@ -662,9 +675,10 @@ func (s *Service) GetMetrics() Metrics { } return Metrics{ - Enabled: true, - IsLeader: isLeader, - RaftTerm: term, - IsVoter: isVoter, + Enabled: true, + IsLeader: isLeader, + RaftTerm: term, + IsVoter: isVoter, + ExpectedNodes: expectedNodes, } } From dababa212ce54e1c3f8249cf83ad6b9801083a7f Mon Sep 17 00:00:00 2001 From: Ante Gulin Date: Tue, 4 Aug 2026 11:04:08 +0200 Subject: [PATCH 2/4] PMM-14956 Harden PMM_HA_PEERS parsing --- managed/cmd/pmm-managed/main.go | 30 ++++++++++++++++++++++++---- managed/cmd/pmm-managed/main_test.go | 27 +++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index 2468e9778a..4becd33867 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -800,10 +800,7 @@ func main() { //nolint:gocognit,maintidx,cyclop ctx = logger.Set(ctx, "main") defer l.Info("Done.") - var nodes []string - if *haPeers != "" { - nodes = strings.Split(*haPeers, ",") - } + nodes := parseHAPeers(*haPeers) haParams := &models.HAParams{ Enabled: *haEnabled, NodeID: *haNodeID, @@ -1240,6 +1237,31 @@ func main() { //nolint:gocognit,maintidx,cyclop wg.Wait() } +// parseHAPeers splits the PMM_HA_PEERS value into node addresses, trimming surrounding +// whitespace and dropping empty and duplicate entries. The peer list is expected to name +// every node in the cluster, including this one, and its length is reported as +// pmm_ha_expected_nodes. A trailing comma or a padded list would otherwise inflate that +// count and make the node-unreachable and quorum alerts fire on a healthy cluster. +func parseHAPeers(peers string) []string { + var nodes []string + seen := make(map[string]struct{}) + + for node := range strings.SplitSeq(peers, ",") { + node = strings.TrimSpace(node) + if node == "" { + continue + } + if _, ok := seen[node]; ok { + logrus.Warnf("Ignoring duplicate entry %q in PMM_HA_PEERS.", node) + continue + } + seen[node] = struct{}{} + nodes = append(nodes, node) + } + + return nodes +} + func parseLoggerConfig(level string, debug, trace bool) logrus.Level { if trace { return logrus.TraceLevel diff --git a/managed/cmd/pmm-managed/main_test.go b/managed/cmd/pmm-managed/main_test.go index 17e536726f..76c33cfaaf 100644 --- a/managed/cmd/pmm-managed/main_test.go +++ b/managed/cmd/pmm-managed/main_test.go @@ -28,6 +28,33 @@ import ( "golang.org/x/tools/go/packages" ) +func TestParseHAPeers(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + peers string + want []string + }{ + {"empty", "", nil}, + {"single node", "node-1", []string{"node-1"}}, + {"three nodes", "node-1,node-2,node-3", []string{"node-1", "node-2", "node-3"}}, + // A trailing comma used to yield an extra empty element, inflating the + // expected node count and firing the quorum alert on a healthy cluster. + {"trailing comma", "node-1,node-2,node-3,", []string{"node-1", "node-2", "node-3"}}, + {"surrounding whitespace", " node-1 , node-2 ", []string{"node-1", "node-2"}}, + {"only separators", ",,,", nil}, + {"duplicates", "node-1,node-2,node-1", []string{"node-1", "node-2"}}, + {"host and port preserved", "node-1:9761,node-2:9761", []string{"node-1:9761", "node-2:9761"}}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, parseHAPeers(tt.peers)) + }) + } +} + func TestPackages(t *testing.T) { cmd := exec.Command("pmm-managed", "-h") b, err := cmd.CombinedOutput() From 16ecd35ee5f54312edfa304c9333e36ef35f5846 Mon Sep 17 00:00:00 2001 From: Ante Gulin Date: Tue, 4 Aug 2026 11:04:10 +0200 Subject: [PATCH 3/4] PMM-14956 Add HA cluster alert templates --- .../alerting-templates/ha_leader_flapping.yml | 33 +++++++++++++++++++ .../data/alerting-templates/ha_no_leader.yml | 22 +++++++++++++ .../ha_node_unreachable.yml | 28 ++++++++++++++++ .../alerting-templates/ha_quorum_at_risk.yml | 28 ++++++++++++++++ .../alerting-templates/ha_split_brain.yml | 28 ++++++++++++++++ 5 files changed, 139 insertions(+) create mode 100644 managed/data/alerting-templates/ha_leader_flapping.yml create mode 100644 managed/data/alerting-templates/ha_no_leader.yml create mode 100644 managed/data/alerting-templates/ha_node_unreachable.yml create mode 100644 managed/data/alerting-templates/ha_quorum_at_risk.yml create mode 100644 managed/data/alerting-templates/ha_split_brain.yml diff --git a/managed/data/alerting-templates/ha_leader_flapping.yml b/managed/data/alerting-templates/ha_leader_flapping.yml new file mode 100644 index 0000000000..255ae0281f --- /dev/null +++ b/managed/data/alerting-templates/ha_leader_flapping.yml @@ -0,0 +1,33 @@ +--- +templates: + - name: pmm_ha_leader_flapping + version: 1 + summary: PMM HA leader is flapping + queries: + - ref_id: A + expr: |- + max by(node_id) (changes(pmm_ha_raft_term[10m])) + expressions: + - ref_id: C + type: math + expression: "$A > [[ .threshold ]]" + condition: C + params: + - name: threshold + summary: Number of Raft term changes within 10 minutes + type: float + range: [1, 100] + value: 5 + for: 5m + severity: warning + annotations: + summary: PMM HA leader is flapping + description: |- + The Raft term changed more than [[ .threshold ]] times in the last 10 minutes. + Frequent re-elections indicate an unstable network, an overloaded leader, or a leader that keeps restarting. + + Remediation steps: + 1. Check network latency and packet loss between PMM Server nodes on the configured Raft port (PMM_HA_RAFT_PORT). + 2. Check whether pmm-managed is restarting on any node. + 3. Check CPU, memory and disk I/O saturation on the PMM Server nodes. A starved leader misses its heartbeats. + 4. Review the pmm-managed logs for repeated transitions between the candidate and follower states. diff --git a/managed/data/alerting-templates/ha_no_leader.yml b/managed/data/alerting-templates/ha_no_leader.yml new file mode 100644 index 0000000000..d30b8d0000 --- /dev/null +++ b/managed/data/alerting-templates/ha_no_leader.yml @@ -0,0 +1,22 @@ +--- +templates: + - name: pmm_ha_no_leader + version: 1 + summary: PMM HA cluster has no active leader + # last_over_time bounds how long a departed leader's final sample can mask a leaderless + # cluster, narrowing the lookback to 2m. max by(node_id) collapses any duplicate series + # for one node, so a node is counted once no matter what labels the scrape adds. + expr: 'sum(max by(node_id) (last_over_time(pmm_ha_leader_status[2m]))) == bool 0' + for: 3m + severity: critical + annotations: + summary: PMM HA cluster has no active leader + description: |- + No node in the PMM High Availability cluster currently holds the Raft leader lease. + Leader-only background work such as advisor checks, backups, telemetry and scheduled tasks is not running. + + Remediation steps: + 1. Check that a majority of PMM Server nodes are running and reachable. + 2. Check network connectivity between nodes on the configured Raft and gossip ports (PMM_HA_RAFT_PORT and PMM_HA_GOSSIP_PORT). + 3. Inspect the pmm-managed logs on each node for repeated election attempts or 'no known peers' messages. + 4. If the cluster has lost a majority of its nodes, restore them. Raft cannot elect a leader without quorum. diff --git a/managed/data/alerting-templates/ha_node_unreachable.yml b/managed/data/alerting-templates/ha_node_unreachable.yml new file mode 100644 index 0000000000..f40a0b0d20 --- /dev/null +++ b/managed/data/alerting-templates/ha_node_unreachable.yml @@ -0,0 +1,28 @@ +--- +templates: + - name: pmm_ha_node_unreachable + version: 1 + summary: PMM HA node unreachable + # A node that goes down stops emitting pmm_ha_up entirely, so it cannot be named from a + # live series. The first term names nodes seen in the last 6h but not the last 2m. The + # second is the count-versus-expected check, guarded by absent() so it only reports when + # no node can be named: one that never reported, or has been gone longer than 6h. + expr: |- + (max by(node_id) (last_over_time(pmm_ha_up[6h])) unless max by(node_id) (last_over_time(pmm_ha_up[2m]))) + or + ((count(max by(node_id) (last_over_time(pmm_ha_up[2m]))) < bool max(last_over_time(pmm_ha_expected_nodes[2m]))) + and absent(max by(node_id) (last_over_time(pmm_ha_up[6h])) unless max by(node_id) (last_over_time(pmm_ha_up[2m])))) + for: 5m + severity: warning + annotations: + summary: "{{ if $labels.node_id }}PMM HA node '{{ $labels.node_id }}' is unreachable{{ else }}A PMM HA node is unreachable, fewer nodes are reporting than configured{{ end }}" + description: |- + {{ if $labels.node_id }}PMM Server node '{{ $labels.node_id }}' has stopped reporting HA metrics.{{ else }}Fewer PMM Server nodes are reporting HA metrics than the number configured for this cluster.{{ end }} + A node is down, restarting repeatedly, or unable to reach the shared metrics storage. + + Remediation steps: + 1. Open the High Availability page to see which nodes are still reporting. + 2. Check the status of the PMM Server container or pod on every node. + 3. Check that the missing node can still write metrics to the shared VictoriaMetrics storage. + 4. Check network connectivity between nodes on the configured Raft and gossip ports (PMM_HA_RAFT_PORT and PMM_HA_GOSSIP_PORT). + 5. If the node was removed on purpose, update PMM_HA_PEERS on the remaining nodes so that the expected node count matches. diff --git a/managed/data/alerting-templates/ha_quorum_at_risk.yml b/managed/data/alerting-templates/ha_quorum_at_risk.yml new file mode 100644 index 0000000000..dec8d18f6c --- /dev/null +++ b/managed/data/alerting-templates/ha_quorum_at_risk.yml @@ -0,0 +1,28 @@ +--- +templates: + - name: pmm_ha_quorum_at_risk + version: 1 + summary: PMM HA quorum at risk + # last_over_time drops nodes that stopped reporting out of the voter count, instead of + # waiting out the metrics database's longer staleness window. + expr: |- + (count(max by(node_id) (last_over_time(pmm_ha_up{role="voter"}[2m]))) <= bool floor(max(last_over_time(pmm_ha_expected_nodes[2m])) / 2 + 1)) + and on() (max(last_over_time(pmm_ha_expected_nodes[2m])) > 2) + for: 3m + severity: critical + annotations: + summary: PMM HA quorum at risk, the cluster has no spare voters left + description: |- + The number of live Raft voters has fallen to or below the smallest majority that still forms a quorum. + At the boundary the cluster still works, but one more node failure will make it impossible to elect a + leader. Below the boundary leader election has already stopped, and leader-only work such as advisor + checks, backups and scheduled tasks has stopped with it. The High Availability page shows which of the + two applies. + + Remediation steps: + 1. If a PMM_HA_PEERS change is being rolled out, this alert can fire while the nodes still disagree on the cluster size. Confirm the rollout has finished before treating it as a failure. + 2. Restore the failed nodes now. Do not restart or drain any of the remaining nodes until the cluster has a spare again. + 3. Check the status of the PMM Server container or pod and the pmm-managed logs on every node. + 4. Check network connectivity between nodes on the configured Raft and gossip ports (PMM_HA_RAFT_PORT and PMM_HA_GOSSIP_PORT). + 5. Check that the shared PostgreSQL, VictoriaMetrics and ClickHouse storage is reachable from the surviving nodes. + 6. Confirm recovery on the High Availability page once all nodes are back. diff --git a/managed/data/alerting-templates/ha_split_brain.yml b/managed/data/alerting-templates/ha_split_brain.yml new file mode 100644 index 0000000000..bfb59b704d --- /dev/null +++ b/managed/data/alerting-templates/ha_split_brain.yml @@ -0,0 +1,28 @@ +--- +templates: + - name: pmm_ha_split_brain + version: 1 + summary: PMM HA split-brain detected + # last_over_time stops a departing leader's final sample being counted alongside the + # new leader. Keep this window shorter than 'for', or an ordinary failover will fire. + # max by(node_id) collapses any duplicate series for one node, so a single node cannot + # be counted twice and raise a split brain on its own. + expr: 'sum(max by(node_id) (last_over_time(pmm_ha_leader_status[2m]))) > bool 1' + for: 3m + severity: critical + annotations: + summary: PMM HA split-brain detected, more than one node claims leadership + description: |- + More than one PMM Server node reports itself as the Raft leader at the same time. + Raft never allows two leaders within one cluster, so this means the nodes have formed + separate clusters instead of one. Every node bootstraps its own single-node cluster at + startup and then relies on peer discovery to merge into the others; when that discovery + fails, each node stays the leader of a cluster of one. + Leader-only work may run more than once and diverging writes are possible. + + Remediation steps: + 1. Open the High Availability page to see which nodes claim leadership, and compare the peers each one knows about. A node that formed its own cluster lists only itself. + 2. Check that PMM_HA_PEERS is identical on every node and that PMM_HA_NODE_ID is unique. A node missing from the peer lists is never discovered. + 3. Check that the nodes could reach each other at startup on the configured gossip and Raft ports (PMM_HA_GOSSIP_PORT and PMM_HA_RAFT_PORT; the Helm chart sets these to values other than the defaults), and look for firewall, security group, iptables or network policy rules blocking them. + 4. Inspect the pmm-managed logs on each node from startup onwards, for failures to join the peer list and for nodes entering the leader state. + 5. Separate Raft clusters do not merge on their own. Once the cause is fixed, restart pmm-managed on the nodes that should not be leaders so they rejoin. If a node keeps re-forming its own cluster, stop it and remove its Raft state directory (/srv/ha/) before restarting; that directory holds only cluster membership state. From 8b2a6813f92248bb58e92d2277316e7be6552881 Mon Sep 17 00:00:00 2001 From: Ante Gulin Date: Tue, 4 Aug 2026 11:04:13 +0200 Subject: [PATCH 4/4] PMM-14956 Document HA alert templates --- documentation/docs/alert/templates_list.md | 35 +++++++++++++++++++ .../docs/install-pmm/install-HA-clustered.md | 8 ++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/documentation/docs/alert/templates_list.md b/documentation/docs/alert/templates_list.md index 387668bdef..1a091eb29b 100644 --- a/documentation/docs/alert/templates_list.md +++ b/documentation/docs/alert/templates_list.md @@ -6,6 +6,7 @@ The table below lists all the alert templates available in Percona Monitoring an - [Operating System templates](#os_alerts) - [PMM templates](#pmm_alerts) +- [PMM High Availability templates](#pmm_ha_alerts) - [MongoDB templates](#mongodb_alerts) - [PBM templates](#pbm_alerts) - [MySQL templates](#mysql_alerts) @@ -29,6 +30,40 @@ The table below lists all the alert templates available in Percona Monitoring an | PMM | **PMM agent down** | Monitors PMM Agent status and alerts when an agent becomes unreachable, indicating potential host or agent issues. | MySQL, MongoDB, PostgreSQL, ProxySQL | | PMM | **Backup failed [Technical Preview]** | Monitors backup processes and raises alerts on failures. Provides details about the failed backup artifact and affected service to ensure data safety and recovery readiness. This template is currently in [Technical Preview](../reference/glossary.md) and is intended for testing purposes only, as it is subject to change. | MySQL, MongoDB, PostgreSQL, ProxySQL | + +### PMM High Availability templates + +These templates monitor a PMM Server [High Availability cluster](../install-pmm/install-HA-clustered.md). They never raise alerts on a standalone (non-HA) PMM installation, because the underlying HA metrics are only exposed when HA mode is enabled. + +| Area | Template name | Description | Database technology | +| :----|:------------- | :---------- | :------------------ | +| PMM HA | **PMM HA cluster has no active leader** | Alerts when no node in the cluster holds the Raft leader lease, which means that leader-only work such as advisor checks, backups, telemetry and scheduled tasks has stopped. | PMM | +| PMM HA | **PMM HA split-brain detected** | Alerts when more than one node claims Raft leadership at the same time, which means the nodes have formed separate Raft clusters instead of one. | PMM | +| PMM HA | **PMM HA leader is flapping** | Alerts when the Raft term on a node changes more than 5 times (default threshold) within 10 minutes, which indicates an unstable network or a leader that keeps restarting. | PMM | +| PMM HA | **PMM HA node unreachable** | Alerts when fewer nodes report HA metrics than the number configured in `PMM_HA_PEERS`, which indicates that at least one PMM Server node is down or isolated. | PMM | +| PMM HA | **PMM HA quorum at risk** | Alerts when the number of live Raft voters has fallen to or below the smallest majority that still forms a quorum. Applies to clusters of three nodes or more. | PMM | + +#### Enable the HA alerts + +These templates are available as soon as PMM Server is installed, but like all other alert templates they do not create alert rules by themselves. To start receiving notifications: + +1. Go to **Alerting > Alert rule templates** and find the template you want to use. +2. Select **New alert rule from template**. +3. Choose a folder and an evaluation group. Percona recommends grouping the HA rules together, for example in a **PMM HA** group. +4. Configure a [contact point](./contact_points.md) so that the alerts reach you. +5. Repeat for each of the five templates. + +#### Coverage limitations + +Keep the following in mind when you rely on these alerts: + +- **A complete cluster outage cannot be detected from inside the cluster.** Each node's metrics are collected only by the monitoring agent running on that same node, so when every node is down there is nothing left to report it and all HA alerts fall silent. Monitor the load balancer endpoint from outside the cluster to cover this case. +- **Split-brain detection requires the isolated node to still reach shared storage.** If a network partition also cuts a node off from the shared VictoriaMetrics storage, its metrics never arrive and the second leader stays invisible. +- **An ordinary network partition does not cause a split brain.** Raft is designed to prevent two leaders: a node in a minority partition cannot win an election. If no side of the partition holds a majority, the cluster is left with no leader at all and *PMM HA no active leader* fires. If a majority survives, it elects a new leader within seconds and that alert stays silent, while *PMM HA quorum at risk* and *PMM HA node unreachable* fire instead. The split-brain alert covers the rarer case where nodes end up in separate clusters, for example when they cannot discover each other at startup and each bootstraps its own. +- **The node unreachable alert names the node only if it reported recently.** A node that stopped within the last 6 hours is named in the alert. A node that has never reported since the cluster started, or that has been down for longer than 6 hours, is still detected but cannot be named, because no metrics remain to identify it. Use the **High Availability** page in that case. +- **The quorum alert does not apply to one- and two-node clusters.** On those the condition would be permanently true, since with two nodes quorum is two and both nodes are always essential, so the template suppresses itself and stays silent. *PMM HA node unreachable* does cover them, so rely on it instead. +- **Changing `PMM_HA_PEERS` raises alerts until the rollout finishes.** The expected node count is the highest value any node reports, so while some nodes carry the new peer list and others still carry the old one, the cluster looks smaller than expected. *PMM HA node unreachable* and *PMM HA quorum at risk* can both fire for the duration of a rolling restart, in either direction. Let the rollout finish before treating either as a real failure. + ### MongoDB templates diff --git a/documentation/docs/install-pmm/install-HA-clustered.md b/documentation/docs/install-pmm/install-HA-clustered.md index 704fe512ed..24ba200e93 100644 --- a/documentation/docs/install-pmm/install-HA-clustered.md +++ b/documentation/docs/install-pmm/install-HA-clustered.md @@ -817,6 +817,12 @@ View detailed role and health information for all PMM nodes in one place. - **Follower** status: which nodes are on standby - **Health** status: whether each node is responding +### Get alerted about cluster problems + +Checking the High Availability page tells you the state of the cluster right now, but it does not notify you when that state changes. PMM ships alert templates that cover the failure modes of an HA cluster: no active leader, split-brain, a flapping leader, an unreachable node, and a quorum at risk. + +Like all alert templates, they do not create alert rules on their own: create a rule from each template and point it at a contact point. For the template list, the setup steps and the coverage limitations, see [PMM High Availability templates](../alert/templates_list.md#pmm_ha_alerts). + ### Scale your deployment #### Scale PMM server replicas @@ -1206,4 +1212,4 @@ This Tech Preview release is designed to gather community feedback before GA. Yo - What works well in your environment? - What's challenging or confusing? - What features are you missing? -- How does performance compare to single-instance deployments? \ No newline at end of file +- How does performance compare to single-instance deployments?