diff --git a/deploy/tests/e2e/cookie-persistence/config/deploy.yml.tmpl b/deploy/tests/e2e/cookie-persistence/config/deploy.yml.tmpl index f75c059e..a4c5b129 100644 --- a/deploy/tests/e2e/cookie-persistence/config/deploy.yml.tmpl +++ b/deploy/tests/e2e/cookie-persistence/config/deploy.yml.tmpl @@ -3,7 +3,7 @@ apiVersion: apps/v1 metadata: name: http-echo spec: - replicas: 1 + replicas: {{ .Replicas }} selector: matchLabels: app: http-echo @@ -34,6 +34,14 @@ metadata: {{ else if .CookiePersistenceNoDynamic }} cookie-persistence-no-dynamic: "mycookie" {{ end }} +{{ if .SourceIPPersistence }} + source-ip-persistence: "true" + source-ip-persistence-size: "{{ .SourceIPPersistenceSize }}" + source-ip-persistence-expire: "{{ .SourceIPPersistenceExpire }}" +{{ end }} +{{ if .LoadBalance }} + load-balance: "{{ .LoadBalance }}" +{{ end }} spec: ports: - name: http diff --git a/deploy/tests/e2e/cookie-persistence/cookie-persistence_test.go b/deploy/tests/e2e/cookie-persistence/cookie-persistence_test.go index f7bb40b9..34a3361e 100644 --- a/deploy/tests/e2e/cookie-persistence/cookie-persistence_test.go +++ b/deploy/tests/e2e/cookie-persistence/cookie-persistence_test.go @@ -17,12 +17,14 @@ package cookiepersistence import ( + "encoding/json" "net/http" "strings" "testing" parser "github.com/haproxytech/client-native/v6/config-parser" "github.com/haproxytech/client-native/v6/config-parser/options" + "github.com/haproxytech/client-native/v6/config-parser/types" "github.com/haproxytech/kubernetes-ingress/deploy/tests/e2e" "github.com/stretchr/testify/suite" @@ -48,6 +50,7 @@ func TestCookiePersistenceTestSuite(t *testing.T) { func (suite *CookiePersistenceTestSuite) Test_CookiePersistence_Dynamic() { //------------------------ // First step : Dynamic + suite.resetTemplateData() suite.tmplData.CookiePersistenceDynamic = true suite.tmplData.CookiePersistenceNoDynamic = false suite.Require().NoError(suite.test.Apply("config/deploy.yml.tmpl", suite.test.GetNS(), suite.tmplData)) @@ -131,6 +134,7 @@ func (suite *CookiePersistenceTestSuite) Test_CookiePersistence_Dynamic() { // ... func (suite *CookiePersistenceTestSuite) Test_CookiePersistence_No_Dynamic() { + suite.resetTemplateData() suite.tmplData.CookiePersistenceNoDynamic = true suite.tmplData.CookiePersistenceDynamic = false suite.Require().NoError(suite.test.Apply("config/deploy.yml.tmpl", suite.test.GetNS(), suite.tmplData)) @@ -209,6 +213,7 @@ func (suite *CookiePersistenceTestSuite) Test_CookiePersistence_No_Dynamic() { func (suite *CookiePersistenceTestSuite) Test_CookiePersistence_Switch() { //--------------------------- // Step 1 : Dynamic + suite.resetTemplateData() suite.tmplData.CookiePersistenceDynamic = true suite.tmplData.CookiePersistenceNoDynamic = false suite.Require().NoError(suite.test.Apply("config/deploy.yml.tmpl", suite.test.GetNS(), suite.tmplData)) @@ -348,3 +353,93 @@ func (suite *CookiePersistenceTestSuite) Test_CookiePersistence_Switch() { return true }, e2e.WaitDuration, e2e.TickDuration) } + +// Expected backend +// backend e2e-tests-cookie-persistence_svc_http-echo_http +// +// ... +// balance source +// stick-table type ip size 1m expire 30m peers localinstance +// stick on src +// ... +func (suite *CookiePersistenceTestSuite) Test_SourceIPPersistence_WithSourceHash() { + suite.resetTemplateData() + suite.tmplData.Replicas = 3 + suite.tmplData.SourceIPPersistence = true + suite.tmplData.LoadBalance = "source" + suite.Require().NoError(suite.test.Apply("config/deploy.yml.tmpl", suite.test.GetNS(), suite.tmplData)) + + var firstHostname string + suite.Eventually(func() bool { + hostname, ok := suite.echoHostname() + if !ok { + return false + } + firstHostname = hostname + return true + }, e2e.WaitDuration, e2e.TickDuration) + + for range 10 { + hostname, ok := suite.echoHostname() + suite.Require().True(ok) + suite.Require().Equal(firstHostname, hostname) + } + + suite.Eventually(func() bool { + cfg, err := suite.test.GetIngressControllerFile("/etc/haproxy/haproxy.cfg") + if err != nil { + suite.T().Logf("Could not get Haproxy config: %v", err) + return false + } + return suite.hasSourceIPPersistenceConfig(cfg) + }, e2e.WaitDuration, e2e.TickDuration) +} + +func (suite *CookiePersistenceTestSuite) hasSourceIPPersistenceConfig(cfg string) bool { + if !strings.Contains(cfg, "balance source") || !strings.Contains(cfg, "stick on src") { + return false + } + reader := strings.NewReader(cfg) + p, err := parser.New(options.Reader(reader)) + if err != nil { + suite.T().Logf("Could not get Haproxy config parser: %v", err) + return false + } + beName := suite.test.GetNS() + "_svc_http-echo_http" + rawTable, err := p.Get(parser.Backends, beName, "stick-table") + if err != nil { + suite.T().Logf("Could not get stick-table for backend %s: %v", beName, err) + return false + } + table, ok := rawTable.(*types.StickTable) + if !ok { + suite.T().Logf("Unexpected stick-table type %T", rawTable) + return false + } + return table.Type == "ip" && + table.Size == "1m" && + (table.Expire == "30m" || table.Expire == "1800000") && + table.Peers == "localinstance" +} + +func (suite *CookiePersistenceTestSuite) echoHostname() (string, bool) { + res, cls, err := suite.client.Do() + if res == nil { + suite.T().Log(err) + return "", false + } + defer cls() + if res.StatusCode != http.StatusOK { + return "", false + } + var body struct { + OS struct { + Hostname string `json:"hostname"` + } `json:"os"` + } + if err := json.NewDecoder(res.Body).Decode(&body); err != nil { + suite.T().Log(err) + return "", false + } + return body.OS.Hostname, body.OS.Hostname != "" +} diff --git a/deploy/tests/e2e/cookie-persistence/suite_test.go b/deploy/tests/e2e/cookie-persistence/suite_test.go index ab0de691..9b0031b7 100644 --- a/deploy/tests/e2e/cookie-persistence/suite_test.go +++ b/deploy/tests/e2e/cookie-persistence/suite_test.go @@ -39,6 +39,11 @@ type CookiePersistenceSuite struct { type tmplData struct { CookiePersistenceDynamic bool CookiePersistenceNoDynamic bool + SourceIPPersistence bool + SourceIPPersistenceSize string + SourceIPPersistenceExpire string + LoadBalance string + Replicas int Host string } @@ -46,7 +51,7 @@ func (suite *CookiePersistenceSuite) SetupSuite() { var err error suite.test, err = e2e.NewTest() suite.Require().NoError(err) - suite.tmplData = tmplData{Host: suite.test.GetNS() + ".test"} + suite.resetTemplateData() suite.client, err = e2e.NewHTTPClient(suite.tmplData.Host) suite.Require().NoError(err) } @@ -59,6 +64,15 @@ func TestCookiePersistenceSuite(t *testing.T) { suite.Run(t, new(CookiePersistenceSuite)) } +func (suite *CookiePersistenceSuite) resetTemplateData() { + suite.tmplData = tmplData{ + Host: suite.test.GetNS() + ".test", + Replicas: 1, + SourceIPPersistenceSize: "1m", + SourceIPPersistenceExpire: "30m", + } +} + // Check that the server serverName for backend backendName // has a "cookie" params with a value equal to the server name func (suite *CookiePersistenceSuite) checkServerCookie(p parser.Parser, backendName, serverName string) { diff --git a/deploy/tests/e2e/dump.go b/deploy/tests/e2e/dump.go index 5d276cb5..2b0d138e 100644 --- a/deploy/tests/e2e/dump.go +++ b/deploy/tests/e2e/dump.go @@ -14,7 +14,10 @@ package e2e -import "strings" +import ( + "fmt" + "strings" +) func (t Test) GetIngressControllerFile(path string) (string, error) { po, err := t.getIngressControllerPod() @@ -30,5 +33,12 @@ func (t Test) GetIngressControllerFile(path string) (string, error) { func (t Test) getIngressControllerPod() (string, error) { out, errExec := t.execute("", "kubectl", "get", "pods", "-n", "haproxy-controller", "-l", "run=haproxy-ingress", "-o", "name", "--field-selector=status.phase==Running", "-l", "run=haproxy-ingress") - return strings.TrimSuffix(out, "\n"), errExec + if errExec != nil { + return "", errExec + } + pods := strings.Fields(out) + if len(pods) == 0 { + return "", fmt.Errorf("no running ingress controller pod found") + } + return pods[0], nil } diff --git a/documentation/annotations.md b/documentation/annotations.md index 4c1bed4e..c0c7bdd2 100644 --- a/documentation/annotations.md +++ b/documentation/annotations.md @@ -41,6 +41,9 @@ more info about custom annotations can be found in [annotations-custom.md](annot | [backend-config-snippet](#config-snippet) | string | | |:large_blue_circle:|:large_blue_circle:|:large_blue_circle:| | [cookie-persistence](#cookie-persistence) | string | | |:large_blue_circle:|:white_circle:|:large_blue_circle:| | [cookie-persistence-no-dynamic](#cookie-persistence-no-dynamic) | string | | |:large_blue_circle:|:white_circle:|:large_blue_circle:| +| [source-ip-persistence](#source-ip-persistence) :construction:(dev) | [bool](#bool) | | |:large_blue_circle:|:white_circle:|:large_blue_circle:| +| [source-ip-persistence-size](#source-ip-persistence) :construction:(dev) | string | "1m" | source-ip-persistence |:large_blue_circle:|:white_circle:|:large_blue_circle:| +| [source-ip-persistence-expire](#source-ip-persistence) :construction:(dev) | string | "30m" | source-ip-persistence |:large_blue_circle:|:white_circle:|:large_blue_circle:| | [dontlognull](#logging) | [bool](#bool) | "true" | |:large_blue_circle:|:white_circle:|:white_circle:| | [src-ip-header](#src-ip-header) | string | "null" | |:large_blue_circle:|:large_blue_circle:|:white_circle:| | [forwarded-for](#x-forwarded-for) | [bool](#bool) | "true" | |:large_blue_circle:|:large_blue_circle:|:large_blue_circle:| @@ -1255,6 +1258,8 @@ rate-limit-status-code: "429" :information_source: To track the http requests rate, a stick-table named "Ratelimit-" will be created. For example, if the `rate-limit-period` is set to *2s*, the name of the table will be *Ratelimit-2000*. + :information_source: Generated rate-limit stick tables use the `localinstance` peer section. When multiple controller pods expose a reachable `--localpeer-port`, the controller reconciles peer entries for pods in the same controller workload so stick-table updates can replicate between those pods. + Possible values: - An integer representing the maximum number of requests to accept @@ -1655,6 +1660,83 @@ set-host: "example.local" *** +#### Source Ip Persistence + +##### `source-ip-persistence` + + + > :construction: this is only available from next version, currently available in dev build + + Enables source IP based persistent connections (sticky sessions) between a client and a pod by creating a backend stick table and sticking on the request source address. + This persistence mode works for HTTP and TCP backends and can be used together with source-hash or MagLev-style balancing so that hash-based placement chooses the initial pod while the stick table preserves the selected pod across later requests and failover events. + + Available on: `configmap` `service` + + :information_source: When enabled, the controller inserts `stick-table type ip size expire peers localinstance` and `stick on src` in the corresponding backend. + + :information_source: Generated source-IP persistence stick tables use the `localinstance` peer section. When multiple controller pods expose a reachable `--localpeer-port`, the controller reconciles peer entries for pods in the same controller workload so stick-table updates can replicate between those pods. + + :information_source: Backend endpoint addresses are assigned to server slots deterministically so every controller replica uses the same `SRV_` identity for source-hash balancing and peer-replicated stick-table entries. + + :information_source: This annotation is resolved at the service level, falling back to the configmap default. As the HAProxy backend is shared by every ingress referencing the same service, setting it on an ingress is ignored to avoid a non-deterministic backend configuration. The service value takes precedence over the configmap default. + +Possible values: + +- true +- false + +Example: + +```yaml +source-ip-persistence: "true" +``` + +##### `source-ip-persistence-size` + + + > :construction: this is only available from next version, currently available in dev build + + Sets the size of the source IP persistence stick table. + + Available on: `configmap` `service` + + :information_source: This value is only used when `source-ip-persistence` is enabled. + +Possible values: + +- A valid HAProxy stick-table size, such as `100k`, `1m`, or a plain integer value + +Example: + +```yaml +source-ip-persistence-size: "1m" +``` + +##### `source-ip-persistence-expire` + + + > :construction: this is only available from next version, currently available in dev build + + Sets how long an idle source IP persistence table entry is kept. + + Available on: `configmap` `service` + + :information_source: This value is only used when `source-ip-persistence` is enabled. + +Possible values: + +- A valid HAProxy time value, such as `30s`, `30m`, or `1h` + +Example: + +```yaml +source-ip-persistence-expire: "30m" +``` + +

:arrow_up_small: back to top

+ +*** + #### Src Ip Header ##### `src-ip-header` diff --git a/documentation/controller.md b/documentation/controller.md index 8e8b29a3..7bcd70dc 100644 --- a/documentation/controller.md +++ b/documentation/controller.md @@ -31,6 +31,7 @@ Image can be run with arguments: | [`--ipv6-bind-address`](#--ipv6-bind-address) | `::` | | [`--http-bind-port`](#--http-bind-port) | `8080` | | [`--https-bind-port`](#--https-bind-port) | `8443` | +| [`--localpeer-port`](#--localpeer-port) | `10000` | | [`--disable-http`](#--disable-http) | `false` | | [`--disable-https`](#--disable-https) | `false` | | [`--sync-period`](#--sync-period) | `5s` | @@ -535,6 +536,24 @@ Example: *** +### `--localpeer-port` + + Sets the TCP port used in HAProxy peer entries managed by the controller's `localinstance` peer section. HAProxy uses this peer section for generated stick tables, including request rate limiting. The controller manages the running pod's peer entry and reconciles entries for pods that belong to the same controller Deployment, DaemonSet, or ReplicaSet. Expose this port between controller pods when peer replication is required. This option is not a general static peer mesh configuration; manually defined peer sections are not reconciled by this controller logic. + +Possible values: + +- A valid port in the range. Default: 10000 + +Example: + +```yaml +--localpeer-port=10000 +``` + +

:arrow_up_small: back to top

+ +*** + ### `--disable-http` Disabling the HTTP frontend. diff --git a/documentation/doc.yaml b/documentation/doc.yaml index d3e269ce..0036fa04 100644 --- a/documentation/doc.yaml +++ b/documentation/doc.yaml @@ -265,6 +265,13 @@ image_arguments: helm: |- helm install haproxy haproxytech/kubernetes-ingress \ --set-string "controller.extraArgs={--https-bind-port=8443}" + - argument: --localpeer-port + description: Sets the TCP port used in HAProxy peer entries managed by the controller's `localinstance` peer section. HAProxy uses this peer section for generated stick tables, including request rate limiting. The controller manages the running pod's peer entry and reconciles entries for pods that belong to the same controller Deployment, DaemonSet, or ReplicaSet. Expose this port between controller pods when peer replication is required. This option is not a general static peer mesh configuration; manually defined peer sections are not reconciled by this controller logic. + values: + - "A valid port in the range. Default: 10000" + default: 10000 + version_min: "1.9" + example: --localpeer-port=10000 - argument: --disable-http description: Disabling the HTTP frontend. values: @@ -946,6 +953,56 @@ annotations: - service version_min: "3.1" example: ['cookie-persistence-no-dynamic: "mycookie"'] + - title: source-ip-persistence + type: bool + description: + - Enables source IP based persistent connections (sticky sessions) between a client and a pod by creating a backend stick table and sticking on the request source address. + - This persistence mode works for HTTP and TCP backends and can be used together with source-hash or MagLev-style balancing so that hash-based placement chooses the initial pod while the stick table preserves the selected pod across later requests and failover events. + tip: + - When enabled, the controller inserts `stick-table type ip size expire peers localinstance` and `stick on src` in the corresponding backend. + - Generated source-IP persistence stick tables use the `localinstance` peer section. When multiple controller pods expose a reachable `--localpeer-port`, the controller reconciles peer entries for pods in the same controller workload so stick-table updates can replicate between those pods. + - Backend endpoint addresses are assigned to server slots deterministically so every controller replica uses the same `SRV_` identity for source-hash balancing and peer-replicated stick-table entries. + - This annotation is resolved at the service level, falling back to the configmap default. As the HAProxy backend is shared by every ingress referencing the same service, setting it on an ingress is ignored to avoid a non-deterministic backend configuration. The service value takes precedence over the configmap default. + values: + - "true" + - "false" + applies_to: + - configmap + - service + version_min: "3.3" + example: ['source-ip-persistence: "true"'] + - title: source-ip-persistence-size + type: string + group: source-ip-persistence + dependencies: "source-ip-persistence" + default: "1m" + description: + - Sets the size of the source IP persistence stick table. + tip: + - This value is only used when `source-ip-persistence` is enabled. + values: + - A valid HAProxy stick-table size, such as `100k`, `1m`, or a plain integer value + applies_to: + - configmap + - service + version_min: "3.3" + example: ['source-ip-persistence-size: "1m"'] + - title: source-ip-persistence-expire + type: string + group: source-ip-persistence + dependencies: "source-ip-persistence" + default: "30m" + description: + - Sets how long an idle source IP persistence table entry is kept. + tip: + - This value is only used when `source-ip-persistence` is enabled. + values: + - A valid HAProxy time value, such as `30s`, `30m`, or `1h` + applies_to: + - configmap + - service + version_min: "3.3" + example: ['source-ip-persistence-expire: "30m"'] - title: dontlognull type: bool group: logging @@ -1316,6 +1373,7 @@ annotations: tip: - If this number is exceeded, HAProxy will deny requests with 403 status code. - To track the http requests rate, a stick-table named "Ratelimit-" will be created. For example, if the `rate-limit-period` is set to *2s*, the name of the table will be *Ratelimit-2000*. + - Generated rate-limit stick tables use the `localinstance` peer section. When multiple controller pods expose a reachable `--localpeer-port`, the controller reconciles peer entries for pods in the same controller workload so stick-table updates can replicate between those pods. values: - An integer representing the maximum number of requests to accept applies_to: diff --git a/pkg/annotations/service/source_ip_persistence.go b/pkg/annotations/service/source_ip_persistence.go new file mode 100644 index 00000000..5e596df9 --- /dev/null +++ b/pkg/annotations/service/source_ip_persistence.go @@ -0,0 +1,105 @@ +// Copyright 2019 HAProxy Technologies LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "fmt" + + "github.com/haproxytech/client-native/v6/models" + + "github.com/haproxytech/kubernetes-ingress/pkg/annotations/common" + "github.com/haproxytech/kubernetes-ingress/pkg/store" + "github.com/haproxytech/kubernetes-ingress/pkg/utils" +) + +const ( + defaultSourceIPPersistenceSize = "1m" + defaultSourceIPPersistenceExpire = "30m" + localPeerSection = "localinstance" +) + +type SourceIPPersistence struct { + backend *models.Backend + name string + nameSize string + nameExpire string +} + +func NewSourceIPPersistence(n string, b *models.Backend) *SourceIPPersistence { + return &SourceIPPersistence{ + name: n, + nameSize: n + "-size", + nameExpire: n + "-expire", + backend: b, + } +} + +func (a *SourceIPPersistence) GetName() string { + return a.name +} + +func (a *SourceIPPersistence) Process(k store.K8s, annotations ...map[string]string) error { + input := common.GetValue(a.GetName(), annotations...) + if input == "" { + a.clear() + return nil + } + + enabled, err := utils.GetBoolValue(input, a.GetName()) + if err != nil { + return fmt.Errorf("%s: %w", a.GetName(), err) + } + if !enabled { + a.clear() + return nil + } + + sizeInput := common.GetValue(a.nameSize, annotations...) + if sizeInput == "" { + sizeInput = defaultSourceIPPersistenceSize + } + size, err := utils.ParseSize(sizeInput) + if err != nil { + return fmt.Errorf("%s: %w", a.nameSize, err) + } + + expireInput := common.GetValue(a.nameExpire, annotations...) + if expireInput == "" { + expireInput = defaultSourceIPPersistenceExpire + } + expire, err := utils.ParseTime(expireInput) + if err != nil { + return fmt.Errorf("%s: %w", a.nameExpire, err) + } + + a.backend.StickTable = &models.ConfigStickTable{ + Type: "ip", + Size: size, + Expire: expire, + Peers: localPeerSection, + } + a.backend.StickRuleList = models.StickRules{ + { + Type: "on", + Pattern: "src", + }, + } + return nil +} + +func (a *SourceIPPersistence) clear() { + a.backend.StickTable = nil + a.backend.StickRuleList = nil +} diff --git a/pkg/controller/builder.go b/pkg/controller/builder.go index b81f933f..de20cfd0 100644 --- a/pkg/controller/builder.go +++ b/pkg/controller/builder.go @@ -16,6 +16,7 @@ package controller import ( "bytes" + "context" "encoding/base64" "errors" "log" @@ -178,7 +179,7 @@ func (builder *Builder) Build() *HAProxyController { logger.Panic(haproxy.UploadFrontends()) - prefix, errPrefix := utils.GetPodPrefix(os.Getenv("POD_NAME")) + prefix, errPrefix := getControllerPodPrefix(context.Background(), builder.clientSet, os.Getenv("POD_NAMESPACE"), os.Getenv("POD_NAME")) logger.Error(errPrefix) builder.store.GatewayControllerName = builder.osArgs.GatewayControllerName diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 927b6302..63c5c0dd 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -43,6 +43,8 @@ import ( var logger = utils.GetLogger() +const localPeerSection = "localinstance" + // HAProxyController is ingress controller type HAProxyController struct { store store.K8s @@ -85,18 +87,11 @@ func (c *HAProxyController) clientAPIClosure(fn func() error) (err error) { // Start initializes and runs HAProxyController func (c *HAProxyController) Start() { logger.Panic(c.clientAPIClosure(func() error { - err := c.haproxy.PeerEntryDelete("localinstance", "local") + err := c.haproxy.PeerEntryDelete(localPeerSection, "local") if err != nil { return err } - return c.haproxy.PeerEntryCreateOrEdit( - "localinstance", - models.PeerEntry{ - Name: c.Hostname, - Address: &c.PodIP, - Port: &c.osArgs.LocalPeerPort, - }, - ) + return c.haproxy.PeerEntryCreateOrEdit(localPeerSection, c.localPeerEntry(c.Hostname, c.PodIP)) })) c.initHandlers() logger.Error(c.setupHAProxyRules()) @@ -135,6 +130,7 @@ func (c *HAProxyController) updateHAProxy() { logger.Trace("HAProxy config sync transaction started") c.handleGlobalConfig() + logger.Error(c.reconcileLocalPeerEntries()) if len(route.CustomRoutes) != 0 { logger.Error(route.CustomRoutesReset(c.haproxy)) diff --git a/pkg/controller/peer_entries.go b/pkg/controller/peer_entries.go new file mode 100644 index 00000000..891dc402 --- /dev/null +++ b/pkg/controller/peer_entries.go @@ -0,0 +1,103 @@ +// Copyright 2026 HAProxy Technologies LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "strings" + + "github.com/haproxytech/client-native/v6/models" + "github.com/haproxytech/kubernetes-ingress/pkg/haproxy/instance" +) + +func (c *HAProxyController) reconcileLocalPeerEntries() error { + currentEntries, err := c.haproxy.PeerEntriesGet(localPeerSection) + if err != nil { + return err + } + + currentByName := map[string]*models.PeerEntry{} + for _, currentEntry := range currentEntries { + if currentEntry != nil { + currentByName[currentEntry.Name] = currentEntry + } + } + + expectedEntries := c.localPeerEntries() + changed := false + for _, currentEntry := range currentEntries { + if currentEntry == nil || !c.managesPeerEntry(currentEntry.Name) { + continue + } + if _, ok := expectedEntries[currentEntry.Name]; ok { + continue + } + if err := c.haproxy.PeerEntryDelete(localPeerSection, currentEntry.Name); err != nil { + return err + } + changed = true + } + + for name, expectedEntry := range expectedEntries { + currentEntry, ok := currentByName[name] + if ok && peerEntryEqual(currentEntry, expectedEntry) { + continue + } + if err := c.haproxy.PeerEntryCreateOrEdit(localPeerSection, expectedEntry); err != nil { + return err + } + changed = true + } + + if changed { + instance.Reload("HAProxy local peer entries changed") + } + return nil +} + +func (c *HAProxyController) localPeerEntries() map[string]models.PeerEntry { + expectedEntries := map[string]models.PeerEntry{} + if c.PodIP != "" { + expectedEntries[c.Hostname] = c.localPeerEntry(c.Hostname, c.PodIP) + } + for podName, podIP := range c.store.HaProxyPods { + if podIP == "" { + continue + } + expectedEntries[podName] = c.localPeerEntry(podName, podIP) + } + return expectedEntries +} + +func (c *HAProxyController) localPeerEntry(name string, address string) models.PeerEntry { + return models.PeerEntry{ + Name: name, + Address: &address, + Port: &c.osArgs.LocalPeerPort, + } +} + +func (c *HAProxyController) managesPeerEntry(name string) bool { + if c.podPrefix == "" { + return name == c.Hostname + } + return name == c.Hostname || strings.HasPrefix(name, c.podPrefix+"-") +} + +func peerEntryEqual(currentEntry *models.PeerEntry, expectedEntry models.PeerEntry) bool { + if currentEntry == nil || currentEntry.Address == nil || expectedEntry.Address == nil || currentEntry.Port == nil || expectedEntry.Port == nil { + return false + } + return currentEntry.Name == expectedEntry.Name && *currentEntry.Address == *expectedEntry.Address && *currentEntry.Port == *expectedEntry.Port +} diff --git a/pkg/controller/pods.go b/pkg/controller/pods.go new file mode 100644 index 00000000..fd58c380 --- /dev/null +++ b/pkg/controller/pods.go @@ -0,0 +1,56 @@ +// Copyright 2026 HAProxy Technologies LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + + "github.com/haproxytech/kubernetes-ingress/pkg/utils" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +func getControllerPodPrefix(ctx context.Context, clientset kubernetes.Interface, namespace, podName string) (string, error) { + fallbackPrefix, fallbackErr := utils.GetPodPrefix(podName) + if clientset == nil || namespace == "" || podName == "" { + return fallbackPrefix, fallbackErr + } + + pod, err := clientset.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return fallbackPrefix, err + } + ownerRef := metav1.GetControllerOf(pod) + if ownerRef == nil { + return fallbackPrefix, fallbackErr + } + + switch ownerRef.Kind { + case "DaemonSet", "Deployment": + return ownerRef.Name, nil + case "ReplicaSet": + replicaSet, err := clientset.AppsV1().ReplicaSets(namespace).Get(ctx, ownerRef.Name, metav1.GetOptions{}) + if err != nil { + return fallbackPrefix, err + } + replicaSetOwnerRef := metav1.GetControllerOf(replicaSet) + if replicaSetOwnerRef == nil || replicaSetOwnerRef.Kind != "Deployment" { + return ownerRef.Name, nil + } + return replicaSetOwnerRef.Name, nil + default: + return fallbackPrefix, fallbackErr + } +} diff --git a/pkg/controller/pods_test.go b/pkg/controller/pods_test.go new file mode 100644 index 00000000..773cffc2 --- /dev/null +++ b/pkg/controller/pods_test.go @@ -0,0 +1,94 @@ +// Copyright 2026 HAProxy Technologies LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + + "github.com/haproxytech/kubernetes-ingress/pkg/utils" +) + +func TestGetControllerPodPrefixResolvesDeploymentName(t *testing.T) { + client := fake.NewSimpleClientset( + controllerReplicaSet("default", "haproxy-ingress-abc123", "rs-1", "haproxy-ingress", "deploy-1"), + controllerPod("default", "haproxy-ingress-abc123-aaaaa", "pod-1", "ReplicaSet", "haproxy-ingress-abc123", "rs-1"), + ) + + prefix, err := getControllerPodPrefix(context.Background(), client, "default", "haproxy-ingress-abc123-aaaaa") + if err != nil { + t.Fatal(err) + } + if prefix != "haproxy-ingress" { + t.Fatalf("expected Deployment name prefix, got %q", prefix) + } +} + +func TestGetControllerPodPrefixResolvesDaemonSetName(t *testing.T) { + client := fake.NewSimpleClientset( + controllerPod("default", "haproxy-ingress-aaaaa", "pod-1", "DaemonSet", "haproxy-ingress", "ds-1"), + ) + + prefix, err := getControllerPodPrefix(context.Background(), client, "default", "haproxy-ingress-aaaaa") + if err != nil { + t.Fatal(err) + } + if prefix != "haproxy-ingress" { + t.Fatalf("expected DaemonSet name prefix, got %q", prefix) + } +} + +func controllerPod(namespace, name, uid, ownerKind, ownerName, ownerUID string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + UID: types.UID(uid), + OwnerReferences: []metav1.OwnerReference{ + controllerOwnerReference(ownerKind, ownerName, types.UID(ownerUID)), + }, + }, + } +} + +func controllerReplicaSet(namespace, name, uid, ownerName, ownerUID string) *appsv1.ReplicaSet { + return &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + UID: types.UID(uid), + OwnerReferences: []metav1.OwnerReference{ + controllerOwnerReference("Deployment", ownerName, types.UID(ownerUID)), + }, + }, + } +} + +func controllerOwnerReference(kind, name string, uid types.UID) metav1.OwnerReference { + return metav1.OwnerReference{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: kind, + Name: name, + UID: uid, + Controller: utils.Ptr(true), + BlockOwnerDeletion: utils.Ptr(true), + } +} diff --git a/pkg/haproxy/api/api.go b/pkg/haproxy/api/api.go index 2aa8323a..ec603dce 100644 --- a/pkg/haproxy/api/api.go +++ b/pkg/haproxy/api/api.go @@ -95,6 +95,7 @@ type HAProxyClient interface { //nolint:interfacebloat PeerEntryDelete(peerSection string, name string) error PeerEntryEdit(peerSection string, peer models.PeerEntry) error PeerEntryCreateOrEdit(peerSection string, peer models.PeerEntry) error + PeerEntriesGet(peerSection string) (models.PeerEntries, error) SetMapContent(mapFile string, payload []string) error SetServerAddrAndState([]RuntimeServerData) error SetAuxCfgFile(auxCfgFile string) diff --git a/pkg/haproxy/api/frontend.go b/pkg/haproxy/api/frontend.go index fb613ae4..88d78af6 100644 --- a/pkg/haproxy/api/frontend.go +++ b/pkg/haproxy/api/frontend.go @@ -502,6 +502,15 @@ func (c *clientNative) PeerEntryCreateOrEdit(peerSection string, peerEntry model return err } +func (c *clientNative) PeerEntriesGet(peerSection string) (models.PeerEntries, error) { + configuration, err := c.nativeAPI.Configuration() + if err != nil { + return nil, err + } + _, peerEntries, err := configuration.GetPeerEntries(peerSection, c.activeTransaction) + return peerEntries, err +} + func (c *clientNative) PeerEntryDelete(peerSection, entry string) error { cfg, err := c.nativeAPI.Configuration() if err != nil { diff --git a/pkg/haproxy/api/runtime.go b/pkg/haproxy/api/runtime.go index 335a59c4..f926def9 100644 --- a/pkg/haproxy/api/runtime.go +++ b/pkg/haproxy/api/runtime.go @@ -186,7 +186,7 @@ func (c *clientNative) SyncBackendSrvs(backend *store.RuntimeBackend) error { } // Configure new Endpoints in available HAProxySrvs - for newEndpoint := range endpoints { + for _, newEndpoint := range store.SortedRuntimeEndpoints(endpoints) { if len(disabled) == 0 { break } diff --git a/pkg/k8s/informers.go b/pkg/k8s/informers.go index eb60a08a..154dfb32 100644 --- a/pkg/k8s/informers.go +++ b/pkg/k8s/informers.go @@ -541,9 +541,8 @@ func (k k8s) getEndpointsInformer(eventChan chan k8ssync.SyncDataEvent, factory return informer } -func (k *k8s) getPodInformer(namespace, podPrefix string, resyncPeriod time.Duration, eventChan chan k8ssync.SyncDataEvent) cache.Controller { //nolint:ireturn - var prefix string - watchlist := cache.NewListWatchFromClient(k.builtInClient.CoreV1().RESTClient(), "pods", namespace, fields.Nothing()) +func (k *k8s) getPodInformer(namespace string, podMatcher controllerPodMatcher, resyncPeriod time.Duration, eventChan chan k8ssync.SyncDataEvent) cache.Controller { //nolint:ireturn + watchlist := cache.NewListWatchFromClient(k.builtInClient.CoreV1().RESTClient(), "pods", namespace, controllerPodFieldSelector()) _, eController := cache.NewInformerWithOptions( cache.InformerOptions{ ListerWatcher: watchlist, @@ -551,19 +550,18 @@ func (k *k8s) getPodInformer(namespace, podPrefix string, resyncPeriod time.Dura ResyncPeriod: resyncPeriod, Handler: cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - meta := obj.(*corev1.Pod).ObjectMeta - prefix, _ = utils.GetPodPrefix(meta.Name) - if prefix != podPrefix { + pod := obj.(*corev1.Pod) + meta := pod.ObjectMeta + if !podMatcher.matches(pod) { return } - item := store.PodEvent{Status: store.ADDED, Name: meta.Name, Namespace: meta.Namespace} + item := store.PodEvent{Status: store.ADDED, Name: meta.Name, Namespace: meta.Namespace, IP: pod.Status.PodIP} eventChan <- ToSyncDataEvent(item, item, meta.UID, meta.ResourceVersion) }, DeleteFunc: func(obj interface{}) { //revive:disable-next-line:unchecked-type-assertion meta := obj.(*corev1.Pod).ObjectMeta - prefix, _ = utils.GetPodPrefix(meta.Name) - if prefix != podPrefix { + if !podMatcher.matches(obj.(*corev1.Pod)) { return } item := store.PodEvent{Status: store.DELETED, Name: meta.Name, Namespace: meta.Namespace} @@ -571,12 +569,12 @@ func (k *k8s) getPodInformer(namespace, podPrefix string, resyncPeriod time.Dura }, UpdateFunc: func(oldObj, newObj interface{}) { //revive:disable-next-line:unchecked-type-assertion - meta := newObj.(*corev1.Pod).ObjectMeta - prefix, _ = utils.GetPodPrefix(meta.Name) - if prefix != podPrefix { + pod := newObj.(*corev1.Pod) + meta := pod.ObjectMeta + if !podMatcher.matches(pod) { return } - item := store.PodEvent{Status: store.MODIFIED, Name: meta.Name, Namespace: meta.Namespace} + item := store.PodEvent{Status: store.MODIFIED, Name: meta.Name, Namespace: meta.Namespace, IP: pod.Status.PodIP} eventChan <- ToSyncDataEvent(item, item, meta.UID, meta.ResourceVersion) }, }, @@ -586,6 +584,10 @@ func (k *k8s) getPodInformer(namespace, podPrefix string, resyncPeriod time.Dura return eController } +func controllerPodFieldSelector() fields.Selector { + return fields.Everything() +} + func (k k8s) addIngressClassHandlers(eventChan chan k8ssync.SyncDataEvent, informer cache.SharedIndexInformer) { errW := informer.SetWatchErrorHandler(func(r *cache.Reflector, err error) { go logger.Debug("IngressClass informer error: %s", err) diff --git a/pkg/k8s/main.go b/pkg/k8s/main.go index 514e8b94..583bdde7 100644 --- a/pkg/k8s/main.go +++ b/pkg/k8s/main.go @@ -95,7 +95,7 @@ type k8s struct { publishSvc *utils.NamespaceValue gatewayClient *gatewayclientset.Clientset crdClient *crdclientset.Clientset - podPrefix string + podMatcher controllerPodMatcher podNamespace string whiteListedNS []string syncPeriod time.Duration @@ -132,7 +132,9 @@ func New(osArgs utils.OSArgs, whitelist map[string]struct{}, publishSvc *utils.N logger.Error("CRD API client not present") } - prefix, _ := utils.GetPodPrefix(os.Getenv("POD_NAME")) + podName := os.Getenv("POD_NAME") + podNamespace := os.Getenv("POD_NAMESPACE") + prefix, _ := utils.GetPodPrefix(podName) k := k8s{ builtInClient: builtInClient, crClientV1: crclientsetv1.NewForConfigOrDie(restconfig), @@ -143,8 +145,8 @@ func New(osArgs utils.OSArgs, whitelist map[string]struct{}, publishSvc *utils.N crsRegisteredOnStart: map[string]struct{}{}, whiteListedNS: getWhitelistedNS(whitelist, osArgs.ConfigMap.Namespace), publishSvc: publishSvc, - podNamespace: os.Getenv("POD_NAMESPACE"), - podPrefix: prefix, + podNamespace: podNamespace, + podMatcher: newControllerPodMatcher(context.Background(), builtInClient, podNamespace, podName, prefix), syncPeriod: osArgs.SyncPeriod, initialSyncPeriod: osArgs.InitialSyncPeriod, cacheResyncPeriod: osArgs.CacheResyncPeriod, @@ -354,8 +356,8 @@ func (k k8s) runInformersGwAPI(eventChan chan k8ssync.SyncDataEvent, stop chan s } func (k k8s) runPodInformer(eventChan chan k8ssync.SyncDataEvent, stop chan struct{}, informersSynced *[]cache.InformerSynced) { - if k.podPrefix != "" { - pi := k.getPodInformer(k.podNamespace, k.podPrefix, k.cacheResyncPeriod, eventChan) + if k.podMatcher.enabled() { + pi := k.getPodInformer(k.podNamespace, k.podMatcher, k.cacheResyncPeriod, eventChan) go pi.Run(stop) *informersSynced = append(*informersSynced, pi.HasSynced) } diff --git a/pkg/k8s/pods.go b/pkg/k8s/pods.go new file mode 100644 index 00000000..940a2ab8 --- /dev/null +++ b/pkg/k8s/pods.go @@ -0,0 +1,113 @@ +// Copyright 2026 HAProxy Technologies LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k8s + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + k8sclientset "k8s.io/client-go/kubernetes" + + "github.com/haproxytech/kubernetes-ingress/pkg/utils" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type controllerOwner struct { + kind string + name string + uid types.UID +} + +type controllerPodMatcher struct { + client k8sclientset.Interface + namespace string + owner controllerOwner + fallbackPrefix string +} + +func newControllerPodMatcher(ctx context.Context, client k8sclientset.Interface, namespace, podName, fallbackPrefix string) controllerPodMatcher { + matcher := controllerPodMatcher{ + client: client, + namespace: namespace, + fallbackPrefix: fallbackPrefix, + } + if client == nil || namespace == "" || podName == "" { + return matcher + } + pod, err := client.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + logger.Warningf("unable to resolve current pod owner for peer discovery: %v", err) + return matcher + } + owner, ok := matcher.resolvePodController(ctx, pod) + if !ok { + return matcher + } + matcher.owner = owner + return matcher +} + +func (m controllerPodMatcher) enabled() bool { + return m.owner.kind != "" || m.fallbackPrefix != "" +} + +func (m controllerPodMatcher) matches(pod *corev1.Pod) bool { + if pod == nil { + return false + } + if m.owner.kind != "" { + owner, ok := m.resolvePodController(context.Background(), pod) + return ok && owner == m.owner + } + if m.fallbackPrefix == "" { + return false + } + prefix, _ := utils.GetPodPrefix(pod.Name) + return prefix == m.fallbackPrefix +} + +func (m controllerPodMatcher) resolvePodController(ctx context.Context, pod *corev1.Pod) (controllerOwner, bool) { + ownerRef := metav1.GetControllerOf(pod) + if ownerRef == nil { + return controllerOwner{}, false + } + switch ownerRef.Kind { + case "DaemonSet": + return controllerOwner{kind: ownerRef.Kind, name: ownerRef.Name, uid: ownerRef.UID}, true + case "ReplicaSet": + return m.resolveReplicaSetOwner(ctx, ownerRef.Name) + case "Deployment": + return controllerOwner{kind: ownerRef.Kind, name: ownerRef.Name, uid: ownerRef.UID}, true + default: + return controllerOwner{}, false + } +} + +func (m controllerPodMatcher) resolveReplicaSetOwner(ctx context.Context, replicaSetName string) (controllerOwner, bool) { + if m.client == nil || m.namespace == "" { + return controllerOwner{}, false + } + replicaSet, err := m.client.AppsV1().ReplicaSets(m.namespace).Get(ctx, replicaSetName, metav1.GetOptions{}) + if err != nil { + logger.Warningf("unable to resolve ReplicaSet owner for peer discovery: %v", err) + return controllerOwner{}, false + } + ownerRef := metav1.GetControllerOf(replicaSet) + if ownerRef == nil || ownerRef.Kind != "Deployment" { + return controllerOwner{kind: "ReplicaSet", name: replicaSet.Name, uid: replicaSet.UID}, true + } + return controllerOwner{kind: ownerRef.Kind, name: ownerRef.Name, uid: ownerRef.UID}, true +} diff --git a/pkg/k8s/pods_test.go b/pkg/k8s/pods_test.go new file mode 100644 index 00000000..93b0acd0 --- /dev/null +++ b/pkg/k8s/pods_test.go @@ -0,0 +1,108 @@ +// Copyright 2026 HAProxy Technologies LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k8s + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + + "github.com/haproxytech/kubernetes-ingress/pkg/utils" +) + +func TestControllerPodMatcherMatchesDeploymentPodsAcrossReplicaSets(t *testing.T) { + client := fake.NewSimpleClientset( + replicaSet("default", "haproxy-ingress-abc123", "rs-1", "haproxy-ingress", "deploy-1"), + replicaSet("default", "haproxy-ingress-def456", "rs-2", "haproxy-ingress", "deploy-1"), + replicaSet("default", "haproxy-ingress-other", "rs-3", "haproxy-ingress-other", "deploy-2"), + pod("default", "haproxy-ingress-abc123-aaaaa", "pod-1", "ReplicaSet", "haproxy-ingress-abc123", "rs-1"), + ) + matcher := newControllerPodMatcher(context.Background(), client, "default", "haproxy-ingress-abc123-aaaaa", "haproxy-ingress") + + if !matcher.matches(pod("default", "haproxy-ingress-def456-bbbbb", "pod-2", "ReplicaSet", "haproxy-ingress-def456", "rs-2")) { + t.Fatal("expected pod from another ReplicaSet owned by the same Deployment to match") + } + if matcher.matches(pod("default", "haproxy-ingress-other-ccccc", "pod-3", "ReplicaSet", "haproxy-ingress-other", "rs-3")) { + t.Fatal("expected pod from another Deployment to be ignored") + } +} + +func TestControllerPodMatcherMatchesDaemonSetPodsExactly(t *testing.T) { + client := fake.NewSimpleClientset( + pod("default", "haproxy-ingress-aaaaa", "pod-1", "DaemonSet", "haproxy-ingress", "ds-1"), + ) + matcher := newControllerPodMatcher(context.Background(), client, "default", "haproxy-ingress-aaaaa", "haproxy") + + if !matcher.matches(pod("default", "haproxy-ingress-bbbbb", "pod-2", "DaemonSet", "haproxy-ingress", "ds-1")) { + t.Fatal("expected pod from the same DaemonSet to match") + } + if matcher.matches(pod("default", "haproxy-other-ccccc", "pod-3", "DaemonSet", "haproxy-other", "ds-2")) { + t.Fatal("expected pod from another DaemonSet to be ignored") + } +} + +func TestControllerPodFieldSelectorMatchesAnyPod(t *testing.T) { + selector := controllerPodFieldSelector() + + if !selector.Matches(fields.Set{"metadata.name": "haproxy-ingress-abc123-aaaaa"}) { + t.Fatal("expected peer pod informer selector to match pod names") + } + if !selector.Matches(fields.Set{"spec.nodeName": "worker-1"}) { + t.Fatal("expected peer pod informer selector to match pods without a name field") + } +} + +func pod(namespace, name, uid, ownerKind, ownerName, ownerUID string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + UID: types.UID(uid), + OwnerReferences: []metav1.OwnerReference{ + newControllerOwnerReference(ownerKind, ownerName, types.UID(ownerUID)), + }, + }, + } +} + +func replicaSet(namespace, name, uid, ownerName, ownerUID string) *appsv1.ReplicaSet { + return &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + UID: types.UID(uid), + OwnerReferences: []metav1.OwnerReference{ + newControllerOwnerReference("Deployment", ownerName, types.UID(ownerUID)), + }, + }, + } +} + +func newControllerOwnerReference(kind, name string, uid types.UID) metav1.OwnerReference { + return metav1.OwnerReference{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: kind, + Name: name, + UID: uid, + Controller: utils.Ptr(true), + BlockOwnerDeletion: utils.Ptr(true), + } +} diff --git a/pkg/service/endpoints.go b/pkg/service/endpoints.go index 5ca5da80..d8a17d81 100644 --- a/pkg/service/endpoints.go +++ b/pkg/service/endpoints.go @@ -132,7 +132,7 @@ func (s *Service) scaleHAProxySrvs(backend *store.RuntimeBackend) { copy(slots, backend.HAProxySrvs) i := len(backend.HAProxySrvs) // ... then add the new slots ... - for endpoint := range backend.Endpoints { + for _, endpoint := range store.SortedRuntimeEndpoints(backend.Endpoints) { srv := &store.HAProxySrv{ Name: fmt.Sprintf("SRV_%d", i+1), Address: endpoint.Address, diff --git a/pkg/service/endpoints_test.go b/pkg/service/endpoints_test.go index b2cdd491..831a2dba 100644 --- a/pkg/service/endpoints_test.go +++ b/pkg/service/endpoints_test.go @@ -17,6 +17,7 @@ package service import ( "testing" + "github.com/haproxytech/client-native/v6/models" "github.com/haproxytech/kubernetes-ingress/pkg/store" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -196,3 +197,30 @@ func TestGetRuntimeBackend_ClusterIP_PortMismatch(t *testing.T) { assert.Error(t, err) assert.Contains(t, err.Error(), "web") } + +func TestScaleHAProxySrvsOrdersEndpointsDeterministically(t *testing.T) { + svc := &Service{ + backend: &models.Backend{BackendBase: models.BackendBase{Name: "backend"}}, + } + backend := &store.RuntimeBackend{ + Endpoints: store.RuntimeEndpoints{ + {Address: "10.244.0.10", Port: 8080}: {}, + {Address: "10.244.0.2", Port: 8080}: {}, + {Address: "10.244.0.2", Port: 8443}: {}, + }, + } + + svc.scaleHAProxySrvs(backend) + + require.Len(t, backend.HAProxySrvs, 42) + assert.Equal(t, "SRV_1", backend.HAProxySrvs[0].Name) + assert.Equal(t, "10.244.0.2", backend.HAProxySrvs[0].Address) + assert.Equal(t, int64(8080), backend.HAProxySrvs[0].Port) + assert.Equal(t, "SRV_2", backend.HAProxySrvs[1].Name) + assert.Equal(t, "10.244.0.2", backend.HAProxySrvs[1].Address) + assert.Equal(t, int64(8443), backend.HAProxySrvs[1].Port) + assert.Equal(t, "SRV_3", backend.HAProxySrvs[2].Name) + assert.Equal(t, "10.244.0.10", backend.HAProxySrvs[2].Address) + assert.Equal(t, int64(8080), backend.HAProxySrvs[2].Port) + assert.Empty(t, backend.Endpoints) +} diff --git a/pkg/service/service.go b/pkg/service/service.go index fb1a2cce..2b2dea6e 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -257,19 +257,23 @@ func (s *Service) getBackendModel(store store.K8s, a annotations.Annotations, cl logger.Errorf("service '%s/%s': annotation '%s': %s", s.resource.Namespace, s.resource.Name, a.GetName(), err) } } - // cookie-persistence is resolved against the Service annotations, falling - // back to the ConfigMap default, but NEVER the ingress annotations. The - // backend is shared across every ingress referencing this service, so - // honoring the ingress source would let two ingresses set divergent cookie - // configs on the same backend, causing non-deterministic config churn. The - // ConfigMap is a single global default and stays safe. Hence it is handled - // here rather than in the generic a.Backend() group, and must run before - // the DynamicCookieKey block below. Service takes precedence over ConfigMap - // (GetValue returns the first source where the key is set). + // Backend session-persistence annotations are resolved against the Service + // annotations, falling back to the ConfigMap default, but NEVER the ingress + // annotations. The backend is shared across every ingress referencing this + // service, so honoring the ingress source would let two ingresses set + // divergent persistence configs on the same backend, causing + // non-deterministic config churn. The ConfigMap is a single global default + // and stays safe. Hence these are handled here rather than in the generic + // a.Backend() group. Service takes precedence over ConfigMap (GetValue + // returns the first source where the key is set). cookieAnn := serviceann.NewCookie("cookie-persistence", &backend.Backend) if cookieErr := cookieAnn.Process(store, s.resource.Annotations, store.ConfigMaps.Main.Annotations); cookieErr != nil { logger.Errorf("service '%s/%s': annotation '%s': %s", s.resource.Namespace, s.resource.Name, cookieAnn.GetName(), cookieErr) } + sourceIPAnn := serviceann.NewSourceIPPersistence("source-ip-persistence", &backend.Backend) + if sourceIPErr := sourceIPAnn.Process(store, s.resource.Annotations, store.ConfigMaps.Main.Annotations); sourceIPErr != nil { + logger.Errorf("service '%s/%s': annotation '%s': %s", s.resource.Namespace, s.resource.Name, sourceIPAnn.GetName(), sourceIPErr) + } } // Manadatory backend params diff --git a/pkg/store/events.go b/pkg/store/events.go index 21057bb4..8cebf20b 100644 --- a/pkg/store/events.go +++ b/pkg/store/events.go @@ -354,10 +354,13 @@ func (k *K8s) EventSecret(ns *Namespace, data *Secret) (updateRequired bool) { func (k *K8s) EventPod(podEvent PodEvent) (updateRequired bool) { switch podEvent.Status { case ADDED, MODIFIED: - if _, ok := k.HaProxyPods[podEvent.Name]; ok { + if podEvent.IP == "" { return false } - k.HaProxyPods[podEvent.Name] = struct{}{} + if currentIP, ok := k.HaProxyPods[podEvent.Name]; ok && currentIP == podEvent.IP { + return false + } + k.HaProxyPods[podEvent.Name] = podEvent.IP case DELETED: if _, ok := k.HaProxyPods[podEvent.Name]; !ok { return false diff --git a/pkg/store/events_pod_test.go b/pkg/store/events_pod_test.go new file mode 100644 index 00000000..16433a2c --- /dev/null +++ b/pkg/store/events_pod_test.go @@ -0,0 +1,58 @@ +// Copyright 2026 HAProxy Technologies LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +import ( + "testing" + + "github.com/haproxytech/kubernetes-ingress/pkg/utils" +) + +func TestEventPodTracksPodIPChanges(t *testing.T) { + store := NewK8sStore(utils.OSArgs{}) + + if !store.EventPod(PodEvent{Status: ADDED, Name: "haproxy-ingress-abcde-fghij", Namespace: "default", IP: "10.0.0.1"}) { + t.Fatal("expected add event to require an update") + } + if got := store.HaProxyPods["haproxy-ingress-abcde-fghij"]; got != "10.0.0.1" { + t.Fatalf("expected pod IP 10.0.0.1, got %q", got) + } + if store.EventPod(PodEvent{Status: MODIFIED, Name: "haproxy-ingress-abcde-fghij", Namespace: "default", IP: "10.0.0.1"}) { + t.Fatal("expected unchanged pod IP to skip update") + } + if !store.EventPod(PodEvent{Status: MODIFIED, Name: "haproxy-ingress-abcde-fghij", Namespace: "default", IP: "10.0.0.2"}) { + t.Fatal("expected changed pod IP to require an update") + } + if got := store.HaProxyPods["haproxy-ingress-abcde-fghij"]; got != "10.0.0.2" { + t.Fatalf("expected pod IP 10.0.0.2, got %q", got) + } + if !store.EventPod(PodEvent{Status: DELETED, Name: "haproxy-ingress-abcde-fghij", Namespace: "default"}) { + t.Fatal("expected delete event to require an update") + } + if _, ok := store.HaProxyPods["haproxy-ingress-abcde-fghij"]; ok { + t.Fatal("expected pod to be removed") + } +} + +func TestEventPodIgnoresEmptyPodIP(t *testing.T) { + store := NewK8sStore(utils.OSArgs{}) + + if store.EventPod(PodEvent{Status: ADDED, Name: "haproxy-ingress-abcde-fghij", Namespace: "default"}) { + t.Fatal("expected empty pod IP to skip update") + } + if len(store.HaProxyPods) != 0 { + t.Fatalf("expected no tracked pods, got %d", len(store.HaProxyPods)) + } +} diff --git a/pkg/store/store.go b/pkg/store/store.go index 3b8a5e4d..d08b42d9 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -37,7 +37,7 @@ type K8s struct { SecretsProcessed map[string]struct{} BackendsProcessed map[string]struct{} GatewayClasses map[string]*GatewayClass - HaProxyPods map[string]struct{} + HaProxyPods map[string]string BackendsWithNoConfigSnippets map[string]struct{} FrontendRC *rc.ResourceCounter GatewayControllerName string @@ -85,7 +85,7 @@ func NewK8sStore(args utils.OSArgs) K8s { BackendsProcessed: map[string]struct{}{}, GatewayClasses: map[string]*GatewayClass{}, BackendsWithNoConfigSnippets: map[string]struct{}{}, - HaProxyPods: map[string]struct{}{}, + HaProxyPods: map[string]string{}, FrontendRC: rc.NewResourceCounter(), IngressesByService: map[string]*utils.OrderedSet[string, *Ingress]{}, } diff --git a/pkg/store/types.go b/pkg/store/types.go index cfb677e3..9cfa4eeb 100644 --- a/pkg/store/types.go +++ b/pkg/store/types.go @@ -16,6 +16,8 @@ package store import ( "fmt" + "net/netip" + "sort" "time" "github.com/haproxytech/client-native/v6/models" @@ -62,6 +64,7 @@ type PodEvent struct { Status Status Name string Namespace string + IP string } // Service is useful data from k8s structures about service @@ -85,6 +88,31 @@ type RuntimeEndpoint struct { // RuntimeEndpoints is a set of different RuntimeEndpoint of a HAProxy backend type RuntimeEndpoints = map[RuntimeEndpoint]struct{} +// SortedRuntimeEndpoints returns endpoints in stable address and port order. +func SortedRuntimeEndpoints(endpoints RuntimeEndpoints) []RuntimeEndpoint { + sortedEndpoints := make([]RuntimeEndpoint, 0, len(endpoints)) + for endpoint := range endpoints { + sortedEndpoints = append(sortedEndpoints, endpoint) + } + sort.SliceStable(sortedEndpoints, func(i, j int) bool { + return runtimeEndpointLess(sortedEndpoints[i], sortedEndpoints[j]) + }) + return sortedEndpoints +} + +func runtimeEndpointLess(a, b RuntimeEndpoint) bool { + addrA, errA := netip.ParseAddr(a.Address) + addrB, errB := netip.ParseAddr(b.Address) + if errA == nil && errB == nil { + if cmp := addrA.Compare(addrB); cmp != 0 { + return cmp < 0 + } + } else if a.Address != b.Address { + return a.Address < b.Address + } + return a.Port < b.Port +} + // RuntimeBackend holds the runtime state of an HAProxy backend type RuntimeBackend struct { Endpoints RuntimeEndpoints diff --git a/pkg/store/types_test.go b/pkg/store/types_test.go new file mode 100644 index 00000000..5d71945b --- /dev/null +++ b/pkg/store/types_test.go @@ -0,0 +1,27 @@ +package store + +import ( + "reflect" + "testing" +) + +func TestSortedRuntimeEndpoints(t *testing.T) { + endpoints := RuntimeEndpoints{ + {Address: "10.244.0.10", Port: 8080}: {}, + {Address: "10.244.0.2", Port: 8443}: {}, + {Address: "10.244.0.2", Port: 8080}: {}, + {Address: "example.internal", Port: 80}: {}, + } + + got := SortedRuntimeEndpoints(endpoints) + want := []RuntimeEndpoint{ + {Address: "10.244.0.2", Port: 8080}, + {Address: "10.244.0.2", Port: 8443}, + {Address: "10.244.0.10", Port: 8080}, + {Address: "example.internal", Port: 80}, + } + + if !reflect.DeepEqual(got, want) { + t.Fatalf("SortedRuntimeEndpoints() = %#v, want %#v", got, want) + } +} diff --git a/test/annotations/source_ip_persistence_test.go b/test/annotations/source_ip_persistence_test.go new file mode 100644 index 00000000..c3314c0d --- /dev/null +++ b/test/annotations/source_ip_persistence_test.go @@ -0,0 +1,150 @@ +package annotations_test + +import ( + "reflect" + "testing" + + "github.com/haproxytech/client-native/v6/models" + + "github.com/haproxytech/kubernetes-ingress/pkg/annotations/service" + "github.com/haproxytech/kubernetes-ingress/pkg/store" + "github.com/haproxytech/kubernetes-ingress/pkg/utils" +) + +func TestSourceIPPersistenceProcess(t *testing.T) { + tests := []struct { + name string + ann []map[string]string + initial *models.Backend + wantTable *models.ConfigStickTable + wantRules models.StickRules + wantErr bool + wantClear bool + }{ + { + name: "enabled with defaults", + ann: []map[string]string{ + {"source-ip-persistence": "true"}, + }, + wantTable: &models.ConfigStickTable{ + Type: "ip", + Size: utils.PtrInt64(1024 * 1024), + Expire: utils.PtrInt64(30 * 60 * 1000), + Peers: "localinstance", + }, + wantRules: models.StickRules{ + { + Type: "on", + Pattern: "src", + }, + }, + }, + { + name: "enabled with custom size and expiry", + ann: []map[string]string{ + { + "source-ip-persistence": "true", + "source-ip-persistence-size": "2m", + "source-ip-persistence-expire": "45s", + }, + }, + wantTable: &models.ConfigStickTable{ + Type: "ip", + Size: utils.PtrInt64(2 * 1024 * 1024), + Expire: utils.PtrInt64(45 * 1000), + Peers: "localinstance", + }, + wantRules: models.StickRules{ + { + Type: "on", + Pattern: "src", + }, + }, + }, + { + name: "service false overrides configmap true", + ann: []map[string]string{ + {"source-ip-persistence": "false"}, + {"source-ip-persistence": "true"}, + }, + initial: &models.Backend{ + BackendBase: models.BackendBase{ + StickTable: &models.ConfigStickTable{Type: "ip"}, + }, + StickRuleList: models.StickRules{{Type: "on", Pattern: "src"}}, + }, + wantClear: true, + }, + { + name: "unset clears generated persistence", + ann: []map[string]string{{}}, + initial: &models.Backend{ + BackendBase: models.BackendBase{ + StickTable: &models.ConfigStickTable{Type: "ip"}, + }, + StickRuleList: models.StickRules{{Type: "on", Pattern: "src"}}, + }, + wantClear: true, + }, + { + name: "invalid boolean", + ann: []map[string]string{ + {"source-ip-persistence": "maybe"}, + }, + wantErr: true, + }, + { + name: "invalid size", + ann: []map[string]string{ + { + "source-ip-persistence": "true", + "source-ip-persistence-size": "large", + }, + }, + wantErr: true, + }, + { + name: "invalid expiry", + ann: []map[string]string{ + { + "source-ip-persistence": "true", + "source-ip-persistence-expire": "soon", + }, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + backend := &models.Backend{} + if tt.initial != nil { + backend = tt.initial + } + ann := service.NewSourceIPPersistence("source-ip-persistence", backend) + + err := ann.Process(store.K8s{}, tt.ann...) + if (err != nil) != tt.wantErr { + t.Fatalf("Process() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + if tt.wantClear { + if backend.StickTable != nil { + t.Fatalf("StickTable = %#v, want nil", backend.StickTable) + } + if len(backend.StickRuleList) != 0 { + t.Fatalf("StickRuleList = %#v, want empty", backend.StickRuleList) + } + return + } + if !reflect.DeepEqual(backend.StickTable, tt.wantTable) { + t.Fatalf("StickTable = %#v, want %#v", backend.StickTable, tt.wantTable) + } + if !reflect.DeepEqual(backend.StickRuleList, tt.wantRules) { + t.Fatalf("StickRuleList = %#v, want %#v", backend.StickRuleList, tt.wantRules) + } + }) + } +}