From 8c5a6fb903833da2de30a94c9edb1fd186b51efb Mon Sep 17 00:00:00 2001 From: Andrew Peabody Date: Thu, 12 Mar 2026 02:42:27 +0000 Subject: [PATCH 1/4] Feature: Add optional 'leases' etcd shard - Allows 'leases' API validation - Maps well-known port assignments - Configures API Server overrides and nodeup DNS mapping - Updates Etcd Manager and TLS Configurations - Implements Node diagnostics, E2E integrations, and log overrides - Introduces mock test fixtures in cluster.go --- cmd/kops/create_cluster.go | 4 +- cmd/kops/integration_test.go | 12 ++++ docs/cli/kops_create_cluster.md | 2 +- k8s/crds/kops.k8s.io_clusters.yaml | 4 +- nodeup/pkg/model/etcd_manager_tls.go | 2 +- nodeup/pkg/model/kube_apiserver.go | 3 + nodeup/pkg/model/logrotate.go | 1 + pkg/apis/kops/cluster.go | 2 +- pkg/apis/kops/status.go | 2 +- pkg/apis/kops/v1alpha2/cluster.go | 2 +- pkg/apis/kops/v1alpha3/cluster.go | 2 +- pkg/apis/kops/validation/validation.go | 2 +- pkg/dump/dumper.go | 1 + pkg/model/awsmodel/autoscalinggroup_test.go | 2 + pkg/model/awsmodel/firewall.go | 60 ++++++++++++------- pkg/model/azuremodel/network.go | 9 ++- pkg/model/bootstrapscript.go | 8 +-- pkg/model/bootstrapscript_test.go | 4 ++ pkg/model/components/apiserver.go | 2 + pkg/model/components/etcdmanager/model.go | 12 ++++ pkg/model/openstackmodel/firewall.go | 14 ++++- pkg/model/openstackmodel/servergroup_test.go | 2 + .../adds-additional-security-groups.yaml | 18 ++++++ .../adds-cloud-labels-from-ClusterSpec.yaml | 18 ++++++ ...s-cloud-labels-from-InstanceGroupSpec.yaml | 18 ++++++ ...llowed-address-pairs-with-annotations.yaml | 18 ++++++ ...erver-group-affinity-with-annotations.yaml | 18 ++++++ ...ithout-bastion-auto-zone-distribution.yaml | 18 ++++++ ...astion-with-API-loadbalancer-dns-none.yaml | 18 ++++++ ...without-bastion-with-API-loadbalancer.yaml | 18 ++++++ ...tup-3-masters-3-nodes-without-bastion.yaml | 18 ++++++ ...sters-3-nodes-without-external-router.yaml | 18 ++++++ .../one-master-one-node-one-bastion-2.yaml | 18 ++++++ .../one-master-one-node-one-bastion.yaml | 18 ++++++ ...hout-bastion-no-public-ip-association.yaml | 18 ++++++ .../servergroup/one-master-one-node.yaml | 18 ++++++ ...astion-with-API-loadbalancer-dns-none.yaml | 18 ++++++ ...uncate-cluster-names-to-42-characters.yaml | 18 ++++++ ...subnet-as-availability-zones-fallback.yaml | 18 ++++++ ...nce-group-zones-as-availability-zones.yaml | 18 ++++++ pkg/nodemodel/nodeupconfigbuilder.go | 2 +- pkg/testutils/cluster.go | 2 +- pkg/wellknownports/wellknownports.go | 11 ++++ protokube/pkg/etcd/cluster_spec.go | 2 +- .../cilium-connectivity-test/run-test.sh | 2 +- 45 files changed, 453 insertions(+), 42 deletions(-) diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index 164005a0d87e3..56c5613f647b9 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -316,9 +316,9 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { }) cmd.Flags().BoolVar(&options.DisableSubnetTags, "disable-subnet-tags", options.DisableSubnetTags, "Disable automatic subnet tagging") - cmd.Flags().StringSliceVar(&options.EtcdClusters, "etcd-clusters", options.EtcdClusters, "Names of the etcd clusters: main, events") + cmd.Flags().StringSliceVar(&options.EtcdClusters, "etcd-clusters", options.EtcdClusters, "Names of the etcd clusters: main, events, leases") cmd.RegisterFlagCompletionFunc("etcd-clusters", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return []string{"main", "events"}, cobra.ShellCompDirectiveNoFileComp + return []string{"main", "events", "leases"}, cobra.ShellCompDirectiveNoFileComp }) cmd.Flags().BoolVar(&encryptEtcdStorage, "encrypt-etcd-storage", false, "Generate key in AWS KMS and use it for encrypt etcd volumes") diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 649df842a3da8..29bd52bdc1607 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -1418,6 +1418,18 @@ func (i *integrationTest) setupCluster(t *testing.T, ctx context.Context, inputY secondaryKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBALE1vJwNk3HlXVk6JfFlK9oWkdHAp4cN9y4xSK12g+2dpUyUxMYN\nYAy4JWYUcUBaiEhjKd6YR6CZmRnXlLsASt8CAwEAAQJABeku812Yj3IBHRrNbTHc\ntpeOIZr1e5HBru7B59dOKzzKrI2SozD+wKmhi2r+8yPkdU1nq4DE1Pboc1BmPh9C\n0QIhAMiAQ+yZRuThl8qOCZ+D9Frmml102DIf5d1NjGGQD84FAiEA4kMJCM194VPV\n2W7QsLH+szbwRHXg1dOlR9WQHJ8rZpMCIF/F7SwyV0vzerdVu8EHngxhxPDJZJAk\n7n8UkO71iqclAiEAypza9z4E7oWDZ507Vi9edJ/K0pN4jiJjzIrq7SZ/1+8CID2K\nAMbqYsKhlMt8zM+hSUg+u8wcWs8CVBb4ozQY2Xyb\n-----END RSA PRIVATE KEY-----", secondaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBeDCCASKgAwIBAgIMFo+bQ+EuVthBfuZvMA0GCSqGSIb3DQEBCwUAMB0xGzAZ\nBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDExNDZaFw0zMTA3\nMDUyMDExNDZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG\nSIb3DQEBAQUAA0sAMEgCQQCxNbycDZNx5V1ZOiXxZSvaFpHRwKeHDfcuMUitdoPt\nnaVMlMTGDWAMuCVmFHFAWohIYynemEegmZkZ15S7AErfAgMBAAGjQjBAMA4GA1Ud\nDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTAjQ8T4HclPIsC\nqipEfUIcLP6jqTANBgkqhkiG9w0BAQsFAANBAJdZ17TN3HlWrH7HQgfR12UBwz8K\nG9DurDznVaBVUYaHY8Sg5AvAXeb+yIF2JMmRR+bK+/G1QYY2D3/P31Ic2Oo=\n-----END CERTIFICATE-----", }) + storeKeyset(t, ctx, keyStore, "etcd-manager-ca-leases", &testingKeyset{ + primaryKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIBOwIBAAJBAMW5A2xmJgkkoaURt6/pc0zhbo8rq7kX4zoWJmUV+MNVLXecut3V\nHPfLI3PRhlGDB3ftJNapf2uPLRoZyujeoycCAwEAAQJBALIOHMEfdB1DubW3MN3f\ns4+Ga1PPFgPHOT9z9vuNP8pWcRWGACXdln4T/VM5LQYrwTQ/i9EMZycl3ISbTUfy\nEPECIQD5RWUR1dF4S2VGFtxhttbZbP6m3Nk/eiOmT3wPv4TJDQIhAMsPY9YgTmfV\nuZwykVu/UopdjVY/vFAiFYwA2Km8b2gDAiB9jdiUnTA++SrvnMAwb5nUNjQl9ANx\nF6IxOMPyYrMNWQIhALb2wANRCrSeq+ak3bqockwALXi4ZwphG78RiCewhUVXAiA+\n4yljHjbbEGQje8VuxmA3ITMeCwAkIqjXY1Z5DUTnDA==\n-----END RSA PRIVATE KEY-----", + primaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBfDCCASagAwIBAgIMFo+bKjm1c3jfv6hIMA0GCSqGSIb3DQEBCwUAMB8xHTAb\nBgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWluMB4XDTIxMDcwNTIwMDk1NloXDTMx\nMDcwNTIwMDk1NlowHzEdMBsGA1UEAxMUZXRjZC1tYW5hZ2VyLWNhLW1haW4wXDAN\nBgkqhkiG9w0BAQEFAANLADBIAkEAxbkDbGYmCSShpRG3r+lzTOFujyuruRfjOhYm\nZRX4w1Utd5y63dUc98sjc9GGUYMHd+0k1ql/a48tGhnK6N6jJwIDAQABo0IwQDAO\nBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUWZLkbBFx\nGAgPU4i62c52unSo7RswDQYJKoZIhvcNAQELBQADQQAj6Pgd0va/8FtkyMlnohLu\nGf4v8RJO6zk3Y6jJ4+cwWziipFM1ielMzSOZfFcCZgH3m5Io40is4hPSqyq2TOA6\n-----END CERTIFICATE-----", + secondaryKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBAMN9483Hf4qLDdOG9Fl2w7ewdHN7Cd2mn3Biz7xt8UQfTeW2K/fq\nmQKt5swBZMbHJ+I9XHuW9fxikwxAApZmYHUCAwEAAQJAOOGfcBe1L52oRz0ESie5\naPBJ4fQR+dFqoOvPYBdpVRV4h8PcLGhH7H0RO0pJf9ni0MxWDMn2R8Nw6/I7zSgr\n/QIhAN432G6YOItNGj0wrNBgZerFIOVdnHe+higgAhJOtNFbAiEA4TXsL5ALyAYI\nVDS66EbriI15z5XxiauBk0zAbqun7m8CIQDUK+Ichn7GkpGRBx6ZvtDQvfNQzHaO\n5nzVZupTbI68rQIgLzkNU1PTBJgvOujroDTuwm1X820vfnyV6PsZBpu71MUCIAPQ\nTjwL4gGtCZtHXHqAUS9vgf4sQ40oBqNb3NhshheB\n-----END RSA PRIVATE KEY-----", + secondaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBfDCCASagAwIBAgIMFo+bQ+Eg8Si30gr4MA0GCSqGSIb3DQEBCwUAMB8xHTAb\nBgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWluMB4XDTIxMDcwNTIwMTE0NloXDTMx\nMDcwNTIwMTE0NlowHzEdMBsGA1UEAxMUZXRjZC1tYW5hZ2VyLWNhLW1haW4wXDAN\nBgkqhkiG9w0BAQEFAANLADBIAkEAw33jzcd/iosN04b0WXbDt7B0c3sJ3aafcGLP\nvG3xRB9N5bYr9+qZAq3mzAFkxscn4j1ce5b1/GKTDEAClmZgdQIDAQABo0IwQDAO\nBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUE/h+3gDP\nDvKwHRyiYlXM8voZ1wowDQYJKoZIhvcNAQELBQADQQBXuimeEoAOu5HN4hG7NqL9\nt40K3ZRhRZv3JQWnRVJCBDjg1rD0GQJR/n+DoWvbeijI5C9pNjr2pWSIYR1eYCvd\n-----END CERTIFICATE-----", + }) + storeKeyset(t, ctx, keyStore, "etcd-peers-ca-leases", &testingKeyset{ + primaryKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIBOwIBAAJBALJFpdanCA3og1CrCz2n8G88SUm/ZGej11VMWGVCoMBpQld7swGa\nI7g0lxbvoSjN4GHnO1Hf/g0TUUzbHxOKxLcCAwEAAQJBAI418S1i4ZH2wYpAaB8v\nMSYLOYuTGk1y7fwlgv6EQCg8esJcMCeDsqT5V5sUicT6jT5m3KdpKA4v4kpZJzHo\nr8ECIQDRtEmpTSmTQ1FAVPu34j6ZU0W5zT8RMaoUFPCXPJ/M9QIhANmg7bTqNNBY\nd7TUxmgm2NW5GDn0yyg1WqoIL4wOJz97AiBvrCad9e1x8qNOMvNpVR4o4GN9MoOn\nUF9WGmCU6T/gEQIgdhnEBdK3eH0Z8TMqvKigMVNyFzmF6jsSCYXJr7qah/MCIQCy\npxPa6cKMC0n9t61B+1f7O2yCvwllormxaFYVm9J4xw==\n-----END RSA PRIVATE KEY-----", + primaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBeDCCASKgAwIBAgIMFo+bKjmuLDDLcDHsMA0GCSqGSIb3DQEBCwUAMB0xGzAZ\nBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDA5NTZaFw0zMTA3\nMDUyMDA5NTZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG\nSIb3DQEBAQUAA0sAMEgCQQCyRaXWpwgN6INQqws9p/BvPElJv2Rno9dVTFhlQqDA\naUJXe7MBmiO4NJcW76EozeBh5ztR3/4NE1FM2x8TisS3AgMBAAGjQjBAMA4GA1Ud\nDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQtE1d49uSvpURf\nOQ25Vlu6liY20DANBgkqhkiG9w0BAQsFAANBAAgLVaetJZcfOA3OIMMvQbz2Ydrt\nuWF9BKkIad8jrcIrm3IkOtR8bKGmDIIaRKuG/ZUOL6NMe2fky3AAfKwleL4=\n-----END CERTIFICATE-----", + secondaryKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBALE1vJwNk3HlXVk6JfFlK9oWkdHAp4cN9y4xSK12g+2dpUyUxMYN\nYAy4JWYUcUBaiEhjKd6YR6CZmRnXlLsASt8CAwEAAQJABeku812Yj3IBHRrNbTHc\ntpeOIZr1e5HBru7B59dOKzzKrI2SozD+wKmhi2r+8yPkdU1nq4DE1Pboc1BmPh9C\n0QIhAMiAQ+yZRuThl8qOCZ+D9Frmml102DIf5d1NjGGQD84FAiEA4kMJCM194VPV\n2W7QsLH+szbwRHXg1dOlR9WQHJ8rZpMCIF/F7SwyV0vzerdVu8EHngxhxPDJZJAk\n7n8UkO71iqclAiEAypza9z4E7oWDZ507Vi9edJ/K0pN4jiJjzIrq7SZ/1+8CID2K\nAMbqYsKhlMt8zM+hSUg+u8wcWs8CVBb4ozQY2Xyb\n-----END RSA PRIVATE KEY-----", + secondaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBeDCCASKgAwIBAgIMFo+bQ+EuVthBfuZvMA0GCSqGSIb3DQEBCwUAMB0xGzAZ\nBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDExNDZaFw0zMTA3\nMDUyMDExNDZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG\nSIb3DQEBAQUAA0sAMEgCQQCxNbycDZNx5V1ZOiXxZSvaFpHRwKeHDfcuMUitdoPt\nnaVMlMTGDWAMuCVmFHFAWohIYynemEegmZkZ15S7AErfAgMBAAGjQjBAMA4GA1Ud\nDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTAjQ8T4HclPIsC\nqipEfUIcLP6jqTANBgkqhkiG9w0BAQsFAANBAJdZ17TN3HlWrH7HQgfR12UBwz8K\nG9DurDznVaBVUYaHY8Sg5AvAXeb+yIF2JMmRR+bK+/G1QYY2D3/P31Ic2Oo=\n-----END CERTIFICATE-----", + }) storeKeyset(t, ctx, keyStore, "service-account", &testingKeyset{ primaryKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----", primaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBZzCCARGgAwIBAgIBAjANBgkqhkiG9w0BAQsFADAaMRgwFgYDVQQDEw9zZXJ2\naWNlLWFjY291bnQwHhcNMjEwNTAyMjAzMDA2WhcNMzEwNTAyMjAzMDA2WjAaMRgw\nFgYDVQQDEw9zZXJ2aWNlLWFjY291bnQwXDANBgkqhkiG9w0BAQEFAANLADBIAkEA\n2JbeF8dNwqfEKKD65aGlVs58fWkA0qZdVLKw8qATzRBJTi1nqbj2kAR4gyy/C8Mx\nouxva/om9d7Sq8Ka55T7+wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T\nAQH/BAUwAwEB/zAdBgNVHQ4EFgQUI5beFHueAGyT1pQ6UTOdbMfj3gQwDQYJKoZI\nhvcNAQELBQADQQBwPLO+Np8o6k3aNBGKE4JTCOs06X72OXNivkWWWP/9XGz6x4DI\nHPU65kbUn/pWXBUVVlpsKsdmWA2Bu8pd/vD+\n-----END CERTIFICATE-----\n", diff --git a/docs/cli/kops_create_cluster.md b/docs/cli/kops_create_cluster.md index 4f5a6fe736c38..8c69b16e62e99 100644 --- a/docs/cli/kops_create_cluster.md +++ b/docs/cli/kops_create_cluster.md @@ -89,7 +89,7 @@ kops create cluster [CLUSTER] [flags] --dns-zone string DNS hosted zone (defaults to longest matching zone) --dry-run If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest. --encrypt-etcd-storage Generate key in AWS KMS and use it for encrypt etcd volumes - --etcd-clusters strings Names of the etcd clusters: main, events (default [main,events]) + --etcd-clusters strings Names of the etcd clusters: main, events, leases (default [main,events]) --etcd-storage-type string The default storage type for etcd members --gce-service-account string Service account with which the GCE VM runs. Warning: if not set, VMs will run as default compute service account. -h, --help help for cluster diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 0e123d442daf1..5fa0976e96ded 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -1444,8 +1444,8 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true name: - description: Name is the name of the etcd cluster (main, events - etc) + description: Name is the name of the etcd cluster (main, events, + leases etc) type: string provider: description: |- diff --git a/nodeup/pkg/model/etcd_manager_tls.go b/nodeup/pkg/model/etcd_manager_tls.go index 66ada7567336f..ab00a00c41eda 100644 --- a/nodeup/pkg/model/etcd_manager_tls.go +++ b/nodeup/pkg/model/etcd_manager_tls.go @@ -46,7 +46,7 @@ func (b *EtcdManagerTLSBuilder) Build(ctx *fi.NodeupModelBuilderContext) error { keys["etcd-clients-ca"] = "etcd-clients-ca-" + etcdClusterName // Because API server can only have a single client certificate for etcd, we need to share a client CA - if etcdClusterName == "main" || etcdClusterName == "events" { + if etcdClusterName == "main" || etcdClusterName == "events" || etcdClusterName == "leases" { keys["etcd-clients-ca"] = "etcd-clients-ca" } diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index 14baebc9389db..2400729447280 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -601,6 +601,9 @@ func (b *KubeAPIServerBuilder) buildPod(ctx context.Context, kubeAPIServer *kops if strings.HasPrefix(kubeAPIServer.EtcdServersOverrides[i], "/events") { kubeAPIServer.EtcdServersOverrides[i] = strings.ReplaceAll(kubeAPIServer.EtcdServersOverrides[i], "127.0.0.1", eventsEtcdDNSName) } + if strings.HasPrefix(kubeAPIServer.EtcdServersOverrides[i], "/registry/leases") { + kubeAPIServer.EtcdServersOverrides[i] = strings.ReplaceAll(kubeAPIServer.EtcdServersOverrides[i], "127.0.0.1", "leases.etcd.internal."+clusterName) + } } } diff --git a/nodeup/pkg/model/logrotate.go b/nodeup/pkg/model/logrotate.go index ac06a7afc758d..5889f0f9c4628 100644 --- a/nodeup/pkg/model/logrotate.go +++ b/nodeup/pkg/model/logrotate.go @@ -55,6 +55,7 @@ func (b *LogrotateBuilder) Build(c *fi.NodeupModelBuilderContext) error { b.addLogRotate(c, "kubelet", "/var/log/kubelet.log", logRotateOptions{}) b.addLogRotate(c, "etcd", "/var/log/etcd.log", logRotateOptions{}) b.addLogRotate(c, "etcd-events", "/var/log/etcd-events.log", logRotateOptions{}) + b.addLogRotate(c, "etcd-leases", "/var/log/etcd-leases.log", logRotateOptions{}) if b.NodeupConfig.UseCiliumEtcd { b.addLogRotate(c, "etcd-cilium", "/var/log/etcd-cilium.log", logRotateOptions{}) } diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index a92373ed7cd37..10cd9a77f3b6b 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -668,7 +668,7 @@ const ( // EtcdClusterSpec is the etcd cluster specification type EtcdClusterSpec struct { - // Name is the name of the etcd cluster (main, events etc) + // Name is the name of the etcd cluster (main, events, leases etc) Name string `json:"name,omitempty"` // Provider is the provider used to run etcd: Manager, Legacy. // Defaults to Manager. diff --git a/pkg/apis/kops/status.go b/pkg/apis/kops/status.go index d1912d0e77e45..313076e5daecf 100644 --- a/pkg/apis/kops/status.go +++ b/pkg/apis/kops/status.go @@ -23,7 +23,7 @@ type ClusterStatus struct { // EtcdClusterStatus represents the status of etcd: because etcd only allows limited reconfiguration, we have to block changes once etcd has been initialized. type EtcdClusterStatus struct { - // Name is the name of the etcd cluster (main, events etc) + // Name is the name of the etcd cluster (main, events, leases etc) Name string `json:"name,omitempty"` // EtcdMember stores the configurations for each member of the cluster (including the data volume) Members []*EtcdMemberStatus `json:"etcdMembers,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index ec26ac410b760..bfb24f053947f 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -648,7 +648,7 @@ const ( // EtcdClusterSpec is the etcd cluster specification type EtcdClusterSpec struct { - // Name is the name of the etcd cluster (main, events etc) + // Name is the name of the etcd cluster (main, events, leases etc) Name string `json:"name,omitempty"` // Provider is the provider used to run etcd: Manager, Legacy. // Defaults to Manager. diff --git a/pkg/apis/kops/v1alpha3/cluster.go b/pkg/apis/kops/v1alpha3/cluster.go index 1a1d6ad16d95d..6933aa8bd9cbd 100644 --- a/pkg/apis/kops/v1alpha3/cluster.go +++ b/pkg/apis/kops/v1alpha3/cluster.go @@ -623,7 +623,7 @@ type ExternalDNSConfig struct { // EtcdClusterSpec is the etcd cluster specification type EtcdClusterSpec struct { - // Name is the name of the etcd cluster (main, events etc) + // Name is the name of the etcd cluster (main, events, leases etc) Name string `json:"name,omitempty"` Provider string `json:"-"` // Members stores the configurations for each member of the cluster (including the data volume) diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index 7b9a50438a653..0e1c82eacb22f 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -1408,7 +1408,7 @@ func validateExternalPolicies(role string, policies []string, fldPath *field.Pat func validateEtcdClusterSpec(spec kops.EtcdClusterSpec, c *kops.Cluster, fieldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} - allErrs = append(allErrs, IsValidValue(fieldPath.Child("name"), &spec.Name, []string{"cilium", "main", "events"})...) + allErrs = append(allErrs, IsValidValue(fieldPath.Child("name"), &spec.Name, []string{"cilium", "main", "events", "leases"})...) if spec.Provider != "" { allErrs = append(allErrs, IsValidValue(fieldPath.Child("provider"), &spec.Provider, []kops.EtcdProviderType{kops.EtcdProviderTypeManager})...) diff --git a/pkg/dump/dumper.go b/pkg/dump/dumper.go index 64f23c8d24688..acf81a2e9b2fa 100644 --- a/pkg/dump/dumper.go +++ b/pkg/dump/dumper.go @@ -78,6 +78,7 @@ func NewLogDumper(bastionAddress string, sshConfig *ssh.ClientConfig, keyRing ag "etcd", "etcd-events", "etcd-cilium", + "etcd-leases", "glbc", "cluster-autoscaler", "kube-addon-manager", diff --git a/pkg/model/awsmodel/autoscalinggroup_test.go b/pkg/model/awsmodel/autoscalinggroup_test.go index 0a64b6b9f4357..cf90f653a00b7 100644 --- a/pkg/model/awsmodel/autoscalinggroup_test.go +++ b/pkg/model/awsmodel/autoscalinggroup_test.go @@ -210,8 +210,10 @@ func TestAPIServerAdditionalSecurityGroupsWithNLB(t *testing.T) { "apiserver-aggregator-ca", "etcd-clients-ca", "etcd-manager-ca-events", + "etcd-manager-ca-leases", "etcd-manager-ca-main", "etcd-peers-ca-events", + "etcd-peers-ca-leases", "etcd-peers-ca-main", "service-account", } { diff --git a/pkg/model/awsmodel/firewall.go b/pkg/model/awsmodel/firewall.go index 3d4fcb42d485d..43f7bd832ace2 100644 --- a/pkg/model/awsmodel/firewall.go +++ b/pkg/model/awsmodel/firewall.go @@ -133,6 +133,14 @@ func (b *FirewallModelBuilder) applyNodeToMasterBlockSpecificPorts(c *fi.Cloudup tcpBlocked[wellknownports.EtcdMainPeerPort] = true tcpBlocked[wellknownports.EtcdEventsPeerPort] = true + for _, c := range b.Cluster.Spec.EtcdClusters { + if c.Name == "leases" { + tcpBlocked[wellknownports.EtcdLeasesClientPort] = true + tcpBlocked[wellknownports.EtcdLeasesPeerPort] = true + break + } + } + udpRanges := []portRange{{From: 1, To: 65535}} protocols := []Protocol{} @@ -307,27 +315,39 @@ func (b *AWSModelContext) GetSecurityGroups(role kops.InstanceGroupRole) ([]Secu name := b.SecurityGroupName(role) switch role { case kops.InstanceGroupRoleControlPlane: + removeExtraRules := []string{ + "port=22", // SSH + "port=443", // k8s api + "port=" + strconv.Itoa(wellknownports.EtcdMainPeerPort), // etcd main peer + "port=" + strconv.Itoa(wellknownports.EtcdEventsPeerPort), // etcd events peer + "port=3988", // kops-controller + "port=" + strconv.Itoa(wellknownports.EtcdMainClientPort), // etcd main + "port=" + strconv.Itoa(wellknownports.EtcdEventsClientPort), // etcd events + "port=4789", // VXLAN + "port=179", // Calico + "port=8443", // k8s api secondary listener + "port=3:4", // ICMP + "port=-1", // ICMPv6 + + // TODO: UDP vs TCP vs ICMP vs ICMPv6 + // TODO: Protocol 4 for calico + } + + for _, c := range b.Cluster.Spec.EtcdClusters { + if c.Name == "leases" { + removeExtraRules = append(removeExtraRules, + "port="+strconv.Itoa(wellknownports.EtcdLeasesPeerPort), // etcd leases peer + "port="+strconv.Itoa(wellknownports.EtcdLeasesClientPort), // etcd leases + ) + break + } + } + baseGroup = &awstasks.SecurityGroup{ - Name: fi.PtrTo(name), - VPC: b.LinkToVPC(), - Description: fi.PtrTo("Security group for masters"), - RemoveExtraRules: []string{ - "port=22", // SSH - "port=443", // k8s api - "port=" + strconv.Itoa(wellknownports.EtcdMainPeerPort), // etcd main peer - "port=" + strconv.Itoa(wellknownports.EtcdEventsPeerPort), // etcd events peer - "port=3988", // kops-controller - "port=" + strconv.Itoa(wellknownports.EtcdMainClientPort), // etcd main - "port=" + strconv.Itoa(wellknownports.EtcdEventsClientPort), // etcd events - "port=4789", // VXLAN - "port=179", // Calico - "port=8443", // k8s api secondary listener - "port=3:4", // ICMP - "port=-1", // ICMPv6 - - // TODO: UDP vs TCP vs ICMP vs ICMPv6 - // TODO: Protocol 4 for calico - }, + Name: fi.PtrTo(name), + VPC: b.LinkToVPC(), + Description: fi.PtrTo("Security group for masters"), + RemoveExtraRules: removeExtraRules, } baseGroup.Tags = b.CloudTags(name, false) case kops.InstanceGroupRoleNode: diff --git a/pkg/model/azuremodel/network.go b/pkg/model/azuremodel/network.go index 767b1621a861b..b719821d85b38 100644 --- a/pkg/model/azuremodel/network.go +++ b/pkg/model/azuremodel/network.go @@ -176,6 +176,13 @@ func (b *NetworkModelBuilder) Build(c *fi.CloudupModelBuilderContext) error { DestinationApplicationSecurityGroupNames: []*string{fi.PtrTo(b.NameForApplicationSecurityGroupNodes())}, DestinationPortRange: fi.PtrTo("*"), }) + etcdPeerMax := wellknownports.EtcdEventsPeerPort + for _, c := range b.Cluster.Spec.EtcdClusters { + if c.Name == "leases" { + etcdPeerMax = wellknownports.EtcdLeasesPeerPort + break + } + } nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{ Name: fi.PtrTo("DenyNodesToEtcdManager"), Priority: fi.PtrTo[int32](1003), @@ -185,7 +192,7 @@ func (b *NetworkModelBuilder) Build(c *fi.CloudupModelBuilderContext) error { SourceApplicationSecurityGroupNames: []*string{fi.PtrTo(b.NameForApplicationSecurityGroupNodes())}, SourcePortRange: fi.PtrTo("*"), DestinationApplicationSecurityGroupNames: []*string{fi.PtrTo(b.NameForApplicationSecurityGroupControlPlane())}, - DestinationPortRange: fi.PtrTo(strconv.Itoa(wellknownports.EtcdMainPeerPort) + "-" + strconv.Itoa(wellknownports.EtcdEventsPeerPort)), + DestinationPortRange: fi.PtrTo(strconv.Itoa(wellknownports.EtcdMainPeerPort) + "-" + strconv.Itoa(etcdPeerMax)), }) nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{ Name: fi.PtrTo("DenyNodesToEtcd"), diff --git a/pkg/model/bootstrapscript.go b/pkg/model/bootstrapscript.go index 5bcfdbf8b227e..55451c19b791f 100644 --- a/pkg/model/bootstrapscript.go +++ b/pkg/model/bootstrapscript.go @@ -165,16 +165,16 @@ func (b *BootstrapScript) kubeEnv(cluster *kops.Cluster, ig *kops.InstanceGroup, func KeypairNamesForInstanceGroup(cluster *kops.Cluster, ig *kops.InstanceGroup) []string { keypairs := []string{"kubernetes-ca"} - // Add keypairs for default etcd clusters (main and events, not cilium) + // Add keypairs for default etcd clusters (main, events, and leases, not cilium) if ig.IsControlPlane() { for _, etcdCluster := range cluster.Spec.EtcdClusters { k := etcdCluster.Name - if k != "events" && k != "main" { + if k != "events" && k != "main" && k != "leases" { // Likely cilium continue } keypairs = append(keypairs, "etcd-manager-ca-"+k, "etcd-peers-ca-"+k) - // The client ca certificate is shared between events and main etcd clusters + // The client ca certificate is shared between events, main, and leases etcd clusters keypairs = append(keypairs, "etcd-clients-ca") } } @@ -195,7 +195,7 @@ func KeypairNamesForInstanceGroup(cluster *kops.Cluster, ig *kops.InstanceGroup) // Add keypairs for cilium etcd clusters (not the default etcd clusters) for _, etcdCluster := range cluster.Spec.EtcdClusters { k := etcdCluster.Name - if k == "events" || k == "main" { + if k == "events" || k == "main" || k == "leases" { // Not cilium continue } diff --git a/pkg/model/bootstrapscript_test.go b/pkg/model/bootstrapscript_test.go index 26305bdba308e..44a573ec9d947 100644 --- a/pkg/model/bootstrapscript_test.go +++ b/pkg/model/bootstrapscript_test.go @@ -155,9 +155,13 @@ func TestBootstrapUserData(t *testing.T) { "apiserver-aggregator-ca", "etcd-clients-ca", "etcd-manager-ca-events", + "etcd-manager-ca-leases", "etcd-manager-ca-main", "etcd-peers-ca-events", + "etcd-peers-ca-leases", "etcd-peers-ca-main", + "kubelet", + "kube-proxy", "service-account", } { task := &fitasks.Keypair{ diff --git a/pkg/model/components/apiserver.go b/pkg/model/components/apiserver.go index 42188b4394ec9..8604e6c0e4e42 100644 --- a/pkg/model/components/apiserver.go +++ b/pkg/model/components/apiserver.go @@ -155,6 +155,8 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(cluster *kops.Cluster) error scheme = "http" } c.EtcdServersOverrides = append(c.EtcdServersOverrides, fmt.Sprintf("/events#%s://127.0.0.1:%d", scheme, wellknownports.EtcdEventsClientPort)) + case "leases": + c.EtcdServersOverrides = append(c.EtcdServersOverrides, fmt.Sprintf("/registry/leases#https://127.0.0.1:%d", wellknownports.EtcdLeasesClientPort)) } } diff --git a/pkg/model/components/etcdmanager/model.go b/pkg/model/components/etcdmanager/model.go index 1f51a7cac39eb..5f79323db1f82 100644 --- a/pkg/model/components/etcdmanager/model.go +++ b/pkg/model/components/etcdmanager/model.go @@ -414,6 +414,10 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance if !featureflag.APIServerNodes.Enabled() { clientHost = b.Cluster.APIInternalName() } + + case "leases": + // ok + default: return nil, fmt.Errorf("unknown etcd cluster key %q", etcdCluster.Name) } @@ -758,6 +762,14 @@ func PortsForCluster(etcdCluster kops.EtcdClusterSpec) (Ports, error) { PeerPort: wellknownports.EtcdCiliumPeerPort, }, nil + case "leases": + return Ports{ + GRPCPort: wellknownports.EtcdLeasesGRPC, + QuarantinedGRPCPort: wellknownports.EtcdLeasesQuarantinedClientPort, + ClientPort: wellknownports.EtcdLeasesClientPort, + PeerPort: wellknownports.EtcdLeasesPeerPort, + }, nil + default: return Ports{}, fmt.Errorf("unknown etcd cluster key %q", etcdCluster.Name) } diff --git a/pkg/model/openstackmodel/firewall.go b/pkg/model/openstackmodel/firewall.go index 61174268f7669..6f1b08790da1f 100644 --- a/pkg/model/openstackmodel/firewall.go +++ b/pkg/model/openstackmodel/firewall.go @@ -152,6 +152,16 @@ func (b *FirewallModelBuilder) addETCDRules(c *fi.CloudupModelBuilderContext, sg nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode) nodeSG := sgMap[nodeName] + etcdClientMax := wellknownports.EtcdEventsClientPort + etcdPeerMax := wellknownports.EtcdEventsPeerPort + for _, c := range b.Cluster.Spec.EtcdClusters { + if c.Name == "leases" { + etcdClientMax = wellknownports.EtcdLeasesClientPort + etcdPeerMax = wellknownports.EtcdLeasesPeerPort + break + } + } + // ETCD Peer Discovery etcdRule := &openstacktasks.SecurityGroupRule{ Lifecycle: b.Lifecycle, @@ -159,7 +169,7 @@ func (b *FirewallModelBuilder) addETCDRules(c *fi.CloudupModelBuilderContext, sg Protocol: s(string(rules.ProtocolTCP)), EtherType: s(IPV4), PortRangeMin: i(wellknownports.EtcdMainClientPort), - PortRangeMax: i(wellknownports.EtcdEventsClientPort), + PortRangeMax: i(etcdClientMax), } etcdPeerRule := &openstacktasks.SecurityGroupRule{ Lifecycle: b.Lifecycle, @@ -167,7 +177,7 @@ func (b *FirewallModelBuilder) addETCDRules(c *fi.CloudupModelBuilderContext, sg Protocol: s(string(rules.ProtocolTCP)), EtherType: s(IPV4), PortRangeMin: i(wellknownports.EtcdMainPeerPort), - PortRangeMax: i(wellknownports.EtcdEventsPeerPort), + PortRangeMax: i(etcdPeerMax), } b.addDirectionalGroupRule(c, masterSG, masterSG, etcdRule) b.addDirectionalGroupRule(c, masterSG, masterSG, etcdPeerRule) diff --git a/pkg/model/openstackmodel/servergroup_test.go b/pkg/model/openstackmodel/servergroup_test.go index d97ebaa4b247e..57e86cad4f722 100644 --- a/pkg/model/openstackmodel/servergroup_test.go +++ b/pkg/model/openstackmodel/servergroup_test.go @@ -1605,8 +1605,10 @@ func RunGoldenTest(t *testing.T, basedir string, testCase serverGroupModelBuilde "apiserver-aggregator-ca", "etcd-clients-ca", "etcd-manager-ca-events", + "etcd-manager-ca-leases", "etcd-manager-ca-main", "etcd-peers-ca-events", + "etcd-peers-ca-leases", "etcd-peers-ca-main", "service-account", } { diff --git a/pkg/model/openstackmodel/tests/servergroup/adds-additional-security-groups.yaml b/pkg/model/openstackmodel/tests/servergroup/adds-additional-security-groups.yaml index aef8feda31144..75e30636b09e7 100644 --- a/pkg/model/openstackmodel/tests/servergroup/adds-additional-security-groups.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/adds-additional-security-groups.yaml @@ -105,6 +105,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -123,6 +132,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-ClusterSpec.yaml b/pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-ClusterSpec.yaml index 40f8bc66b1c64..ff5490c7914b2 100644 --- a/pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-ClusterSpec.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-ClusterSpec.yaml @@ -104,6 +104,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -122,6 +131,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-InstanceGroupSpec.yaml b/pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-InstanceGroupSpec.yaml index 40f8bc66b1c64..ff5490c7914b2 100644 --- a/pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-InstanceGroupSpec.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-InstanceGroupSpec.yaml @@ -104,6 +104,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -122,6 +131,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/configures-allowed-address-pairs-with-annotations.yaml b/pkg/model/openstackmodel/tests/servergroup/configures-allowed-address-pairs-with-annotations.yaml index f17c6f31f6561..fab942cd45091 100644 --- a/pkg/model/openstackmodel/tests/servergroup/configures-allowed-address-pairs-with-annotations.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/configures-allowed-address-pairs-with-annotations.yaml @@ -106,6 +106,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -124,6 +133,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/configures-server-group-affinity-with-annotations.yaml b/pkg/model/openstackmodel/tests/servergroup/configures-server-group-affinity-with-annotations.yaml index 2dfef35f1200b..687ea3c1d76b6 100644 --- a/pkg/model/openstackmodel/tests/servergroup/configures-server-group-affinity-with-annotations.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/configures-server-group-affinity-with-annotations.yaml @@ -103,6 +103,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -121,6 +130,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-auto-zone-distribution.yaml b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-auto-zone-distribution.yaml index 91606bbaee60f..4dd02ef15b753 100644 --- a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-auto-zone-distribution.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-auto-zone-distribution.yaml @@ -593,6 +593,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -611,6 +620,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer-dns-none.yaml b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer-dns-none.yaml index 5e7f8833665a4..57ba8c994ef5e 100644 --- a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer-dns-none.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer-dns-none.yaml @@ -523,6 +523,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -541,6 +550,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer.yaml b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer.yaml index c399722226022..e1aa496071789 100644 --- a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer.yaml @@ -520,6 +520,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -538,6 +547,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion.yaml b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion.yaml index 73a3a139a0f57..41ecf44590e45 100644 --- a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion.yaml @@ -605,6 +605,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -623,6 +632,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-external-router.yaml b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-external-router.yaml index d49fb297e10c1..49d435bafd729 100644 --- a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-external-router.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-external-router.yaml @@ -515,6 +515,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -533,6 +542,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion-2.yaml b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion-2.yaml index b1d0412146e0c..df7da9035d047 100644 --- a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion-2.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion-2.yaml @@ -264,6 +264,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -282,6 +291,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion.yaml b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion.yaml index dcd87b992efbc..45f3d18adb748 100644 --- a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion.yaml @@ -277,6 +277,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -295,6 +304,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-without-bastion-no-public-ip-association.yaml b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-without-bastion-no-public-ip-association.yaml index b7fcf86d4a7dd..181fa06b6ada6 100644 --- a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-without-bastion-no-public-ip-association.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-without-bastion-no-public-ip-association.yaml @@ -189,6 +189,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -207,6 +216,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node.yaml b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node.yaml index 8f46e402e87d1..764a257a0d27f 100644 --- a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node.yaml @@ -219,6 +219,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -237,6 +246,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/single-zone-setup-3-masters-1-node-without-bastion-with-API-loadbalancer-dns-none.yaml b/pkg/model/openstackmodel/tests/servergroup/single-zone-setup-3-masters-1-node-without-bastion-with-API-loadbalancer-dns-none.yaml index 367f91af9fc0c..7025bd5fe842c 100644 --- a/pkg/model/openstackmodel/tests/servergroup/single-zone-setup-3-masters-1-node-without-bastion-with-API-loadbalancer-dns-none.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/single-zone-setup-3-masters-1-node-without-bastion-with-API-loadbalancer-dns-none.yaml @@ -375,6 +375,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -393,6 +402,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/truncate-cluster-names-to-42-characters.yaml b/pkg/model/openstackmodel/tests/servergroup/truncate-cluster-names-to-42-characters.yaml index 90f2278a7c79f..b51f531ad2152 100644 --- a/pkg/model/openstackmodel/tests/servergroup/truncate-cluster-names-to-42-characters.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/truncate-cluster-names-to-42-characters.yaml @@ -219,6 +219,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -237,6 +246,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/uses-instance-group-subnet-as-availability-zones-fallback.yaml b/pkg/model/openstackmodel/tests/servergroup/uses-instance-group-subnet-as-availability-zones-fallback.yaml index a57f000dd62c0..f8cd834629394 100644 --- a/pkg/model/openstackmodel/tests/servergroup/uses-instance-group-subnet-as-availability-zones-fallback.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/uses-instance-group-subnet-as-availability-zones-fallback.yaml @@ -105,6 +105,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -123,6 +132,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/model/openstackmodel/tests/servergroup/uses-instance-group-zones-as-availability-zones.yaml b/pkg/model/openstackmodel/tests/servergroup/uses-instance-group-zones-as-availability-zones.yaml index 5f79d1f6f7956..e5a5495c52467 100644 --- a/pkg/model/openstackmodel/tests/servergroup/uses-instance-group-zones-as-availability-zones.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/uses-instance-group-zones-as-availability-zones.yaml @@ -105,6 +105,15 @@ subject: cn=etcd-manager-ca-events type: ca --- Lifecycle: "" +Name: etcd-manager-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-manager-ca-main Signer: null alternateNames: null @@ -123,6 +132,15 @@ subject: cn=etcd-peers-ca-events type: ca --- Lifecycle: "" +Name: etcd-peers-ca-leases +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-leases +type: ca +--- +Lifecycle: "" Name: etcd-peers-ca-main Signer: null alternateNames: null diff --git a/pkg/nodemodel/nodeupconfigbuilder.go b/pkg/nodemodel/nodeupconfigbuilder.go index a85dd8859a04d..597830aa476c2 100644 --- a/pkg/nodemodel/nodeupconfigbuilder.go +++ b/pkg/nodemodel/nodeupconfigbuilder.go @@ -269,7 +269,7 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, wellKnownAddre if err := loadCertificates(keysets, "etcd-peers-ca-"+k, config, true); err != nil { return nil, nil, err } - if k != "events" && k != "main" { + if k != "events" && k != "main" && k != "leases" { if err := loadCertificates(keysets, "etcd-clients-ca-"+k, config, true); err != nil { return nil, nil, err } diff --git a/pkg/testutils/cluster.go b/pkg/testutils/cluster.go index 1bbdb9b7ef172..3208c8b359a22 100644 --- a/pkg/testutils/cluster.go +++ b/pkg/testutils/cluster.go @@ -52,7 +52,7 @@ func addEtcdClusters(c *kops.Cluster) { } etcdZones := subnetNames.List() - for _, etcdCluster := range []string{"main", "events"} { + for _, etcdCluster := range []string{"main", "events", "leases"} { etcd := kops.EtcdClusterSpec{} etcd.Name = etcdCluster for _, zone := range etcdZones { diff --git a/pkg/wellknownports/wellknownports.go b/pkg/wellknownports/wellknownports.go index f814596dc63ab..5d32194a471fb 100644 --- a/pkg/wellknownports/wellknownports.go +++ b/pkg/wellknownports/wellknownports.go @@ -47,12 +47,18 @@ const ( // EtcdEventsQuarantinedClientPort is the port used by etcd when quarantined, for the events etcd EtcdEventsQuarantinedClientPort = 3995 + // EtcdLeasesQuarantinedClientPort is the port used by etcd when quarantined, for the leases etcd + EtcdLeasesQuarantinedClientPort = 4005 + // EtcdMainGRPC is the GRPC port used by etcd-manager, for the main etcd EtcdMainGRPC = 3996 // EtcdEventsGRPC is the GRPC port used by etcd-manager, for the events etcd EtcdEventsGRPC = 3997 + // EtcdLeasesGRPC is the GRPC port used by etcd-manager, for the leases etcd + EtcdLeasesGRPC = 4006 + // DNSControllerGossipWeaveMesh is the port where dns-controller listens for the weave-mesh backend gossip DNSControllerGossipWeaveMesh = 3998 @@ -68,6 +74,8 @@ const ( EtcdEventsClientPort = 4002 // EtcdCiliumClientPort is the port were the Cilium etcd cluster listens EtcdCiliumClientPort = 4003 + // EtcdLeasesClientPort is the client port for the leases etcd cluster + EtcdLeasesClientPort = 4004 // EtcdMainPeerPort is the peer port for the main etcd cluster EtcdMainPeerPort = 2380 @@ -75,6 +83,8 @@ const ( EtcdEventsPeerPort = 2381 // EtcdCiliumPeerPort is the peer port for the cilium etcd cluster EtcdCiliumPeerPort = 2382 + // EtcdLeasesPeerPort is the peer port for the leases etcd cluster + EtcdLeasesPeerPort = 2383 // CiliumOperatorPrometheusPort is the port the Cilium Operator exposes metrics CiliumPrometheusOperatorPort = 6942 @@ -120,5 +130,6 @@ func DNSGossipPortRanges() []PortRange { func ETCDPortRanges() []PortRange { return []PortRange{ {Min: 3994, Max: 3997}, + {Min: 4005, Max: 4006}, } } diff --git a/protokube/pkg/etcd/cluster_spec.go b/protokube/pkg/etcd/cluster_spec.go index 63e063d8fa05e..2902ab5fade99 100644 --- a/protokube/pkg/etcd/cluster_spec.go +++ b/protokube/pkg/etcd/cluster_spec.go @@ -23,7 +23,7 @@ import ( // EtcdClusterSpec is configuration for the etcd cluster type EtcdClusterSpec struct { - // ClusterKey is a key that identifies the etcd cluster (main or events) + // ClusterKey is a key that identifies the etcd cluster (main, events, or leases) ClusterKey string `json:"clusterKey,omitempty"` // NodeName is my nodename in the cluster NodeName string `json:"nodeName,omitempty"` diff --git a/tests/e2e/scenarios/cilium-connectivity-test/run-test.sh b/tests/e2e/scenarios/cilium-connectivity-test/run-test.sh index 570245606221b..c2f237f3f7410 100755 --- a/tests/e2e/scenarios/cilium-connectivity-test/run-test.sh +++ b/tests/e2e/scenarios/cilium-connectivity-test/run-test.sh @@ -29,7 +29,7 @@ else KUBETEST2_ARGS+=("--kops-version-marker=${KOPS_VERSION_MARKER:-https://storage.googleapis.com/k8s-staging-kops/kops/releases/markers/master/latest-ci.txt}") fi -CREATE_ARGS="--networking cilium --set=cluster.spec.networking.cilium.hubble.enabled=true --set=cluster.spec.certManager.enabled=true" +CREATE_ARGS="--networking cilium --set=cluster.spec.networking.cilium.hubble.enabled=true --set=cluster.spec.certManager.enabled=true --etcd-clusters main,events,cilium,leases" if [[ "${1:-}" == "kube-proxy" ]]; then CREATE_ARGS="${CREATE_ARGS} --set=cluster.spec.networking.cilium.enableNodePort=false --set=cluster.spec.kubeProxy.enabled=true" From a929f42f2188d46dde01601eaf7657586908ae29 Mon Sep 17 00:00:00 2001 From: Andrew Peabody Date: Fri, 13 Mar 2026 22:15:07 +0000 Subject: [PATCH 2/4] docs: document optional leases etcd shard Add documentation about the optional leases etcd shard for megaclusters, including an example configuration. --- docs/cluster_spec.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index b0369d5d3f407..a59e3ee06509e 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -181,6 +181,38 @@ etcdClusters: The etcd version used by kOps follows the recommended etcd version for the given kubernetes version. It is possible to override this by adding the `version` key to each of the etcd clusters. +### The `leases` etcd shard for megaclusters + +kOps supports configuring an optional `leases` etcd shard. This shard is specifically designed for megaclusters to isolate the massive volume of high-frequency write traffic generated by node heartbeats. By redirecting this traffic to a dedicated `leases` shard, it prevents the primary etcd database from being overwhelmed, ensuring overall control plane stability. + +```yaml +etcdClusters: +- etcdMembers: + - instanceGroup: master0-az0 + name: a-1 + - instanceGroup: master1-az0 + name: a-2 + - instanceGroup: master0-az1 + name: b-1 + name: main +- etcdMembers: + - instanceGroup: master0-az0 + name: a-1 + - instanceGroup: master1-az0 + name: a-2 + - instanceGroup: master0-az1 + name: b-1 + name: events +- etcdMembers: + - instanceGroup: master0-az0 + name: a-1 + - instanceGroup: master1-az0 + name: a-2 + - instanceGroup: master0-az1 + name: b-1 + name: leases +``` + By default, the Volumes created for the etcd clusters are `gp3` and 20GB each. The volume size, type (`gp2`, `gp3`, `io1`, `io2`), iops( for `io1`, `io2`, `gp3`) and throughput (`gp3`) can be configured via their parameters. As of kOps 1.12.0 it is also possible to modify the requests for your etcd cluster members using the `cpuRequest` and `memoryRequest` parameters. From 5b25da96dea965238958b58a2f01da6ab509972b Mon Sep 17 00:00:00 2001 From: Andrew Peabody Date: Fri, 13 Mar 2026 23:34:52 +0000 Subject: [PATCH 3/4] Fix leases override format --- nodeup/pkg/model/kube_apiserver.go | 2 +- pkg/model/components/apiserver.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index 2400729447280..aa2e71fe38bdd 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -601,7 +601,7 @@ func (b *KubeAPIServerBuilder) buildPod(ctx context.Context, kubeAPIServer *kops if strings.HasPrefix(kubeAPIServer.EtcdServersOverrides[i], "/events") { kubeAPIServer.EtcdServersOverrides[i] = strings.ReplaceAll(kubeAPIServer.EtcdServersOverrides[i], "127.0.0.1", eventsEtcdDNSName) } - if strings.HasPrefix(kubeAPIServer.EtcdServersOverrides[i], "/registry/leases") { + if strings.HasPrefix(kubeAPIServer.EtcdServersOverrides[i], "coordination.k8s.io/leases") { kubeAPIServer.EtcdServersOverrides[i] = strings.ReplaceAll(kubeAPIServer.EtcdServersOverrides[i], "127.0.0.1", "leases.etcd.internal."+clusterName) } } diff --git a/pkg/model/components/apiserver.go b/pkg/model/components/apiserver.go index 8604e6c0e4e42..bcb3cbf39d1c5 100644 --- a/pkg/model/components/apiserver.go +++ b/pkg/model/components/apiserver.go @@ -156,7 +156,7 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(cluster *kops.Cluster) error } c.EtcdServersOverrides = append(c.EtcdServersOverrides, fmt.Sprintf("/events#%s://127.0.0.1:%d", scheme, wellknownports.EtcdEventsClientPort)) case "leases": - c.EtcdServersOverrides = append(c.EtcdServersOverrides, fmt.Sprintf("/registry/leases#https://127.0.0.1:%d", wellknownports.EtcdLeasesClientPort)) + c.EtcdServersOverrides = append(c.EtcdServersOverrides, fmt.Sprintf("coordination.k8s.io/leases#https://127.0.0.1:%d", wellknownports.EtcdLeasesClientPort)) } } From 7451418a6485c3b38a215436adca1e3e9d1e61a4 Mon Sep 17 00:00:00 2001 From: Andrew Peabody Date: Sat, 14 Mar 2026 00:06:36 +0000 Subject: [PATCH 4/4] Add leasesEtcdDNSName variable --- nodeup/pkg/model/kube_apiserver.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index aa2e71fe38bdd..fd3d6312da1db 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -594,6 +594,7 @@ func (b *KubeAPIServerBuilder) buildPod(ctx context.Context, kubeAPIServer *kops clusterName := b.NodeupConfig.ClusterName mainEtcdDNSName := "main.etcd.internal." + clusterName eventsEtcdDNSName := "events.etcd.internal." + clusterName + leasesEtcdDNSName := "leases.etcd.internal." + clusterName for i := range kubeAPIServer.EtcdServers { kubeAPIServer.EtcdServers[i] = strings.ReplaceAll(kubeAPIServer.EtcdServers[i], "127.0.0.1", mainEtcdDNSName) } @@ -602,7 +603,7 @@ func (b *KubeAPIServerBuilder) buildPod(ctx context.Context, kubeAPIServer *kops kubeAPIServer.EtcdServersOverrides[i] = strings.ReplaceAll(kubeAPIServer.EtcdServersOverrides[i], "127.0.0.1", eventsEtcdDNSName) } if strings.HasPrefix(kubeAPIServer.EtcdServersOverrides[i], "coordination.k8s.io/leases") { - kubeAPIServer.EtcdServersOverrides[i] = strings.ReplaceAll(kubeAPIServer.EtcdServersOverrides[i], "127.0.0.1", "leases.etcd.internal."+clusterName) + kubeAPIServer.EtcdServersOverrides[i] = strings.ReplaceAll(kubeAPIServer.EtcdServersOverrides[i], "127.0.0.1", leasesEtcdDNSName) } } }