diff --git a/.golangci.yml b/.golangci.yml index 5424dd79be..376daca06d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -367,7 +367,8 @@ linters: text: "s.scope.ControlPlaneLoadBalancer is deprecated" - linters: - staticcheck - text: "SA1019: .*EKSConfig is deprecated" + path: bootstrap/eks/ + text: 'SA1019: (.*)EKSConfig is deprecated' paths: - third_party$ - builtin$ diff --git a/Makefile b/Makefile index 5b3b4a3806..182804fbe4 100644 --- a/Makefile +++ b/Makefile @@ -467,7 +467,7 @@ test-e2e: $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) generate-test-flavors e2e-image ## .PHONY: test-e2e-eks ## Run EKS e2e tests using clusterctl test-e2e-eks: generate-test-flavors $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) e2e-image ## Run eks e2e tests - time go run github.com/onsi/ginkgo/v2/ginkgo -tags=e2e $(GINKGO_ARGS) ./test/e2e/suites/managed/... -- -config-path="$(E2E_EKS_CONF_PATH)" --source-template="$(EKS_SOURCE_TEMPLATE)" $(E2E_ARGS) $(EKS_E2E_ARGS) + time go run github.com/onsi/ginkgo/v2/ginkgo -tags=e2e $(GINKGO_ARGS) -nodes 2 ./test/e2e/suites/managed/... -- -config-path="$(E2E_EKS_CONF_PATH)" --source-template="$(EKS_SOURCE_TEMPLATE)" $(E2E_ARGS) $(EKS_E2E_ARGS) CONFORMANCE_E2E_ARGS ?= -kubetest.config-file=$(KUBETEST_CONF_PATH) CONFORMANCE_E2E_ARGS += $(E2E_ARGS) diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigs.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigs.yaml index 7c6b948992..84cd3f1298 100644 --- a/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigs.yaml +++ b/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigs.yaml @@ -194,8 +194,12 @@ spec: name: v1beta2 schema: openAPIV3Schema: - description: EKSConfig is the schema for the Amazon EKS Machine Bootstrap - Configuration API. + description: |- + EKSConfig is the schema for the Amazon EKS Machine Bootstrap Configuration API. + + Deprecated: EKSConfig is deprecated and will be removed in a future release. + Amazon Linux 2 (AL2) reaches end-of-life in June 2026 see: https://aws.amazon.com/amazon-linux-2/faqs/ + Please use NodeadmConfig with Amazon Linux 2023 (AL2023) instead. properties: apiVersion: description: |- diff --git a/docs/book/src/topics/eks/cluster-upgrades.md b/docs/book/src/topics/eks/cluster-upgrades.md index ed0bb7015a..a8514ad867 100644 --- a/docs/book/src/topics/eks/cluster-upgrades.md +++ b/docs/book/src/topics/eks/cluster-upgrades.md @@ -4,4 +4,81 @@ Upgrading the Kubernetes version of the control plane is supported by the provider. To perform an upgrade you need to update the `version` in the spec of the `AWSManagedControlPlane`. Once the version has changed the provider will handle the upgrade for you. -You can only upgrade a EKS cluster by 1 minor version at a time. If you attempt to upgrade the version by more then 1 minor version the provider will ensure the upgrade is done in multiple steps of 1 minor version. For example upgrading from v1.15 to v1.17 would result in your cluster being upgraded v1.15 -> v1.16 first and then v1.16 to v1.17. \ No newline at end of file +You can only upgrade a EKS cluster by 1 minor version at a time. If you attempt to upgrade the version by more then 1 minor version the provider will ensure the upgrade is done in multiple steps of 1 minor version. For example upgrading from v1.15 to v1.17 would result in your cluster being upgraded v1.15 -> v1.16 first and then v1.16 to v1.17. + +## Upgrading Nodes from AL2 (EKSConfig) to AL2023 (NodeadmConfig) + +Amazon Linux 2 (AL2) AMIs are only supported up to Kubernetes v1.32. To upgrade cluster nodes to v1.33 or newer, you **must** migrate them to Amazon Linux 2023 (AL2023) AMIs. This migration also requires changing the bootstrap provider from `EKSConfig` to the new `NodeadmConfig`. + +The upgrade process follows the standard Cluster API rolling update strategy. You will create a new bootstrap template (using `NodeadmConfigTemplate`) and update your `MachineDeployment` or `MachinePool` to reference it, along with the new Kubernetes version and an AL2023-based AMI. + +### MachineDeployment Upgrade Example + +Here is an example of upgrading a `MachineDeployment` from Kubernetes v1.32 (using `EKSConfig`) to v1.33 (using `NodeadmConfig`). + +**Before (v1.32 with `EKSConfigTemplate`):** + +```yaml +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: EKSConfigTemplate +metadata: + name: default132 +spec: + template: + spec: + postBootstrapCommands: + - "echo \"bye world\"" +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: default +spec: + clusterName: default + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: EKSConfigTemplate + name: default132 + infrastructureRef: + kind: AWSMachineTemplate + name: default132 + version: v1.32.0 +``` + +After (v1.33 with NodeadmConfigTemplate): + +A new NodeadmConfigTemplate is created, and the MachineDeployment is updated to reference it and the new version. +YAML + +```yaml +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: NodeadmConfigTemplate +metadata: + name: default +spec: + template: + spec: + preNodeadmCommands: + - "echo \"hello world\"" +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: default +spec: + clusterName: default + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: NodeadmConfigTemplate + name: default + infrastructureRef: + kind: AWSMachineTemplate + name: default + version: v1.33.0 +``` diff --git a/docs/book/src/topics/eks/creating-a-cluster.md b/docs/book/src/topics/eks/creating-a-cluster.md index 7ec523837f..27c26634f1 100644 --- a/docs/book/src/topics/eks/creating-a-cluster.md +++ b/docs/book/src/topics/eks/creating-a-cluster.md @@ -17,6 +17,68 @@ NOTE: When creating an EKS cluster only the **MAJOR.MINOR** of the `-kubernetes- By default CAPA relies on the default EKS cluster upgrade policy, which at the moment of writing is EXTENDED support. See more info about [cluster upgrade policy](https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html) +## Choosing a Bootstrap Provider: EKSConfig vs. NodeadmConfig + +With the introduction of Amazon Linux 2023 (AL2023), the bootstrapping method for EKS nodes has changed. Cluster API Provider AWS (CAPA) supports two bootstrap providers for EKS: + +1. **`EKSConfig`**: The original bootstrap provider. It uses the legacy `bootstrap.sh` script and is intended for use with **Amazon Linux 2 (AL2)** AMIs. +2. **`NodeadmConfig`**: The new bootstrap provider. It uses the modern `nodeadm` tool and is **required** for **Amazon Linux 2023 (AL2023)** AMIs. + +### When to use which provider + +The provider you must use depends on the Amazon Machine Image (AMI) and Kubernetes version you are targeting. Amazon Linux 2 AMIs are only supported for Kubernetes v1.32 and older. + +| Bootstrap Provider | AMI Type | Kubernetes Version | +| --- | --- | --- | +| `EKSConfig` | Amazon Linux 2 (AL2) | $\le$ v1.32 | +| `NodeadmConfig` | Amazon Linux 2023 (AL2023) | $\ge$ v1.33 | + +When you generate a cluster, you will need to ensure your `MachineDeployment` or `MachinePool` references the correct bootstrap template `kind`. + +NOTE: + +- [The EKS team stopped publishing Al2 AMIs for Kubernetes versions 1.33 and higher.](https://awslabs.github.io/amazon-eks-ami/usage/al2/) +- [Amazon Linux 2 end of support date (End of Life, or EOL) will be on 2026-06-30.](https://aws.amazon.com/amazon-linux-2/faqs/) + +**For AL2 / K8s $\le$ v1.32, use `EKSConfigTemplate`:** +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: default +spec: + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: EKSConfigTemplate # <-- Uses bootstrap.sh + name: default-132 + version: v1.32.0 +``` + +### Secrets Manager + +Amazon Linux 2023 does not have the proper tooling to use the secrets manager flow for bootstrapping. CAPA uses a [custom cloud-init datasource](https://github.com/kubernetes-sigs/image-builder/pull/1583) to fetch the secure contents like the `kubeadm` tokens from secrets manager. Crucially, there is no current support for publishing CAPA-compatible AL2023 AMIs that include this necessary custom cloud-init datasource. + +Therefore, whenever creating `AWSMachineTemplate` objects `insecureSkipSecretsManager` must be set to true. + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: default +spec: + template: + spec: + cloudInit: + insecureSkipSecretsManager: true + ami: + eksLookupType: AmazonLinux2023 + iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io + instanceType: m5a.16xlarge +``` + ## Kubeconfig When creating an EKS cluster 2 kubeconfigs are generated and stored as secrets in the management cluster. This is different to when you create a non-managed cluster using the AWS provider. diff --git a/main.go b/main.go index caf3d04e5b..4df05575bf 100644 --- a/main.go +++ b/main.go @@ -496,7 +496,7 @@ func setupEKSReconcilersAndWebhooks(ctx context.Context, mgr ctrl.Manager, Client: mgr.GetClient(), WatchFilterValue: watchFilterValue, }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: awsClusterConcurrency, RecoverPanic: ptr.To[bool](true)}); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "EKSConfig") + setupLog.Error(err, "unable to create controller", "controller", "NodeadmConfig") os.Exit(1) } diff --git a/templates/cluster-template-eks-nodeadm-clusterclass.yaml b/templates/cluster-template-eks-nodeadm-clusterclass.yaml new file mode 100644 index 0000000000..21ff5722ff --- /dev/null +++ b/templates/cluster-template-eks-nodeadm-clusterclass.yaml @@ -0,0 +1,70 @@ +apiVersion: cluster.x-k8s.io/v1beta2 +kind: ClusterClass +metadata: + name: ${CLUSTER_CLASS_NAME} +spec: + controlPlane: + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: AWSManagedControlPlaneTemplate + name: "${CLUSTER_CLASS_NAME}-control-plane" + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSManagedClusterTemplate + name: "${CLUSTER_CLASS_NAME}" + workers: + machineDeployments: + - class: default-worker + template: + bootstrap: + ref: + name: "${CLUSTER_CLASS_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: EKSConfigTemplate + infrastructure: + ref: + name: "${CLUSTER_CLASS_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate +--- +kind: AWSManagedClusterTemplate +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +metadata: + name: "${CLUSTER_CLASS_NAME}" +spec: + template: + spec: {} +--- +kind: AWSManagedControlPlaneTemplate +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +metadata: + name: "${CLUSTER_CLASS_NAME}-control-plane" +spec: + template: + spec: + region: "${AWS_REGION}" + sshKeyName: "${AWS_SSH_KEY_NAME}" + version: "${KUBERNETES_VERSION}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: "${CLUSTER_CLASS_NAME}-md-0" +spec: + template: + spec: + ami: + eksLookupType: AmazonLinux2023 + cloudInit: + insecureSkipSecretsManager: true + instanceType: "${AWS_NODE_MACHINE_TYPE}" + iamInstanceProfile: "nodes.cluster-api-provider-aws.sigs.k8s.io" + sshKeyName: "${AWS_SSH_KEY_NAME}" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: NodeadmConfigTemplate +metadata: + name: "${CLUSTER_CLASS_NAME}-md-0" +spec: + template: {} diff --git a/test/e2e/suites/managed/eks_nodeadm_clusterclass_test.go b/test/e2e/suites/managed/eks_nodeadm_clusterclass_test.go new file mode 100644 index 0000000000..4bce73664a --- /dev/null +++ b/test/e2e/suites/managed/eks_nodeadm_clusterclass_test.go @@ -0,0 +1,79 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package managed + +import ( + "context" + "fmt" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/utils/ptr" + + ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2" + "sigs.k8s.io/cluster-api-provider-aws/v2/test/e2e/shared" + capi_e2e "sigs.k8s.io/cluster-api/test/e2e" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/util" +) + +var _ = ginkgo.Describe("[managed] [general] [nodeadm] EKS nodeadm ClusterClass tests", func() { + const specName = "cluster" + var ( + ctx context.Context + clusterName string + ) + + ginkgo.BeforeEach(func() { + ctx = context.TODO() + + if !runGeneralTests() { + ginkgo.Skip("skipping due to unmet condition") + } + + ginkgo.By("should have a valid test configuration") + Expect(e2eCtx.Environment.BootstrapClusterProxy).ToNot(BeNil(), "BootstrapClusterProxy can't be nil") + Expect(e2eCtx.E2EConfig).ToNot(BeNil(), "E2EConfig can't be nil") + Expect(e2eCtx.E2EConfig.Variables).To(HaveKey(shared.KubernetesVersion)) + Expect(e2eCtx.E2EConfig.Variables).To(HaveKey(shared.CNIAddonVersion)) + + clusterName = fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + + ginkgo.By("default iam role should exist") + VerifyRoleExistsAndOwned(ctx, ekscontrolplanev1.DefaultEKSControlPlaneRole, "", false, e2eCtx.AWSSession) + }) + + capi_e2e.QuickStartSpec(context.TODO(), func() capi_e2e.QuickStartSpecInput { + return capi_e2e.QuickStartSpecInput{ + E2EConfig: e2eCtx.E2EConfig, + ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath, + BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy, + ArtifactFolder: e2eCtx.Settings.ArtifactFolder, + SkipCleanup: e2eCtx.Settings.SkipCleanup, + Flavor: ptr.To(EKSNodeadmClusterClassFlavor), + ClusterName: ptr.To(clusterName), + WorkerMachineCount: ptr.To(int64(3)), + ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{ + WaitForControlPlaneInitialized: WaitForEKSControlPlaneInitialized, + WaitForControlPlaneMachinesReady: WaitForEKSControlPlaneMachinesReady, + }, + } + }) +}) diff --git a/test/e2e/suites/unmanaged/unmanaged_CAPI_test.go b/test/e2e/suites/unmanaged/unmanaged_CAPI_test.go index a15e9d2d3d..5f22a2b9d7 100644 --- a/test/e2e/suites/unmanaged/unmanaged_CAPI_test.go +++ b/test/e2e/suites/unmanaged/unmanaged_CAPI_test.go @@ -113,7 +113,7 @@ var _ = ginkgo.Context("[unmanaged] [Cluster API Framework]", func() { }) }) - ginkgo.PDescribe("Clusterctl Upgrade Spec [from latest v1beta1 release to v1beta2]", func() { + ginkgo.Describe("Clusterctl Upgrade Spec [from latest v1beta1 release to v1beta2]", func() { ginkgo.BeforeEach(func() { if !e2eCtx.Settings.SkipQuotas { // As the resources cannot be defined by the It() clause in CAPI tests, using the largest values required for all It() tests in this CAPI test. diff --git a/test/e2e/suites/unmanaged/unmanaged_functional_test.go b/test/e2e/suites/unmanaged/unmanaged_functional_test.go index 1bd31fab25..3fe553af0f 100644 --- a/test/e2e/suites/unmanaged/unmanaged_functional_test.go +++ b/test/e2e/suites/unmanaged/unmanaged_functional_test.go @@ -119,7 +119,6 @@ var _ = ginkgo.Context("[unmanaged] [functional]", func() { ginkgo.Describe("GPU-enabled cluster test", func() { ginkgo.It("should create cluster with single worker", func() { - ginkgo.Skip("Args field of clusterctl.ApplyClusterTemplateAndWaitInput was removed, need to add support for server-side filtering.") specName := "functional-gpu-cluster" namespace := shared.SetupSpecNamespace(ctx, specName, e2eCtx) if !e2eCtx.Settings.SkipQuotas { @@ -151,13 +150,8 @@ var _ = ginkgo.Context("[unmanaged] [functional]", func() { WaitForClusterIntervals: e2eCtx.E2EConfig.GetIntervals(specName, "wait-cluster"), WaitForControlPlaneIntervals: e2eCtx.E2EConfig.GetIntervals(specName, "wait-control-plane"), WaitForMachineDeployments: e2eCtx.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), - // nvidia-gpu flavor creates a config map as part of a crs, that exceeds the annotations size limit when we do kubectl apply. - // This is because the entire config map is stored in `last-applied` annotation for tracking. - // The workaround is to use server side apply by passing `--server-side` flag to kubectl apply. - // More on server side apply here: https://kubernetes.io/docs/reference/using-api/server-side-apply/ - // TODO: Need a PR to re-add argument support to this type. - // It was removed in https://github.com/kubernetes-sigs/cluster-api/commit/b4349fecaa626865e71b058a8b01e0377fb9e444 - // Args: []string{"--server-side"}, + // GPU operator components are deployed via ClusterResourceSet which handles large ConfigMaps + // without the kubectl annotation size limit issues that occur with client-side apply. }, result) shared.AWSGPUSpec(ctx, e2eCtx, shared.AWSGPUSpecInput{