From 7d65eca225b22cb8e03b02aaca27839cff68ec55 Mon Sep 17 00:00:00 2001 From: shafinhasnat Date: Thu, 30 Oct 2025 00:51:38 +0600 Subject: [PATCH 1/2] Annotate delete candidate node and cleanup orphan node object --- .../hyperstack/hyperstack_kubernetes.go | 66 +++++++++++++++++++ .../hyperstack/hyperstack_manager.go | 1 + .../hyperstack/hyperstack_node_group.go | 11 +++- 3 files changed, 75 insertions(+), 3 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/hyperstack/hyperstack_kubernetes.go b/cluster-autoscaler/cloudprovider/hyperstack/hyperstack_kubernetes.go index fc884d70423c..50227cc47a45 100644 --- a/cluster-autoscaler/cloudprovider/hyperstack/hyperstack_kubernetes.go +++ b/cluster-autoscaler/cloudprovider/hyperstack/hyperstack_kubernetes.go @@ -23,6 +23,7 @@ import ( "io" "net/http" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -130,6 +131,7 @@ func DeleteNodeObject(nodeNames []string) error { return nil } +// GetNodeObjectCountByLabel returns the number of nodes with the given label. func GetNodeObjectCountByLabel(labelKey string) (int, error) { config, err := rest.InClusterConfig() if err != nil { @@ -147,3 +149,67 @@ func GetNodeObjectCountByLabel(labelKey string) (int, error) { } return len(nodeList.Items), nil } + +// AnnotateNodeObject annotates a node object with the given annotations. +func AnnotateNodeObject(nodeName string, annotationKey string, annotationValue string) error { + config, err := rest.InClusterConfig() + if err != nil { + return fmt.Errorf("failed to get in-cluster config: %v", err) + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("failed to create kubernetes client: %v", err) + } + node, err := clientset.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get node %s: %v", nodeName, err) + } + existingAnnotations := node.GetAnnotations() + if existingAnnotations == nil { + existingAnnotations = map[string]string{} + } + existingAnnotations[annotationKey] = annotationValue + node.SetAnnotations(existingAnnotations) + _, err = clientset.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to annotate node %s: %v", nodeName, err) + } + return nil +} + +// CleanUpOrphanNodeObject cleans up orphan nodes by deleting the delete-candidate annotation. +func CleanUpOrphanNodeObject() error { + config, err := rest.InClusterConfig() + if err != nil { + return fmt.Errorf("failed to get in-cluster config: %v", err) + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("failed to create kubernetes client: %v", err) + } + nodeList, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list nodes: %v", err) + } + for _, node := range nodeList.Items { + annotations := node.GetAnnotations() + nodeReady := true + conditionStatus := node.Status.Conditions + for _, condition := range conditionStatus { + if condition.Type == v1.NodeReady { + if condition.Status != v1.ConditionTrue { + nodeReady = false + } + } + } + deleteCandidate := annotations[deleteCandidateAnnotation] + if !nodeReady && deleteCandidate == "true" { + klog.Infof("Cleaning up orphan node %s: node-ready: %t, delete-candidate: %s", node.Name, nodeReady, deleteCandidate) + err := DeleteNodeObject([]string{node.Name}) + if err != nil { + return fmt.Errorf("failed to delete node object%s: %v", node.Name, err) + } + } + } + return nil +} diff --git a/cluster-autoscaler/cloudprovider/hyperstack/hyperstack_manager.go b/cluster-autoscaler/cloudprovider/hyperstack/hyperstack_manager.go index f5795052abac..f5ccfa89f0b8 100644 --- a/cluster-autoscaler/cloudprovider/hyperstack/hyperstack_manager.go +++ b/cluster-autoscaler/cloudprovider/hyperstack/hyperstack_manager.go @@ -386,6 +386,7 @@ func (h *Hyperstack) GetClusterNodesWithResponse(ctx context.Context, clusterId // Refresh updates manager node groups from the provider state. func (m *Manager) Refresh() error { + defer CleanUpOrphanNodeObject() ctx := context.Background() clusterId, err := GetNodeLabel(clusterIdLabel) if err != nil { diff --git a/cluster-autoscaler/cloudprovider/hyperstack/hyperstack_node_group.go b/cluster-autoscaler/cloudprovider/hyperstack/hyperstack_node_group.go index 87e89833d743..27642166635f 100644 --- a/cluster-autoscaler/cloudprovider/hyperstack/hyperstack_node_group.go +++ b/cluster-autoscaler/cloudprovider/hyperstack/hyperstack_node_group.go @@ -30,9 +30,10 @@ import ( ) const ( - nodeIdLabel = "hyperstack.cloud/node-id" - nodeRoleLabel = "node-role.kubernetes.io/worker" - nodeGroupLabel = "hyperstack.cloud/node-group-id" + nodeIdLabel = "hyperstack.cloud/node-id" + nodeRoleLabel = "node-role.kubernetes.io/worker" + nodeGroupLabel = "hyperstack.cloud/node-group-id" + deleteCandidateAnnotation = "hyperstack.cloud/delete-candidate" ) // NodeGroup represents a Hyperstack node group managed by the autoscaler. @@ -121,6 +122,10 @@ func (n *NodeGroup) DeleteNodes(nodes []*apiv1.Node) error { klog.V(4).Infof("[DeleteNodes] Node %s is not a worker node, skipping", node.Name) continue } + err := AnnotateNodeObject(node.Name, deleteCandidateAnnotation, "true") + if err != nil { + return fmt.Errorf("failed to annotate node %s: %v", node.Name, err) + } klog.V(4).Info("[DeleteNodes] Deleting node with arguments ", nodeID) nodeIDInt, err := strconv.Atoi(nodeID) if err != nil { From 3e6349e60edd1e07e5cd4eb250aee16befc6e59f Mon Sep 17 00:00:00 2001 From: shafinhasnat Date: Thu, 30 Oct 2025 22:28:38 +0600 Subject: [PATCH 2/2] fix: issue: missing whitespace in object%s --- .../cloudprovider/hyperstack/hyperstack_kubernetes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-autoscaler/cloudprovider/hyperstack/hyperstack_kubernetes.go b/cluster-autoscaler/cloudprovider/hyperstack/hyperstack_kubernetes.go index 50227cc47a45..83ff9a78e4d9 100644 --- a/cluster-autoscaler/cloudprovider/hyperstack/hyperstack_kubernetes.go +++ b/cluster-autoscaler/cloudprovider/hyperstack/hyperstack_kubernetes.go @@ -207,7 +207,7 @@ func CleanUpOrphanNodeObject() error { klog.Infof("Cleaning up orphan node %s: node-ready: %t, delete-candidate: %s", node.Name, nodeReady, deleteCandidate) err := DeleteNodeObject([]string{node.Name}) if err != nil { - return fmt.Errorf("failed to delete node object%s: %v", node.Name, err) + return fmt.Errorf("failed to delete node object %s: %v", node.Name, err) } } }