Cleanup Orphan node objects

cluster-autoscaler/cloudprovider/hyperstack/hyperstack_kubernetes.go

@@ -23,6 +23,7 @@ import (
 	"io"
 	"net/http"
 
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -130,6 +131,7 @@ func DeleteNodeObject(nodeNames []string) error {
 	return nil
 }
 
+// GetNodeObjectCountByLabel returns the number of nodes with the given label.
 func GetNodeObjectCountByLabel(labelKey string) (int, error) {
 	config, err := rest.InClusterConfig()
 	if err != nil {
@@ -147,3 +149,67 @@ func GetNodeObjectCountByLabel(labelKey string) (int, error) {
 	}
 	return len(nodeList.Items), nil
 }
+
+// AnnotateNodeObject annotates a node object with the given annotations.
+func AnnotateNodeObject(nodeName string, annotationKey string, annotationValue string) error {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return fmt.Errorf("failed to get in-cluster config: %v", err)
+	}
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return fmt.Errorf("failed to create kubernetes client: %v", err)
+	}
+	node, err := clientset.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to get node %s: %v", nodeName, err)
+	}
+	existingAnnotations := node.GetAnnotations()
+	if existingAnnotations == nil {
+		existingAnnotations = map[string]string{}
+	}
+	existingAnnotations[annotationKey] = annotationValue
+	node.SetAnnotations(existingAnnotations)
+	_, err = clientset.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to annotate node %s: %v", nodeName, err)
+	}
+	return nil
+}
+
+// CleanUpOrphanNodeObject cleans up orphan nodes by deleting the delete-candidate annotation.
+func CleanUpOrphanNodeObject() error {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return fmt.Errorf("failed to get in-cluster config: %v", err)
+	}
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return fmt.Errorf("failed to create kubernetes client: %v", err)
+	}
+	nodeList, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to list nodes: %v", err)
+	}
+	for _, node := range nodeList.Items {
+		annotations := node.GetAnnotations()
+		nodeReady := true
+		conditionStatus := node.Status.Conditions
+		for _, condition := range conditionStatus {
+			if condition.Type == v1.NodeReady {
+				if condition.Status != v1.ConditionTrue {
+					nodeReady = false
+				}
+			}
+		}
+		deleteCandidate := annotations[deleteCandidateAnnotation]
+		if !nodeReady && deleteCandidate == "true" {
+			klog.Infof("Cleaning up orphan node %s: node-ready: %t, delete-candidate: %s", node.Name, nodeReady, deleteCandidate)
+			err := DeleteNodeObject([]string{node.Name})
+			if err != nil {
+				return fmt.Errorf("failed to delete node object %s: %v", node.Name, err)
+			}
+		}
+	}
+	return nil
+}

cluster-autoscaler/cloudprovider/hyperstack/hyperstack_manager.go

@@ -386,6 +386,7 @@ func (h *Hyperstack) GetClusterNodesWithResponse(ctx context.Context, clusterId
 
 // Refresh updates manager node groups from the provider state.
 func (m *Manager) Refresh() error {
+	defer CleanUpOrphanNodeObject()
 	ctx := context.Background()
 	clusterId, err := GetNodeLabel(clusterIdLabel)
 	if err != nil {

cluster-autoscaler/cloudprovider/hyperstack/hyperstack_node_group.go

@@ -30,9 +30,10 @@ import (
 )
 
 const (
-	nodeIdLabel    = "hyperstack.cloud/node-id"
-	nodeRoleLabel  = "node-role.kubernetes.io/worker"
-	nodeGroupLabel = "hyperstack.cloud/node-group-id"
+	nodeIdLabel               = "hyperstack.cloud/node-id"
+	nodeRoleLabel             = "node-role.kubernetes.io/worker"
+	nodeGroupLabel            = "hyperstack.cloud/node-group-id"
+	deleteCandidateAnnotation = "hyperstack.cloud/delete-candidate"
 )
 
 // NodeGroup represents a Hyperstack node group managed by the autoscaler.
@@ -121,6 +122,10 @@ func (n *NodeGroup) DeleteNodes(nodes []*apiv1.Node) error {
 			klog.V(4).Infof("[DeleteNodes] Node %s is not a worker node, skipping", node.Name)
 			continue
 		}
+		err := AnnotateNodeObject(node.Name, deleteCandidateAnnotation, "true")
+		if err != nil {
+			return fmt.Errorf("failed to annotate node %s: %v", node.Name, err)
+		}
 		klog.V(4).Info("[DeleteNodes] Deleting node with arguments ", nodeID)
 		nodeIDInt, err := strconv.Atoi(nodeID)
 		if err != nil {