only report Progressing for active network rollouts

Keep pod-based Progressing tied to an actual CNO rollout instead of
temporary unavailability during node reboot churn. Persist the rollout
generation in status manager state so Progressing stays true until the
rollout is both observed and fully available.

For machine config status, stop treating generic MCP node convergence as
a CNO rollout signal. Check whether the CNO machine config is still
present in the pool's rendered source list so routine MCO updates do not
flip network Progressing to true.

Signed-off-by: Jamo Luhrsen <jluhrsen@gmail.com>
Co-Authored-by: Claude Code and Codex

Author: jluhrsen

pkg/controller/statusmanager/machineconfig_status.go

@@ -9,7 +9,6 @@ import (
 	configv1 "github.com/openshift/api/config/v1"
 	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
 	"github.com/openshift/cluster-network-operator/pkg/names"
-	mcutil "github.com/openshift/cluster-network-operator/pkg/util/machineconfig"
 	mcomcfgv1 "github.com/openshift/machine-config-operator/pkg/apihelpers"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -140,18 +139,11 @@ func (status *StatusManager) SetFromMachineConfigPool(mcPools []mcfgv1.MachineCo
 	// No degraded pools, so clear degraded status
 	status.setNotDegraded(MachineConfig)
 
-	// Now check for progressing and process machine configs
 	for role, machineConfigs := range status.renderedMachineConfigs {
 		pools, err := status.findMachineConfigPoolsForLabel(mcPools, map[string]string{names.MachineConfigLabelRoleKey: role})
 		if err != nil {
 			klog.Errorf("failed to get machine config pools for the role %s: %v", role, err)
 		}
-
-		progressingPool := status.isAnyMachineConfigPoolProgressing(pools)
-		if progressingPool != "" {
-			status.setProgressing(MachineConfig, "MachineConfig", fmt.Sprintf("%s machine config pool in progressing state", progressingPool))
-			return nil
-		}
 		for _, pool := range pools {
 			if pool.Spec.Paused {
 				// When a machine config pool is in paused state, then it is expected that mco doesn't process any machine configs for the pool.
@@ -165,7 +157,7 @@ func (status *StatusManager) SetFromMachineConfigPool(mcPools []mcfgv1.MachineCo
 				mcSet := sets.Set[string]{}
 				mcSet.Insert(machineConfig)
 				if mcsBeingRemoved, ok := status.machineConfigsBeingRemoved[role]; ok && mcsBeingRemoved.Has(machineConfig) {
-					removed = mcutil.AreMachineConfigsRemovedFromPool(pool.Status, mcSet)
+					removed = areMachineConfigsRemovedFromPoolSource(pool.Status, mcSet)
 					if removed {
 						status.machineConfigsBeingRemoved[role].Delete(machineConfig)
 						// Delete map entry from status cache if role doesn't have machine configs. By deleting the entry,
@@ -183,7 +175,7 @@ func (status *StatusManager) SetFromMachineConfigPool(mcPools []mcfgv1.MachineCo
 						}
 					}
 				} else {
-					added = mcutil.AreMachineConfigsRenderedOnPool(pool.Status, mcSet)
+					added = areMachineConfigsRenderedOnPoolSource(pool.Status, mcSet)
 				}
 				if !added || !removed {
 					status.setProgressing(MachineConfig, "MachineConfig",
@@ -239,6 +231,22 @@ func (status *StatusManager) setLastRenderedMachineConfigState(renderedMachineCo
 	return status.setAnnotation(context.TODO(), co, renderedMachineConfigAnnotation, &anno)
 }
 
+func areMachineConfigsRenderedOnPoolSource(status mcfgv1.MachineConfigPoolStatus, machineConfigs sets.Set[string]) bool {
+	sourceNames := sets.New[string]()
+	for _, source := range status.Configuration.Source {
+		sourceNames.Insert(source.Name)
+	}
+	return sourceNames.IsSuperset(machineConfigs)
+}
+
+func areMachineConfigsRemovedFromPoolSource(status mcfgv1.MachineConfigPoolStatus, machineConfigs sets.Set[string]) bool {
+	sourceNames := sets.New[string]()
+	for _, source := range status.Configuration.Source {
+		sourceNames.Insert(source.Name)
+	}
+	return !sourceNames.HasAny(machineConfigs.UnsortedList()...)
+}
+
 func (status *StatusManager) isAnyMachineConfigPoolDegraded(pools []mcfgv1.MachineConfigPool) string {
 	var degradedPool string
 	for _, pool := range pools {
@@ -250,17 +258,6 @@ func (status *StatusManager) isAnyMachineConfigPoolDegraded(pools []mcfgv1.Machi
 	return degradedPool
 }
 
-func (status *StatusManager) isAnyMachineConfigPoolProgressing(pools []mcfgv1.MachineConfigPool) string {
-	var progressingPool string
-	for _, pool := range pools {
-		if mcomcfgv1.IsMachineConfigPoolConditionTrue(pool.Status.Conditions, mcfgv1.MachineConfigPoolUpdating) {
-			progressingPool = pool.Name
-			break
-		}
-	}
-	return progressingPool
-}
-
 func (status *StatusManager) findMachineConfigPoolsForLabel(mcPools []mcfgv1.MachineConfigPool, mcLabel labels.Set) ([]mcfgv1.MachineConfigPool, error) {
 	var mcps []mcfgv1.MachineConfigPool
 	for _, mcPool := range mcPools {

pkg/controller/statusmanager/pod_status.go

@@ -45,24 +45,27 @@ type podState struct {
 type daemonsetState struct {
 	ClusteredName
 
-	LastSeenStatus appsv1.DaemonSetStatus
-	LastChangeTime time.Time
+	LastSeenStatus    appsv1.DaemonSetStatus
+	LastChangeTime    time.Time
+	RolloutGeneration int64
 }
 
 // deploymentState is the same as daemonsetState.. but for deployments!
 type deploymentState struct {
 	ClusteredName
 
-	LastSeenStatus appsv1.DeploymentStatus
-	LastChangeTime time.Time
+	LastSeenStatus    appsv1.DeploymentStatus
+	LastChangeTime    time.Time
+	RolloutGeneration int64
 }
 
 // statefulsetState is the same as daemonsetState.. but for statefulsets!
 type statefulsetState struct {
 	ClusteredName
 
-	LastSeenStatus appsv1.StatefulSetStatus
-	LastChangeTime time.Time
+	LastSeenStatus    appsv1.StatefulSetStatus
+	LastChangeTime    time.Time
+	RolloutGeneration int64
 }
 
 // SetFromPods sets the operator Degraded/Progressing/Available status, based on
@@ -87,6 +90,24 @@ func (status *StatusManager) SetFromPods() {
 
 	for _, ds := range daemonSets {
 		dsName := NewClusteredName(ds)
+		dsState, exists := daemonsetStates[dsName]
+		currentRolloutGeneration := rolloutGeneration(ds.Generation, ds.Status.ObservedGeneration)
+		if exists && dsState.RolloutGeneration == 0 {
+			dsState.RolloutGeneration = currentRolloutGeneration
+		}
+		if !status.installComplete || ds.Generation > ds.Status.ObservedGeneration || ds.Status.UpdatedNumberScheduled < ds.Status.DesiredNumberScheduled {
+			if dsState.RolloutGeneration < currentRolloutGeneration {
+				dsState.RolloutGeneration = currentRolloutGeneration
+			}
+		}
+		if dsState.RolloutGeneration != 0 &&
+			ds.Status.ObservedGeneration >= dsState.RolloutGeneration &&
+			ds.Status.NumberUnavailable == 0 &&
+			(ds.Status.DesiredNumberScheduled == 0 || ds.Status.UpdatedNumberScheduled >= ds.Status.DesiredNumberScheduled) &&
+			(ds.Status.DesiredNumberScheduled == 0 || ds.Status.NumberAvailable >= ds.Status.DesiredNumberScheduled) {
+			dsState.RolloutGeneration = 0
+		}
+		dsRolloutActive := dsState.RolloutGeneration != 0
 
 		dsProgressing := false
 
@@ -97,13 +118,14 @@ func (status *StatusManager) SetFromPods() {
 			progressing = append(progressing, fmt.Sprintf("DaemonSet %q update is rolling out (%d out of %d updated)", dsName.String(), ds.Status.UpdatedNumberScheduled, ds.Status.DesiredNumberScheduled))
 			dsProgressing = true
 		} else if ds.Status.NumberUnavailable > 0 {
-			progressing = append(progressing, fmt.Sprintf("DaemonSet %q is not available (awaiting %d nodes)", dsName.String(), ds.Status.NumberUnavailable))
-			dsProgressing = true
-			// Check for any pods in CrashLoopBackOff state and mark the operator as degraded if so.
+			if dsRolloutActive {
+				progressing = append(progressing, fmt.Sprintf("DaemonSet %q is not available (awaiting %d nodes)", dsName.String(), ds.Status.NumberUnavailable))
+				dsProgressing = true
+			}
 			if !isNonCritical(ds) {
 				hung = append(hung, status.CheckCrashLoopBackOffPods(dsName, ds.Spec.Selector.MatchLabels, "DaemonSet")...)
 			}
-		} else if ds.Status.NumberAvailable == 0 && ds.Status.DesiredNumberScheduled > 0 {
+		} else if ds.Status.NumberAvailable == 0 && dsRolloutActive {
 			progressing = append(progressing, fmt.Sprintf("DaemonSet %q is not yet scheduled on any nodes", dsName.String()))
 			dsProgressing = true
 		} else if ds.Generation > ds.Status.ObservedGeneration {
@@ -120,12 +142,11 @@ func (status *StatusManager) SetFromPods() {
 		if dsProgressing && !isNonCritical(ds) {
 			reachedAvailableLevel = false
 
-			dsState, exists := daemonsetStates[dsName]
 			if !exists || !reflect.DeepEqual(dsState.LastSeenStatus, ds.Status) {
 				dsState.LastChangeTime = time.Now()
 				ds.Status.DeepCopyInto(&dsState.LastSeenStatus)
-				daemonsetStates[dsName] = dsState
 			}
+			daemonsetStates[dsName] = dsState
 
 			// Catch hung rollouts
 			if exists && (time.Since(dsState.LastChangeTime)) > ProgressTimeout {
@@ -143,6 +164,23 @@ func (status *StatusManager) SetFromPods() {
 
 	for _, ss := range statefulSets {
 		ssName := NewClusteredName(ss)
+		ssState, exists := statefulsetStates[ssName]
+		currentRolloutGeneration := rolloutGeneration(ss.Generation, ss.Status.ObservedGeneration)
+		if exists && ssState.RolloutGeneration == 0 {
+			ssState.RolloutGeneration = currentRolloutGeneration
+		}
+		if !status.installComplete || ss.Generation > ss.Status.ObservedGeneration || ss.Status.UpdatedReplicas < ss.Status.Replicas {
+			if ssState.RolloutGeneration < currentRolloutGeneration {
+				ssState.RolloutGeneration = currentRolloutGeneration
+			}
+		}
+		if ssState.RolloutGeneration != 0 &&
+			ss.Status.ObservedGeneration >= ssState.RolloutGeneration &&
+			ss.Status.UpdatedReplicas >= ss.Status.Replicas &&
+			ss.Status.ReadyReplicas >= ss.Status.Replicas {
+			ssState.RolloutGeneration = 0
+		}
+		ssRolloutActive := ssState.RolloutGeneration != 0
 
 		ssProgressing := false
 
@@ -153,13 +191,15 @@ func (status *StatusManager) SetFromPods() {
 			progressing = append(progressing, fmt.Sprintf("StatefulSet %q update is rolling out (%d out of %d updated)", ssName.String(), ss.Status.UpdatedReplicas, ss.Status.Replicas))
 			ssProgressing = true
 		} else if ss.Status.ReadyReplicas > 0 && ss.Status.ReadyReplicas < ss.Status.Replicas {
-			progressing = append(progressing, fmt.Sprintf("StatefulSet %q is not available (awaiting %d nodes)", ssName.String(), (ss.Status.Replicas-ss.Status.ReadyReplicas)))
-			ssProgressing = true
+			if ssRolloutActive {
+				progressing = append(progressing, fmt.Sprintf("StatefulSet %q is not available (awaiting %d nodes)", ssName.String(), (ss.Status.Replicas-ss.Status.ReadyReplicas)))
+				ssProgressing = true
+			}
 			// Check for any pods in CrashLoopBackOff state and mark the operator as degraded if so.
 			if !isNonCritical(ss) {
 				hung = append(hung, status.CheckCrashLoopBackOffPods(ssName, ss.Spec.Selector.MatchLabels, "StatefulSet")...)
 			}
-		} else if ss.Status.AvailableReplicas == 0 {
+		} else if ss.Status.AvailableReplicas == 0 && ssRolloutActive {
 			progressing = append(progressing, fmt.Sprintf("StatefulSet %q is not yet scheduled on any nodes", ssName.String()))
 			ssProgressing = true
 		} else if ss.Status.ObservedGeneration < ss.Generation {
@@ -176,12 +216,11 @@ func (status *StatusManager) SetFromPods() {
 		if ssProgressing && !isNonCritical(ss) {
 			reachedAvailableLevel = false
 
-			ssState, exists := statefulsetStates[ssName]
 			if !exists || !reflect.DeepEqual(ssState.LastSeenStatus, ss.Status) {
 				ssState.LastChangeTime = time.Now()
 				ss.Status.DeepCopyInto(&ssState.LastSeenStatus)
-				statefulsetStates[ssName] = ssState
 			}
+			statefulsetStates[ssName] = ssState
 
 			// Catch hung rollouts
 			if exists && (time.Since(ssState.LastChangeTime)) > ProgressTimeout {
@@ -199,6 +238,24 @@ func (status *StatusManager) SetFromPods() {
 
 	for _, dep := range deployments {
 		depName := NewClusteredName(dep)
+		depState, exists := deploymentStates[depName]
+		currentRolloutGeneration := rolloutGeneration(dep.Generation, dep.Status.ObservedGeneration)
+		if exists && depState.RolloutGeneration == 0 {
+			depState.RolloutGeneration = currentRolloutGeneration
+		}
+		if !status.installComplete || dep.Generation > dep.Status.ObservedGeneration || dep.Status.UpdatedReplicas < dep.Status.Replicas {
+			if depState.RolloutGeneration < currentRolloutGeneration {
+				depState.RolloutGeneration = currentRolloutGeneration
+			}
+		}
+		if depState.RolloutGeneration != 0 &&
+			dep.Status.ObservedGeneration >= depState.RolloutGeneration &&
+			dep.Status.UpdatedReplicas >= dep.Status.Replicas &&
+			dep.Status.UnavailableReplicas == 0 &&
+			(dep.Status.Replicas == 0 || dep.Status.AvailableReplicas >= dep.Status.Replicas) {
+			depState.RolloutGeneration = 0
+		}
+		depRolloutActive := depState.RolloutGeneration != 0
 		depProgressing := false
 
 		if isNonCritical(dep) && dep.Status.UnavailableReplicas > 0 && !status.installComplete {
@@ -208,13 +265,15 @@ func (status *StatusManager) SetFromPods() {
 			progressing = append(progressing, fmt.Sprintf("Deployment %q update is rolling out (%d out of %d updated)", depName.String(), dep.Status.UpdatedReplicas, dep.Status.Replicas))
 			depProgressing = true
 		} else if dep.Status.UnavailableReplicas > 0 {
-			progressing = append(progressing, fmt.Sprintf("Deployment %q is not available (awaiting %d nodes)", depName.String(), dep.Status.UnavailableReplicas))
-			depProgressing = true
+			if depRolloutActive {
+				progressing = append(progressing, fmt.Sprintf("Deployment %q is not available (awaiting %d nodes)", depName.String(), dep.Status.UnavailableReplicas))
+				depProgressing = true
+			}
 			// Check for any pods in CrashLoopBackOff state and mark the operator as degraded if so.
 			if !isNonCritical(dep) {
 				hung = append(hung, status.CheckCrashLoopBackOffPods(depName, dep.Spec.Selector.MatchLabels, "Deployment")...)
 			}
-		} else if dep.Status.AvailableReplicas == 0 {
+		} else if dep.Status.AvailableReplicas == 0 && depRolloutActive {
 			progressing = append(progressing, fmt.Sprintf("Deployment %q is not yet scheduled on any nodes", depName.String()))
 			depProgressing = true
 		} else if dep.Status.ObservedGeneration < dep.Generation {
@@ -231,12 +290,11 @@ func (status *StatusManager) SetFromPods() {
 		if depProgressing && !isNonCritical(dep) {
 			reachedAvailableLevel = false
 
-			depState, exists := deploymentStates[depName]
 			if !exists || !reflect.DeepEqual(depState.LastSeenStatus, dep.Status) {
 				depState.LastChangeTime = time.Now()
 				dep.Status.DeepCopyInto(&depState.LastSeenStatus)
-				deploymentStates[depName] = depState
 			}
+			deploymentStates[depName] = depState
 
 			// Catch hung rollouts
 			if exists && (time.Since(depState.LastChangeTime)) > ProgressTimeout {
@@ -280,6 +338,17 @@ func (status *StatusManager) SetFromPods() {
 	}
 }
 
+func rolloutGeneration(generation, observedGeneration int64) int64 {
+	switch {
+	case generation > 0:
+		return generation
+	case observedGeneration > 0:
+		return observedGeneration
+	default:
+		return 1
+	}
+}
+
 // getLastPodState reads the last-seen daemonset + deployment + statefulset
 // states from the clusteroperator annotation and parses it. On error, it
 // returns an empty state, since this should not block updating operator status.