Skip to content

Commit 2277bea

Browse files
djoshydjoshy
committed
operator: verify boot image controller state
This commit ensures that the boot image controller state is acceptable before checking the skew. This check is only done in Automatic mode.
1 parent a16d81c commit 2277bea

2 files changed

Lines changed: 151 additions & 0 deletions

File tree

‎pkg/operator/status.go‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -615,6 +615,31 @@ func taskFailed(task string) string {
615615
return task + "Failed"
616616
}
617617

618+
// checkBootImageControllerReady checks if the boot image controller is ready to have its status used for skew enforcement.
619+
// Returns (statusReady, controllerError).
620+
// If statusReady is false and controllerError is not nil, the controller is degraded and upgrades should be blocked.
621+
// If statusReady is false and controllerError is nil, the controller is still initializing/progressing.
622+
func checkBootImageControllerReady(mcop *opv1.MachineConfiguration) (bool, error) {
623+
// If boot image controller is degraded, return Upgradable=false and an error
624+
if meta.IsStatusConditionTrue(mcop.Status.Conditions, opv1.MachineConfigurationBootImageUpdateDegraded) {
625+
degradedCondition := meta.FindStatusCondition(mcop.Status.Conditions, opv1.MachineConfigurationBootImageUpdateDegraded)
626+
return false, fmt.Errorf("Boot image controller is degraded: %s", degradedCondition.Message)
627+
}
628+
629+
// If boot image controller is still progressing or hasn't completed its first pass, return true for statusReady
630+
progressingCondition := meta.FindStatusCondition(mcop.Status.Conditions, opv1.MachineConfigurationBootImageUpdateProgressing)
631+
if progressingCondition == nil {
632+
klog.V(4).Infof("Boot image controller hasn't completed its first pass, skipping boot image skew enforcement check")
633+
return false, nil
634+
}
635+
if progressingCondition.Status == metav1.ConditionTrue {
636+
klog.V(4).Infof("Boot image controller is still progressing, skipping boot image skew enforcement check")
637+
return false, nil
638+
}
639+
640+
return true, nil
641+
}
642+
618643
// checkBootImageSkewUpgradeableGuard checks if the boot image version is within acceptable limits.
619644
// It returns an error if there is no skew enforcement opinion specified. If one is specified,
620645
// it checks if boot image skew is within the expected limit.
@@ -640,6 +665,15 @@ func (optr *Operator) checkBootImageSkewUpgradeableGuard() (bool, string, error)
640665

641666
switch mcop.Status.BootImageSkewEnforcementStatus.Mode {
642667
case opv1.BootImageSkewEnforcementModeStatusAutomatic:
668+
// Check if boot image controller is ready before using its status
669+
statusReady, controllerError := checkBootImageControllerReady(mcop)
670+
if !statusReady {
671+
if controllerError != nil {
672+
return true, fmt.Sprintf("Upgrades have been disabled due to boot image update failures: %s", controllerError.Error()), nil
673+
}
674+
return false, "", nil
675+
}
676+
643677
skewLimitExceeded, skewLimitExceededMessage = checkBootImageSkew(
644678
mcop.Status.BootImageSkewEnforcementStatus.Automatic.OCPVersion,
645679
mcop.Status.BootImageSkewEnforcementStatus.Automatic.RHCOSVersion,

‎pkg/operator/status_test.go‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -855,13 +855,88 @@ func TestCheckBootImageSkewUpgradeableGuard(t *testing.T) {
855855
expectMessage: "",
856856
expectError: false,
857857
},
858+
// Boot image controller readiness tests
859+
{
860+
name: "boot image controller is degraded in Automatic mode",
861+
featureGateEnabled: true,
862+
mcop: &opv1.MachineConfiguration{
863+
ObjectMeta: metav1.ObjectMeta{Name: ctrlcommon.MCOOperatorKnobsObjectName},
864+
Status: opv1.MachineConfigurationStatus{
865+
Conditions: []metav1.Condition{
866+
{
867+
Type: opv1.MachineConfigurationBootImageUpdateDegraded,
868+
Status: metav1.ConditionTrue,
869+
Message: "failed to update boot images",
870+
},
871+
},
872+
BootImageSkewEnforcementStatus: opv1.BootImageSkewEnforcementStatus{
873+
Mode: opv1.BootImageSkewEnforcementModeStatusAutomatic,
874+
Automatic: opv1.ClusterBootImageAutomatic{
875+
OCPVersion: "4.12.0", // Out of skew, but degraded check should happen first
876+
},
877+
},
878+
},
879+
},
880+
expectUpgradeBlock: true,
881+
expectMessage: "Upgrades have been disabled due to boot image update failures: Boot image controller is degraded: failed to update boot images",
882+
expectError: false,
883+
},
884+
{
885+
name: "boot image controller is progressing in Automatic mode",
886+
featureGateEnabled: true,
887+
mcop: &opv1.MachineConfiguration{
888+
ObjectMeta: metav1.ObjectMeta{Name: ctrlcommon.MCOOperatorKnobsObjectName},
889+
Status: opv1.MachineConfigurationStatus{
890+
Conditions: []metav1.Condition{
891+
{
892+
Type: opv1.MachineConfigurationBootImageUpdateProgressing,
893+
Status: metav1.ConditionTrue,
894+
},
895+
},
896+
BootImageSkewEnforcementStatus: opv1.BootImageSkewEnforcementStatus{
897+
Mode: opv1.BootImageSkewEnforcementModeStatusAutomatic,
898+
Automatic: opv1.ClusterBootImageAutomatic{
899+
OCPVersion: "4.12.0", // Out of skew, but progressing check should skip skew evaluation
900+
},
901+
},
902+
},
903+
},
904+
expectUpgradeBlock: false,
905+
expectMessage: "",
906+
expectError: false,
907+
},
908+
{
909+
name: "boot image controller hasn't completed first pass in Automatic mode",
910+
featureGateEnabled: true,
911+
mcop: &opv1.MachineConfiguration{
912+
ObjectMeta: metav1.ObjectMeta{Name: ctrlcommon.MCOOperatorKnobsObjectName},
913+
Status: opv1.MachineConfigurationStatus{
914+
Conditions: []metav1.Condition{},
915+
BootImageSkewEnforcementStatus: opv1.BootImageSkewEnforcementStatus{
916+
Mode: opv1.BootImageSkewEnforcementModeStatusAutomatic,
917+
Automatic: opv1.ClusterBootImageAutomatic{
918+
OCPVersion: "4.12.0", // Out of skew, but readiness check should skip skew evaluation
919+
},
920+
},
921+
},
922+
},
923+
expectUpgradeBlock: false,
924+
expectMessage: "",
925+
expectError: false,
926+
},
858927
// OCP version tests in automatic mode
859928
{
860929
name: "mode is Automatic with OCP version within limit",
861930
featureGateEnabled: true,
862931
mcop: &opv1.MachineConfiguration{
863932
ObjectMeta: metav1.ObjectMeta{Name: ctrlcommon.MCOOperatorKnobsObjectName},
864933
Status: opv1.MachineConfigurationStatus{
934+
Conditions: []metav1.Condition{
935+
{
936+
Type: opv1.MachineConfigurationBootImageUpdateProgressing,
937+
Status: metav1.ConditionFalse,
938+
},
939+
},
865940
BootImageSkewEnforcementStatus: opv1.BootImageSkewEnforcementStatus{
866941
Mode: opv1.BootImageSkewEnforcementModeStatusAutomatic,
867942
Automatic: opv1.ClusterBootImageAutomatic{
@@ -880,6 +955,12 @@ func TestCheckBootImageSkewUpgradeableGuard(t *testing.T) {
880955
mcop: &opv1.MachineConfiguration{
881956
ObjectMeta: metav1.ObjectMeta{Name: ctrlcommon.MCOOperatorKnobsObjectName},
882957
Status: opv1.MachineConfigurationStatus{
958+
Conditions: []metav1.Condition{
959+
{
960+
Type: opv1.MachineConfigurationBootImageUpdateProgressing,
961+
Status: metav1.ConditionFalse,
962+
},
963+
},
883964
BootImageSkewEnforcementStatus: opv1.BootImageSkewEnforcementStatus{
884965
Mode: opv1.BootImageSkewEnforcementModeStatusAutomatic,
885966
Automatic: opv1.ClusterBootImageAutomatic{
@@ -937,6 +1018,12 @@ func TestCheckBootImageSkewUpgradeableGuard(t *testing.T) {
9371018
mcop: &opv1.MachineConfiguration{
9381019
ObjectMeta: metav1.ObjectMeta{Name: ctrlcommon.MCOOperatorKnobsObjectName},
9391020
Status: opv1.MachineConfigurationStatus{
1021+
Conditions: []metav1.Condition{
1022+
{
1023+
Type: opv1.MachineConfigurationBootImageUpdateProgressing,
1024+
Status: metav1.ConditionFalse,
1025+
},
1026+
},
9401027
BootImageSkewEnforcementStatus: opv1.BootImageSkewEnforcementStatus{
9411028
Mode: opv1.BootImageSkewEnforcementModeStatusAutomatic,
9421029
Automatic: opv1.ClusterBootImageAutomatic{
@@ -956,6 +1043,12 @@ func TestCheckBootImageSkewUpgradeableGuard(t *testing.T) {
9561043
mcop: &opv1.MachineConfiguration{
9571044
ObjectMeta: metav1.ObjectMeta{Name: ctrlcommon.MCOOperatorKnobsObjectName},
9581045
Status: opv1.MachineConfigurationStatus{
1046+
Conditions: []metav1.Condition{
1047+
{
1048+
Type: opv1.MachineConfigurationBootImageUpdateProgressing,
1049+
Status: metav1.ConditionFalse,
1050+
},
1051+
},
9591052
BootImageSkewEnforcementStatus: opv1.BootImageSkewEnforcementStatus{
9601053
Mode: opv1.BootImageSkewEnforcementModeStatusAutomatic,
9611054
Automatic: opv1.ClusterBootImageAutomatic{
@@ -974,6 +1067,12 @@ func TestCheckBootImageSkewUpgradeableGuard(t *testing.T) {
9741067
mcop: &opv1.MachineConfiguration{
9751068
ObjectMeta: metav1.ObjectMeta{Name: ctrlcommon.MCOOperatorKnobsObjectName},
9761069
Status: opv1.MachineConfigurationStatus{
1070+
Conditions: []metav1.Condition{
1071+
{
1072+
Type: opv1.MachineConfigurationBootImageUpdateProgressing,
1073+
Status: metav1.ConditionFalse,
1074+
},
1075+
},
9771076
BootImageSkewEnforcementStatus: opv1.BootImageSkewEnforcementStatus{
9781077
Mode: opv1.BootImageSkewEnforcementModeStatusAutomatic,
9791078
Automatic: opv1.ClusterBootImageAutomatic{
@@ -992,6 +1091,12 @@ func TestCheckBootImageSkewUpgradeableGuard(t *testing.T) {
9921091
mcop: &opv1.MachineConfiguration{
9931092
ObjectMeta: metav1.ObjectMeta{Name: ctrlcommon.MCOOperatorKnobsObjectName},
9941093
Status: opv1.MachineConfigurationStatus{
1094+
Conditions: []metav1.Condition{
1095+
{
1096+
Type: opv1.MachineConfigurationBootImageUpdateProgressing,
1097+
Status: metav1.ConditionFalse,
1098+
},
1099+
},
9951100
BootImageSkewEnforcementStatus: opv1.BootImageSkewEnforcementStatus{
9961101
Mode: opv1.BootImageSkewEnforcementModeStatusAutomatic,
9971102
Automatic: opv1.ClusterBootImageAutomatic{
@@ -1010,6 +1115,12 @@ func TestCheckBootImageSkewUpgradeableGuard(t *testing.T) {
10101115
mcop: &opv1.MachineConfiguration{
10111116
ObjectMeta: metav1.ObjectMeta{Name: ctrlcommon.MCOOperatorKnobsObjectName},
10121117
Status: opv1.MachineConfigurationStatus{
1118+
Conditions: []metav1.Condition{
1119+
{
1120+
Type: opv1.MachineConfigurationBootImageUpdateProgressing,
1121+
Status: metav1.ConditionFalse,
1122+
},
1123+
},
10131124
BootImageSkewEnforcementStatus: opv1.BootImageSkewEnforcementStatus{
10141125
Mode: opv1.BootImageSkewEnforcementModeStatusAutomatic,
10151126
Automatic: opv1.ClusterBootImageAutomatic{
@@ -1029,6 +1140,12 @@ func TestCheckBootImageSkewUpgradeableGuard(t *testing.T) {
10291140
mcop: &opv1.MachineConfiguration{
10301141
ObjectMeta: metav1.ObjectMeta{Name: ctrlcommon.MCOOperatorKnobsObjectName},
10311142
Status: opv1.MachineConfigurationStatus{
1143+
Conditions: []metav1.Condition{
1144+
{
1145+
Type: opv1.MachineConfigurationBootImageUpdateProgressing,
1146+
Status: metav1.ConditionFalse,
1147+
},
1148+
},
10321149
BootImageSkewEnforcementStatus: opv1.BootImageSkewEnforcementStatus{
10331150
Mode: opv1.BootImageSkewEnforcementModeStatusAutomatic,
10341151
Automatic: opv1.ClusterBootImageAutomatic{

0 commit comments

Comments
 (0)