Skip to content

Commit 2fc2f3c

Browse files
QiWang19QiWang19
committed
test api
1 parent 00b8722 commit 2fc2f3c

2 files changed

Lines changed: 203 additions & 0 deletions

File tree

‎pkg/controller/container-runtime-config/container_runtime_config_controller.go‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package containerruntimeconfig
33
import (
44
"context"
55
"fmt"
6+
"path/filepath"
67
"reflect"
78
"strconv"
89
"strings"
@@ -18,6 +19,7 @@ import (
1819
configinformers "github.com/openshift/client-go/config/informers/externalversions"
1920
cligoinformersv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1"
2021
cligolistersv1 "github.com/openshift/client-go/config/listers/config/v1"
22+
cligolistersv1alpha1 "github.com/openshift/client-go/config/listers/config/v1alpha1"
2123
runtimeutils "github.com/openshift/runtime-utils/pkg/registries"
2224

2325
operatorinformersv1alpha1 "github.com/openshift/client-go/operator/informers/externalversions/operator/v1alpha1"
@@ -87,6 +89,7 @@ type Controller struct {
8789

8890
syncHandler func(mcp string) error
8991
syncImgHandler func(mcp string) error
92+
syncCRIOCPHandler func(key string) error
9093
enqueueContainerRuntimeConfig func(*mcfgv1.ContainerRuntimeConfig)
9194

9295
ccLister mcfglistersv1.ControllerConfigLister
@@ -107,6 +110,10 @@ type Controller struct {
107110
itmsLister cligolistersv1.ImageTagMirrorSetLister
108111
itmsListerSynced cache.InformerSynced
109112

113+
criocpLister cligolistersv1alpha1.CRIOCredentialProviderConfigLister
114+
criocpListerSynced cache.InformerSynced
115+
addedCRIOCPObservers bool
116+
110117
configInformerFactory configinformers.SharedInformerFactory
111118
clusterImagePolicyLister cligolistersv1.ClusterImagePolicyLister
112119
clusterImagePolicyListerSynced cache.InformerSynced
@@ -125,6 +132,7 @@ type Controller struct {
125132

126133
queue workqueue.TypedRateLimitingInterface[string]
127134
imgQueue workqueue.TypedRateLimitingInterface[string]
135+
// criocpQueue workqueue.TypedRateLimitingInterface[string]
128136
}
129137

130138
// New returns a new container runtime config controller
@@ -192,6 +200,7 @@ func New(
192200

193201
ctrl.syncHandler = ctrl.syncContainerRuntimeConfig
194202
ctrl.syncImgHandler = ctrl.syncImageConfig
203+
// ctrl.syncCRIOCPHandler = ctrl.syncCRIOCredentialProviderConfig
195204
ctrl.enqueueContainerRuntimeConfig = ctrl.enqueue
196205

197206
ctrl.mcpLister = mcpInformer.Lister()
@@ -231,6 +240,7 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) {
231240
defer utilruntime.HandleCrash()
232241
defer ctrl.queue.ShutDown()
233242
defer ctrl.imgQueue.ShutDown()
243+
// defer ctrl.criocpQueue.ShutDown()
234244
listerCaches := []cache.InformerSynced{ctrl.mcpListerSynced, ctrl.mccrListerSynced, ctrl.ccListerSynced,
235245
ctrl.imgListerSynced, ctrl.icspListerSynced, ctrl.idmsListerSynced, ctrl.itmsListerSynced, ctrl.clusterVersionListerSynced}
236246

@@ -242,6 +252,14 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) {
242252
ctrl.addedPolicyObservers = true
243253
}
244254

255+
if ctrl.criocpEnabled() {
256+
ctrl.addCRIOCPObservers()
257+
klog.Info("added CRIOCredentialProviderConfig observers with CRIOCredentialProviderConfig featuregate enabled")
258+
ctrl.configInformerFactory.Start(stopCh)
259+
listerCaches = append(listerCaches, ctrl.criocpListerSynced)
260+
ctrl.addedCRIOCPObservers = true
261+
}
262+
245263
if !cache.WaitForCacheSync(stopCh, listerCaches...) {
246264
return
247265
}
@@ -256,6 +274,9 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) {
256274
// Just need one worker for the image config
257275
go wait.Until(ctrl.imgWorker, time.Second, stopCh)
258276

277+
// Just need one worker for the CRIOCredentialProviderConfig
278+
// go wait.Until(ctrl.criocpWorker, time.Second, stopCh)
279+
259280
<-stopCh
260281
}
261282

@@ -317,6 +338,31 @@ func (ctrl *Controller) itmsConfDeleted(_ interface{}) {
317338
ctrl.imgQueue.Add("openshift-config")
318339
}
319340

341+
func (ctrl *Controller) addCRIOCPObservers() {
342+
ctrl.configInformerFactory.Config().V1alpha1().CRIOCredentialProviderConfigs().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
343+
AddFunc: ctrl.criocpConfAdded,
344+
UpdateFunc: ctrl.criocpConfUpdated,
345+
DeleteFunc: ctrl.criocpConfDeleted,
346+
})
347+
ctrl.criocpLister = ctrl.configInformerFactory.Config().V1alpha1().CRIOCredentialProviderConfigs().Lister()
348+
ctrl.criocpListerSynced = ctrl.configInformerFactory.Config().V1alpha1().CRIOCredentialProviderConfigs().Informer().HasSynced
349+
}
350+
351+
func (ctrl *Controller) criocpConfAdded(_ interface{}) {
352+
ctrl.imgQueue.Add("openshift-config")
353+
// ctrl.criocpQueue.Add("openshift-config")
354+
}
355+
356+
func (ctrl *Controller) criocpConfUpdated(_, _ interface{}) {
357+
ctrl.imgQueue.Add("openshift-config")
358+
// ctrl.criocpQueue.Add("openshift-config")
359+
}
360+
361+
func (ctrl *Controller) criocpConfDeleted(_ interface{}) {
362+
ctrl.imgQueue.Add("openshift-config")
363+
// ctrl.criocpQueue.Add("openshift-config")
364+
}
365+
320366
func (ctrl *Controller) addImagePolicyObservers() {
321367
ctrl.configInformerFactory.Config().V1().ClusterImagePolicies().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
322368
AddFunc: ctrl.clusterImagePolicyAdded,
@@ -363,6 +409,10 @@ func (ctrl *Controller) sigstoreAPIEnabled() bool {
363409
return ctrl.fgHandler.Enabled(features.FeatureGateSigstoreImageVerification)
364410
}
365411

412+
func (ctrl *Controller) criocpEnabled() bool {
413+
return ctrl.fgHandler.Enabled(features.FeatureGateCRIOCredentialProviderConfig)
414+
}
415+
366416
func (ctrl *Controller) updateContainerRuntimeConfig(oldObj, newObj interface{}) {
367417
oldCtrCfg := oldObj.(*mcfgv1.ContainerRuntimeConfig)
368418
newCtrCfg := newObj.(*mcfgv1.ContainerRuntimeConfig)
@@ -442,6 +492,11 @@ func (ctrl *Controller) imgWorker() {
442492
}
443493
}
444494

495+
// func (ctrl *Controller) criocpWorker() {
496+
// for ctrl.processNextCRIOCPWorkItem() {
497+
// }
498+
// }
499+
445500
func (ctrl *Controller) processNextWorkItem() bool {
446501
key, quit := ctrl.queue.Get()
447502
if quit {
@@ -468,6 +523,19 @@ func (ctrl *Controller) processNextImgWorkItem() bool {
468523
return true
469524
}
470525

526+
// func (ctrl *Controller) processNextCRIOCPWorkItem() bool {
527+
// key, quit := ctrl.criocpQueue.Get()
528+
// if quit {
529+
// return false
530+
// }
531+
// defer ctrl.criocpQueue.Done(key)
532+
533+
// err := ctrl.syncCRIOCPHandler(key)
534+
// ctrl.handleCRIOCPErr(err, key)
535+
536+
// return true
537+
// }
538+
471539
func (ctrl *Controller) handleErr(err error, key string) {
472540
if err == nil {
473541
ctrl.queue.Forget(key)
@@ -504,6 +572,24 @@ func (ctrl *Controller) handleImgErr(err error, key string) {
504572
ctrl.imgQueue.AddAfter(key, 1*time.Minute)
505573
}
506574

575+
// func (ctrl *Controller) handleCRIOCPErr(err error, key string) {
576+
// if err == nil {
577+
// ctrl.criocpQueue.Forget(key)
578+
// return
579+
// }
580+
581+
// if ctrl.criocpQueue.NumRequeues(key) < maxRetries {
582+
// klog.V(2).Infof("Error syncing CRIOCredentialProviderConfig %v: %v", key, err)
583+
// ctrl.criocpQueue.AddRateLimited(key)
584+
// return
585+
// }
586+
587+
// utilruntime.HandleError(err)
588+
// klog.V(2).Infof("Dropping CRIOCredentialProviderConfig %q out of the queue: %v", key, err)
589+
// ctrl.criocpQueue.Forget(key)
590+
// ctrl.criocpQueue.AddAfter(key, 1*time.Minute)
591+
// }
592+
507593
// generateOriginalContainerRuntimeConfigs returns rendered default storage, registries and policy config files
508594
func generateOriginalContainerRuntimeConfigs(templateDir string, cc *mcfgv1.ControllerConfig, role string) (*ign3types.File, *ign3types.File, *ign3types.File, error) {
509595
// Render the default templates
@@ -551,6 +637,53 @@ func generateOriginalContainerRuntimeConfigs(templateDir string, cc *mcfgv1.Cont
551637
return gmcStorageConfig, gmcRegistriesConfig, gmcPolicyJSON, nil
552638
}
553639

640+
func generateOriginalCredentialProviderConfig(templateDir string, cc *mcfgv1.ControllerConfig, role string) (*ign3types.File, error) {
641+
642+
// Render the default templates
643+
rc := &mtmpl.RenderConfig{
644+
ControllerConfigSpec: &cc.Spec,
645+
}
646+
generatedConfigs, err := mtmpl.GenerateMachineConfigsForRole(rc, role, templateDir)
647+
if err != nil {
648+
return nil, fmt.Errorf("generateMachineConfigsforRole failed with error %w", err)
649+
}
650+
// Find generated provider.yaml
651+
var (
652+
config, gmcCredProviderConfig *ign3types.File
653+
errCredProvider error
654+
credProviderConfigPath string
655+
)
656+
657+
// Determine credential provider config path based on platform
658+
// staying consistent with path used in pkg/controller/template/render.go
659+
credProviderConfigPathFormat := filepath.FromSlash("/etc/kubernetes/credential-providers/%s-credential-provider.yaml")
660+
switch cc.Spec.Infra.Status.PlatformStatus.Type {
661+
case apicfgv1.AWSPlatformType:
662+
credProviderConfigPath = fmt.Sprintf(credProviderConfigPathFormat, "ecr")
663+
case apicfgv1.GCPPlatformType:
664+
credProviderConfigPath = fmt.Sprintf(credProviderConfigPathFormat, "gcr")
665+
case apicfgv1.AzurePlatformType:
666+
credProviderConfigPath = fmt.Sprintf(credProviderConfigPathFormat, "acr")
667+
default:
668+
return nil, fmt.Errorf("unsupported platform type: %s", cc.Spec.Infra.Status.PlatformStatus.Type)
669+
}
670+
klog.Infof("credential provider config path set to: %s", credProviderConfigPath)
671+
672+
// Find credential provider config
673+
for _, gmc := range generatedConfigs {
674+
config, errCredProvider = findCredProviderConfig(gmc, credProviderConfigPath)
675+
if errCredProvider != nil {
676+
klog.Infof("could not find credential provider config in generated config %s: %v", gmc.Name, errCredProvider)
677+
return nil, fmt.Errorf("could not generate original credential provider configs: %w", errCredProvider)
678+
}
679+
680+
gmcCredProviderConfig = config
681+
682+
}
683+
684+
return gmcCredProviderConfig, nil
685+
}
686+
554687
func (ctrl *Controller) syncStatusOnly(cfg *mcfgv1.ContainerRuntimeConfig, err error, args ...interface{}) error {
555688
statusUpdateErr := retry.RetryOnConflict(updateBackoff, func() error {
556689
newcfg, getErr := ctrl.mccrLister.Get(cfg.Name)
@@ -1016,6 +1149,9 @@ func (ctrl *Controller) syncImageConfig(key string) error {
10161149
if err != nil {
10171150
return fmt.Errorf("could not sync registries Ignition config: %w", err)
10181151
}
1152+
1153+
crioCredentialProviderConfigIgnition(ctrl.templatesDir, controllerConfig, role, releaseImage)
1154+
10191155
return err
10201156
}); err != nil {
10211157
return fmt.Errorf("could not Create/Update MachineConfig: %w", err)
@@ -1028,6 +1164,45 @@ func (ctrl *Controller) syncImageConfig(key string) error {
10281164
return nil
10291165
}
10301166

1167+
// func (ctrl *Controller) syncCRIOCredentialProviderConfig(key string) error {
1168+
// startTime := time.Now()
1169+
// klog.V(4).Infof("Started syncing CRIOCredentialProvider config %q (%v)", key, startTime)
1170+
// defer func() {
1171+
// klog.V(4).Infof("Finished syncing CRIOCredentialProvider config %q (%v)", key, time.Since(startTime))
1172+
// }()
1173+
1174+
// // Get ControllerConfig
1175+
// controllerConfig, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName)
1176+
// if err != nil {
1177+
// return fmt.Errorf("could not get ControllerConfig %w", err)
1178+
// }
1179+
1180+
// sel, err := metav1.LabelSelectorAsSelector(metav1.AddLabelToSelector(&metav1.LabelSelector{}, builtInLabelKey, ""))
1181+
// if err != nil {
1182+
// return err
1183+
// }
1184+
// // Find all the MCO built in MachineConfigPools
1185+
// mcpPools, err := ctrl.mcpLister.List(sel)
1186+
// if err != nil {
1187+
// return err
1188+
// }
1189+
1190+
// for _, pool := range mcpPools {
1191+
// role := pool.Name
1192+
// credProviderConfigIgn, err := generateOriginalCredentialProviderConfig(ctrl.templatesDir, controllerConfig, role)
1193+
// if err != nil {
1194+
// return fmt.Errorf("could not generate original CRIO credential provider config for role %s: %w", role, err)
1195+
// }
1196+
// contents, err := ctrlcommon.DecodeIgnitionFileContents(credProviderConfigIgn.Contents.Source, credProviderConfigIgn.Contents.Compression)
1197+
// if err != nil {
1198+
// return fmt.Errorf("could not decode CRIO credential provider config for role %s: %w", role, err)
1199+
// }
1200+
// klog.Infof("Decoded CRIO credential provider config contents successfully for role %s: %s", role, string(contents))
1201+
// }
1202+
1203+
// return nil
1204+
// }
1205+
10311206
func (ctrl *Controller) syncIgnitionConfig(managedKey string, ignFile *ign3types.Config, pool *mcfgv1.MachineConfigPool, ownerRef metav1.OwnerReference) (bool, error) {
10321207
rawIgn, err := json.Marshal(ignFile)
10331208
if err != nil {
@@ -1372,3 +1547,16 @@ func (ctrl *Controller) getPoolsForContainerRuntimeConfig(config *mcfgv1.Contain
13721547

13731548
return pools, nil
13741549
}
1550+
1551+
func crioCredentialProviderConfigIgnition(templateDir string, controllerConfig *mcfgv1.ControllerConfig, role, releaseImage string) error {
1552+
credProviderConfigIgn, err := generateOriginalCredentialProviderConfig(templateDir, controllerConfig, role)
1553+
if err != nil {
1554+
return fmt.Errorf("could not generate original CRIO credential provider config for role %s: %w", role, err)
1555+
}
1556+
contents, err := ctrlcommon.DecodeIgnitionFileContents(credProviderConfigIgn.Contents.Source, credProviderConfigIgn.Contents.Compression)
1557+
if err != nil {
1558+
return fmt.Errorf("could not decode CRIO credential provider config for role %s: %w", role, err)
1559+
}
1560+
klog.Infof("Decoded CRIO credential provider config contents successfully for role %s: %s", role, string(contents))
1561+
return nil
1562+
}

‎pkg/controller/container-runtime-config/helpers.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,21 @@ func findPolicyJSON(mc *mcfgv1.MachineConfig) (*ign3types.File, error) {
200200
return nil, fmt.Errorf("could not find Policy JSON")
201201
}
202202

203+
func findCredProviderConfig(mc *mcfgv1.MachineConfig, credProviderConfigPath string) (*ign3types.File, error) {
204+
ignCfg, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw)
205+
if err != nil {
206+
return nil, fmt.Errorf("parsing Credential Provider Ignition config failed with error: %w", err)
207+
}
208+
for _, c := range ignCfg.Storage.Files {
209+
klog.Infof("Checking file path : %s", c.Path)
210+
if c.Path == credProviderConfigPath {
211+
c := c
212+
return &c, nil
213+
}
214+
}
215+
return nil, fmt.Errorf("could not find Credential Provider Config")
216+
}
217+
203218
// Deprecated: use getManagedKeyCtrCfg
204219
func getManagedKeyCtrCfgDeprecated(pool *mcfgv1.MachineConfigPool) string {
205220
return fmt.Sprintf("99-%s-%s-containerruntime", pool.Name, pool.ObjectMeta.UID)

0 commit comments

Comments
 (0)