Skip to content

Commit 36dcbbc

Browse files
Merge pull request #5519 from pablintino/mco-2033
MCO-2033: Refactor bootstrap dependency handling
2 parents db346c5 + e22760f commit 36dcbbc

3 files changed

Lines changed: 353 additions & 186 deletions

File tree

cmd/machine-config-operator/bootstrap.go

Lines changed: 14 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -24,42 +24,31 @@ var (
2424
}
2525

2626
bootstrapOpts struct {
27+
destinationDir string
2728
baremetalRuntimeCfgImage string
28-
cloudConfigFile string
29-
configFile string
30-
cloudProviderCAFile string
3129
corednsImage string
32-
destinationDir string
3330
haproxyImage string
34-
imagesConfigMapFile string
35-
infraConfigFile string
3631
infraImage string
3732
releaseImage string
3833
keepalivedImage string
39-
kubeCAFile string
4034
mcoImage string
4135
oauthProxyImage string
4236
kubeRbacProxyImage string
4337
dockerRegistryImage string
44-
networkConfigFile string
4538
oscontentImage string
46-
pullSecretFile string
47-
mcsCAFile string
48-
proxyConfigFile string
49-
additionalTrustBundleFile string
50-
dnsConfigFile string
5139
imageReferences string
5240
baseOSContainerImage string
5341
baseOSExtensionsContainerImage string
42+
dependencyFiles operator.BootstrapDependenciesFiles
5443
}
5544
)
5645

5746
func init() {
5847
rootCmd.AddCommand(bootstrapCmd)
5948
// See https://docs.openshift.com/container-platform/4.13/security/certificate_types_descriptions/machine-config-operator-certificates.html
60-
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.mcsCAFile, "root-ca", "/etc/ssl/kubernetes/ca.crt", "Path to installer-generated root MCS CA")
61-
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.kubeCAFile, "kube-ca", "", "path to kube-apiserver serving-ca bundle")
62-
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.pullSecretFile, "pull-secret", "/assets/manifests/pull.json", "path to secret manifest that contains pull secret.")
49+
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.MCSCAFile, "root-ca", "/etc/ssl/kubernetes/ca.crt", "Path to installer-generated root MCS CA")
50+
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.KubeAPIServerServingCA, "kube-ca", "", "path to kube-apiserver serving-ca bundle")
51+
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.PullSecret, "pull-secret", "/assets/manifests/pull.json", "path to secret manifest that contains pull secret.")
6352
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.destinationDir, "dest-dir", "", "The destination directory where MCO writes the manifests.")
6453
bootstrapCmd.MarkFlagRequired("dest-dir")
6554
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.mcoImage, "machine-config-operator-image", "", "Image for Machine Config Operator.")
@@ -70,14 +59,14 @@ func init() {
7059
bootstrapCmd.MarkFlagRequired("infra-image")
7160
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.releaseImage, "release-image", "", "Release image used for cluster installation.")
7261
bootstrapCmd.MarkFlagRequired("release-image")
73-
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.configFile, "config-file", "", "ClusterConfig ConfigMap file.")
62+
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.ClusterConfig, "config-file", "", "ClusterConfig ConfigMap file.")
7463
bootstrapCmd.MarkFlagRequired("config-file")
75-
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.infraConfigFile, "infra-config-file", "/assets/manifests/cluster-infrastructure-02-config.yml", "File containing infrastructure.config.openshift.io manifest.")
76-
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.networkConfigFile, "network-config-file", "/assets/manifests/cluster-network-02-config.yml", "File containing network.config.openshift.io manifest.")
77-
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.cloudConfigFile, "cloud-config-file", "", "File containing the config map that contains the cloud config for cloudprovider.")
78-
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.proxyConfigFile, "proxy-config-file", "/assets/manifests/cluster-proxy-01-config.yaml", "File containing proxy.config.openshift.io manifest.")
79-
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dnsConfigFile, "dns-config-file", "/assets/manifests/cluster-dns-02-config.yml", "File containing dns.config.openshift.io manifest.")
80-
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.additionalTrustBundleFile, "additional-trust-bundle-config-file", "/assets/manifests/user-ca-bundle-config.yaml", "File containing the additional user provided CA bundle manifest.")
64+
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.Infrastructure, "infra-config-file", "/assets/manifests/cluster-infrastructure-02-config.yml", "File containing infrastructure.config.openshift.io manifest.")
65+
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.Network, "network-config-file", "/assets/manifests/cluster-network-02-config.yml", "File containing network.config.openshift.io manifest.")
66+
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.CloudConfig, "cloud-config-file", "", "File containing the config map that contains the cloud config for cloudprovider.")
67+
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.Proxy, "proxy-config-file", "/assets/manifests/cluster-proxy-01-config.yaml", "File containing proxy.config.openshift.io manifest.")
68+
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.DNS, "dns-config-file", "/assets/manifests/cluster-dns-02-config.yml", "File containing dns.config.openshift.io manifest.")
69+
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.AdditionalTrustBundle, "additional-trust-bundle-config-file", "/assets/manifests/user-ca-bundle-config.yaml", "File containing the additional user provided CA bundle manifest.")
8170
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.keepalivedImage, "keepalived-image", "", "Image for Keepalived.")
8271
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.corednsImage, "coredns-image", "", "Image for CoreDNS.")
8372
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.haproxyImage, "haproxy-image", "", "Image for haproxy.")
@@ -88,7 +77,7 @@ func init() {
8877
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.kubeRbacProxyImage, "kube-rbac-proxy-image", "", "Image for origin kube-rbac proxy.")
8978
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dockerRegistryImage, "docker-registry-image", "", "Image for docker-registry.")
9079
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.imageReferences, "image-references", "", "File containing imagestreams (from cluster-version-operator)")
91-
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.cloudProviderCAFile, "cloud-provider-ca-file", "", "path to cloud provider CA certificate")
80+
bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.CloudProviderCA, "cloud-provider-ca-file", "", "path to cloud provider CA certificate")
9281

9382
}
9483

@@ -178,15 +167,7 @@ func runBootstrapCmd(_ *cobra.Command, _ []string) {
178167
}
179168

180169
if err := operator.RenderBootstrap(
181-
bootstrapOpts.additionalTrustBundleFile,
182-
bootstrapOpts.proxyConfigFile,
183-
bootstrapOpts.configFile,
184-
bootstrapOpts.infraConfigFile,
185-
bootstrapOpts.networkConfigFile,
186-
bootstrapOpts.dnsConfigFile,
187-
bootstrapOpts.cloudConfigFile,
188-
bootstrapOpts.cloudProviderCAFile,
189-
bootstrapOpts.mcsCAFile, bootstrapOpts.kubeCAFile, bootstrapOpts.pullSecretFile,
170+
bootstrapOpts.dependencyFiles,
190171
&imgs,
191172
bootstrapOpts.destinationDir,
192173
bootstrapOpts.releaseImage,

pkg/operator/bootstrap.go

Lines changed: 61 additions & 153 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,6 @@ import (
99
"k8s.io/klog/v2"
1010

1111
configv1 "github.com/openshift/api/config/v1"
12-
configscheme "github.com/openshift/client-go/config/clientset/versioned/scheme"
13-
corev1 "k8s.io/api/core/v1"
14-
"k8s.io/apimachinery/pkg/runtime"
15-
"k8s.io/client-go/kubernetes/scheme"
16-
1712
ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
1813
templatectrl "github.com/openshift/machine-config-operator/pkg/controller/template"
1914
)
@@ -26,137 +21,20 @@ type manifest struct {
2621

2722
// RenderBootstrap writes to destinationDir static Pods.
2823
func RenderBootstrap(
29-
additionalTrustBundleFile,
30-
proxyFile,
31-
clusterConfigConfigMapFile,
32-
infraFile, networkFile, dnsFile,
33-
cloudConfigFile, cloudProviderCAFile,
34-
mcsCAFile, kubeAPIServerServingCA, pullSecretFile string,
24+
dependenciesFiles BootstrapDependenciesFiles,
3525
imgs *ctrlcommon.Images,
3626
destinationDir, releaseImage string,
3727
) error {
38-
filesData := map[string][]byte{}
39-
files := []string{
40-
proxyFile,
41-
clusterConfigConfigMapFile,
42-
infraFile,
43-
networkFile,
44-
mcsCAFile,
45-
pullSecretFile,
46-
dnsFile,
47-
}
48-
if kubeAPIServerServingCA != "" {
49-
files = append(files, kubeAPIServerServingCA)
50-
}
51-
if cloudProviderCAFile != "" {
52-
files = append(files, cloudProviderCAFile)
53-
}
54-
for _, file := range files {
55-
data, err := os.ReadFile(file)
56-
if err != nil {
57-
return err
58-
}
59-
filesData[file] = data
60-
}
61-
62-
// create ControllerConfigSpec
63-
obji, err := runtime.Decode(configscheme.Codecs.UniversalDecoder(configv1.SchemeGroupVersion), filesData[infraFile])
64-
if err != nil {
65-
return err
66-
}
67-
infra, ok := obji.(*configv1.Infrastructure)
68-
if !ok {
69-
return fmt.Errorf("expected *configv1.Infrastructure found %T", obji)
70-
}
71-
72-
obji, err = runtime.Decode(configscheme.Codecs.UniversalDecoder(configv1.SchemeGroupVersion), filesData[proxyFile])
73-
if err != nil {
74-
return err
75-
}
76-
proxy, ok := obji.(*configv1.Proxy)
77-
if !ok {
78-
return fmt.Errorf("expected *configv1.Proxy found %T", obji)
79-
}
80-
81-
obji, err = runtime.Decode(configscheme.Codecs.UniversalDecoder(configv1.SchemeGroupVersion), filesData[networkFile])
82-
if err != nil {
83-
return err
84-
}
85-
network, ok := obji.(*configv1.Network)
86-
if !ok {
87-
return fmt.Errorf("expected *configv1.Network found %T", obji)
88-
}
89-
90-
obji, err = runtime.Decode(configscheme.Codecs.UniversalDecoder(configv1.SchemeGroupVersion), filesData[dnsFile])
28+
dependencies, err := NewBootstrapDependencies(dependenciesFiles)
9129
if err != nil {
92-
return err
93-
}
94-
dns, ok := obji.(*configv1.DNS)
95-
if !ok {
96-
return fmt.Errorf("expected *configv1.DNS found %T", obji)
30+
return fmt.Errorf("error parsing dependencies for MCO bootstrap: %w", err)
9731
}
9832

99-
spec, err := createDiscoveredControllerConfigSpec(infra, network, proxy, dns)
33+
config, err := buildSpec(dependencies, imgs, releaseImage)
10034
if err != nil {
101-
return err
102-
}
103-
104-
additionalTrustBundleData, err := os.ReadFile(additionalTrustBundleFile)
105-
if err != nil && !os.IsNotExist(err) {
106-
return err
107-
}
108-
if additionalTrustBundleData != nil {
109-
obji, err := runtime.Decode(scheme.Codecs.UniversalDecoder(corev1.SchemeGroupVersion), additionalTrustBundleData)
110-
if err != nil {
111-
return err
112-
}
113-
additionalTrustBundle, ok := obji.(*corev1.ConfigMap)
114-
if !ok {
115-
return fmt.Errorf("expected *corev1.ConfigMap found %T", obji)
116-
}
117-
spec.AdditionalTrustBundle = []byte(additionalTrustBundle.Data["ca-bundle.crt"])
118-
}
119-
120-
// if the cloudConfig is set in infra read the cloudConfigFile
121-
if infra.Spec.CloudConfig.Name != "" {
122-
cloudConf, err := loadBootstrapCloudProviderConfig(infra, cloudConfigFile)
123-
if err != nil {
124-
return fmt.Errorf("failed to load the cloud provider config: %w", err)
125-
}
126-
spec.CloudProviderConfig = cloudConf
127-
}
128-
129-
bundle := make([]byte, 0)
130-
bundle = append(bundle, filesData[mcsCAFile]...)
131-
// Append the kube-ca if given.
132-
if _, ok := filesData[kubeAPIServerServingCA]; ok {
133-
spec.KubeAPIServerServingCAData = filesData[kubeAPIServerServingCA]
134-
}
135-
// Set the cloud-provider CA if given.
136-
if data, ok := filesData[cloudProviderCAFile]; ok {
137-
spec.CloudProviderCAData = data
35+
return fmt.Errorf("error building spec for MCO bootstrap: %w", err)
13836
}
13937

140-
spec.RootCAData = bundle
141-
spec.PullSecret = nil
142-
spec.BaseOSContainerImage = imgs.BaseOSContainerImage
143-
spec.BaseOSExtensionsContainerImage = imgs.BaseOSExtensionsContainerImage
144-
spec.ReleaseImage = releaseImage
145-
spec.Images = map[string]string{
146-
templatectrl.MachineConfigOperatorKey: imgs.MachineConfigOperator,
147-
148-
templatectrl.APIServerWatcherKey: imgs.MachineConfigOperator,
149-
templatectrl.InfraImageKey: imgs.InfraImage,
150-
templatectrl.KeepalivedKey: imgs.Keepalived,
151-
templatectrl.CorednsKey: imgs.Coredns,
152-
templatectrl.HaproxyKey: imgs.Haproxy,
153-
templatectrl.BaremetalRuntimeCfgKey: imgs.BaremetalRuntimeCfg,
154-
templatectrl.KubeRbacProxyKey: imgs.KubeRbacProxy,
155-
templatectrl.DockerRegistryKey: imgs.DockerRegistry,
156-
}
157-
158-
config := getRenderConfig("", string(filesData[kubeAPIServerServingCA]), spec, &imgs.RenderConfigImages, infra, nil, nil, "2")
159-
16038
manifests := []manifest{
16139
{
16240
name: "manifests/machineconfigcontroller/controllerconfig.yaml",
@@ -171,7 +49,7 @@ func RenderBootstrap(
17149
name: "manifests/bootstrap-pod-v2.yaml",
17250
filename: "bootstrap/machineconfigoperator-bootstrap-pod.yaml",
17351
}, {
174-
data: filesData[pullSecretFile],
52+
data: []byte(dependencies.PullSecret),
17553
filename: "bootstrap/manifests/machineconfigcontroller-pull-secret",
17654
}, {
17755
name: "manifests/machineconfigserver/csr-bootstrap-role-binding.yaml",
@@ -182,14 +60,14 @@ func RenderBootstrap(
18260
},
18361
}
18462

185-
if infra.Status.ControlPlaneTopology == configv1.HighlyAvailableArbiterMode {
63+
if dependencies.Infrastructure.Status.ControlPlaneTopology == configv1.HighlyAvailableArbiterMode {
18664
manifests = append(manifests, manifest{
18765
name: "manifests/arbiter.machineconfigpool.yaml",
18866
filename: "bootstrap/manifests/arbiter.machineconfigpool.yaml",
18967
})
19068
}
19169

192-
manifests = appendManifestsByPlatform(manifests, *infra)
70+
manifests = appendManifestsByPlatform(manifests, dependencies.Infrastructure)
19371

19472
for _, m := range manifests {
19573
var b []byte
@@ -222,7 +100,59 @@ func RenderBootstrap(
222100
return nil
223101
}
224102

225-
func appendManifestsByPlatform(manifests []manifest, infra configv1.Infrastructure) []manifest {
103+
func buildSpec(dependencies *BootstrapDependencies, imgs *ctrlcommon.Images, releaseImage string) (*renderConfig, error) {
104+
105+
// create ControllerConfigSpec
106+
spec, err := createDiscoveredControllerConfigSpec(
107+
dependencies.Infrastructure,
108+
dependencies.Network,
109+
dependencies.Proxy,
110+
dependencies.DNS)
111+
if err != nil {
112+
return nil, err
113+
}
114+
115+
if dependencies.AdditionalTrustBundle != "" {
116+
spec.AdditionalTrustBundle = []byte(dependencies.AdditionalTrustBundle)
117+
}
118+
119+
if dependencies.CloudConfig != "" {
120+
spec.CloudProviderConfig = dependencies.CloudConfig
121+
}
122+
123+
// Append the kube-ca if given.
124+
if dependencies.KubeAPIServerServingCA != "" {
125+
spec.KubeAPIServerServingCAData = []byte(dependencies.KubeAPIServerServingCA)
126+
}
127+
// Set the cloud-provider CA if given.
128+
if dependencies.CloudProviderCA != "" {
129+
spec.CloudProviderCAData = []byte(dependencies.CloudProviderCA)
130+
}
131+
132+
spec.RootCAData = []byte(dependencies.MCSCA)
133+
spec.PullSecret = nil
134+
spec.BaseOSContainerImage = imgs.BaseOSContainerImage
135+
spec.BaseOSExtensionsContainerImage = imgs.BaseOSExtensionsContainerImage
136+
spec.ReleaseImage = releaseImage
137+
spec.Images = map[string]string{
138+
templatectrl.MachineConfigOperatorKey: imgs.MachineConfigOperator,
139+
140+
templatectrl.APIServerWatcherKey: imgs.MachineConfigOperator,
141+
templatectrl.InfraImageKey: imgs.InfraImage,
142+
templatectrl.KeepalivedKey: imgs.Keepalived,
143+
templatectrl.CorednsKey: imgs.Coredns,
144+
templatectrl.HaproxyKey: imgs.Haproxy,
145+
templatectrl.BaremetalRuntimeCfgKey: imgs.BaremetalRuntimeCfg,
146+
templatectrl.KubeRbacProxyKey: imgs.KubeRbacProxy,
147+
templatectrl.DockerRegistryKey: imgs.DockerRegistry,
148+
}
149+
150+
config := getRenderConfig("", dependencies.KubeAPIServerServingCA, spec,
151+
&imgs.RenderConfigImages, dependencies.Infrastructure, nil, nil, "2")
152+
return config, nil
153+
}
154+
155+
func appendManifestsByPlatform(manifests []manifest, infra *configv1.Infrastructure) []manifest {
226156
lbType := configv1.LoadBalancerTypeOpenShiftManagedDefault
227157
if infra.Status.PlatformStatus.BareMetal != nil {
228158
if infra.Status.PlatformStatus.BareMetal.LoadBalancer != nil {
@@ -300,28 +230,6 @@ func appendManifestsByPlatform(manifests []manifest, infra configv1.Infrastructu
300230
return manifests
301231
}
302232

303-
// loadBootstrapCloudProviderConfig reads the cloud provider config from cloudConfigFile based on infra object.
304-
func loadBootstrapCloudProviderConfig(infra *configv1.Infrastructure, cloudConfigFile string) (string, error) {
305-
data, err := os.ReadFile(cloudConfigFile)
306-
if err != nil {
307-
return "", err
308-
}
309-
obji, err := runtime.Decode(scheme.Codecs.UniversalDecoder(corev1.SchemeGroupVersion), data)
310-
if err != nil {
311-
return "", err
312-
}
313-
cm, ok := obji.(*corev1.ConfigMap)
314-
if !ok {
315-
return "", fmt.Errorf("expected *corev1.ConfigMap found %T", obji)
316-
}
317-
cloudConf, ok := cm.Data["cloud.conf"]
318-
if !ok {
319-
klog.Infof("falling back to reading cloud provider config from user specified key %s", infra.Spec.CloudConfig.Key)
320-
cloudConf = cm.Data[infra.Spec.CloudConfig.Key]
321-
}
322-
return cloudConf, nil
323-
}
324-
325233
func getPlatformManifests(manifests []manifest, platformName string, lbType configv1.PlatformLoadBalancerType) []manifest {
326234
var corednsName string
327235
var corefileName string

0 commit comments

Comments
 (0)