diff --git a/build/configs/controller.yaml b/build/configs/controller.yaml index e31a44a8396..0d3152eec6f 100644 --- a/build/configs/controller.yaml +++ b/build/configs/controller.yaml @@ -15,9 +15,6 @@ tracerProvider: serviceName: "controller" zipkin: url: "http://localhost:9411/api/v2/spans" -server: - host: "0.0.0.0" - port: 8083 workerServer: port: 7073 ucp: diff --git a/build/scripts/start-radius.sh b/build/scripts/start-radius.sh index 10063a93e0a..ebb2653f82a 100755 --- a/build/scripts/start-radius.sh +++ b/build/scripts/start-radius.sh @@ -328,7 +328,7 @@ fi # Start Controller with dlv echo "Starting Controller with dlv on port 40002..." -dlv exec "$DEBUG_ROOT/bin/controller" --listen=127.0.0.1:40002 --headless=true --api-version=2 --accept-multiclient --continue -- --config-file="$SCRIPT_DIR/../configs/controller.yaml" --cert-dir="" > "$DEBUG_ROOT/logs/controller.log" 2>&1 & +dlv exec "$DEBUG_ROOT/bin/controller" --listen=127.0.0.1:40002 --headless=true --api-version=2 --accept-multiclient --continue -- --config-file="$SCRIPT_DIR/../configs/controller.yaml" > "$DEBUG_ROOT/logs/controller.log" 2>&1 & echo $! > "$DEBUG_ROOT/logs/controller.pid" # Wait for Controller to start (check health endpoint) diff --git a/cmd/controller/cmd/root.go b/cmd/controller/cmd/root.go index 4ac4c42f5a3..f8df4d14137 100644 --- a/cmd/controller/cmd/root.go +++ b/cmd/controller/cmd/root.go @@ -42,7 +42,6 @@ var rootCmd = &cobra.Command{ Long: `Server process for Radius Kubernetes interoperability (controller).`, RunE: func(cmd *cobra.Command, args []string) error { configFilePath := cmd.Flag("config-file").Value.String() - tlsCertDir := cmd.Flag("cert-dir").Value.String() options, err := hostoptions.NewHostOptionsFromEnvironment(configFilePath) if err != nil { @@ -63,7 +62,7 @@ var rootCmd = &cobra.Command{ logger.Info("Loaded options", "configfile", configFilePath) services := []hosting.Service{ - &controller.Service{Options: options, TLSCertDir: tlsCertDir}, + &controller.Service{Options: options}, } if options.Config.TracerProvider.Enabled { @@ -78,7 +77,6 @@ var rootCmd = &cobra.Command{ func Execute() { // Let users override the configuration via `--config-file`. rootCmd.Flags().String("config-file", fmt.Sprintf("controller-%s.yaml", hostoptions.Environment()), "The service configuration file.") - rootCmd.Flags().String("cert-dir", "/var/tls/cert", "The directory containing the TLS certificates.") cobra.CheckErr(rootCmd.ExecuteContext(context.Background())) } diff --git a/cmd/controller/controller-dev.yaml b/cmd/controller/controller-dev.yaml index b3f398d1585..aed6c52412f 100644 --- a/cmd/controller/controller-dev.yaml +++ b/cmd/controller/controller-dev.yaml @@ -15,9 +15,6 @@ tracerProvider: serviceName: "controller" zipkin: url: "http://localhost:9411/api/v2/spans" -server: - host: "0.0.0.0" - port: 8083 # workerServer port specifies port set for Health Checks workerServer: diff --git a/deploy/Chart/README.md b/deploy/Chart/README.md index 346ebc2a305..0b509a840c2 100644 --- a/deploy/Chart/README.md +++ b/deploy/Chart/README.md @@ -242,9 +242,7 @@ When enabled, three policies are applied: - `radius-allow-internal` — re-permits east-west traffic between Radius components (intra-namespace), matched by the immutable `kubernetes.io/metadata.name` namespace label. -- `radius-allow-control-plane` — allows the Kubernetes API server to reach UCP - (APIService aggregation) and the controller (admission webhook) on port `9443`, - from the CIDRs in `networkPolicies.controlPlaneCIDRs`. +- `radius-allow-control-plane` — allows the Kubernetes API server to reach UCP (APIService aggregation) on port `9443`, from the CIDRs in `networkPolicies.controlPlaneCIDRs`. Only ingress is restricted; egress is left open so UCP can reach the Kubernetes API server and pods can resolve DNS. @@ -256,14 +254,7 @@ API server and pods can resolve DNS. #### Setting `controlPlaneCIDRs` -The kube-apiserver reaches UCP (APIService aggregation) and the controller -(admission webhook) over the host network, so this traffic arrives with the -**node's** IP rather than a pod IP and cannot be matched by a namespace/pod -selector. You must supply the source CIDR(s) via -`networkPolicies.controlPlaneCIDRs` — **this is required when -`networkPolicies.enabled=true`; Helm rendering fails if it is empty** — otherwise -the default-deny policy would block API aggregation and webhooks and break the -control plane. +The kube-apiserver reaches UCP (APIService aggregation) over the host network, so this traffic arrives with the **node's** IP rather than a pod IP and cannot be matched by a namespace/pod selector. You must supply the source CIDR(s) via `networkPolicies.controlPlaneCIDRs` — **this is required when `networkPolicies.enabled=true`; Helm rendering fails if it is empty** — otherwise the default-deny policy would block API aggregation and break the control plane. Use your cluster's node/control-plane subnet(s), **not** individual node IPs (a `/32` would exclude other control-plane addresses): diff --git a/deploy/Chart/crds/radius/radapp.io_recipes.yaml b/deploy/Chart/crds/radius/radapp.io_recipes.yaml deleted file mode 100644 index 7cbbc4806d5..00000000000 --- a/deploy/Chart/crds/radius/radapp.io_recipes.yaml +++ /dev/null @@ -1,163 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.21.0 - name: recipes.radapp.io -spec: - group: radapp.io - names: - categories: - - all - - radius - kind: Recipe - listKind: RecipeList - plural: recipes - singular: recipe - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Type of resource the recipe should create - jsonPath: .spec.type - name: Type - type: string - - description: Name of the secret to create - jsonPath: .spec.secretName - name: Secret - type: string - - description: Status of the resource - jsonPath: .status.phrase - name: Status - type: string - name: v1alpha3 - schema: - openAPIV3Schema: - description: Recipe is the Schema for the recipes API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: RecipeSpec defines the desired state of Recipe - properties: - application: - description: |- - Application is the name of the Radius application to use. If unset the namespace of the - Recipe will be used as the application name. - type: string - environment: - description: |- - Environment is the name of the Radius environment to use. If unset the value 'default' will be - used as the environment name. - type: string - secretName: - description: SecretName is the name of a Kubernetes secret to create - once the resource is created. - type: string - type: - description: 'Type is the type of resource to create. eg: ''Applications.Datastores/redisCaches''.' - type: string - required: - - type - type: object - status: - description: RecipeStatus defines the observed state of Recipe - properties: - application: - description: Application is the resource ID of the application. - type: string - environment: - description: Environment is the resource ID of the environment. - type: string - observedGeneration: - description: |- - ObservedGeneration is the most recent generation observed for this Recipe. It corresponds to the - Recipe's generation, which is updated on mutation by the API Server. - type: integer - operation: - description: Operation tracks the status of an in-progress provisioning - operation. - properties: - operationKind: - description: OperationKind describes the type of operation being - performed. - type: string - resumeToken: - description: ResumeToken is a token that can be used to resume - an in-progress provisioning operation. - type: string - type: object - phrase: - description: Phrase indicates the current status of the Recipe. - type: string - resource: - description: Resource is the resource ID of the resource. - type: string - scope: - description: Scope is the resource ID of the scope. - type: string - secret: - description: Secret specifies a reference to the secret being managed - by this Recipe. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids - type: string - type: object - x-kubernetes-map-type: atomic - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/deploy/Chart/templates/controller/configmaps.yaml b/deploy/Chart/templates/controller/configmaps.yaml index d2dcd51e7d2..4f19b0d804d 100644 --- a/deploy/Chart/templates/controller/configmaps.yaml +++ b/deploy/Chart/templates/controller/configmaps.yaml @@ -8,10 +8,6 @@ metadata: app.kubernetes.io/part-of: radius data: controller-config.yaml: |- - server: - host: "0.0.0.0" - port: 9443 - databaseProvider: provider: "apiserver" apiserver: diff --git a/deploy/Chart/templates/controller/deployment.yaml b/deploy/Chart/templates/controller/deployment.yaml index 47c8f6f2912..c409af9170a 100644 --- a/deploy/Chart/templates/controller/deployment.yaml +++ b/deploy/Chart/templates/controller/deployment.yaml @@ -64,10 +64,6 @@ spec: - '--config-file' - '/etc/config/controller-config.yaml' env: - - name: TLS_CERT_DIR - value: '/var/tls/cert' - - name: PORT - value: '9443' - name: BICEP value: '/usr/local/bin/bicep' {{- if .Values.global.rootCA.cert }} @@ -76,9 +72,6 @@ spec: value: {{ .Values.global.rootCA.mountPath }} {{- end}} ports: - - containerPort: 9443 - name: controller - protocol: TCP {{- if eq .Values.global.prometheus.enabled true }} - containerPort: {{ .Values.global.prometheus.port }} name: metrics @@ -100,9 +93,6 @@ spec: subPath: bicepconfig.json - name: config-volume mountPath: /etc/config - - name: cert - mountPath: '/var/tls/cert' - readOnly: true {{- if .Values.global.rootCA.cert }} - name: {{ .Values.global.rootCA.volumeName }} mountPath: {{ .Values.global.rootCA.mountPath }} @@ -121,9 +111,6 @@ spec: - name: config-volume configMap: name: controller-config - - name: cert - secret: - secretName: controller-cert {{- if .Values.global.rootCA.cert }} - name: {{ .Values.global.rootCA.volumeName }} secret: diff --git a/deploy/Chart/templates/controller/rbac.yaml b/deploy/Chart/templates/controller/rbac.yaml index 113691dd359..99fffa2aed7 100644 --- a/deploy/Chart/templates/controller/rbac.yaml +++ b/deploy/Chart/templates/controller/rbac.yaml @@ -10,7 +10,6 @@ rules: - "" resources: - namespaces - - secrets - events verbs: - create @@ -20,24 +19,9 @@ rules: - patch - update - watch -- apiGroups: - - apps - resources: - - deployments - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - radapp.io resources: - - recipes - - recipes/status - deploymenttemplates - deploymenttemplates/status - deploymentresources diff --git a/deploy/Chart/templates/controller/service.yaml b/deploy/Chart/templates/controller/service.yaml deleted file mode 100644 index a3702063b68..00000000000 --- a/deploy/Chart/templates/controller/service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: controller - namespace: "{{ .Release.Namespace }}" - labels: - app.kubernetes.io/name: controller - app.kubernetes.io/part-of: radius -spec: - ports: - - port: 443 - name: https - protocol: TCP - targetPort: 9443 - selector: - app.kubernetes.io/name: controller diff --git a/deploy/Chart/templates/controller/validating-webhook-configuration.yaml b/deploy/Chart/templates/controller/validating-webhook-configuration.yaml deleted file mode 100644 index 75fd766b3f6..00000000000 --- a/deploy/Chart/templates/controller/validating-webhook-configuration.yaml +++ /dev/null @@ -1,47 +0,0 @@ -{{- $ca := genCA "controller-ca" 3650 }} -{{- $cn := printf "controller" }} -{{- $altName1 := printf "controller.%s" .Release.Namespace }} -{{- $altName2 := printf "controller.%s.svc" .Release.Namespace }} -{{- $altName3 := printf "controller.%s.svc.cluster" .Release.Namespace }} -{{- $altName4 := printf "controller.%s.svc.cluster.local" .Release.Namespace }} -{{- $cert := genSignedCert $cn nil (list $altName1 $altName2 $altName3 $altName4) 3650 $ca }} -apiVersion: v1 -kind: Secret -metadata: - name: controller-cert - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: controller - app.kubernetes.io/part-of: radius -data: - tls.crt: {{ include "secrets.lookup" (dict "secret" "controller-cert" "namespace" .Release.Namespace "key" "tls.crt" "defaultValue" $cert.Cert) }} - tls.key: {{ include "secrets.lookup" (dict "secret" "controller-cert" "namespace" .Release.Namespace "key" "tls.key" "defaultValue" $cert.Key) }} - ca.crt: {{ include "secrets.lookup" (dict "secret" "controller-cert" "namespace" .Release.Namespace "key" "ca.crt" "defaultValue" $ca.Cert) }} ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: recipe-webhook.radapp.io -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - caBundle: {{ include "secrets.lookup" (dict "secret" "controller-cert" "namespace" .Release.Namespace "key" "ca.crt" "defaultValue" $ca.Cert) }} - service: - name: controller - namespace: {{ .Release.Namespace }} - path: /validate-radapp-io-v1alpha3-recipe - failurePolicy: Fail - matchPolicy: Equivalent - name: recipe-webhook.radapp.io - rules: - - apiGroups: - - radapp.io - apiVersions: - - v1alpha3 - operations: - - CREATE - - UPDATE - resources: - - recipes - sideEffects: None diff --git a/deploy/Chart/templates/networkpolicies.yaml b/deploy/Chart/templates/networkpolicies.yaml index 0343c3c4402..fcc593c996c 100644 --- a/deploy/Chart/templates/networkpolicies.yaml +++ b/deploy/Chart/templates/networkpolicies.yaml @@ -1,6 +1,6 @@ {{- if .Values.networkPolicies.enabled }} {{- if not .Values.networkPolicies.controlPlaneCIDRs }} -{{- fail "networkPolicies.enabled=true requires networkPolicies.controlPlaneCIDRs to be set to the cluster's node/control-plane CIDR(s) so the Kubernetes API server can reach UCP and the controller on :9443. Set networkPolicies.controlPlaneCIDRs, or set networkPolicies.enabled=false to install without these policies." }} +{{- fail "networkPolicies.enabled=true requires networkPolicies.controlPlaneCIDRs to be set to the cluster's node/control-plane CIDR(s) so the Kubernetes API server can reach UCP on :9443. Set networkPolicies.controlPlaneCIDRs, or set networkPolicies.enabled=false to install without these policies." }} {{- end }} # Optional ingress lockdown for the Radius control-plane namespace. # @@ -55,13 +55,12 @@ spec: # Allow ingress from the Kubernetes API server. It reaches the control plane over # the host network, so this traffic arrives at the destination pod with the NODE's # IP rather than a pod IP and cannot be matched by a pod/namespace selector: -# - kube-apiserver -> ucp:9443 (APIService aggregation for api.ucp.dev) -# - kube-apiserver -> controller:9443 (recipe admission webhook) -# Both arrive on container port 9443, so the rule is scoped to that port to limit +# - kube-apiserver -> ucp:9443 (APIService aggregation for api.ucp.dev) +# The rule is scoped to UCP on container port 9443 to limit # what host-network sources in these CIDRs can reach. The UCP HTTPS readiness # probe also originates from the node on port 9443 and is covered by this rule. # The dynamic-rp probes are exec probes and need no network allowance. -# Without this policy the default-deny above blocks API aggregation and webhooks +# Without this policy the default-deny above blocks API aggregation # and breaks the control plane. The CIDRs are cluster-specific (typically the node # subnet) and come from networkPolicies.controlPlaneCIDRs. apiVersion: networking.k8s.io/v1 @@ -72,7 +71,9 @@ metadata: labels: app.kubernetes.io/part-of: radius spec: - podSelector: {} + podSelector: + matchLabels: + app.kubernetes.io/name: ucp policyTypes: - Ingress ingress: diff --git a/deploy/Chart/tests/controller_test.yaml b/deploy/Chart/tests/controller_test.yaml new file mode 100644 index 00000000000..45e192642d3 --- /dev/null +++ b/deploy/Chart/tests/controller_test.yaml @@ -0,0 +1,136 @@ +suite: test controller without legacy Kubernetes onboarding +templates: + - controller/* +release: + name: radius + namespace: radius-system +tests: + - it: renders only the deployment, configuration, service account, and RBAC + asserts: + - hasDocuments: + count: 1 + template: controller/deployment.yaml + - hasDocuments: + count: 1 + template: controller/configmaps.yaml + - hasDocuments: + count: 1 + template: controller/serviceaccount.yaml + - hasDocuments: + count: 2 + template: controller/rbac.yaml + - notMatchRegex: + path: kind + pattern: ^(Service|Secret|ValidatingWebhookConfiguration)$ + + - it: retains Bicep and health without webhook ports or certificates + set: + global.prometheus.enabled: false + templates: + - controller/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].args + value: + - --config-file + - /etc/config/controller-config.yaml + - equal: + path: spec.template.spec.containers[0].env + value: + - name: BICEP + value: /usr/local/bin/bicep + - equal: + path: spec.template.spec.containers[0].ports + value: + - containerPort: 3000 + name: healthz + protocol: TCP + - equal: + path: spec.template.spec.volumes + value: + - name: bicep + emptyDir: {} + - name: bicepconfig + emptyDir: {} + - name: config-volume + configMap: + name: controller-config + - equal: + path: spec.template.spec.containers[0].volumeMounts + value: + - name: bicep + mountPath: /usr/local/bin + - name: bicepconfig + mountPath: /bicepconfig.json + subPath: bicepconfig.json + - name: config-volume + mountPath: /etc/config + + - it: retains the optional metrics port + set: + global.prometheus.enabled: true + global.prometheus.port: 9090 + templates: + - controller/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].ports + value: + - containerPort: 9090 + name: metrics + protocol: TCP + - containerPort: 3000 + name: healthz + protocol: TCP + + - it: removes webhook server configuration + templates: + - controller/configmaps.yaml + asserts: + - notMatchRegex: + path: data["controller-config.yaml"] + pattern: '(?m)^server:' + - matchRegex: + path: data["controller-config.yaml"] + pattern: 'workerServer:\s+port: 3000' + + - it: retains deployment and Flux permissions without legacy resource permissions + templates: + - controller/rbac.yaml + documentIndex: 0 + asserts: + - lengthEqual: + path: rules + count: 6 + - equal: + path: rules[0].resources + value: + - namespaces + - events + - equal: + path: rules[1].apiGroups + value: + - radapp.io + - equal: + path: rules[1].resources + value: + - deploymenttemplates + - deploymenttemplates/status + - deploymentresources + - deploymentresources/status + - equal: + path: rules[2].apiGroups + value: + - api.ucp.dev + - equal: + path: rules[3].resources + value: + - gitrepositories + - equal: + path: rules[4].resources + value: + - gitrepositories/status + - equal: + path: rules[5].resources + value: + - customresourcedefinitions diff --git a/deploy/Chart/tests/networkpolicies_test.yaml b/deploy/Chart/tests/networkpolicies_test.yaml index 17321675041..56d95594e4e 100644 --- a/deploy/Chart/tests/networkpolicies_test.yaml +++ b/deploy/Chart/tests/networkpolicies_test.yaml @@ -15,7 +15,7 @@ tests: networkPolicies.enabled: true asserts: - failedTemplate: - errorMessage: "networkPolicies.enabled=true requires networkPolicies.controlPlaneCIDRs to be set to the cluster's node/control-plane CIDR(s) so the Kubernetes API server can reach UCP and the controller on :9443. Set networkPolicies.controlPlaneCIDRs, or set networkPolicies.enabled=false to install without these policies." + errorMessage: "networkPolicies.enabled=true requires networkPolicies.controlPlaneCIDRs to be set to the cluster's node/control-plane CIDR(s) so the Kubernetes API server can reach UCP on :9443. Set networkPolicies.controlPlaneCIDRs, or set networkPolicies.enabled=false to install without these policies." - it: renders default-deny and allow-internal as the first two policies when enabled set: @@ -57,7 +57,7 @@ tests: value: radius-system documentIndex: 1 - - it: renders the control-plane allow policy scoped to port 9443 with one ipBlock per CIDR + - it: renders the control-plane allow policy scoped to UCP port 9443 with one ipBlock per CIDR set: networkPolicies.enabled: true networkPolicies.controlPlaneCIDRs: @@ -70,6 +70,11 @@ tests: path: metadata.name value: radius-allow-control-plane documentIndex: 2 + - equal: + path: spec.podSelector.matchLabels + value: + app.kubernetes.io/name: ucp + documentIndex: 2 - equal: path: spec.ingress[0].from[0].ipBlock.cidr value: 172.18.0.0/16 diff --git a/deploy/Chart/values.yaml b/deploy/Chart/values.yaml index be323c1f3cc..cf58f0f426f 100644 --- a/deploy/Chart/values.yaml +++ b/deploy/Chart/values.yaml @@ -259,13 +259,12 @@ networkPolicies: # Source CIDRs the Kubernetes API server uses to reach the Radius control plane. # This traffic is host-network, so it arrives with the NODE's IP rather than a # pod IP and cannot be matched by a namespace/pod selector: - # - kube-apiserver -> ucp:9443 (APIService aggregation for api.ucp.dev) - # - kube-apiserver -> controller:9443 (recipe admission webhook) - # The allow policy is scoped to port 9443 (the only host-network destination). + # - kube-apiserver -> ucp:9443 (APIService aggregation for api.ucp.dev) + # The allow policy is scoped to UCP port 9443 (the only host-network destination). # Set this to your cluster's node/control-plane source CIDR(s) -- e.g. the VPC or # subnet CIDR on a managed cluster, or the Docker network subnet for KinD. This # is REQUIRED when enabled (rendering fails if empty); without it the default-deny - # policy would block API aggregation and webhooks and break the control plane. + # policy would block API aggregation and break the control plane. # Example: ["10.0.0.0/16"] controlPlaneCIDRs: [] diff --git a/docs/architecture/controller.md b/docs/architecture/controller.md index 13d4ed110b5..a1621c9aecd 100644 --- a/docs/architecture/controller.md +++ b/docs/architecture/controller.md @@ -4,8 +4,7 @@ The `controller` binary runs the Kubernetes controller-manager-based workflows for Radius. It watches cluster resources, reconciles them, and drives Radius APIs where Kubernetes-native automation is required. -The controller owns reconciliation and webhook behavior. It is not the primary -home of resource-type authoring or UCP routing logic. +The controller owns Kubernetes reconciliation for template deployment and Flux GitOps. It is not the primary home of resource-type authoring or UCP routing logic. ## Entry Points @@ -17,36 +16,33 @@ home of resource-type authoring or UCP routing logic. ## Quick Reference -| Topic | Start Here | -|------|------------| -| Startup | `cmd/controller/cmd/root.go` | -| Manager setup | `pkg/controller/service.go` | -| Reconciler logic | `pkg/controller/reconciler` | -| CRD types | `pkg/controller/api` | +| Topic | Start Here | +|------------------|------------------------------| +| Startup | `cmd/controller/cmd/root.go` | +| Manager setup | `pkg/controller/service.go` | +| Reconciler logic | `pkg/controller/reconciler` | +| CRD types | `pkg/controller/api` | -| Test Focus | Packages | -|-----------|----------| -| Reconcile and webhook behavior | `./pkg/controller/reconciler/...` | -| Broad safety check | `./pkg/controller/...` | +| Test Focus | Packages | +|--------------------|-----------------------------------| +| Reconcile behavior | `./pkg/controller/reconciler/...` | +| Broad safety check | `./pkg/controller/...` | ## Core Packages -| Package | Responsibility | -|--------|----------------| -| `pkg/controller/service.go` | controller manager bootstrap | -| `pkg/controller/reconciler` | reconcilers and webhook wiring | -| `pkg/controller/api` | CRD-backed Kubernetes API types | -| `pkg/sdk` | clients used to call back into Radius APIs | +| Package | Responsibility | +|-----------------------------|----------------------------------------------------------------| +| `pkg/controller/service.go` | controller manager bootstrap | +| `pkg/controller/reconciler` | deployment template, deployment resource, and Flux reconcilers | +| `pkg/controller/api` | CRD-backed Kubernetes API types | +| `pkg/sdk` | clients used to call back into Radius APIs | ## How It Works The root command builds shared host options, creates the logger, and starts a single `controller.Service` through shared hosting. -Inside [pkg/controller/service.go](../../pkg/controller/service.go), the service -creates a controller-runtime manager, registers API schemes, configures metrics -and health probes, then registers reconcilers for recipe, deployment, -deployment template, deployment resource, and Flux-oriented behavior. +Inside [pkg/controller/service.go](../../pkg/controller/service.go), the service creates a controller-runtime manager, registers API schemes, configures metrics and health probes, then registers `DeploymentTemplateReconciler`, `DeploymentResourceReconciler`, and `FluxController`. The shared operation types in [operation_types.go](../../pkg/controller/api/radapp.io/v1alpha3/operation_types.go) preserve asynchronous operation tracking for both deployment CRDs. Some reconcilers call back into Radius APIs using SDK clients configured with the current UCP connection. That is the main architectural bridge between the @@ -72,8 +68,7 @@ controller and the rest of the control plane. ### Suggested Test Scope - `go test ./pkg/controller/...` -- Pay particular attention to reconciler and webhook tests in - `pkg/controller/reconciler/...` +- Pay particular attention to deployment template, deployment resource, and Flux tests in `pkg/controller/reconciler/...`. ## Package Dependency View @@ -147,4 +142,4 @@ back into Kubernetes resources. ## Related Docs - [service-interaction-map.md](service-interaction-map.md) -- [rad-cli.md](rad-cli.md) \ No newline at end of file +- [rad-cli.md](rad-cli.md) diff --git a/docs/architecture/service-interaction-map.md b/docs/architecture/service-interaction-map.md index e6491e1518a..08a96ada63a 100644 --- a/docs/architecture/service-interaction-map.md +++ b/docs/architecture/service-interaction-map.md @@ -47,8 +47,7 @@ graph TD portable resource providers (Dapr, Datastores, Messaging) in the same process. - **`dynamic-rp`** is the main authoring surface for Radius resource types and generic resource lifecycle behavior. -- **`controller`** runs Kubernetes reconcilers and webhooks for Radius custom - resources and related workflows. +- **`controller`** runs Kubernetes reconcilers for Radius deployment templates, deployment resources, and Flux GitOps workflows. - **Deployment Engine** is not implemented in this repository, but several flows cross that boundary. UCP proxies deployment requests to the deployment engine, and the deployment engine calls back to UCP for each resource it @@ -143,7 +142,7 @@ sequenceDiagram start in UCP. - If the change is about **authoring or handling Radius resource types**, start in `dynamic-rp`. -- If the change is about **Kubernetes watch/reconcile/webhook behavior**, +- If the change is about **Kubernetes watch/reconcile behavior**, start in `controller`. - If the change is about **user experience, config, or command orchestration**, start in `rad`. @@ -159,4 +158,4 @@ sequenceDiagram - [cmd/controller/main.go](../../cmd/controller/main.go) - [cmd/controller/cmd/root.go](../../cmd/controller/cmd/root.go) - [cmd/rad/main.go](../../cmd/rad/main.go) -- [cmd/rad/cmd/root.go](../../cmd/rad/cmd/root.go) \ No newline at end of file +- [cmd/rad/cmd/root.go](../../cmd/rad/cmd/root.go) diff --git a/pkg/controller/api/radapp.io/v1alpha3/groupversion_info_test.go b/pkg/controller/api/radapp.io/v1alpha3/groupversion_info_test.go index 31ef2ca0173..34185ff3dd3 100644 --- a/pkg/controller/api/radapp.io/v1alpha3/groupversion_info_test.go +++ b/pkg/controller/api/radapp.io/v1alpha3/groupversion_info_test.go @@ -39,8 +39,6 @@ func TestAddToScheme(t *testing.T) { obj runtime.Object kind string }{ - {&Recipe{}, "Recipe"}, - {&RecipeList{}, "RecipeList"}, {&DeploymentTemplate{}, "DeploymentTemplate"}, {&DeploymentTemplateList{}, "DeploymentTemplateList"}, {&DeploymentResource{}, "DeploymentResource"}, @@ -57,6 +55,13 @@ func TestAddToScheme(t *testing.T) { }) } + for _, kind := range []string{"Recipe", "RecipeList"} { + t.Run(kind+"NotRegistered", func(t *testing.T) { + _, err := scheme.New(GroupVersion.WithKind(kind)) + require.True(t, runtime.IsNotRegisteredError(err), "legacy %s must not be registered", kind) + }) + } + // metav1.AddToGroupVersion must be invoked so list/watch machinery // (ListOptions, etc.) works for this GroupVersion. gvks, _, err := scheme.ObjectKinds(&metav1.ListOptions{}) diff --git a/pkg/controller/api/radapp.io/v1alpha3/operation_types.go b/pkg/controller/api/radapp.io/v1alpha3/operation_types.go new file mode 100644 index 00000000000..fef857d65f3 --- /dev/null +++ b/pkg/controller/api/radapp.io/v1alpha3/operation_types.go @@ -0,0 +1,39 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha3 + +import "net/http" + +// ResourceOperation describes the status of an in-progress provisioning operation. +type ResourceOperation struct { + // ResumeToken is a token that can be used to resume an in-progress provisioning operation. + ResumeToken string `json:"resumeToken,omitempty"` + + // OperationKind describes the type of operation being performed. + OperationKind OperationKind `json:"operationKind,omitempty"` +} + +// OperationKind is the type of operation being performed. +type OperationKind string + +const ( + // OperationKindPut is a PUT (create or update) operation. + OperationKindPut = http.MethodPut + + // OperationKindDelete is a DELETE operation. + OperationKindDelete = http.MethodDelete +) diff --git a/pkg/controller/api/radapp.io/v1alpha3/recipe_types.go b/pkg/controller/api/radapp.io/v1alpha3/recipe_types.go deleted file mode 100644 index cad48d595f0..00000000000 --- a/pkg/controller/api/radapp.io/v1alpha3/recipe_types.go +++ /dev/null @@ -1,153 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha3 - -import ( - "net/http" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -// RecipeSpec defines the desired state of Recipe -type RecipeSpec struct { - // Type is the type of resource to create. eg: 'Applications.Datastores/redisCaches'. - // +kubebuilder:validation:Required - Type string `json:"type,omitempty"` - - // SecretName is the name of a Kubernetes secret to create once the resource is created. - // +kubebuilder:validation:Optional - SecretName string `json:"secretName,omitempty"` - - // Environment is the name of the Radius environment to use. If unset the value 'default' will be - // used as the environment name. - Environment string `json:"environment,omitempty"` - - // Application is the name of the Radius application to use. If unset the namespace of the - // Recipe will be used as the application name. - Application string `json:"application,omitempty"` -} - -// RecipePhrase is a string representation of the current status of a Recipe. -type RecipePhrase string - -const ( - // PhraseUpdating indicates that the Recipe is being updated. - PhraseUpdating RecipePhrase = "Updating" - - // PhraseReady indicates that the Recipe is ready. - PhraseReady RecipePhrase = "Ready" - - // PhraseFailed indicates that the Recipe has failed. - PhraseFailed RecipePhrase = "Failed" - - // PhraseDeleting indicates that the Recipe is being deleted. - PhraseDeleting RecipePhrase = "Deleting" - - // PhraseDeleted indicates that the Recipe has been deleted. - PhraseDeleted RecipePhrase = "Deleted" -) - -// RecipeStatus defines the observed state of Recipe -type RecipeStatus struct { - // ObservedGeneration is the most recent generation observed for this Recipe. It corresponds to the - // Recipe's generation, which is updated on mutation by the API Server. - // +kubebuilder:validation:Optional - // +kubebuilder:validation:Format="" - ObservedGeneration int64 `json:"observedGeneration,omitempty" protobuf:"varint,1,opt,name=observedGeneration"` - - // Application is the resource ID of the application. - // +kubebuilder:validation:Optional - Application string `json:"application,omitempty"` - - // Environment is the resource ID of the environment. - // +kubebuilder:validation:Optional - Environment string `json:"environment,omitempty"` - - // Scope is the resource ID of the scope. - // +kubebuilder:validation:Optional - Scope string `json:"scope,omitempty"` - - // Resource is the resource ID of the resource. - // +kubebuilder:validation:Optional - Resource string `json:"resource,omitempty"` - - // Operation tracks the status of an in-progress provisioning operation. - // +kubebuilder:validation:Optional - Operation *ResourceOperation `json:"operation,omitempty"` - - // Phrase indicates the current status of the Recipe. - // +kubebuilder:validation:Optional - Phrase RecipePhrase `json:"phrase,omitempty"` - - // Secret specifies a reference to the secret being managed by this Recipe. - // +kubebuilder:validation:Optional - Secret corev1.ObjectReference `json:"secret,omitempty"` -} - -// ResourceOperation describes the status of an in-progress provisioning operation. -type ResourceOperation struct { - // ResumeToken is a token that can be used to resume an in-progress provisioning operation. - ResumeToken string `json:"resumeToken,omitempty"` - - // OperationKind describes the type of operation being performed. - OperationKind OperationKind `json:"operationKind,omitempty"` -} - -// OperationKind is the type of operation being performed. -type OperationKind string - -const ( - // OperationKindPut is a PUT (create or update) operation. - OperationKindPut = http.MethodPut - - // OperationKindDelete is a DELETE operation. - OperationKindDelete = http.MethodDelete -) - -//+kubebuilder:object:root=true -//+kubebuilder:resource:categories={"all","radius"} -//+kubebuilder:printcolumn:name="Type",type="string",JSONPath=".spec.type",description="Type of resource the recipe should create" -//+kubebuilder:printcolumn:name="Secret",type="string",JSONPath=".spec.secretName",description="Name of the secret to create" -//+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phrase",description="Status of the resource" -//+kubebuilder:subresource:status - -// Recipe is the Schema for the recipes API -type Recipe struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec RecipeSpec `json:"spec,omitempty"` - Status RecipeStatus `json:"status,omitempty"` -} - -//+kubebuilder:object:root=true - -// RecipeList contains a list of Recipe -type RecipeList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []Recipe `json:"items"` -} - -func init() { - SchemeBuilder.Register(func(s *runtime.Scheme) error { - s.AddKnownTypes(GroupVersion, &Recipe{}, &RecipeList{}) - return nil - }) -} diff --git a/pkg/controller/api/radapp.io/v1alpha3/zz_generated.deepcopy.go b/pkg/controller/api/radapp.io/v1alpha3/zz_generated.deepcopy.go index 01df9346377..4c6317c7f54 100644 --- a/pkg/controller/api/radapp.io/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/controller/api/radapp.io/v1alpha3/zz_generated.deepcopy.go @@ -224,101 +224,6 @@ func (in *DeploymentTemplateStatus) DeepCopy() *DeploymentTemplateStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Recipe) DeepCopyInto(out *Recipe) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Recipe. -func (in *Recipe) DeepCopy() *Recipe { - if in == nil { - return nil - } - out := new(Recipe) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Recipe) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RecipeList) DeepCopyInto(out *RecipeList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]Recipe, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RecipeList. -func (in *RecipeList) DeepCopy() *RecipeList { - if in == nil { - return nil - } - out := new(RecipeList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *RecipeList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RecipeSpec) DeepCopyInto(out *RecipeSpec) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RecipeSpec. -func (in *RecipeSpec) DeepCopy() *RecipeSpec { - if in == nil { - return nil - } - out := new(RecipeSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RecipeStatus) DeepCopyInto(out *RecipeStatus) { - *out = *in - if in.Operation != nil { - in, out := &in.Operation, &out.Operation - *out = new(ResourceOperation) - **out = **in - } - out.Secret = in.Secret -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RecipeStatus. -func (in *RecipeStatus) DeepCopy() *RecipeStatus { - if in == nil { - return nil - } - out := new(RecipeStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceOperation) DeepCopyInto(out *ResourceOperation) { *out = *in diff --git a/pkg/controller/reconciler/annotations.go b/pkg/controller/reconciler/annotations.go deleted file mode 100644 index 1cbaa5bc23d..00000000000 --- a/pkg/controller/reconciler/annotations.go +++ /dev/null @@ -1,265 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package reconciler - -import ( - "encoding/json" - "fmt" - "strings" - - radappiov1alpha3 "github.com/radius-project/radius/pkg/controller/api/radapp.io/v1alpha3" - "github.com/radius-project/radius/pkg/hashutil" - "github.com/radius-project/radius/pkg/ucp/resources" - appsv1 "k8s.io/api/apps/v1" -) - -type deploymentPhrase string - -const ( - deploymentPhraseWaiting deploymentPhrase = "Waiting" - deploymentPhraseUpdating deploymentPhrase = "Updating" - deploymentPhraseReady deploymentPhrase = "Ready" - deploymentPhraseDeleting deploymentPhrase = "Deleting" - deploymentPhraseFailed deploymentPhrase = "Failed" -) - -// deploymentAnnotations represents the user-provided configuration and the status (Radius related status) -// of the Deployment. -type deploymentAnnotations struct { - // Configuration is the configuration of the Deployment provided by the user via annotations. - // This will be nil if Radius is not enabled for the Deployment. - Configuration *deploymentConfiguration - - // ConfigurationHash is the hash of the user-provided configuration. - // This will be used to diff the configuration and determine if the Deployment needs to be updated. - ConfigurationHash string - - // Status is the status of the Deployment (Radius related status). - Status *deploymentStatus -} - -// There are 4 cases that is possible based on the previous state and the current state of the Deployment: -// Case 1: Previous State: Enabled - Current State: Disabled -// Case 2: Previous State: Disabled - Current State: Enabled -// Case 3: Previous State: Enabled - Current State: Enabled -// Case 4: Previous State: Disabled - Current State: Disabled -// -// How to understand the previous state: -// 1. If "radapp.io/status" annotation is set, then the previous state is Enabled. -// -// Ways to disable Radius: -// 1. "radapp.io/enabled" annotation is set to "false". -// 2. "radapp.io/enabled" annotation is not set. -// -// Ways to enable Radius: -// 1. "radapp.io/enabled" annotation is set to "true". - -// deploymentConfiguration is the configuration of the Deployment provided by the user via annotations. -type deploymentConfiguration struct { - Application string `json:"application,omitempty"` - Environment string `json:"environment,omitempty"` - Connections map[string]string `json:"connections,omitempty"` -} - -func (c *deploymentConfiguration) computeHash() (string, error) { - b, err := json.Marshal(c) - if err != nil { - return "", err - } - - return hashutil.Hex(b), nil -} - -// matchesHash reports whether stored matches the current configuration hash. -// -// It accepts both the current SHA-256 hash and the legacy SHA-1 hash so that -// upgrading Radius (which changes the hash algorithm) does not flag an otherwise -// unchanged configuration as out-of-date and trigger an unnecessary update. -func (c *deploymentConfiguration) matchesHash(stored string) bool { - b, err := json.Marshal(c) - if err != nil { - return false - } - - return stored == hashutil.Hex(b) || stored == hashutil.LegacyHex(b) -} - -type deploymentStatus struct { - Scope string `json:"scope,omitempty"` - Application string `json:"application,omitempty"` - Environment string `json:"environment,omitempty"` - Container string `json:"container,omitempty"` - Operation *radappiov1alpha3.ResourceOperation `json:"operation,omitempty"` - Phrase deploymentPhrase `json:"phrase,omitempty"` -} - -// readAnnotations reads the annotations from a Deployment. -// -// This includes the configuration specified by the user, the hash of the configuration, and the status. -func readAnnotations(deployment *appsv1.Deployment) (deploymentAnnotations, error) { - if deployment.Annotations == nil { - return deploymentAnnotations{}, nil - } - - result := deploymentAnnotations{ - ConfigurationHash: deployment.Annotations[AnnotationRadiusConfigurationHash], - } - - s := deploymentStatus{} - status := deployment.Annotations[AnnotationRadiusStatus] - if status != "" { - err := json.Unmarshal([]byte(status), &s) - if err != nil { - return result, fmt.Errorf("failed to unmarshal status annotation: %w", err) - } - - err = validateDeploymentStatus(&s) - if err != nil { - return result, fmt.Errorf("invalid status annotation: %w", err) - } - - result.Status = &s - } - - // Note: we need to read and return the configuration even if Radius is not enabled for the Deployment. - // This is important so that can clean up previously created connections when Radius is disabled. - enabled := deployment.Annotations[AnnotationRadiusEnabled] - if !strings.EqualFold(enabled, "true") { - return result, nil - } - - result.Configuration = &deploymentConfiguration{ - Environment: deployment.Annotations[AnnotationRadiusEnvironment], - Application: deployment.Annotations[AnnotationRadiusApplication], - Connections: map[string]string{}, - } - - for k, v := range deployment.Annotations { - if after, ok := strings.CutPrefix(k, AnnotationRadiusConnectionPrefix); ok { - result.Configuration.Connections[after] = v - } - } - - return result, nil -} - -func validateDeploymentStatus(status *deploymentStatus) error { - if status == nil { - return nil - } - - // The reconciler state machine may persist status.scope (and an in-progress operation) before - // status.container is known, so scope-only status is valid. When the environment or application - // changes, status.scope is updated to the new scope while status.container still references the - // previous container (which the reconciler then deletes), so we do not require status.scope to - // match the container's root scope. - if status.Scope != "" { - if _, err := resources.ParseScope(status.Scope); err != nil { - return fmt.Errorf("invalid status.scope: %w", err) - } - } - - if status.Container == "" { - return nil - } - - if status.Scope == "" { - return fmt.Errorf("status.scope must be set when status.container is set") - } - - parsedContainer, err := resources.ParseResource(status.Container) - if err != nil { - return fmt.Errorf("invalid status.container: %w", err) - } - if !strings.EqualFold(parsedContainer.Type(), applicationsCoreContainersResourceType) { - return fmt.Errorf("status.container type %q is not %q", parsedContainer.Type(), applicationsCoreContainersResourceType) - } - - return nil -} - -// ApplyToDeployment applies the configuration and status to a Deployment. -// -// This should be used before saving the Deployment's state. -func (annotations *deploymentAnnotations) ApplyToDeployment(deployment *appsv1.Deployment) error { - if deployment.Annotations == nil { - deployment.Annotations = map[string]string{} - } - - status := "" - if annotations.Status != nil { - b, err := json.Marshal(annotations.Status) - if err != nil { - return err - } - - status = string(b) - } - - deployment.Annotations[AnnotationRadiusStatus] = status - - if annotations.Configuration == nil { - deployment.Annotations[AnnotationRadiusEnabled] = "false" - return nil - } - - hash, err := annotations.Configuration.computeHash() - if err != nil { - return err - } - - deployment.Annotations[AnnotationRadiusConfigurationHash] = hash - deployment.Annotations[AnnotationRadiusEnabled] = "true" - - for k, v := range annotations.Configuration.Connections { - deployment.Annotations[AnnotationRadiusConnectionPrefix+k] = v - } - - return nil -} - -// IsUpToDate returns true if the Deployment is up to date with the configuration. -// -// This should be used to determine if the Radius container needs to be updated based -// on a change made by the user. -func (annotations *deploymentAnnotations) IsUpToDate() bool { - if annotations.ConfigurationHash == "" { - return false - } - - if annotations.Status == nil { - return false - } - - return annotations.Configuration.matchesHash(annotations.ConfigurationHash) -} - -// OperationInProgress returns true if there is an operation in progress for the given deployment. -func (annotations *deploymentAnnotations) OperationInProgress() bool { - return annotations.Status != nil && annotations.Status.Operation != nil -} - -// isRadiusEnabled returns true if Radius is enabled for the given deployment. -func (annotations *deploymentAnnotations) isRadiusEnabled() bool { - return annotations.Configuration != nil -} - -// needsCleanup returns true if Radius was previously enabled on the deployment and now is disabled. -// This means that we need to clean up the resources created by Radius. -func (annotations *deploymentAnnotations) needsCleanup() bool { - return annotations.Configuration == nil && annotations.Status != nil -} diff --git a/pkg/controller/reconciler/annotations_test.go b/pkg/controller/reconciler/annotations_test.go deleted file mode 100644 index 7a07d44436a..00000000000 --- a/pkg/controller/reconciler/annotations_test.go +++ /dev/null @@ -1,239 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package reconciler - -import ( - "encoding/json" - "fmt" - "testing" - - "github.com/radius-project/radius/pkg/ucp/resources" - "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func Test_readAnnotations(t *testing.T) { - testDeploymentStatus := &deploymentStatus{ - Scope: "/planes/radius/local/resourceGroups/controller-test", - Application: "test-application", - Environment: "test-environment", - Container: "/planes/radius/local/resourceGroups/controller-test/providers/Applications.Core/containers/test-container", - Operation: nil, - Phrase: deploymentPhraseReady, - } - - dsm, err := json.Marshal(testDeploymentStatus) - require.NoError(t, err) - - // invalidDeploymentStatus is missing a curly brace at the end of the JSON - // so that an unmarshaling error can be triggered. - invalidDeploymentStatus := []byte(`{"invalid": "json"`) - - _, invalidContainerIDErr := resources.ParseResource("not-a-resource-id") - _, invalidScopeIDErr := resources.ParseScope("not-a-scope") - - tests := []struct { - name string - deployment *appsv1.Deployment - annotations deploymentAnnotations - err error - }{ - { - name: "radius-disabled-with-annotation", - deployment: &appsv1.Deployment{ - Annotations: map[string]string{ - AnnotationRadiusEnabled: "false", - }, - }, - annotations: deploymentAnnotations{ - Configuration: nil, - ConfigurationHash: "", - Status: nil, - }, - err: nil, - }, - { - name: "radius-disabled-empty-annotation-map", - deployment: &appsv1.Deployment{ - Annotations: map[string]string{}, - }, - annotations: deploymentAnnotations{ - Configuration: nil, - ConfigurationHash: "", - Status: nil, - }, - err: nil, - }, - { - name: "radius-disabled-no-annotations", - deployment: &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{}, - }, - annotations: deploymentAnnotations{ - Configuration: nil, - ConfigurationHash: "", - Status: nil, - }, - err: nil, - }, - { - name: "radius-was-enabled-now-disabled-with-annotations", - deployment: &appsv1.Deployment{ - Annotations: map[string]string{ - AnnotationRadiusEnabled: "false", - AnnotationRadiusConfigurationHash: "configuration-hash", - AnnotationRadiusStatus: string(dsm), - }, - }, - annotations: deploymentAnnotations{ - Configuration: nil, - ConfigurationHash: "configuration-hash", - Status: testDeploymentStatus, - }, - err: nil, - }, - { - name: "radius-enabled-with-annotations", - deployment: &appsv1.Deployment{ - Annotations: map[string]string{ - AnnotationRadiusEnabled: "true", - AnnotationRadiusConfigurationHash: "configuration-hash", - AnnotationRadiusStatus: string(dsm), - AnnotationRadiusApplication: "test-application", - AnnotationRadiusEnvironment: "test-environment", - AnnotationRadiusConnectionPrefix + "test-connection": "test-connection-value", - }, - }, - annotations: deploymentAnnotations{ - Configuration: &deploymentConfiguration{ - Environment: "test-environment", - Application: "test-application", - Connections: map[string]string{ - "test-connection": "test-connection-value", - }, - }, - ConfigurationHash: "configuration-hash", - Status: testDeploymentStatus, - }, - err: nil, - }, - { - name: "status-unmarshal-error", - deployment: &appsv1.Deployment{ - Annotations: map[string]string{ - AnnotationRadiusEnabled: "true", - AnnotationRadiusConfigurationHash: "configuration-hash", - AnnotationRadiusStatus: string(invalidDeploymentStatus), - AnnotationRadiusApplication: "test-application", - AnnotationRadiusEnvironment: "test-environment", - AnnotationRadiusConnectionPrefix + "test-connection": "test-connection-value", - }, - }, - annotations: deploymentAnnotations{ - ConfigurationHash: "configuration-hash", - }, - err: fmt.Errorf("failed to unmarshal status annotation: %w", - json.Unmarshal(invalidDeploymentStatus, &deploymentStatus{})), - }, - { - name: "status-invalid-container-id", - deployment: &appsv1.Deployment{ - Annotations: map[string]string{ - AnnotationRadiusStatus: `{"scope":"/planes/radius/local/resourceGroups/controller-test","container":"not-a-resource-id"}`, - }, - }, - annotations: deploymentAnnotations{ConfigurationHash: ""}, - err: fmt.Errorf("invalid status annotation: invalid status.container: %w", invalidContainerIDErr), - }, - { - name: "status-scope-container-mismatch-allowed", - deployment: &appsv1.Deployment{ - Annotations: map[string]string{ - AnnotationRadiusStatus: `{"scope":"/planes/radius/local/resourceGroups/controller-test","container":"/planes/radius/local/resourceGroups/other/providers/Applications.Core/containers/test-container"}`, - }, - }, - // The reconciler intentionally produces this transitional state when the environment or - // application changes: status.scope advances to the new scope while status.container still - // references the previous container until it is deleted. - annotations: deploymentAnnotations{ - ConfigurationHash: "", - Status: &deploymentStatus{ - Scope: "/planes/radius/local/resourceGroups/controller-test", - Container: "/planes/radius/local/resourceGroups/other/providers/Applications.Core/containers/test-container", - }, - }, - err: nil, - }, - { - name: "status-container-wrong-resource-type", - deployment: &appsv1.Deployment{ - Annotations: map[string]string{ - AnnotationRadiusStatus: `{"scope":"/planes/radius/local/resourceGroups/controller-test","container":"/planes/radius/local/resourceGroups/controller-test/providers/Applications.Core/applications/test-app"}`, - }, - }, - annotations: deploymentAnnotations{ConfigurationHash: ""}, - err: fmt.Errorf("invalid status annotation: status.container type %q is not %q", "Applications.Core/applications", applicationsCoreContainersResourceType), - }, - { - name: "status-only-scope-set", - deployment: &appsv1.Deployment{ - Annotations: map[string]string{ - AnnotationRadiusStatus: `{"scope":"/planes/radius/local/resourceGroups/controller-test"}`, - }, - }, - annotations: deploymentAnnotations{ - ConfigurationHash: "", - Status: &deploymentStatus{ - Scope: "/planes/radius/local/resourceGroups/controller-test", - }, - }, - err: nil, - }, - { - name: "status-only-container-set", - deployment: &appsv1.Deployment{ - Annotations: map[string]string{ - AnnotationRadiusStatus: `{"container":"/planes/radius/local/resourceGroups/controller-test/providers/Applications.Core/containers/test-container"}`, - }, - }, - annotations: deploymentAnnotations{ConfigurationHash: ""}, - err: fmt.Errorf("invalid status annotation: status.scope must be set when status.container is set"), - }, - { - name: "status-invalid-scope-only", - deployment: &appsv1.Deployment{ - Annotations: map[string]string{ - AnnotationRadiusStatus: `{"scope":"not-a-scope"}`, - }, - }, - annotations: deploymentAnnotations{ConfigurationHash: ""}, - err: fmt.Errorf("invalid status annotation: invalid status.scope: %w", invalidScopeIDErr), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - annotations, err := readAnnotations(tt.deployment) - if tt.err == nil { - require.NoError(t, err) - } else { - require.EqualError(t, err, tt.err.Error()) - } - require.Equal(t, tt.annotations, annotations) - }) - } -} diff --git a/pkg/controller/reconciler/connections.go b/pkg/controller/reconciler/connections.go deleted file mode 100644 index d4d96348f57..00000000000 --- a/pkg/controller/reconciler/connections.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package reconciler - -import ( - "fmt" - "reflect" - "strings" - - "github.com/radius-project/radius/pkg/cli/clients_new/generated" -) - -// resourceToConnectionValues converts a resource to a map of connection values. This will filter out any -// properties that should not be considered as env-vars or secrets. -func resourceToConnectionEnvVars(name string, resource generated.GenericResource, secrets generated.GenericResourcesClientListSecretsResponse) (map[string]string, error) { - values, err := resourceToConnectionValues(resource) - if err != nil { - return nil, err - } - - for k, v := range secrets.Value { - values[k] = *v - } - - results := map[string]string{} - - for k, v := range values { - key := fmt.Sprintf("CONNECTION_%s_%s", strings.ToUpper(name), strings.ToUpper(k)) - results[key] = v - } - - for k, v := range secrets.Value { - key := fmt.Sprintf("CONNECTION_%s_%s", strings.ToUpper(name), strings.ToUpper(k)) - results[key] = *v - } - - return results, nil -} - -// resourceToConnectionValues converts a resource to a map of connection values. This will filter out any -// properties that should not be considered as env-vars or secrets. -func resourceToConnectionValues(resource generated.GenericResource) (map[string]string, error) { - values := map[string]string{} - - for k, v := range resource.Properties { - switch k { - case "application": - case "environment": - case "provisioningState": - case "resourceProvisioning": - case "recipe": - case "resources": - case "status": - default: - // Ignore composite types. Values are scalars. - kind := reflect.TypeOf(v).Kind() - if kind == reflect.Map { - break - } else if kind == reflect.Slice { - break - } else if kind == reflect.Struct { - break - } - - switch v := v.(type) { - case string: - values[k] = v - case bool: - values[k] = fmt.Sprintf("%t", v) - case float64: - values[k] = fmt.Sprintf("%v", v) - default: - return nil, fmt.Errorf("unsupported type for property %s: %T", k, v) - } - } - } - - return values, nil -} diff --git a/pkg/controller/reconciler/const.go b/pkg/controller/reconciler/const.go index 96090680949..45ec6cf8f07 100644 --- a/pkg/controller/reconciler/const.go +++ b/pkg/controller/reconciler/const.go @@ -28,37 +28,6 @@ const ( // which climbs to ~16m and extends total drain time. DeleteRetryDelay time.Duration = 30 * time.Second - // AnnotationRadiusEnabled is the name of the annotation that indicates if a Deployment has Radius enabled. - AnnotationRadiusEnabled = "radapp.io/enabled" - - // AnnotationRadiusConnectionPrefix is the name of the annotation that indicates the name of the connection to use. - AnnotationRadiusConnectionPrefix = "radapp.io/connection-" - - // AnnotationRadiusStatus is the name of the annotation that indicates the status of a Deployment. - AnnotationRadiusStatus = "radapp.io/status" - - // AnnotationRadiusConfigurationHash is the name of the annotation that indicates the hash of the configuration. - AnnotationRadiusConfigurationHash = "radapp.io/configuration-hash" - - // AnnotationRadiusEnvironment is the name of the annotation that indicates the name of the environment. If unset, - // the value 'default' will be used as the environment name. - AnnotationRadiusEnvironment = "radapp.io/environment" - - // AnnotationRadiusApplication is the name of the annotation that indicates the name of the application. If unset, - // the namespace of the Deployment will be used as the application name. - AnnotationRadiusApplication = "radapp.io/application" - - // DeploymentFinalizer is the name of the finalizer added to Deployments. - DeploymentFinalizer = "radapp.io/deployment-finalizer" - - // EventScopeMismatch is emitted when a deployment annotation references a scope outside - // of the deployment's ownership boundary. - EventScopeMismatch = "ScopeMismatch" - - // EventContainerOwnershipMismatch is emitted when a container does not reference - // the source Kubernetes deployment resource. - EventContainerOwnershipMismatch = "ContainerOwnershipMismatch" - // EventDeploymentResourceDeleteSkipped is emitted when the controller skips deleting the // resource referenced by a DeploymentResource because the object is not a controller-owned // child of a DeploymentTemplate whose deployment scope covers Spec.Id. @@ -68,11 +37,6 @@ const ( // controller owner reference on a DeploymentResource. deploymentTemplateKind = "DeploymentTemplate" - applicationsCoreContainersResourceType = "Applications.Core/containers" - - // RecipeFinalizer is the name of the finalizer added to Recipes. - RecipeFinalizer = "radapp.io/recipe-finalizer" - // DeploymentTemplateFinalizer is the name of the finalizer added to DeploymentTemplates. DeploymentTemplateFinalizer = "radapp.io/deployment-template-finalizer" diff --git a/pkg/controller/reconciler/deployment_reconciler.go b/pkg/controller/reconciler/deployment_reconciler.go deleted file mode 100644 index 04a8935cdfc..00000000000 --- a/pkg/controller/reconciler/deployment_reconciler.go +++ /dev/null @@ -1,722 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package reconciler - -import ( - "context" - "fmt" - "strings" - "time" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/tools/record" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - "github.com/go-logr/logr" - "github.com/radius-project/radius/pkg/cli/clients" - radappiov1alpha3 "github.com/radius-project/radius/pkg/controller/api/radapp.io/v1alpha3" - "github.com/radius-project/radius/pkg/corerp/api/v20231001preview" - "github.com/radius-project/radius/pkg/kubernetes" - sdkclients "github.com/radius-project/radius/pkg/sdk/clients" - "github.com/radius-project/radius/pkg/to" - "github.com/radius-project/radius/pkg/ucp/resources" - "github.com/radius-project/radius/pkg/ucp/ucplog" -) - -// DeploymentReconciler reconciles a Deployment object. -type DeploymentReconciler struct { - // Client is the Kubernetes client. - Client client.Client - - // Scheme is the Kubernetes scheme. - Scheme *runtime.Scheme - - // EventRecorder is the Kubernetes event recorder. - EventRecorder record.EventRecorder - - // Radius is the Radius client. - Radius RadiusClient - - // DelayInterval is the amount of time to wait between operations. - DelayInterval time.Duration -} - -// Reconcile is the main reconciliation loop for the Deployment resource. -func (r *DeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := ucplog.FromContextOrDiscard(ctx).WithValues("kind", "Deployment", "name", req.Name, "namespace", req.Namespace) - ctx = logr.NewContext(ctx, logger) - - deployment := appsv1.Deployment{} - err := r.Client.Get(ctx, req.NamespacedName, &deployment) - if apierrors.IsNotFound(err) { - // This can happen due to a data-race if the deployment is created and then deleted before we can - // reconcile it. There's nothing to do here. - logger.Info("Deployment has already been deleted.") - return ctrl.Result{}, nil - } else if err != nil { - logger.Error(err, "Unable to fetch resource.") - return ctrl.Result{}, err - } - - // Our algorithm is as follows: - // - // 1. Check if we have an "operation" in progress. If so, check it's status. - // a. If the operation is still in progress, then queue another reconcile (polling). - // b. If the operation completed successfully then update the status and continue processing (happy-path). - // c. If the operation failed then update the status and continue processing (retry). - // 2. If the deployment is being deleted then process deletion. - // a. This may require us to start a DELETE operation. After that we can continue polling. - // 3. If the deployment is not being deleted then process this as a creation or update. - // a. This may require us to start a PUT operation. After that we can continue polling. - // - // We do it this way because it guarantees that we only have one operation going at a time. - - // Since Deployment is a built-in type in Kubernetes we can't add our own status field to it. - // We have to store our status in an annotation. - annotations, err := readAnnotations(&deployment) - if err != nil { - logger.Error(err, "Failed to read deployment status.") - deployment.Annotations[AnnotationRadiusStatus] = "" - - // This could happen if someone manually edited the annotations. We can reset it to empty - // and repair it on the next reconcile. - } - - // If there is an operation in progress, then we need to continue polling it. - if annotations.OperationInProgress() { - // NOTE: if reconcileOperation completes successfully, then it will return a "zero" result, - // this means the operation has completed and we should continue processing. - result, err := r.reconcileOperation(ctx, &deployment, &annotations) - if err != nil { - logger.Error(err, "Unable to reconcile in-progress operation.") - return ctrl.Result{}, err - } else if result.IsZero() { - // NOTE: if reconcileOperation completes successfully, then it will return a "zero" result, - // this means the operation has completed and we should continue processing. - logger.Info("Operation completed successfully.") - } else { - logger.Info("Requeueing to continue operation.") - return result, nil - } - } - - // If the Deployment is being deleted **or** if Radius is no longer enabled, then we should - // clean up any Radius state. - // - // If the DeletionTimestamp is not nil, then it means that the Deployment is being deleted. - // - // If the Configuration is nil, which means that Radius is disabled, and Status is not nil, - // then we need to reconcile the deletion of the Deployment. - if deployment.DeletionTimestamp != nil || annotations.needsCleanup() { - return r.reconcileDelete(ctx, &deployment, &annotations) - } - - // When do we need to reconcile update? When Radius is enabled, no matter - // if it was previously enabled or disabled. - if annotations.isRadiusEnabled() { - return r.reconcileUpdate(ctx, &deployment, &annotations) - } - - r.EventRecorder.Event(&deployment, corev1.EventTypeNormal, "NoOp", fmt.Sprintf("Radius is not enabled for %s", deployment.Name)) - return ctrl.Result{}, nil -} - -// reconcileOperation reconciles a Deployment that has an operation in progress. -func (r *DeploymentReconciler) reconcileOperation(ctx context.Context, deployment *appsv1.Deployment, annotations *deploymentAnnotations) (ctrl.Result, error) { - logger := ucplog.FromContextOrDiscard(ctx) - - // NOTE: the pollers are actually different types, so we have to duplicate the code - // for the PUT and DELETE handling. This makes me sad :( but there isn't a great - // solution besides duplicating the code. - // - // The only difference between these two codepaths is how they handle success. - if annotations.Status.Operation.OperationKind == radappiov1alpha3.OperationKindPut { - poller, err := r.Radius.Containers(annotations.Status.Scope).ContinueCreateOperation(ctx, annotations.Status.Operation.ResumeToken) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to continue PUT operation: %w", err) - } - - _, err = poller.Poll(ctx) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to poll operation status: %w", err) - } - - if !poller.Done() { - return ctrl.Result{RequeueAfter: r.requeueDelay()}, nil - } - - // If we get here, the operation is complete. - _, err = poller.Result(ctx) - if err != nil { - // Operation failed, reset state and retry. - r.EventRecorder.Event(deployment, corev1.EventTypeWarning, "ResourceError", err.Error()) - logger.Error(err, "Update failed.") - - annotations.Status.Operation = nil - annotations.Status.Phrase = deploymentPhraseFailed - - err = r.saveState(ctx, deployment, annotations) - if err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{RequeueAfter: r.requeueDelay()}, nil - } - - // If we get here, the operation was a success. Update the status and continue. - // - // NOTE: we don't need to save the status here, because we're going to continue reconciling. - annotations.Status.Operation = nil - annotations.Status.Container = annotations.Status.Scope + "/providers/Applications.Core/containers/" + deployment.Name - return ctrl.Result{}, nil - } else if annotations.Status.Operation.OperationKind == radappiov1alpha3.OperationKindDelete { - poller, err := r.Radius.Containers(annotations.Status.Scope).ContinueDeleteOperation(ctx, annotations.Status.Operation.ResumeToken) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to continue DELETE operation: %w", err) - } - - _, err = poller.Poll(ctx) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to poll operation status: %w", err) - } - - if !poller.Done() { - return ctrl.Result{RequeueAfter: r.requeueDelay()}, nil - } - - // If we get here, the operation is complete. - _, err = poller.Result(ctx) - if err != nil { - // Operation failed, reset state and retry. - r.EventRecorder.Event(deployment, corev1.EventTypeWarning, "ResourceError", err.Error()) - logger.Error(err, "Delete failed.") - - annotations.Status.Operation = nil - annotations.Status.Phrase = deploymentPhraseFailed - - err = r.saveState(ctx, deployment, annotations) - if err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{RequeueAfter: r.requeueDelay()}, nil - } - - // If we get here, the operation was a success. Update the status and continue. - // - // NOTE: we don't need to save the status here, because we're going to continue reconciling. - annotations.Status.Operation = nil - annotations.Status.Container = "" - return ctrl.Result{}, nil - } - - // If we get here, this was an unknown operation kind. This is a bug in our code, or someone - // tampered with the status of the object. Just reset the state and move on. - logger.Error(fmt.Errorf("unknown operation kind: %s", annotations.Status.Operation.OperationKind), "Unknown operation kind.") - - annotations.Status.Operation = nil - annotations.Status.Phrase = deploymentPhraseFailed - - err := r.saveState(ctx, deployment, annotations) - if err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil -} - -func (r *DeploymentReconciler) reconcileUpdate(ctx context.Context, deployment *appsv1.Deployment, annotations *deploymentAnnotations) (ctrl.Result, error) { - logger := ucplog.FromContextOrDiscard(ctx) - - // Ensure that our finalizer is present before we start any operations. - if controllerutil.AddFinalizer(deployment, DeploymentFinalizer) { - err := r.Client.Update(ctx, deployment) - if err != nil { - return ctrl.Result{}, err - } - } - - environmentName := "default" - if annotations.Configuration.Environment != "" { - environmentName = annotations.Configuration.Environment - } - - applicationName := deployment.Namespace - if annotations.Configuration.Application != "" { - applicationName = annotations.Configuration.Application - } - - resourceGroupID, environmentID, applicationID, err := resolveDependencies(ctx, r.Radius, "/planes/radius/local", environmentName, applicationName) - if err != nil { - r.EventRecorder.Event(deployment, corev1.EventTypeWarning, "DependencyError", err.Error()) - logger.Error(err, "Unable to resolve dependencies.") - return ctrl.Result{}, fmt.Errorf("failed to resolve dependencies: %w", err) - } - - if annotations.Status == nil { - annotations.Status = &deploymentStatus{} - } - - annotations.Status.Scope = resourceGroupID - annotations.Status.Environment = environmentID - annotations.Status.Application = applicationID - - // There are three possible states returned here: - // - // 1) err != nil - an error happened, this will be retried next reconcile. - // 2) waiting == true - we're waiting on dependencies, this will be retried next reconcile. - // 3) updatePoller != nil - we've started a PUT operation, this will be checked next reconcile. - // 4) deletePoller != nil - we've started a DELETE operation, this will be checked next reconcile. - updatePoller, deletePoller, waiting, err := r.startPutOrDeleteOperationIfNeeded(ctx, deployment, annotations) - if err != nil { - logger.Error(err, "Unable to create or update resource.") - r.EventRecorder.Event(deployment, corev1.EventTypeWarning, "ResourceError", err.Error()) - return ctrl.Result{}, err - } else if waiting { - logger.Info("Waiting on dependencies.") - r.EventRecorder.Event(deployment, corev1.EventTypeNormal, "DependencyNotReady", "Waiting on dependencies.") - - annotations.Status.Phrase = deploymentPhraseWaiting - err = r.saveState(ctx, deployment, annotations) - if err != nil { - return ctrl.Result{}, err - } - - // We don't need to requeue here because we watch Recipes and will be notified when - // the state changes. - return ctrl.Result{}, nil - } else if updatePoller != nil { - // We've successfully started an operation. Update the status and requeue. - token, err := updatePoller.ResumeToken() - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to get operation token: %w", err) - } - - annotations.Status.Operation = &radappiov1alpha3.ResourceOperation{ResumeToken: token, OperationKind: radappiov1alpha3.OperationKindPut} - annotations.Status.Phrase = deploymentPhraseUpdating - err = r.saveState(ctx, deployment, annotations) - if err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{RequeueAfter: r.requeueDelay()}, nil - } else if deletePoller != nil { - // We've successfully started an operation. Update the status and requeue. - token, err := deletePoller.ResumeToken() - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to get operation token: %w", err) - } - - annotations.Status.Operation = &radappiov1alpha3.ResourceOperation{ResumeToken: token, OperationKind: radappiov1alpha3.OperationKindDelete} - annotations.Status.Phrase = deploymentPhraseDeleting - err = r.saveState(ctx, deployment, annotations) - if err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{RequeueAfter: r.requeueDelay()}, nil - } - - // If we get here then it means we can process the result of the operation. - logger.Info("Resource is in desired state.", "resourceId", annotations.Status.Container) - - annotations.Status.Phrase = deploymentPhraseReady - err = r.updateDeployment(ctx, deployment, annotations) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to update deployment: %w", err) - } - - err = r.saveState(ctx, deployment, annotations) - if err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil -} - -func (r *DeploymentReconciler) reconcileDelete(ctx context.Context, deployment *appsv1.Deployment, annotations *deploymentAnnotations) (ctrl.Result, error) { - logger := ucplog.FromContextOrDiscard(ctx) - - poller, err := r.startDeleteOperationIfNeeded(ctx, deployment, annotations) - if err != nil { - logger.Error(err, "Unable to delete resource.") - r.EventRecorder.Event(deployment, corev1.EventTypeWarning, "ResourceError", err.Error()) - return ctrl.Result{}, err - } else if poller != nil { - // We've successfully started an operation. Update the status and requeue. - token, err := poller.ResumeToken() - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to get operation token: %w", err) - } - - annotations.Status.Operation = &radappiov1alpha3.ResourceOperation{ResumeToken: token, OperationKind: radappiov1alpha3.OperationKindDelete} - annotations.Status.Phrase = deploymentPhraseDeleting - err = r.saveState(ctx, deployment, annotations) - if err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{RequeueAfter: r.requeueDelay()}, nil - } - - logger.Info("Resource is deleted.") - - err = r.cleanupDeployment(ctx, deployment) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to cleanup deployment: %w", err) - } - - // At this point we've cleaned up everything. We can remove the finalizer which will allow deletion of the - // recipe. - controllerutil.RemoveFinalizer(deployment, DeploymentFinalizer) - err = r.Client.Update(ctx, deployment) - if err != nil { - return ctrl.Result{}, err - } - - r.EventRecorder.Event(deployment, corev1.EventTypeNormal, "Reconciled", "Successfully reconciled resource.") - return ctrl.Result{}, nil -} - -func (r *DeploymentReconciler) startPutOrDeleteOperationIfNeeded(ctx context.Context, deployment *appsv1.Deployment, annotations *deploymentAnnotations) (sdkclients.Poller[v20231001preview.ContainersClientCreateOrUpdateResponse], sdkclients.Poller[v20231001preview.ContainersClientDeleteResponse], bool, error) { - logger := ucplog.FromContextOrDiscard(ctx) - - resourceID := annotations.Status.Scope + "/providers/Applications.Core/containers/" + deployment.Name - - // Check the annotations first to see how the current configuration compares to the desired configuration. - if annotations.Status.Container != "" && !strings.EqualFold(annotations.Status.Container, resourceID) { - // If we get here it means that the environment or application changed, so we should delete - // the old resource and create a new one. - logger.Info("Container is already created but is out-of-date") - - poller, err := r.startDeleteOperationIfNeeded(ctx, deployment, annotations) - if err != nil { - return nil, nil, false, err - } else if poller != nil { - return nil, poller, false, nil - } - - // Deletion completed synchronously. - annotations.Status.Container = "" - } - - // Note: we separate this check from the previous block, because it could complete synchronously. - if !annotations.IsUpToDate() { - logger.Info("Container configuration is out-of-date.") - } else if annotations.Status.Container != "" { - logger.Info("Container is already created and is up-to-date.") - return nil, nil, false, nil - } - - logger.Info("Starting PUT operation.") - properties := v20231001preview.ContainerProperties{ - Application: new(annotations.Status.Application), - ResourceProvisioning: to.Ptr(v20231001preview.ContainerResourceProvisioningManual), - Connections: map[string]*v20231001preview.ConnectionProperties{}, - Container: &v20231001preview.Container{ - Image: new("none"), - }, - Resources: []*v20231001preview.ResourceReference{ - { - ID: new("/planes/kubernetes/local/namespaces/" + deployment.Namespace + "/providers/apps/Deployment/" + deployment.Name), - }, - }, - } - - for name, source := range annotations.Configuration.Connections { - recipe := radappiov1alpha3.Recipe{} - err := r.Client.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: source}, &recipe) - if apierrors.IsNotFound(err) { - logger.Info("Recipe does not exist.", "recipe", source) - return nil, nil, true, nil - } else if err != nil { - return nil, nil, false, fmt.Errorf("failed to fetch recipe %s: %w", source, err) - } else if recipe.Status.Resource == "" { - logger.Info("Recipe is not ready.", "recipe", source) - return nil, nil, true, nil - } - - properties.Connections[name] = &v20231001preview.ConnectionProperties{ - Source: new(recipe.Status.Resource), - } - } - - poller, err := createOrUpdateContainer(ctx, r.Radius, resourceID, &properties) - if err != nil { - return nil, nil, false, err - } else if poller != nil { - return poller, nil, false, nil - } - - // Update completed synchronously - annotations.Status.Container = resourceID - return poller, nil, false, nil -} - -func (r *DeploymentReconciler) startDeleteOperationIfNeeded(ctx context.Context, deployment *appsv1.Deployment, annotations *deploymentAnnotations) (sdkclients.Poller[v20231001preview.ContainersClientDeleteResponse], error) { - logger := ucplog.FromContextOrDiscard(ctx) - - if annotations == nil || annotations.Status == nil { - logger.Info("Container is already deleted (or was never created).") - return nil, nil - } - - if annotations.Status.Container == "" { - logger.Info("Container is already deleted (or was never created).") - return nil, nil - } - - expectedDeploymentResourceID := makeKubernetesDeploymentResourceID(deployment.Namespace, deployment.Name) - container, err := fetchContainerResource(ctx, r.Radius, annotations.Status.Container) - if clients.Is404Error(err) { - logger.Info("Container was already deleted before cleanup began.", "container", annotations.Status.Container) - annotations.Status.Container = "" - return nil, nil - } else if err != nil { - return nil, fmt.Errorf("failed to fetch container before delete: %w", err) - } - - if !containerHasResourceReference(&container, expectedDeploymentResourceID) { - logger.Info("Refusing cross-ownership delete attempt.", - "container", annotations.Status.Container, - "expectedDeploymentResource", expectedDeploymentResourceID, - ) - r.EventRecorder.Event(deployment, corev1.EventTypeWarning, EventContainerOwnershipMismatch, - fmt.Sprintf("Container %q does not reference deployment resource %q", annotations.Status.Container, expectedDeploymentResourceID)) - - // Clear status to avoid a persistent reconcile loop on a tampered annotation. - annotations.Status.Container = "" - return nil, nil - } - - logger.Info("Starting DELETE operation.") - poller, err := deleteContainer(ctx, r.Radius, annotations.Status.Container) - if err != nil { - return nil, err - } else if poller != nil { - return poller, nil - } - - // Deletion completed synchronously. - annotations.Status.Container = "" - return nil, nil -} - -func (r *DeploymentReconciler) updateDeployment(ctx context.Context, deployment *appsv1.Deployment, annotations *deploymentAnnotations) error { - // We store the connection values in a Kubernetes secret and then use the secret to populate environment variables. - secretName := client.ObjectKey{Namespace: deployment.Namespace, Name: fmt.Sprintf("%s-connections", deployment.Name)} - - if len(annotations.Configuration.Connections) == 0 { - // No need for a secret if there are no connections. - removeSecretReference(deployment, fmt.Sprintf("%s-connections", deployment.Name)) - delete(deployment.Spec.Template.ObjectMeta.Annotations, kubernetes.AnnotationSecretHash) - - err := r.Client.Delete(ctx, &corev1.Secret{Namespace: secretName.Namespace, Name: secretName.Name}) - if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("failed to delete secret %s: %w", secretName.Name, err) - } - - return nil - } - - // First retrieve the secret. - createSecret := false - secret := corev1.Secret{} - err := r.Client.Get(ctx, secretName, &secret) - if apierrors.IsNotFound(err) { - // It's OK if the secret doesn't exist yet. We'll create it below. - createSecret = true - secret.Name = secretName.Name - secret.Namespace = secretName.Namespace - secret.OwnerReferences = []metav1.OwnerReference{ - *metav1.NewControllerRef(deployment, appsv1.SchemeGroupVersion.WithKind("Deployment")), - } - } else if err != nil { - return fmt.Errorf("failed to fetch secret %s: %w", secretName, err) - } - - // envtest has some quirky behavior around StringData which makes it hard to test. So we're - // using Data directly. - secret.Data = map[string][]byte{} - - for name, source := range annotations.Configuration.Connections { - recipe := radappiov1alpha3.Recipe{} - err := r.Client.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: source}, &recipe) - if err != nil { - return fmt.Errorf("failed to fetch recipe %s: %w", source, err) - } - - if recipe.Status.Resource == "" { - return fmt.Errorf("recipe %s is not ready", source) - } - - id, err := resources.Parse(recipe.Status.Resource) - if err != nil { - return err - } - - response, err := r.Radius.Resources(id.RootScope(), id.Type()).Get(ctx, id.Name()) - if err != nil { - return fmt.Errorf("failed to fetch resource %s: %w", id, err) - } - - secrets, err := r.Radius.Resources(id.RootScope(), id.Type()).ListSecrets(ctx, id.Name()) - if clients.Is404Error(err) { - // This is fine. The resource doesn't have any secrets. - secrets.Value = map[string]*string{} - } else if err != nil { - return fmt.Errorf("failed to fetch secrets for resource %s: %w", id, err) - } - - values, err := resourceToConnectionEnvVars(name, response.GenericResource, secrets) - if err != nil { - return fmt.Errorf("failed to read values resource %s: %w", id, err) - } - - for k, v := range values { - secret.Data[k] = []byte(v) - } - } - - // Add the hash of the secret data to the Pod definition. This will force a rollout when the secrets - // change. - hash := kubernetes.HashSecretData(secret.Data) - if deployment.Spec.Template.ObjectMeta.Annotations == nil { - deployment.Spec.Template.ObjectMeta.Annotations = map[string]string{} - } - deployment.Spec.Template.ObjectMeta.Annotations[kubernetes.AnnotationSecretHash] = hash - - addSecretReference(deployment, secretName.Name) - - if createSecret { - err = r.Client.Create(ctx, &secret) - if err != nil { - return fmt.Errorf("failed to create secret %s: %w", secretName, err) - } - } else { - err = r.Client.Update(ctx, &secret) - if err != nil { - return fmt.Errorf("failed to update secret %s: %w", secretName, err) - } - } - - return nil -} - -func (r *DeploymentReconciler) cleanupDeployment(ctx context.Context, deployment *appsv1.Deployment) error { - delete(deployment.Annotations, AnnotationRadiusStatus) - delete(deployment.Annotations, AnnotationRadiusConfigurationHash) - delete(deployment.Spec.Template.ObjectMeta.Annotations, kubernetes.AnnotationSecretHash) - - secretName := client.ObjectKey{Namespace: deployment.Namespace, Name: fmt.Sprintf("%s-connections", deployment.Name)} - err := r.Client.Delete(ctx, &corev1.Secret{Namespace: secretName.Namespace, Name: secretName.Name}) - if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("failed to delete secret %s: %w", secretName.Name, err) - } - - removeSecretReference(deployment, secretName.Name) - return nil -} - -func (r *DeploymentReconciler) saveState(ctx context.Context, deployment *appsv1.Deployment, annotations *deploymentAnnotations) error { - err := annotations.ApplyToDeployment(deployment) - if err != nil { - return fmt.Errorf("unable to apply annotations: %w", err) - } - - err = r.Client.Update(ctx, deployment) - if err != nil { - return err - } - - return nil -} - -func (r *DeploymentReconciler) findDeploymentsForRecipe(ctx context.Context, obj client.Object) []reconcile.Request { - recipe, ok := obj.(*radappiov1alpha3.Recipe) - if !ok { - return []reconcile.Request{} - } - - deployments := &appsv1.DeploymentList{} - options := &client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(indexField, recipe.Name), - Namespace: recipe.Namespace, - } - err := r.Client.List(ctx, deployments, options) - if err != nil { - return []reconcile.Request{} - } - - requests := []reconcile.Request{} - for _, item := range deployments.Items { - requests = append(requests, reconcile.Request{ - Name: item.GetName(), - Namespace: item.GetNamespace(), - }) - } - return requests -} - -func (r *DeploymentReconciler) requeueDelay() time.Duration { - delay := r.DelayInterval - if delay == 0 { - delay = PollingDelay - } - - return delay -} - -const indexField = "spec.recipe-reference" - -func (r *DeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error { - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &appsv1.Deployment{}, indexField, func(rawObj client.Object) []string { - deployment := rawObj.(*appsv1.Deployment) - annotations, err := readAnnotations(deployment) - if err != nil { - return []string{} - } else if annotations.Configuration == nil { - return []string{} - } - - recipes := []string{} - for _, recipe := range annotations.Configuration.Connections { - recipes = append(recipes, recipe) - } - - return recipes - }); err != nil { - return err - } - - return ctrl.NewControllerManagedBy(mgr). - For(&appsv1.Deployment{}). - Watches(&radappiov1alpha3.Recipe{}, handler.EnqueueRequestsFromMapFunc(r.findDeploymentsForRecipe), builder.WithPredicates(predicate.ResourceVersionChangedPredicate{})). - Owns(&corev1.Secret{}). - Complete(r) -} diff --git a/pkg/controller/reconciler/deployment_reconciler_test.go b/pkg/controller/reconciler/deployment_reconciler_test.go deleted file mode 100644 index aec4127ed6b..00000000000 --- a/pkg/controller/reconciler/deployment_reconciler_test.go +++ /dev/null @@ -1,806 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package reconciler - -import ( - "context" - "fmt" - "net/http" - "testing" - "time" - - "github.com/radius-project/radius/pkg/cli/clients_new/generated" - radappiov1alpha3 "github.com/radius-project/radius/pkg/controller/api/radapp.io/v1alpha3" - "github.com/radius-project/radius/pkg/corerp/api/v20231001preview" - "github.com/radius-project/radius/pkg/kubernetes" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - crconfig "sigs.k8s.io/controller-runtime/pkg/config" - "sigs.k8s.io/controller-runtime/pkg/metrics/server" -) - -const ( - deploymentTestWaitDuration = time.Second * 10 - deploymentTestWaitInterval = time.Millisecond * 200 - deploymentTestControllerDelayInterval = time.Millisecond * 100 -) - -func SetupDeploymentTest(t *testing.T) (*mockRadiusClient, client.Client) { - SkipWithoutEnvironment(t) - - // Shut down the manager when the test exits. - ctx, cancel := context.WithCancel(t.Context()) - - mgr, err := ctrl.NewManager(config, ctrl.Options{ - Scheme: scheme, - Controller: crconfig.Controller{ - SkipNameValidation: new(true), - }, - - // Suppress metrics in tests to avoid conflicts. - Metrics: server.Options{ - BindAddress: "0", - }, - }) - require.NoError(t, err) - - radius := NewMockRadiusClient() - //nolint:staticcheck // SA1019: GetEventRecorderFor is deprecated but migration to new events API requires significant refactoring - err = (&DeploymentReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - EventRecorder: mgr.GetEventRecorderFor("deployment-controller"), - Radius: radius, - DelayInterval: deploymentTestControllerDelayInterval, - }).SetupWithManager(mgr) - require.NoError(t, err) - - startManager(t, mgr, ctx, cancel) - - return radius, mgr.GetClient() -} - -func Test_DeploymentReconciler_StartDeleteOperationIfNeeded_OwnershipMismatch_BlocksDelete(t *testing.T) { - ctx := t.Context() - radius := NewMockRadiusClient() - reconciler := &DeploymentReconciler{ - Radius: radius, - EventRecorder: record.NewFakeRecorder(10), - } - - deployment := &appsv1.Deployment{Name: "current-app", Namespace: "current-namespace"} - containerID := "/planes/radius/local/resourceGroups/tenant-b/providers/Applications.Core/containers/other-container" - - radius.Update(func() { - radius.containers[containerID] = v20231001preview.ContainerResource{ - Properties: &v20231001preview.ContainerProperties{ - Resources: []*v20231001preview.ResourceReference{{ - ID: new("/planes/kubernetes/local/namespaces/other-namespace/providers/apps/Deployment/other-app"), - }}, - }, - } - }) - - annotations := &deploymentAnnotations{Status: &deploymentStatus{Container: containerID}} - poller, err := reconciler.startDeleteOperationIfNeeded(ctx, deployment, annotations) - require.NoError(t, err) - require.Nil(t, poller) - require.Empty(t, annotations.Status.Container) - requireNoDeleteOperation(t, radius, containerID) - - _, err = radius.Containers("/planes/radius/local/resourceGroups/tenant-b").Get(ctx, "other-container", nil) - require.NoError(t, err) -} - -func Test_DeploymentReconciler_StartPutOrDeleteOperationIfNeeded_OwnershipMismatch_BlocksDelete(t *testing.T) { - ctx := t.Context() - radius := NewMockRadiusClient() - reconciler := &DeploymentReconciler{ - Radius: radius, - EventRecorder: record.NewFakeRecorder(10), - } - - deployment := &appsv1.Deployment{Name: "current-app", Namespace: "current-namespace"} - otherContainerID := "/planes/radius/local/resourceGroups/tenant-b/providers/Applications.Core/containers/other-container" - - radius.Update(func() { - radius.containers[otherContainerID] = v20231001preview.ContainerResource{ - Properties: &v20231001preview.ContainerProperties{ - Resources: []*v20231001preview.ResourceReference{{ - ID: new("/planes/kubernetes/local/namespaces/other-namespace/providers/apps/Deployment/other-container"), - }}, - }, - } - }) - - annotations := &deploymentAnnotations{ - Configuration: &deploymentConfiguration{}, - Status: &deploymentStatus{ - Scope: "/planes/radius/local/resourceGroups/tenant-a", - Application: "/planes/radius/local/resourceGroups/tenant-a/providers/Applications.Core/applications/current-app", - Container: otherContainerID, - }, - } - - updatePoller, deletePoller, waiting, err := reconciler.startPutOrDeleteOperationIfNeeded(ctx, deployment, annotations) - require.NoError(t, err) - require.NotNil(t, updatePoller) - require.Nil(t, deletePoller) - require.False(t, waiting) - requireNoDeleteOperation(t, radius, otherContainerID) - - _, err = radius.Containers("/planes/radius/local/resourceGroups/tenant-b").Get(ctx, "other-container", nil) - require.NoError(t, err) -} - -func Test_DeploymentReconciler_StartDeleteOperationIfNeeded_OwnershipMatch_AllowsDelete(t *testing.T) { - ctx := t.Context() - radius := NewMockRadiusClient() - reconciler := &DeploymentReconciler{ - Radius: radius, - EventRecorder: record.NewFakeRecorder(10), - } - - deployment := &appsv1.Deployment{Name: "current-app", Namespace: "current-namespace"} - containerID := "/planes/radius/local/resourceGroups/tenant-a/providers/Applications.Core/containers/current-app" - - radius.Update(func() { - radius.containers[containerID] = v20231001preview.ContainerResource{ - Properties: &v20231001preview.ContainerProperties{ - Resources: []*v20231001preview.ResourceReference{{ - ID: new(makeKubernetesDeploymentResourceID("current-namespace", "current-app")), - }}, - }, - } - }) - - annotations := &deploymentAnnotations{Status: &deploymentStatus{Container: containerID}} - poller, err := reconciler.startDeleteOperationIfNeeded(ctx, deployment, annotations) - require.NoError(t, err) - require.NotNil(t, poller) - require.Equal(t, containerID, annotations.Status.Container) -} - -func requireNoDeleteOperation(t *testing.T, radius *mockRadiusClient, resourceID string) { - t.Helper() - - radius.lock.Lock() - defer radius.lock.Unlock() - - for _, operation := range radius.operations { - require.False(t, operation.Kind == http.MethodDelete && operation.ResourceID == resourceID, "unexpected delete operation for %s", resourceID) - } -} - -// Creates a deployment with Radius enabled. -// -// Then exercises the cleanup path by deleting the deployment. -func Test_DeploymentReconciler_RadiusEnabled_ThenDeploymentDeleted(t *testing.T) { - ctx := t.Context() - radius, client := SetupDeploymentTest(t) - - name := types.NamespacedName{Namespace: "deployment-enabled-deleted", Name: "test-deployment-enabled-deleted"} - err := client.Create(ctx, &corev1.Namespace{Name: name.Namespace}) - require.NoError(t, err) - - deployment := makeDeployment(name) - deployment.Annotations[AnnotationRadiusEnabled] = "true" - err = client.Create(ctx, deployment) - require.NoError(t, err) - - // Deployment will be waiting for environment to be created. - createEnvironment(radius, "default", "default") - - // Deployment will be waiting for container to complete deployment. - annotations := waitForStateUpdating(t, client, name) - - radius.CompleteOperation(annotations.Status.Operation.ResumeToken, nil) - - // Deployment will update after operation completes - annotations = waitForStateReady(t, client, name) - - err = client.Get(ctx, name, deployment) - require.NoError(t, err) - - // We should not have created a secret reference since there are no connections. - require.False(t, removeSecretReference(deployment, deployment.Name+"-connections")) - - container, err := radius.Containers(annotations.Status.Scope).Get(ctx, deployment.Name, nil) - require.NoError(t, err) - require.Equal(t, "manual", string(*container.Properties.ResourceProvisioning)) - require.Equal(t, []*v20231001preview.ResourceReference{{ID: new("/planes/kubernetes/local/namespaces/deployment-enabled-deleted/providers/apps/Deployment/" + deployment.Name)}}, container.Properties.Resources) - - err = client.Delete(ctx, deployment) - require.NoError(t, err) - - // Deletion of the container is in progress. - annotations = waitForStateDeleting(t, client, name) - radius.CompleteOperation(annotations.Status.Operation.ResumeToken, nil) - - // Now deleting of the deployment object can complete. - waitForDeploymentDeleted(t, client, name) -} - -func Test_DeploymentReconciler_ChangeEnvironmentAndApplication(t *testing.T) { - ctx := t.Context() - radius, client := SetupDeploymentTest(t) - - name := types.NamespacedName{Namespace: "deployment-change-envapp", Name: "test-deployment-change-envapp"} - err := client.Create(ctx, &corev1.Namespace{Name: name.Namespace}) - require.NoError(t, err) - - deployment := makeDeployment(name) - deployment.Annotations[AnnotationRadiusEnabled] = "true" - err = client.Create(ctx, deployment) - require.NoError(t, err) - - // Deployment will be waiting for environment to be created. - createEnvironment(radius, "default", "default") - - // Deployment will be waiting for container to complete deployment. - annotations := waitForStateUpdating(t, client, name) - require.Equal(t, "/planes/radius/local/resourcegroups/default-deployment-change-envapp", annotations.Status.Scope) - require.Equal(t, "/planes/radius/local/resourceGroups/default/providers/Applications.Core/environments/default", annotations.Status.Environment) - require.Equal(t, "/planes/radius/local/resourcegroups/default-deployment-change-envapp/providers/Applications.Core/applications/deployment-change-envapp", annotations.Status.Application) - radius.CompleteOperation(annotations.Status.Operation.ResumeToken, nil) - - // Deployment will update after operation completes - annotations = waitForStateReady(t, client, name) - require.Equal(t, "/planes/radius/local/resourcegroups/default-deployment-change-envapp/providers/Applications.Core/containers/test-deployment-change-envapp", annotations.Status.Container) - - createEnvironment(radius, "new-environment", "new-environment") - - // Now update the deployment to change the environment and application. - err = client.Get(ctx, name, deployment) - require.NoError(t, err) - - deployment.Annotations[AnnotationRadiusEnvironment] = "new-environment" - deployment.Annotations[AnnotationRadiusApplication] = "new-application" - - err = client.Update(ctx, deployment) - require.NoError(t, err) - - // Now the deployment will delete and re-create the resource. - - // Deletion of the container is in progress. - annotations = waitForStateDeleting(t, client, name) - radius.CompleteOperation(annotations.Status.Operation.ResumeToken, nil) - - // Resource should be gone. - _, err = radius.Containers(annotations.Status.Scope).Get(ctx, name.Name, nil) - require.Error(t, err) - - // Recipe will be waiting for extender to complete provisioning. - annotations = waitForStateUpdating(t, client, name) - require.Equal(t, "/planes/radius/local/resourcegroups/new-environment-new-application", annotations.Status.Scope) - require.Equal(t, "/planes/radius/local/resourceGroups/new-environment/providers/Applications.Core/environments/new-environment", annotations.Status.Environment) - require.Equal(t, "/planes/radius/local/resourcegroups/new-environment-new-application/providers/Applications.Core/applications/new-application", annotations.Status.Application) - radius.CompleteOperation(annotations.Status.Operation.ResumeToken, nil) - - // Recipe will update after operation completes - annotations = waitForStateReady(t, client, name) - require.Equal(t, "/planes/radius/local/resourcegroups/new-environment-new-application/providers/Applications.Core/containers/test-deployment-change-envapp", annotations.Status.Container) - - err = client.Delete(ctx, deployment) - require.NoError(t, err) - - // Deletion of the container is in progress. - annotations = waitForStateDeleting(t, client, name) - radius.CompleteOperation(annotations.Status.Operation.ResumeToken, nil) - - // Now deleting of the deployment object can complete. - waitForDeploymentDeleted(t, client, name) -} - -// Creates a deployment with Radius enabled. -// -// Then exercises the cleanup path by disabling Radius. -func Test_DeploymentReconciler_RadiusEnabled_ThenRadiusDisabled(t *testing.T) { - ctx := t.Context() - radius, client := SetupDeploymentTest(t) - - name := types.NamespacedName{Namespace: "deployment-enabled-disabled", Name: "test-deployment-enabled-disabled"} - err := client.Create(ctx, &corev1.Namespace{Name: name.Namespace}) - require.NoError(t, err) - - deployment := makeDeployment(name) - deployment.Annotations[AnnotationRadiusEnabled] = "true" - err = client.Create(ctx, deployment) - require.NoError(t, err) - - // Deployment will be waiting for environment to be created. - createEnvironment(radius, "default", "default") - - // Deployment will be waiting for container to complete deployment. - annotations := waitForStateUpdating(t, client, name) - - radius.CompleteOperation(annotations.Status.Operation.ResumeToken, nil) - - // Deployment will update after operation completes - annotations = waitForStateReady(t, client, name) - - container, err := radius.Containers(annotations.Status.Scope).Get(ctx, deployment.Name, nil) - require.NoError(t, err) - require.Equal(t, "manual", string(*container.Properties.ResourceProvisioning)) - require.Equal(t, []*v20231001preview.ResourceReference{{ID: new("/planes/kubernetes/local/namespaces/deployment-enabled-disabled/providers/apps/Deployment/" + deployment.Name)}}, container.Properties.Resources) - - // Trigger cleanup by disabling Radius. - err = client.Get(ctx, name, deployment) - require.NoError(t, err) - deployment.Annotations[AnnotationRadiusEnabled] = "false" - err = client.Update(ctx, deployment) - require.NoError(t, err) - - // Deletion of the container is in progress. - annotations = waitForStateDeleting(t, client, name) - radius.CompleteOperation(annotations.Status.Operation.ResumeToken, nil) - - waitForRadiusContainerDeleted(t, client, name) -} - -// Creates a deployment with Radius enabled and connections to two recipes. -// -// Then makes those recipes Ready so connections can be enabled. -// -// Then changes the configuration to *drop* one of the connections. -// -// Then exercises the cleanup path by disabling Radius - and shows that we can revert -// the changes Radius made to the deployment. -func Test_DeploymentReconciler_Connections(t *testing.T) { - ctx := t.Context() - radius, client := SetupDeploymentTest(t) - - name := types.NamespacedName{Namespace: "deployment-connections", Name: "test-deployment-connections"} - secretName := types.NamespacedName{Namespace: name.Namespace, Name: fmt.Sprintf("%s-connections", name.Name)} - err := client.Create(ctx, &corev1.Namespace{Name: name.Namespace}) - require.NoError(t, err) - - deployment := makeDeployment(name) - deployment.Annotations[AnnotationRadiusEnabled] = "true" - deployment.Annotations[AnnotationRadiusConnectionPrefix+"a"] = "recipe-a" - deployment.Annotations[AnnotationRadiusConnectionPrefix+"b"] = "recipe-b" - - err = client.Create(ctx, deployment) - require.NoError(t, err) - - // Deployment will be waiting for environment to be created. - createEnvironment(radius, "default", "default") - - // Deployment will be waiting for recipe resources to be created - _ = waitForStateWaiting(t, client, name) - - // Create the recipes, but don't mark them as provisioned yet. - recipeA := makeRecipe(types.NamespacedName{Namespace: name.Namespace, Name: "recipe-a"}, "Applications.Core/extenders") - recipeB := makeRecipe(types.NamespacedName{Namespace: name.Namespace, Name: "recipe-b"}, "Applications.Core/extenders") - - err = client.Create(ctx, recipeA) - require.NoError(t, err) - err = client.Create(ctx, recipeB) - require.NoError(t, err) - - // Deployment will be waiting for recipe resources to be created. - annotations := waitForStateWaiting(t, client, name) - - // Create the radius resources associated with the recipes - extenderA := generated.GenericResource{ - Properties: map[string]any{ - "a-value": "a", - "secrets": map[string]string{ - "a-secret": "a", - }, - }, - } - poller, err := radius.Resources(annotations.Status.Scope, "Applications.Core/extenders").BeginCreateOrUpdate(ctx, recipeA.Name, extenderA, nil) - require.NoError(t, err) - token, err := poller.ResumeToken() - require.NoError(t, err) - radius.CompleteOperation(token, nil) - - extenderB := generated.GenericResource{ - Properties: map[string]any{ - "b-value": "b", - "secrets": map[string]string{ - "b-secret": "b", - }, - }, - } - poller, err = radius.Resources(annotations.Status.Scope, "Applications.Core/extenders").BeginCreateOrUpdate(ctx, recipeB.Name, extenderB, nil) - require.NoError(t, err) - token, err = poller.ResumeToken() - require.NoError(t, err) - radius.CompleteOperation(token, nil) - - recipeA.Status = radappiov1alpha3.RecipeStatus{ - Resource: annotations.Status.Scope + "/providers/Applications.Core/extenders/" + recipeA.Name, - } - recipeB.Status = radappiov1alpha3.RecipeStatus{ - Resource: annotations.Status.Scope + "/providers/Applications.Core/extenders/" + recipeB.Name, - } - - // Mark the recipes as provisioned. - err = client.Status().Update(ctx, recipeA) - require.NoError(t, err) - err = client.Status().Update(ctx, recipeB) - require.NoError(t, err) - - // Now we can create the container - annotations = waitForStateUpdating(t, client, name) - - radius.CompleteOperation(annotations.Status.Operation.ResumeToken, nil) - - // Deployment will update after operation completes - annotations = waitForStateReady(t, client, name) - - container, err := radius.Containers(annotations.Status.Scope).Get(ctx, deployment.Name, nil) - require.NoError(t, err) - require.Equal(t, "manual", string(*container.Properties.ResourceProvisioning)) - require.Equal(t, map[string]*v20231001preview.ConnectionProperties{ - "a": { - Source: new(annotations.Status.Scope + "/providers/Applications.Core/extenders/" + recipeA.Name), - }, - "b": { - Source: new(annotations.Status.Scope + "/providers/Applications.Core/extenders/" + recipeB.Name), - }, - }, container.Properties.Connections) - require.Equal(t, []*v20231001preview.ResourceReference{{ID: new("/planes/kubernetes/local/namespaces/deployment-connections/providers/apps/Deployment/" + deployment.Name)}}, container.Properties.Resources) - - err = client.Get(ctx, name, deployment) - require.NoError(t, err) - - // Secret should have been created. - secret := corev1.Secret{} - err = client.Get(ctx, secretName, &secret) - require.NoError(t, err) - - expectedSecretData := map[string][]byte{ - "CONNECTION_A_A-SECRET": []byte("a"), - "CONNECTION_A_A-VALUE": []byte("a"), - "CONNECTION_B_B-SECRET": []byte("b"), - "CONNECTION_B_B-VALUE": []byte("b"), - } - require.Equal(t, expectedSecretData, secret.Data) - - // Secret should be mapped as env-vars - expectedEnvFrom := []corev1.EnvFromSource{ - { - SecretRef: &corev1.SecretEnvSource{ - Name: fmt.Sprintf("%s-connections", deployment.Name), - Optional: new(false), - }, - }, - } - - require.Equal(t, expectedEnvFrom, deployment.Spec.Template.Spec.Containers[0].EnvFrom) - - // Deployment should have the hash of the secret data. - hash := deployment.Spec.Template.Annotations[kubernetes.AnnotationSecretHash] - require.NotEmpty(t, hash) - - // Trigger a change by removing one of the connections. - err = client.Get(ctx, name, deployment) - require.NoError(t, err) - delete(deployment.Annotations, AnnotationRadiusConnectionPrefix+"a") - err = client.Update(ctx, deployment) - require.NoError(t, err) - - // Container will be updated. - annotations = waitForStateUpdating(t, client, name) - - radius.CompleteOperation(annotations.Status.Operation.ResumeToken, nil) - - // Deployment will update after operation completes - _ = waitForStateReady(t, client, name) - - err = client.Get(ctx, name, deployment) - require.NoError(t, err) - - // Secret should have been updated. - err = client.Get(ctx, secretName, &secret) - require.NoError(t, err) - - expectedSecretData = map[string][]byte{ - "CONNECTION_B_B-SECRET": []byte("b"), - "CONNECTION_B_B-VALUE": []byte("b"), - } - require.Equal(t, expectedSecretData, secret.Data) - - // Secret should be mapped as env-vars - require.Equal(t, expectedEnvFrom, deployment.Spec.Template.Spec.Containers[0].EnvFrom) - - // Deployment should have a DIFFERENT hash of the secret data. - require.NotEqual(t, hash, deployment.Spec.Template.Annotations[kubernetes.AnnotationSecretHash]) - - // Trigger cleanup by disabling Radius. - err = client.Get(ctx, name, deployment) - require.NoError(t, err) - deployment.Annotations[AnnotationRadiusEnabled] = "false" - err = client.Update(ctx, deployment) - require.NoError(t, err) - - // Deletion of the container is in progress. - annotations = waitForStateDeleting(t, client, name) - radius.CompleteOperation(annotations.Status.Operation.ResumeToken, nil) - - waitForRadiusContainerDeleted(t, client, name) - - // Deployment should have Radius changes reverted. - err = client.Get(ctx, name, deployment) - require.NoError(t, err) - require.Empty(t, deployment.Spec.Template.Spec.Containers[0].EnvFrom) - - // Secret should be gone - err = client.Get(ctx, secretName, &secret) - require.Error(t, err) - require.True(t, apierrors.IsNotFound(err)) -} - -// Creates a deployment with Radius disabled. -// -// Then checks the Events for Disabled-Disabled. -func Test_DeploymentReconciler_RadiusDisabled_ThenRadiusDisabled_ByAnnotation(t *testing.T) { - ctx := t.Context() - _, client := SetupDeploymentTest(t) - - name := types.NamespacedName{ - Namespace: "deployment-disabled-disabled-by-annotation", - Name: "test-deployment-disabled-disabled-by-annotation", - } - err := client.Create(ctx, &corev1.Namespace{Name: name.Namespace}) - require.NoError(t, err) - - deployment := makeDeployment(name) - err = client.Create(ctx, deployment) - require.NoError(t, err) - - waitForEvent(t, client, - expectedEvent{ - EventType: "Normal", - Reason: "NoOp", - Message: fmt.Sprintf("Radius is not enabled for %s", deployment.Name), - Count: 1, - }, - ) - - // Explicitly setting Radius disabled - deployment.Annotations = map[string]string{ - AnnotationRadiusEnabled: "false", - } - err = client.Update(ctx, deployment) - require.NoError(t, err) - - // We expect the same event to be generated. - // Because Radius was disabled and is still disabled for the deployment. - waitForEvent(t, client, - expectedEvent{ - EventType: "Normal", - Reason: "NoOp", - Message: fmt.Sprintf("Radius is not enabled for %s", deployment.Name), - Count: 2, - }, - ) -} - -// Creates a deployment with Radius disabled. -// -// Then checks the Events for Disabled-Disabled. -func Test_DeploymentReconciler_RadiusDisabled_ThenRadiusDisabled(t *testing.T) { - ctx := t.Context() - _, client := SetupDeploymentTest(t) - - name := types.NamespacedName{ - Namespace: "deployment-disabled-disabled", - Name: "test-deployment-disabled-disabled", - } - err := client.Create(ctx, &corev1.Namespace{Name: name.Namespace}) - require.NoError(t, err) - - deployment := makeDeployment(name) - err = client.Create(ctx, deployment) - require.NoError(t, err) - - waitForEvent(t, client, - expectedEvent{ - EventType: "Normal", - Reason: "NoOp", - Message: fmt.Sprintf("Radius is not enabled for %s", deployment.Name), - Count: 1, - }, - ) - - // Update Labels of the Deployment so that the Reconciler can detect a change. - deployment.Labels = map[string]string{"foo": "bar"} - err = client.Update(ctx, deployment) - require.NoError(t, err) - - // We expect the same event to be generated. - // Because Radius was disabled and is still disabled for the deployment. - waitForEvent(t, client, - expectedEvent{ - EventType: "Normal", - Reason: "NoOp", - Message: fmt.Sprintf("Radius is not enabled for %s", deployment.Name), - Count: 2, - }, - ) -} - -func waitForStateWaiting(t *testing.T, client client.Client, name types.NamespacedName) *deploymentAnnotations { - ctx := t.Context() - - logger := t - var annotations deploymentAnnotations - require.EventuallyWithTf(t, func(t *assert.CollectT) { - logger.Logf("Fetching Deployment: %+v", name) - current := &appsv1.Deployment{} - err := client.Get(ctx, name, current) - require.NoError(t, err) - - annotations, err = readAnnotations(current) - require.NoError(t, err) - assert.NotNil(t, annotations) - logger.Logf("Annotations.Status: %+v", annotations.Status) - - if assert.NotNil(t, annotations.Status) && assert.Equal(t, deploymentPhraseWaiting, annotations.Status.Phrase) { - assert.Empty(t, annotations.Status.Operation) - } - }, deploymentTestWaitDuration, deploymentTestWaitInterval, "waiting for state to be Waiting") - - return &annotations -} - -func waitForStateUpdating(t *testing.T, client client.Client, name types.NamespacedName) *deploymentAnnotations { - ctx := t.Context() - - logger := t - var annotations deploymentAnnotations - require.EventuallyWithTf(t, func(t *assert.CollectT) { - logger.Logf("Fetching Deployment: %+v", name) - current := &appsv1.Deployment{} - err := client.Get(ctx, name, current) - require.NoError(t, err) - - annotations, err = readAnnotations(current) - require.NoError(t, err) - assert.NotNil(t, annotations) - logger.Logf("Annotations.Status: %+v", annotations.Status) - - if assert.NotNil(t, annotations.Status) && assert.Equal(t, deploymentPhraseUpdating, annotations.Status.Phrase) { - assert.NotEmpty(t, annotations.Status.Operation) - } - }, deploymentTestWaitDuration, deploymentTestWaitInterval, "waiting for state to be Updating") - - return &annotations -} - -func waitForStateReady(t *testing.T, client client.Client, name types.NamespacedName) *deploymentAnnotations { - ctx := t.Context() - - logger := t - var annotations deploymentAnnotations - require.EventuallyWithTf(t, func(t *assert.CollectT) { - logger.Logf("Fetching Deployment: %+v", name) - current := &appsv1.Deployment{} - err := client.Get(ctx, name, current) - require.NoError(t, err) - - annotations, err = readAnnotations(current) - require.NoError(t, err) - assert.NotNil(t, annotations) - logger.Logf("Annotations.Status: %+v", annotations.Status) - - if assert.NotNil(t, annotations.Status) && assert.Equal(t, deploymentPhraseReady, annotations.Status.Phrase) { - assert.Empty(t, annotations.Status.Operation) - } - }, deploymentTestWaitDuration, deploymentTestWaitInterval, "waiting for state to be Ready") - - return &annotations -} - -func waitForStateDeleting(t *testing.T, client client.Client, name types.NamespacedName) *deploymentAnnotations { - ctx := t.Context() - - logger := t - var annotations deploymentAnnotations - require.EventuallyWithTf(t, func(t *assert.CollectT) { - logger.Logf("Fetching Deployment: %+v", name) - current := &appsv1.Deployment{} - err := client.Get(ctx, name, current) - require.NoError(t, err) - - annotations, err = readAnnotations(current) - require.NoError(t, err) - assert.NotNil(t, annotations) - logger.Logf("Annotations.Status: %+v", annotations.Status) - - if assert.NotNil(t, annotations.Status) && assert.Equal(t, deploymentPhraseDeleting, annotations.Status.Phrase) { - assert.NotEmpty(t, annotations.Status.Operation) - } - }, deploymentTestWaitDuration, deploymentTestWaitInterval, "waiting for state to be Deleting") - - return &annotations -} - -type expectedEvent struct { - EventType string - Reason string - Message string - Count int -} - -// waitForEvent waits for the expected event to occur in the cluster. -// -// We can have multiple events as the result of the List function but we are only interested in the expected event. -func waitForEvent(t *testing.T, client client.Client, event expectedEvent) { - ctx := t.Context() - logger := t - - require.EventuallyWithTf(t, func(t *assert.CollectT) { - logger.Log("Fetching Events") - - events := &corev1.EventList{} - err := client.List(ctx, events) - require.NoError(t, err) - - found := false - for _, e := range events.Items { - // If the event is the one we are looking for, check the count. - if e.Type == event.EventType && e.Reason == event.Reason && - e.Message == event.Message && assert.Equal(t, event.Count, int(e.Count)) { - found = true - } - } - assert.True(t, found) - }, deploymentTestWaitDuration, deploymentTestWaitInterval, "Waiting for the expected event") -} - -func waitForRadiusContainerDeleted(t *testing.T, client client.Client, name types.NamespacedName) *deploymentAnnotations { - ctx := t.Context() - - logger := t - var annotations *deploymentAnnotations - require.EventuallyWithTf(t, func(t *assert.CollectT) { - logger.Logf("Fetching Deployment: %+v", name) - current := &appsv1.Deployment{} - err := client.Get(ctx, name, current) - require.NoError(t, err) - - logger.Logf("Annotations: %+v", current.Annotations) - assert.NotContains(t, current.Annotations, AnnotationRadiusStatus) - assert.NotContains(t, current.Annotations, AnnotationRadiusConfigurationHash) - }, deploymentTestWaitDuration, deploymentTestWaitInterval, "waiting for state to be Deleting") - - return annotations -} - -func waitForDeploymentDeleted(t *testing.T, client client.Client, name types.NamespacedName) { - ctx := t.Context() - - logger := t - require.Eventuallyf(t, func() bool { - logger.Logf("Fetching Deployment: %+v", name) - err := client.Get(ctx, name, &appsv1.Deployment{}) - return apierrors.IsNotFound(err) - }, deploymentTestWaitDuration, deploymentTestWaitInterval, "waiting for deployment to be deleted") -} diff --git a/pkg/controller/reconciler/deployment_util.go b/pkg/controller/reconciler/deployment_util.go deleted file mode 100644 index dbfb1f88900..00000000000 --- a/pkg/controller/reconciler/deployment_util.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package reconciler - -import ( - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" -) - -// addSecretReference adds a secret reference to the deployment. Returns true if the secret was added, and false if it already exists. -// -// This function is idempotent and will not add the secret reference if it already exists. -func addSecretReference(deployment *appsv1.Deployment, secretName string) bool { - // For now we're just interested in the first container. - container := &deployment.Spec.Template.Spec.Containers[0] - - index := -1 - for i := range deployment.Spec.Template.Spec.Containers[0].EnvFrom { - if container.EnvFrom[i].SecretRef != nil && container.EnvFrom[i].SecretRef.Name == secretName { - index = i - break - } - } - - if index != -1 { - return false // Already present - } - - from := corev1.EnvFromSource{ - SecretRef: &corev1.SecretEnvSource{ - Name: secretName, - Optional: new(false), - }, - } - - container.EnvFrom = append(container.EnvFrom, from) - return true -} - -// removeSecretReference removes the secret reference from the deployment. Returns true if the secret was removed, and false if it was not found. -func removeSecretReference(deployment *appsv1.Deployment, secretName string) bool { - - // For now we're just interested in the first container. - container := &deployment.Spec.Template.Spec.Containers[0] - - index := -1 - for i := range deployment.Spec.Template.Spec.Containers[0].EnvFrom { - if container.EnvFrom[i].SecretRef != nil && container.EnvFrom[i].SecretRef.Name == secretName { - index = i - break - } - } - - if index == -1 { - return false - } - - // Remove the secret from the deployment. - container.EnvFrom = append(container.EnvFrom[0:index], container.EnvFrom[index+1:]...) - return true -} diff --git a/pkg/controller/reconciler/deployment_util_test.go b/pkg/controller/reconciler/deployment_util_test.go deleted file mode 100644 index 8fae0ca7735..00000000000 --- a/pkg/controller/reconciler/deployment_util_test.go +++ /dev/null @@ -1,121 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package reconciler - -import ( - "testing" - - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" -) - -func Test_addSecretReference_AlreadyPresent(t *testing.T) { - expected := []corev1.EnvFromSource{ - { - SecretRef: &corev1.SecretEnvSource{Name: "secret"}, - }, - { - SecretRef: &corev1.SecretEnvSource{Name: "another"}, - }, - { - ConfigMapRef: &corev1.ConfigMapEnvSource{Name: "config"}, - }, - } - - deployment := makeDeployment(types.NamespacedName{}) - deployment.Spec.Template.Spec.Containers[0].EnvFrom = expected - - result := addSecretReference(deployment, "secret") - require.False(t, result) - require.Equal(t, expected, deployment.Spec.Template.Spec.Containers[0].EnvFrom) -} - -func Test_addSecretReference_ReferenceAdded(t *testing.T) { - expected := []corev1.EnvFromSource{ - - { - SecretRef: &corev1.SecretEnvSource{Name: "another"}, - }, - { - ConfigMapRef: &corev1.ConfigMapEnvSource{Name: "config"}, - }, - { - SecretRef: &corev1.SecretEnvSource{Name: "secret", Optional: new(false)}, - }, - } - - deployment := makeDeployment(types.NamespacedName{}) - deployment.Spec.Template.Spec.Containers[0].EnvFrom = []corev1.EnvFromSource{ - { - SecretRef: &corev1.SecretEnvSource{Name: "another"}, - }, - { - ConfigMapRef: &corev1.ConfigMapEnvSource{Name: "config"}, - }, - } - - result := addSecretReference(deployment, "secret") - require.True(t, result) - require.Equal(t, expected, deployment.Spec.Template.Spec.Containers[0].EnvFrom) -} - -func Test_removeSecretReference_AlreadyRemoved(t *testing.T) { - expected := []corev1.EnvFromSource{ - { - SecretRef: &corev1.SecretEnvSource{Name: "another"}, - }, - { - ConfigMapRef: &corev1.ConfigMapEnvSource{Name: "config"}, - }, - } - - deployment := makeDeployment(types.NamespacedName{}) - deployment.Spec.Template.Spec.Containers[0].EnvFrom = expected - - result := removeSecretReference(deployment, "secret") - require.False(t, result) - require.Equal(t, expected, deployment.Spec.Template.Spec.Containers[0].EnvFrom) -} - -func Test_removeSecretReference_ReferenceRemoved(t *testing.T) { - expected := []corev1.EnvFromSource{ - { - SecretRef: &corev1.SecretEnvSource{Name: "another"}, - }, - { - ConfigMapRef: &corev1.ConfigMapEnvSource{Name: "config"}, - }, - } - - deployment := makeDeployment(types.NamespacedName{}) - deployment.Spec.Template.Spec.Containers[0].EnvFrom = []corev1.EnvFromSource{ - { - SecretRef: &corev1.SecretEnvSource{Name: "secret"}, - }, - { - SecretRef: &corev1.SecretEnvSource{Name: "another"}, - }, - { - ConfigMapRef: &corev1.ConfigMapEnvSource{Name: "config"}, - }, - } - - result := removeSecretReference(deployment, "secret") - require.True(t, result) - require.Equal(t, expected, deployment.Spec.Template.Spec.Containers[0].EnvFrom) -} diff --git a/pkg/controller/reconciler/main_test.go b/pkg/controller/reconciler/main_test.go index 4e55af23401..565569c2427 100644 --- a/pkg/controller/reconciler/main_test.go +++ b/pkg/controller/reconciler/main_test.go @@ -24,7 +24,6 @@ import ( sourcev1 "github.com/fluxcd/source-controller/api/v1" "github.com/go-logr/logr" radappiov1alpha3 "github.com/radius-project/radius/pkg/controller/api/radapp.io/v1alpha3" - admissionv1 "k8s.io/api/admissionregistration/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -39,15 +38,6 @@ var config *rest.Config // scheme holds a reference to the scheme for the test environment. var scheme *runtime.Scheme -// testOptions holds a reference to the webhook install options for the test environment. -var testOptions *testWebhookOptions - -type testWebhookOptions struct { - LocalServingHost string - LocalServingPort int - LocalServingCertDir string -} - // TestMain will be called before running any tests in the package. // // We're using this to ensure that one (and only one) copy of env-test is booted up. @@ -74,7 +64,6 @@ func TestMain(m *testing.M) { ErrorIfCRDPathMissing: true, } - initializeWebhookInEnvironment(env) cfg, err := env.Start() if err != nil { panic("failed to start envtest: " + err.Error()) @@ -88,11 +77,6 @@ func TestMain(m *testing.M) { config = cfg scheme = s - testOptions = &testWebhookOptions{ - LocalServingHost: env.WebhookInstallOptions.LocalServingHost, - LocalServingPort: env.WebhookInstallOptions.LocalServingPort, - LocalServingCertDir: env.WebhookInstallOptions.LocalServingCertDir, - } exitCode := m.Run() @@ -111,47 +95,3 @@ func SkipWithoutEnvironment(t *testing.T) { return } } - -// initializeWebhookInEnvironment initializes the webhook installation options and validating configuration in the given environment for validating webhooks. -func initializeWebhookInEnvironment(env *envtest.Environment) { - defaultScopeV1 := admissionv1.AllScopes - failedTypeV1 := admissionv1.Ignore - equivalentTypeV1 := admissionv1.Equivalent - noSideEffectsV1 := admissionv1.SideEffectClassNone - recipeWebhookPathV1 := "/validate-radapp-io-v1alpha3-recipe" - - env.WebhookInstallOptions = envtest.WebhookInstallOptions{ - ValidatingWebhooks: []*admissionv1.ValidatingWebhookConfiguration{ - { - Name: "recipe-webhook-config", - Kind: "ValidatingWebhookConfiguration", - APIVersion: "admissionregistration.k8s.io/v1", - Webhooks: []admissionv1.ValidatingWebhook{ - { - Name: "recipe-webhook.radapp.io", - Rules: []admissionv1.RuleWithOperations{ - { - Operations: []admissionv1.OperationType{"CREATE", "UPDATE"}, - APIGroups: []string{"radapp.io"}, - APIVersions: []string{"v1alpha3"}, - Resources: []string{"recipes"}, - Scope: &defaultScopeV1, - }, - }, - FailurePolicy: &failedTypeV1, - MatchPolicy: &equivalentTypeV1, - SideEffects: &noSideEffectsV1, - ClientConfig: admissionv1.WebhookClientConfig{ - Service: &admissionv1.ServiceReference{ - Name: "controller", - Namespace: "default", - Path: &recipeWebhookPathV1, - }, - }, - AdmissionReviewVersions: []string{"v1"}, - }, - }, - }, - }, - } -} diff --git a/pkg/controller/reconciler/recipe_reconciler.go b/pkg/controller/reconciler/recipe_reconciler.go deleted file mode 100644 index c1220422946..00000000000 --- a/pkg/controller/reconciler/recipe_reconciler.go +++ /dev/null @@ -1,565 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package reconciler - -import ( - "context" - "fmt" - "strings" - "time" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/tools/record" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - "github.com/go-logr/logr" - "github.com/radius-project/radius/pkg/cli/clients" - "github.com/radius-project/radius/pkg/cli/clients_new/generated" - radappiov1alpha3 "github.com/radius-project/radius/pkg/controller/api/radapp.io/v1alpha3" - sdkclients "github.com/radius-project/radius/pkg/sdk/clients" - "github.com/radius-project/radius/pkg/ucp/ucplog" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// RecipeReconciler reconciles a Recipe object. -type RecipeReconciler struct { - // Client is the Kubernetes client. - Client client.Client - - // Scheme is the Kubernetes scheme. - Scheme *runtime.Scheme - - // EventRecorder is the Kubernetes event recorder. - EventRecorder record.EventRecorder - - // Radius is the Radius client. - Radius RadiusClient - - // DelayInterval is the amount of time to wait between operations. - DelayInterval time.Duration -} - -// Reconcile is the main reconciliation loop for the Recipe resource. -func (r *RecipeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := ucplog.FromContextOrDiscard(ctx).WithValues("kind", "Recipe", "name", req.Name, "namespace", req.Namespace) - ctx = logr.NewContext(ctx, logger) - - recipe := radappiov1alpha3.Recipe{} - err := r.Client.Get(ctx, req.NamespacedName, &recipe) - if apierrors.IsNotFound(err) { - // This can happen due to a data-race if the recipe is created and then deleted before we can - // reconcile it. There's nothing to do here. - logger.Info("Recipe is being deleted.") - return ctrl.Result{}, nil - } else if err != nil { - logger.Error(err, "Unable to fetch resource.") - return ctrl.Result{}, err - } - - // Our algorithm is as follows: - // - // 1. Check if we have an "operation" in progress. If so, check it's status. - // a. If the operation is still in progress, then queue another reconcile (polling). - // b. If the operation completed successfully then update the status and continue processing (happy-path). - // c. If the operation failed then update the status and continue processing (retry). - // 2. If the recipe is being deleted then process deletion. - // a. This may require us to start a DELETE operation. After that we can continue polling. - // 3. If the recipe is not being deleted then process this as a creation or update. - // a. This may require us to start a PUT operation. After that we can continue polling. - // - // We do it this way because it guarantees that we only have one operation going at a time. - - if recipe.Status.Operation != nil { - // NOTE: if reconcileOperation completes successfully, then it will return a "zero" result, - // this means the operation has completed and we should continue processing. - result, err := r.reconcileOperation(ctx, &recipe) - if err != nil { - logger.Error(err, "Unable to reconcile in-progress operation.") - return ctrl.Result{}, err - } else if result.IsZero() { - // NOTE: if reconcileOperation completes successfully, then it will return a "zero" result, - // this means the operation has completed and we should continue processing. - logger.Info("Operation completed successfully.") - } else { - logger.Info("Requeueing to continue operation.") - return result, nil - } - } - - if recipe.DeletionTimestamp != nil { - return r.reconcileDelete(ctx, &recipe) - } - - return r.reconcileUpdate(ctx, &recipe) -} - -// ReconileOperation reconciles a Recipe that has an operation in progress. -func (r *RecipeReconciler) reconcileOperation(ctx context.Context, recipe *radappiov1alpha3.Recipe) (ctrl.Result, error) { - logger := ucplog.FromContextOrDiscard(ctx) - - // NOTE: the pollers are actually different types, so we have to duplicate the code - // for the PUT and DELETE handling. This makes me sad :( but there isn't a great - // solution besides duplicating the code. - // - // The only difference between these two codepaths is how they handle success. - if recipe.Status.Operation.OperationKind == radappiov1alpha3.OperationKindPut { - poller, err := r.Radius.Resources(recipe.Status.Scope, recipe.Spec.Type).ContinueCreateOperation(ctx, recipe.Status.Operation.ResumeToken) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to continue PUT operation: %w", err) - } - - _, err = poller.Poll(ctx) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to poll operation status: %w", err) - } - - if !poller.Done() { - return ctrl.Result{RequeueAfter: r.requeueDelay()}, nil - } - - // If we get here, the operation is complete. - _, err = poller.Result(ctx) - if err != nil { - // Operation failed, reset state and retry. - r.EventRecorder.Event(recipe, corev1.EventTypeWarning, "ResourceError", err.Error()) - logger.Error(err, "Update failed.") - - recipe.Status.Operation = nil - recipe.Status.Phrase = radappiov1alpha3.PhraseFailed - - err = r.Client.Status().Update(ctx, recipe) - if err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{RequeueAfter: r.requeueDelay()}, nil - } - - // If we get here, the operation was a success. Update the status and continue. - // - // NOTE: we don't need to save the status here, because we're going to continue reconciling. - recipe.Status.Operation = nil - recipe.Status.Resource = recipe.Status.Scope + "/providers/" + recipe.Spec.Type + "/" + recipe.Name - return ctrl.Result{}, nil - - } else if recipe.Status.Operation.OperationKind == radappiov1alpha3.OperationKindDelete { - poller, err := r.Radius.Resources(recipe.Status.Scope, recipe.Spec.Type).ContinueDeleteOperation(ctx, recipe.Status.Operation.ResumeToken) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to continue DELETE operation: %w", err) - } - - _, err = poller.Poll(ctx) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to poll operation status: %w", err) - } - - if !poller.Done() { - return ctrl.Result{RequeueAfter: r.requeueDelay()}, nil - } - - // If we get here, the operation is complete. - _, err = poller.Result(ctx) - if err != nil { - // Operation failed, reset state and retry. - r.EventRecorder.Event(recipe, corev1.EventTypeWarning, "ResourceError", err.Error()) - logger.Error(err, "Delete failed.") - - recipe.Status.Operation = nil - recipe.Status.Phrase = radappiov1alpha3.PhraseFailed - - err = r.Client.Status().Update(ctx, recipe) - if err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{RequeueAfter: r.requeueDelay()}, nil - } - - // If we get here, the operation was a success. Update the status and continue. - // - // NOTE: we don't need to save the status here, because we're going to continue reconciling. - recipe.Status.Operation = nil - recipe.Status.Resource = "" - return ctrl.Result{}, nil - } - - // If we get here, this was an unknown operation kind. This is a bug in our code, or someone - // tampered with the status of the object. Just reset the state and move on. - logger.Error(fmt.Errorf("unknown operation kind: %s", recipe.Status.Operation.OperationKind), "Unknown operation kind.") - - recipe.Status.Operation = nil - recipe.Status.Phrase = radappiov1alpha3.PhraseFailed - - err := r.Client.Status().Update(ctx, recipe) - if err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil -} - -func (r *RecipeReconciler) reconcileUpdate(ctx context.Context, recipe *radappiov1alpha3.Recipe) (ctrl.Result, error) { - logger := ucplog.FromContextOrDiscard(ctx) - - // Ensure that our finalizer is present before we start any operations. - if controllerutil.AddFinalizer(recipe, RecipeFinalizer) { - err := r.Client.Update(ctx, recipe) - if err != nil { - return ctrl.Result{}, err - } - } - - // Since we're going to reconcile, update the observed generation. - // - // We don't want to do this if we're in the middle of an operation, because we haven't - // fully processed any status changes until the async operation completes. - recipe.Status.ObservedGeneration = recipe.Generation - - environmentName := "default" - if recipe.Spec.Environment != "" { - environmentName = recipe.Spec.Environment - } - - applicationName := recipe.Namespace - if recipe.Spec.Application != "" { - applicationName = recipe.Spec.Application - } - - resourceGroupID, environmentID, applicationID, err := resolveDependencies(ctx, r.Radius, "/planes/radius/local", environmentName, applicationName) - if err != nil { - r.EventRecorder.Event(recipe, corev1.EventTypeWarning, "DependencyError", err.Error()) - logger.Error(err, "Unable to resolve dependencies.") - return ctrl.Result{}, fmt.Errorf("failed to resolve dependencies: %w", err) - } - - recipe.Status.Scope = resourceGroupID - recipe.Status.Environment = environmentID - recipe.Status.Application = applicationID - - updatePoller, deletePoller, err := r.startPutOrDeleteOperationIfNeeded(ctx, recipe) - if err != nil { - logger.Error(err, "Unable to create or update resource.") - r.EventRecorder.Event(recipe, corev1.EventTypeWarning, "ResourceError", err.Error()) - return ctrl.Result{}, err - } else if updatePoller != nil { - // We've successfully started an operation. Update the status and requeue. - token, err := updatePoller.ResumeToken() - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to get operation token: %w", err) - } - - recipe.Status.Operation = &radappiov1alpha3.ResourceOperation{ResumeToken: token, OperationKind: radappiov1alpha3.OperationKindPut} - recipe.Status.Phrase = radappiov1alpha3.PhraseUpdating - err = r.Client.Status().Update(ctx, recipe) - if err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{RequeueAfter: r.requeueDelay()}, nil - } else if deletePoller != nil { - // We've successfully started an operation. Update the status and requeue. - token, err := deletePoller.ResumeToken() - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to get operation token: %w", err) - } - - recipe.Status.Operation = &radappiov1alpha3.ResourceOperation{ResumeToken: token, OperationKind: radappiov1alpha3.OperationKindDelete} - recipe.Status.Phrase = radappiov1alpha3.PhraseDeleting - err = r.Client.Status().Update(ctx, recipe) - if err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{RequeueAfter: r.requeueDelay()}, nil - } - - // If we get here then it means we can process the result of the operation. - logger.Info("Resource is in desired state.", "resourceId", recipe.Status.Resource) - - err = r.updateSecret(ctx, recipe) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to process secret %s: %w", recipe.Spec.SecretName, err) - } - - recipe.Status.Phrase = radappiov1alpha3.PhraseReady - err = r.Client.Status().Update(ctx, recipe) - if err != nil { - return ctrl.Result{}, err - } - - r.EventRecorder.Event(recipe, corev1.EventTypeNormal, "Reconciled", "Successfully reconciled resource.") - return ctrl.Result{}, nil -} - -func (r *RecipeReconciler) reconcileDelete(ctx context.Context, recipe *radappiov1alpha3.Recipe) (ctrl.Result, error) { - logger := ucplog.FromContextOrDiscard(ctx) - - // Since we're going to reconcile, update the observed generation. - // - // We don't want to do this if we're in the middle of an operation, because we haven't - // fully processed any status changes until the async operation completes. - recipe.Status.ObservedGeneration = recipe.Generation - - poller, err := r.startDeleteOperationIfNeeded(ctx, recipe) - if err != nil { - logger.Error(err, "Unable to delete resource.") - r.EventRecorder.Event(recipe, corev1.EventTypeWarning, "ResourceError", err.Error()) - return ctrl.Result{}, err - } else if poller != nil { - // We've successfully started an operation. Update the status and requeue. - token, err := poller.ResumeToken() - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to get operation token: %w", err) - } - - recipe.Status.Operation = &radappiov1alpha3.ResourceOperation{ResumeToken: token, OperationKind: radappiov1alpha3.OperationKindDelete} - recipe.Status.Phrase = radappiov1alpha3.PhraseDeleting - err = r.Client.Status().Update(ctx, recipe) - if err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{RequeueAfter: r.requeueDelay()}, nil - } - - logger.Info("Resource is deleted.") - - err = r.deleteSecret(ctx, recipe) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to process secret %s: %w", recipe.Spec.SecretName, err) - } - - // At this point we've cleaned up everything. We can remove the finalizer which will allow deletion of the - // recipe. - if controllerutil.RemoveFinalizer(recipe, RecipeFinalizer) { - err := r.Client.Update(ctx, recipe) - if err != nil { - return ctrl.Result{}, err - } - - recipe.Status.ObservedGeneration = recipe.Generation - } - - recipe.Status.Phrase = radappiov1alpha3.PhraseDeleted - err = r.Client.Status().Update(ctx, recipe) - if err != nil { - return ctrl.Result{}, err - } - - r.EventRecorder.Event(recipe, corev1.EventTypeNormal, "Reconciled", "Successfully reconciled resource.") - return ctrl.Result{}, nil -} - -func (r *RecipeReconciler) startPutOrDeleteOperationIfNeeded(ctx context.Context, recipe *radappiov1alpha3.Recipe) (sdkclients.Poller[generated.GenericResourcesClientCreateOrUpdateResponse], sdkclients.Poller[generated.GenericResourcesClientDeleteResponse], error) { - logger := ucplog.FromContextOrDiscard(ctx) - - resourceID := recipe.Status.Scope + "/providers/" + recipe.Spec.Type + "/" + recipe.Name - if recipe.Status.Resource != "" && !strings.EqualFold(recipe.Status.Resource, resourceID) { - // If we get here it means that the environment or application changed, so we should delete - // the old resource and create a new one. - logger.Info("Resource is already created but is out-of-date") - - logger.Info("Starting DELETE operation.") - poller, err := deleteResource(ctx, r.Radius, recipe.Status.Resource) - if err != nil { - return nil, nil, err - } else if poller != nil { - return nil, poller, nil - } - - // Deletion was synchronous - recipe.Status.Resource = "" - } - - // Note: we separate this check from the previous block, because it could complete synchronously. - if recipe.Status.Resource != "" { - logger.Info("Resource is already created and is up-to-date.") - return nil, nil, nil - } - - logger.Info("Starting PUT operation.") - properties := map[string]any{ - "application": recipe.Status.Application, - "environment": recipe.Status.Environment, - "resourceProvisioning": "recipe", - } - - poller, err := createOrUpdateResource(ctx, r.Radius, resourceID, properties) - if err != nil { - return nil, nil, err - } else if poller != nil { - return poller, nil, nil - } - - // Update was synchronous - recipe.Status.Resource = resourceID - return nil, nil, nil -} - -func (r *RecipeReconciler) startDeleteOperationIfNeeded(ctx context.Context, recipe *radappiov1alpha3.Recipe) (sdkclients.Poller[generated.GenericResourcesClientDeleteResponse], error) { - logger := ucplog.FromContextOrDiscard(ctx) - if recipe.Status.Resource == "" { - logger.Info("Resource is already deleted (or was never created).") - return nil, nil - } - - logger.Info("Starting DELETE operation.") - poller, err := deleteResource(ctx, r.Radius, recipe.Status.Resource) - if err != nil { - return nil, err - } else if poller != nil { - return poller, err - } - - // Deletion was synchronous - - recipe.Status.Resource = "" - return nil, nil -} - -func (r *RecipeReconciler) updateSecret(ctx context.Context, recipe *radappiov1alpha3.Recipe) error { - logger := ucplog.FromContextOrDiscard(ctx) - - // If the secret name changed, delete the old secret. - if recipe.Spec.SecretName != recipe.Status.Secret.Name && recipe.Status.Secret.Name != "" { - logger.Info("Deleting stale secret", "secret", recipe.Status.Secret.Name) - err := r.Client.Delete(ctx, &corev1.Secret{ - Name: recipe.Status.Secret.Name, - Namespace: recipe.Namespace, - }) - if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("failed to delete stale secret %s: %w", recipe.Status.Secret.Name, err) - } - } - - if recipe.Spec.SecretName == "" { - logger.Info("No secret name specified, skipping secret creation") - recipe.Status.Secret = corev1.ObjectReference{} - return nil - } - - logger.Info("Creating or updating secret.", "secret", recipe.Spec.SecretName) - result, err := fetchResource(ctx, r.Radius, recipe.Status.Resource) - if err != nil { - return fmt.Errorf("failed to read resource: %w", err) - } - - secret := &corev1.Secret{} - err = r.Client.Get(ctx, client.ObjectKey{Namespace: recipe.Namespace, Name: recipe.Spec.SecretName}, secret) - if apierrors.IsNotFound(err) { - // This is OK, we'll create it next. - secret = nil - } else if err != nil { - return fmt.Errorf("failed to fetch secret %s: %w", recipe.Spec.SecretName, err) - } - - // Initialize the secret if it doesn't exist. - if secret == nil { - secret = &corev1.Secret{ - Name: recipe.Spec.SecretName, - Namespace: recipe.Namespace, - OwnerReferences: []metav1.OwnerReference{ - *metav1.NewControllerRef(recipe, radappiov1alpha3.GroupVersion.WithKind("Recipe")), - }, - } - - err = r.Client.Create(ctx, secret) - if err != nil { - return fmt.Errorf("failed to create secret %s: %w", secret.Name, err) - } - } - - // envtest has some quirky behavior around StringData which makes it hard to test. So we're - // using Data directly. - if secret.Data == nil { - secret.Data = map[string][]byte{} - } - - values, err := resourceToConnectionValues(result.GenericResource) - if err != nil { - return fmt.Errorf("failed to read connection values: %w", err) - } - - for k, v := range values { - secret.Data[k] = []byte(v) - } - - secrets, err := r.Radius.Resources(recipe.Status.Scope, recipe.Spec.Type).ListSecrets(ctx, recipe.Name) - if clients.Is404Error(err) { - // Safe to ignore. Not everything implements this. - } else if err != nil { - return fmt.Errorf("failed to list secrets: %w", err) - } else { - for k, v := range secrets.Value { - secret.Data[k] = []byte(*v) - } - } - - err = r.Client.Update(ctx, secret) - if err != nil { - return fmt.Errorf("failed to update secret %s: %w", secret.Name, err) - } - - recipe.Status.Secret = corev1.ObjectReference{ - APIVersion: "v1", - Kind: "Secret", - Namespace: secret.Namespace, - Name: secret.Name, - UID: secret.UID, - } - - return nil -} - -func (r *RecipeReconciler) deleteSecret(ctx context.Context, recipe *radappiov1alpha3.Recipe) error { - logger := ucplog.FromContextOrDiscard(ctx) - - if recipe.Status.Secret.Name != "" { - logger.Info("Deleting secret.", "secret", recipe.Status.Secret.Name) - err := r.Client.Delete(ctx, &corev1.Secret{ - Name: recipe.Status.Secret.Name, - Namespace: recipe.Namespace, - }) - if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("failed to delete secret %s: %w", recipe.Status.Secret.Name, err) - } - } - - recipe.Status.Secret = corev1.ObjectReference{} - return nil -} - -func (r *RecipeReconciler) requeueDelay() time.Duration { - delay := r.DelayInterval - if delay == 0 { - delay = PollingDelay - } - - return delay -} - -// SetupWithManager sets up the controller with the Manager. -func (r *RecipeReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&radappiov1alpha3.Recipe{}). - Owns(&corev1.Secret{}). - Complete(r) -} diff --git a/pkg/controller/reconciler/recipe_reconciler_test.go b/pkg/controller/reconciler/recipe_reconciler_test.go deleted file mode 100644 index ae63d002961..00000000000 --- a/pkg/controller/reconciler/recipe_reconciler_test.go +++ /dev/null @@ -1,339 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package reconciler - -import ( - "context" - "errors" - "testing" - - "github.com/radius-project/radius/pkg/cli/clients_new/generated" - sdkclients "github.com/radius-project/radius/pkg/sdk/clients" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - crconfig "sigs.k8s.io/controller-runtime/pkg/config" - "sigs.k8s.io/controller-runtime/pkg/metrics/server" -) - -func SetupRecipeTest(t *testing.T) (*mockRadiusClient, client.Client) { - SkipWithoutEnvironment(t) - - // Shut down the manager when the test exits. - ctx, cancel := context.WithCancel(t.Context()) - - mgr, err := ctrl.NewManager(config, ctrl.Options{ - Scheme: scheme, - Controller: crconfig.Controller{ - SkipNameValidation: new(true), - }, - - // Suppress metrics in tests to avoid conflicts. - Metrics: server.Options{ - BindAddress: "0", - }, - }) - require.NoError(t, err) - - radius := NewMockRadiusClient() - //nolint:staticcheck // SA1019: GetEventRecorderFor is deprecated but migration to new events API requires significant refactoring - err = (&RecipeReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - EventRecorder: mgr.GetEventRecorderFor("recipe-controller"), - Radius: radius, - DelayInterval: recipeTestControllerDelayInterval, - }).SetupWithManager(mgr) - require.NoError(t, err) - - startManager(t, mgr, ctx, cancel) - - return radius, mgr.GetClient() -} - -func Test_RecipeReconciler_WithoutSecret(t *testing.T) { - ctx := t.Context() - radius, client := SetupRecipeTest(t) - - name := types.NamespacedName{Namespace: "recipe-without-secret", Name: "test-recipe-withoutsecret"} - err := client.Create(ctx, &corev1.Namespace{Name: name.Namespace}) - require.NoError(t, err) - - recipe := makeRecipe(name, "Applications.Core/extenders") - err = client.Create(ctx, recipe) - require.NoError(t, err) - - // Recipe will be waiting for environment to be created. - createEnvironment(radius, "default", "default") - - // Recipe will be waiting for extender to complete provisioning. - status := waitForRecipeStateUpdating(t, client, name, nil) - require.Equal(t, "/planes/radius/local/resourcegroups/default-recipe-without-secret", status.Scope) - require.Equal(t, "/planes/radius/local/resourceGroups/default/providers/Applications.Core/environments/default", status.Environment) - require.Equal(t, "/planes/radius/local/resourcegroups/default-recipe-without-secret/providers/Applications.Core/applications/recipe-without-secret", status.Application) - - radius.CompleteOperation(status.Operation.ResumeToken, nil) - - // Recipe will update after operation completes - status = waitForRecipeStateReady(t, client, name) - require.Equal(t, "/planes/radius/local/resourcegroups/default-recipe-without-secret/providers/Applications.Core/extenders/test-recipe-withoutsecret", status.Resource) - - extender, err := radius.Resources(status.Scope, "Applications.Core/extenders").Get(ctx, name.Name) - require.NoError(t, err) - require.Equal(t, "recipe", extender.Properties["resourceProvisioning"]) - - err = client.Delete(ctx, recipe) - require.NoError(t, err) - - // Deletion of the recipe is in progress. - status = waitForRecipeStateDeleting(t, client, name, nil) - radius.CompleteOperation(status.Operation.ResumeToken, nil) - - // Now deleting of the deployment object can complete. - waitForRecipeDeleted(t, client, name) -} - -func Test_RecipeReconciler_ChangeEnvironmentAndApplication(t *testing.T) { - ctx := t.Context() - radius, client := SetupRecipeTest(t) - - name := types.NamespacedName{Namespace: "recipe-change-envapp", Name: "test-recipe-change-envapp"} - err := client.Create(ctx, &corev1.Namespace{Name: name.Namespace}) - require.NoError(t, err) - - recipe := makeRecipe(name, "Applications.Core/extenders") - err = client.Create(ctx, recipe) - require.NoError(t, err) - - // Recipe will be waiting for environment to be created. - createEnvironment(radius, "default", "default") - - // Recipe will be waiting for extender to complete provisioning. - status := waitForRecipeStateUpdating(t, client, name, nil) - require.Equal(t, "/planes/radius/local/resourcegroups/default-recipe-change-envapp", status.Scope) - require.Equal(t, "/planes/radius/local/resourceGroups/default/providers/Applications.Core/environments/default", status.Environment) - require.Equal(t, "/planes/radius/local/resourcegroups/default-recipe-change-envapp/providers/Applications.Core/applications/recipe-change-envapp", status.Application) - - radius.CompleteOperation(status.Operation.ResumeToken, nil) - - // Recipe will update after operation completes - status = waitForRecipeStateReady(t, client, name) - require.Equal(t, "/planes/radius/local/resourcegroups/default-recipe-change-envapp/providers/Applications.Core/extenders/test-recipe-change-envapp", status.Resource) - - createEnvironment(radius, "new-environment", "new-environment") - - // Now update the recipe to change the environment and application. - err = client.Get(ctx, name, recipe) - require.NoError(t, err) - - recipe.Spec.Environment = "new-environment" - recipe.Spec.Application = "new-application" - - err = client.Update(ctx, recipe) - require.NoError(t, err) - - // Now the recipe will delete and re-create the resource. - - // Deletion of the resource is in progress. - status = waitForRecipeStateDeleting(t, client, name, nil) - radius.CompleteOperation(status.Operation.ResumeToken, nil) - - // Resource should be gone. - _, err = radius.Resources(status.Scope, "Applications.Core/extenders").Get(ctx, name.Name) - require.Error(t, err) - - // Recipe will be waiting for extender to complete provisioning. - status = waitForRecipeStateUpdating(t, client, name, nil) - require.Equal(t, "/planes/radius/local/resourcegroups/new-environment-new-application", status.Scope) - require.Equal(t, "/planes/radius/local/resourceGroups/new-environment/providers/Applications.Core/environments/new-environment", status.Environment) - require.Equal(t, "/planes/radius/local/resourcegroups/new-environment-new-application/providers/Applications.Core/applications/new-application", status.Application) - radius.CompleteOperation(status.Operation.ResumeToken, nil) - - // Recipe will update after operation completes - status = waitForRecipeStateReady(t, client, name) - require.Equal(t, "/planes/radius/local/resourcegroups/new-environment-new-application/providers/Applications.Core/extenders/test-recipe-change-envapp", status.Resource) - - // Now delete the recipe. - err = client.Delete(ctx, recipe) - require.NoError(t, err) - - // Deletion of the resource is in progress. - status = waitForRecipeStateDeleting(t, client, name, nil) - radius.CompleteOperation(status.Operation.ResumeToken, nil) - - // Now deleting of the deployment object can complete. - waitForRecipeDeleted(t, client, name) -} - -func Test_RecipeReconciler_FailureRecovery(t *testing.T) { - // This test tests our ability to recover from failed operations inside Radius. - // - // We use the mock client to simulate the failure of update and delete operations - // and verify that the controller will (eventually) retry these operations. - - ctx := t.Context() - radius, client := SetupRecipeTest(t) - - name := types.NamespacedName{Namespace: "recipe-failure-recovery", Name: "test-recipe-failure-recovery"} - err := client.Create(ctx, &corev1.Namespace{Name: name.Namespace}) - require.NoError(t, err) - - recipe := makeRecipe(name, "Applications.Core/extenders") - err = client.Create(ctx, recipe) - require.NoError(t, err) - - // Recipe will be waiting for environment to be created. - createEnvironment(radius, "default", "default") - - // Recipe will be waiting for extender to complete provisioning. - status := waitForRecipeStateUpdating(t, client, name, nil) - - // Complete the operation, but make it fail. - operation := status.Operation - radius.CompleteOperation(status.Operation.ResumeToken, func(state *sdkclients.OperationState) { - state.Err = errors.New("oops") - - resource, ok := radius.resources[state.ResourceID] - require.True(t, ok, "failed to find resource") - - resource.Properties["provisioningState"] = "Failed" - state.Value = generated.GenericResourcesClientCreateOrUpdateResponse{GenericResource: resource} - }) - - // Recipe should (eventually) start a new provisioning operation - status = waitForRecipeStateUpdating(t, client, name, operation) - - // Complete the operation, successfully this time. - radius.CompleteOperation(status.Operation.ResumeToken, nil) - _ = waitForRecipeStateReady(t, client, name) - - err = client.Delete(ctx, recipe) - require.NoError(t, err) - - // Deletion of the recipe is in progress. - status = waitForRecipeStateDeleting(t, client, name, nil) - - // Complete the operation, but make it fail. - operation = status.Operation - radius.CompleteOperation(status.Operation.ResumeToken, func(state *sdkclients.OperationState) { - state.Err = errors.New("oops") - - resource, ok := radius.resources[state.ResourceID] - require.True(t, ok, "failed to find resource") - - resource.Properties["provisioningState"] = "Failed" - }) - - // Recipe should (eventually) start a new provisioning operation - status = waitForRecipeStateDeleting(t, client, name, operation) - - // Complete the operation, successfully this time. - radius.CompleteOperation(status.Operation.ResumeToken, nil) - - // Now deleting of the deployment object can complete. - waitForRecipeDeleted(t, client, name) -} - -func Test_RecipeReconciler_WithSecret(t *testing.T) { - ctx := t.Context() - radius, client := SetupRecipeTest(t) - - name := types.NamespacedName{Namespace: "recipe-withsecret", Name: "test-recipe-withsecret"} - err := client.Create(ctx, &corev1.Namespace{Name: name.Namespace}) - require.NoError(t, err) - - recipe := makeRecipe(name, "Applications.Core/extenders") - recipe.Spec.SecretName = name.Name - - err = client.Create(ctx, recipe) - require.NoError(t, err) - - // Recipe will be waiting for environment to be created. - createEnvironment(radius, "default", "default") - - // Recipe will be waiting for extender to complete provisioning. - status := waitForRecipeStateUpdating(t, client, name, nil) - - // Update the resource with computed values as part of completing the operation. - radius.CompleteOperation(status.Operation.ResumeToken, func(state *sdkclients.OperationState) { - resource, ok := radius.resources[state.ResourceID] - require.True(t, ok, "failed to find resource") - - resource.Properties["a-value"] = "a" - resource.Properties["secrets"] = map[string]string{ - "b-secret": "b", - } - state.Value = generated.GenericResourcesClientCreateOrUpdateResponse{GenericResource: resource} - }) - - // Recipe will update after operation completes - status = waitForRecipeStateReady(t, client, name) - - expectedData := map[string][]byte{ - "a-value": []byte("a"), - "b-secret": []byte("b"), - } - - waitForSecretData(t, client, name, expectedData) - - extender, err := radius.Resources(status.Scope, "Applications.Core/extenders").Get(ctx, name.Name) - require.NoError(t, err) - require.Equal(t, "recipe", extender.Properties["resourceProvisioning"]) - - // Now we'll change the secret name. - err = client.Get(ctx, name, recipe) - require.NoError(t, err) - - recipe.Spec.SecretName = "new-secret-name" - err = client.Update(ctx, recipe) - require.NoError(t, err) - - // Recipe will update after operation completes - _ = waitForRecipeStateReady(t, client, name) - - // The old secret should be (eventually) deleted - the client reads from - // the informer cache so the deletion may not be visible immediately. - require.Eventuallyf(t, func() bool { - old := corev1.Secret{} - err := client.Get(ctx, name, &old) - return apierrors.IsNotFound(err) - }, recipeTestWaitDuration, recipeTestWaitInterval, "old secret should be deleted") - - waitForSecretData(t, client, types.NamespacedName{Namespace: name.Namespace, Name: "new-secret-name"}, expectedData) - - // Now we'll delete the recipe. - err = client.Delete(ctx, recipe) - require.NoError(t, err) - - // Deletion of the recipe is in progress. - status = waitForRecipeStateDeleting(t, client, name, nil) - radius.CompleteOperation(status.Operation.ResumeToken, nil) - - // Now deleting of the deployment object can complete. - waitForRecipeDeleted(t, client, name) - - // The secret should be (eventually) deleted - the client reads from - // the informer cache so the deletion may not be visible immediately. - require.Eventuallyf(t, func() bool { - s := corev1.Secret{} - err := client.Get(ctx, name, &s) - return apierrors.IsNotFound(err) - }, recipeTestWaitDuration, recipeTestWaitInterval, "secret should be deleted") -} diff --git a/pkg/controller/reconciler/recipe_webhook.go b/pkg/controller/reconciler/recipe_webhook.go deleted file mode 100644 index dd0f5035228..00000000000 --- a/pkg/controller/reconciler/recipe_webhook.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package reconciler - -import ( - "context" - "strings" - - radappiov1alpha3 "github.com/radius-project/radius/pkg/controller/api/radapp.io/v1alpha3" - "github.com/radius-project/radius/pkg/ucp/ucplog" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/validation/field" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -// SetupWebhookWithManager sets up the webhook for the Recipe type with the provided manager. -// It configures the webhook to watch for changes on the Recipe resource and uses the provided validator. -// Returns an error if there was a problem setting up the webhook. -func (r *RecipeWebhook) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr, &radappiov1alpha3.Recipe{}). - WithValidator(r). - Complete() -} - -// RecipeWebhook implements the validating webhook functions for the Recipe type. -type RecipeWebhook struct{} - -// ValidateCreate validates the creation of a Recipe object. -func (r *RecipeWebhook) ValidateCreate(ctx context.Context, recipe *radappiov1alpha3.Recipe) (admission.Warnings, error) { - logger := ucplog.FromContextOrDiscard(ctx) - - logger.Info("Validating Create Recipe %s", recipe.Name) - return r.validateRecipeType(ctx, recipe) -} - -// ValidateUpdate validates the update of a Recipe object. -func (r *RecipeWebhook) ValidateUpdate(ctx context.Context, oldRecipe, newRecipe *radappiov1alpha3.Recipe) (admission.Warnings, error) { - logger := ucplog.FromContextOrDiscard(ctx) - - logger.Info("Validating Update Recipe %s", newRecipe.Name) - return r.validateRecipeType(ctx, newRecipe) -} - -// ValidateDelete validates the deletion of a Recipe object. -func (r *RecipeWebhook) ValidateDelete(ctx context.Context, recipe *radappiov1alpha3.Recipe) (admission.Warnings, error) { - logger := ucplog.FromContextOrDiscard(ctx) - logger.Info("Validating Delete Recipe") - - // currently there is no validation when deleting Recipe - return nil, nil -} - -// validateRecipeType validates Recipe object. -func (r *RecipeWebhook) validateRecipeType(ctx context.Context, recipe *radappiov1alpha3.Recipe) (admission.Warnings, error) { - logger := ucplog.FromContextOrDiscard(ctx) - var errList field.ErrorList - flPath := field.NewPath("spec").Child("type") - - logger.Info("Validating Recipe Type %s in Recipe %s", recipe.Spec.Type, recipe.Name) - if recipe.Spec.Type == "" || strings.Count(recipe.Spec.Type, "/") != 1 { - errList = append(errList, field.Invalid(flPath, recipe.Spec.Type, "must be in the format 'ResourceProvider.Namespace/resourceType'")) - - return nil, apierrors.NewInvalid( - schema.GroupKind{Group: "radapp.io", Kind: "Recipe"}, - recipe.Name, - errList) - } - - return nil, nil -} diff --git a/pkg/controller/reconciler/recipe_webhook_test.go b/pkg/controller/reconciler/recipe_webhook_test.go deleted file mode 100644 index 2beabef5de4..00000000000 --- a/pkg/controller/reconciler/recipe_webhook_test.go +++ /dev/null @@ -1,341 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package reconciler - -import ( - "context" - "crypto/tls" - "fmt" - "net" - "net/http" - "testing" - "time" - - radappiov1alpha3 "github.com/radius-project/radius/pkg/controller/api/radapp.io/v1alpha3" - - "github.com/stretchr/testify/require" - admissionv1 "k8s.io/api/admissionregistration/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/retry" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -const ( - defaultNamespace = "default" - validResourceType = "Applications.Core/extenders" - invalidResourceType = "invalidType" - webhookConfigName = "recipe-webhook-config" -) - -// Test_ValidateRecipe_Type tests a recipe with valid and invalid types. -func Test_ValidateRecipe_Type(t *testing.T) { - ctx := t.Context() - radius, client := setupWebhookTest(t) - - // Environment is created. - createEnvironment(radius, "default", "default") - - t.Run("test recipe for invalid type", func(t *testing.T) { - recipeName := "test-recipe-invalidtype" - namespace := types.NamespacedName{Namespace: defaultNamespace, Name: recipeName} - recipe := makeRecipe(namespace, invalidResourceType) - - err := client.Create(ctx, &corev1.Namespace{Name: namespace.Name}) - require.NoError(t, err) - - // Webhook is expected to trigger during this call and return an error. - err = client.Create(ctx, recipe) - require.True(t, apierrors.IsInvalid(err)) - - // Convert the error to a *apierrors.StatusError to get the status code - statusError, ok := err.(*apierrors.StatusError) - require.True(t, ok) - - // Check for expected status code - require.Equal(t, int32(http.StatusUnprocessableEntity), statusError.ErrStatus.Code) - }) - - t.Run("test recipe for valid type", func(t *testing.T) { - recipeName := "test-recipe-validtype" - namespace := types.NamespacedName{Namespace: defaultNamespace, Name: recipeName} - recipe := makeRecipe(namespace, validResourceType) - - err := client.Create(ctx, &corev1.Namespace{Name: namespace.Name}) - require.NoError(t, err) - - err = client.Create(ctx, recipe) - require.NoError(t, err) - - // Recipe will be waiting for extender to complete provisioning. - status := waitForRecipeStateUpdating(t, client, namespace, nil) - - radius.CompleteOperation(status.Operation.ResumeToken, nil) - _, err = radius.Resources(status.Scope, validResourceType).Get(ctx, namespace.Name) - require.NoError(t, err) - - err = client.Delete(ctx, recipe) - require.NoError(t, err) - - // Deletion of the resource is in progress. - status = waitForRecipeStateDeleting(t, client, namespace, nil) - radius.CompleteOperation(status.Operation.ResumeToken, nil) - - // Now deleting of the deployment object can complete. - waitForRecipeDeleted(t, client, namespace) - }) - - t.Run("test recipe update from valid to invalid type", func(t *testing.T) { - // Create a recipe with a valid type - recipeName := "test-recipe-update" - namespace := types.NamespacedName{Namespace: defaultNamespace, Name: recipeName} - recipe := makeRecipe(namespace, validResourceType) - - err := client.Create(ctx, &corev1.Namespace{Name: namespace.Name}) - require.NoError(t, err) - - err = client.Create(ctx, recipe) - require.NoError(t, err) - - // Recipe will be waiting for extender to complete provisioning. - status := waitForRecipeStateUpdating(t, client, namespace, nil) - - radius.CompleteOperation(status.Operation.ResumeToken, nil) - _, err = radius.Resources(status.Scope, validResourceType).Get(ctx, namespace.Name) - require.NoError(t, err) - - // Using RetryOnConflict to avoid catching a conflict error when updating the recipe. - err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - // Retrieve the latest version of the recipe - recipe = &radappiov1alpha3.Recipe{} - if err := client.Get(ctx, namespace, recipe); err != nil { - return err - } - - // Update the recipe to have an invalid type - recipe.Spec.Type = invalidResourceType - return client.Update(ctx, recipe) - }) - // The webhook should reject the update and return an Invalid error - require.True(t, apierrors.IsInvalid(err)) - - // Convert the error to a *apierrors.StatusError to get the status code - statusError, ok := err.(*apierrors.StatusError) - require.True(t, ok) - - // Check for expected status code - require.Equal(t, int32(http.StatusUnprocessableEntity), statusError.ErrStatus.Code) - - err = client.Delete(ctx, recipe) - require.NoError(t, err) - - // Deletion of the resource is in progress. - status = waitForRecipeStateDeleting(t, client, namespace, nil) - radius.CompleteOperation(status.Operation.ResumeToken, nil) - - // Now deleting of the deployment object can complete. - waitForRecipeDeleted(t, client, namespace) - }) - - // NOTE: We are updating the FailurePolicy of the webhook to Ignore after running webhook tests. - // This is to ensure that the webhook does not interfere with other tests in the reconciler package. - // This approach may be updated in the future. - failurePolicy := admissionv1.Ignore - updateWebhookFailurePolicy(t, webhookConfigName, &failurePolicy) -} - -// Test_Webhook_ValidateFunctions tests webhook functions ValidateCreate, ValidateUpdate, and ValidateDelete -// for a recipe with valid and invalid resource types. -func Test_Webhook_ValidateFunctions(t *testing.T) { - tests := []struct { - name string - recipeName string - typeName string - function string - wantErr bool - }{ - { - name: "create recipe with valid type", - recipeName: "create-recipe-validtype", - typeName: validResourceType, - function: "create", - wantErr: false, - }, - { - name: "create recipe with invalid type", - recipeName: "create-recipe-invalidtype", - typeName: invalidResourceType, - function: "create", - wantErr: true, - }, - { - name: "update recipe with valid type", - recipeName: "update-recipe-validtype", - typeName: validResourceType, - function: "update", - wantErr: false, - }, - { - name: "update recipe with invalid type", - recipeName: "update-recipe-invalidtype", - typeName: invalidResourceType, - function: "update", - wantErr: true, - }, - { - name: "delete recipe with valid type", - recipeName: "delete-recipe-validtype", - typeName: validResourceType, - function: "delete", - wantErr: false, - }, - { - name: "delete recipe with invalid type", - recipeName: "delete-recipe-invalidtype", - typeName: invalidResourceType, - function: "delete", - wantErr: false, - }, - } - for _, tr := range tests { - t.Run(tr.name, func(t *testing.T) { - ctx := t.Context() - var err error - namespace := types.NamespacedName{Namespace: defaultNamespace, Name: tr.recipeName} - recipe := makeRecipe(namespace, tr.typeName) - recipeWebhook := &RecipeWebhook{} - - if tr.function == "create" { - _, err = recipeWebhook.ValidateCreate(ctx, recipe) - } else if tr.function == "update" { - _, err = recipeWebhook.ValidateUpdate(ctx, nil, recipe) - } else { - _, err = recipeWebhook.ValidateDelete(ctx, recipe) - } - - if tr.wantErr { - expectedError := fmt.Sprintf("Recipe.radapp.io \"%s\" is invalid: spec.type: Invalid value: \"%s\": must be in the format 'ResourceProvider.Namespace/resourceType'", tr.recipeName, tr.typeName) - require.True(t, apierrors.IsInvalid(err)) - require.EqualError(t, err, expectedError) - - } else { - require.NoError(t, err) - } - }) - } -} - -// setupWebhookTest sets up a webhook test environment. -func setupWebhookTest(t *testing.T) (*mockRadiusClient, client.Client) { - SkipWithoutEnvironment(t) - - // Shut down the manager when the test exits. - ctx, cancel := context.WithCancel(t.Context()) - - mgr, err := ctrl.NewManager(config, ctrl.Options{ - Scheme: scheme, - WebhookServer: webhook.NewServer(webhook.Options{ - Host: testOptions.LocalServingHost, - Port: testOptions.LocalServingPort, - CertDir: testOptions.LocalServingCertDir, - }), - LeaderElection: false, - Metrics: server.Options{ - BindAddress: "0", - }, - }) - require.NoError(t, err) - - radius := NewMockRadiusClient() - //nolint:staticcheck // SA1019: GetEventRecorderFor is deprecated but migration to new events API requires significant refactoring - err = (&RecipeReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - EventRecorder: mgr.GetEventRecorderFor("recipe-controller"), - Radius: radius, - DelayInterval: recipeTestControllerDelayInterval, - }).SetupWithManager(mgr) - require.NoError(t, err) - - err = (&RecipeWebhook{}).SetupWebhookWithManager(mgr) - require.NoError(t, err) - - startManager(t, mgr, ctx, cancel) - - // wait for the webhook server to get ready - var dialErr error - dialer := &net.Dialer{Timeout: time.Second} - addrPort := fmt.Sprintf("%s:%d", testOptions.LocalServingHost, testOptions.LocalServingPort) - require.Eventuallyf(t, func() bool { - if conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}); err == nil { - conn.Close() - return true - } else if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - dialErr = err - return false - } - - return false - - }, time.Second*5, time.Millisecond*200, "Failed to connect: %v", dialErr) - - // NOTE: The default FailurePolicy of webhook is set to Ignore (main_test.go). - // We are updating the FailurePolicy of the webhook to Fail before running webhook tests to ensure the webhook will return an error when validation fails. - // This approach may be updated in the future. - failurePolicy := admissionv1.Fail - updateWebhookFailurePolicy(t, webhookConfigName, &failurePolicy) - - return radius, mgr.GetClient() -} - -// updateWebhookFailurePolicy updates the failure policy of a ValidatingWebhookConfiguration object. -// The function retrieves the ValidatingWebhookConfiguration object for the given webhookConfigName, -// updates its failure policy with the provided webhookfailurePolicy value, and then updates the object in the Kubernetes cluster. -// If any error occurs during the retrieval or update process, the function fails the test. -func updateWebhookFailurePolicy(t *testing.T, webhookConfigName string, webhookfailurePolicy *admissionv1.FailurePolicyType) { - SkipWithoutEnvironment(t) - - // Shut down the manager when the test exits. - ctx, cancel := context.WithCancel(t.Context()) - t.Cleanup(cancel) - - // Define the object key (name) - key := client.ObjectKey{ - Name: webhookConfigName, // "recipe-webhook-config", - } - - // Create a ValidatingWebhookConfiguration object to receive the data - webhook := &admissionv1.ValidatingWebhookConfiguration{} - - // Get the ValidatingWebhookConfiguration - k8sClient, err := client.New(config, client.Options{}) - require.NoError(t, err) - - err = k8sClient.Get(ctx, key, webhook) - require.NoError(t, err) - - // Update the failure policy of the webhook - webhook.Webhooks[0].FailurePolicy = webhookfailurePolicy - - // Update the ValidatingWebhookConfiguration - err = k8sClient.Update(ctx, webhook) - require.NoError(t, err) -} diff --git a/pkg/controller/reconciler/shared_test.go b/pkg/controller/reconciler/shared_test.go index 9eb1f5901df..4d3d2924d2c 100644 --- a/pkg/controller/reconciler/shared_test.go +++ b/pkg/controller/reconciler/shared_test.go @@ -23,26 +23,12 @@ import ( "testing" "time" - v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" radappiov1alpha3 "github.com/radius-project/radius/pkg/controller/api/radapp.io/v1alpha3" - "github.com/radius-project/radius/pkg/corerp/api/v20231001preview" - "github.com/radius-project/radius/pkg/to" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" ) const ( - recipeTestWaitDuration = time.Second * 10 - recipeTestWaitInterval = time.Millisecond * 200 - recipeTestControllerDelayInterval = time.Millisecond * 100 - // managerShutdownTimeout bounds how long test cleanup waits for a controller-runtime manager to // exit after its context is cancelled. It is comfortably above controller-runtime's default 30s // graceful shutdown timeout, so a clean shutdown always finishes first. If it is exceeded, cleanup @@ -90,165 +76,6 @@ func waitForManagerShutdown(t *testing.T, stopped <-chan struct{}) { } } -func createEnvironment(radius *mockRadiusClient, resourceGroup, name string) { - id := fmt.Sprintf("/planes/radius/local/resourceGroups/%s/providers/Applications.Core/environments/%s", resourceGroup, name) - radius.Update(func() { - radius.environments[id] = v20231001preview.EnvironmentResource{ - ID: new(id), - Name: new(name), - Location: to.Ptr(v1.LocationGlobal), - } - }) -} - -func makeRecipe(name types.NamespacedName, resourceType string) *radappiov1alpha3.Recipe { - return &radappiov1alpha3.Recipe{ - Namespace: name.Namespace, - Name: name.Name, - Spec: radappiov1alpha3.RecipeSpec{ - Type: resourceType, - }, - } -} - -func waitForRecipeStateUpdating(t *testing.T, client client.Client, name types.NamespacedName, oldOperation *radappiov1alpha3.ResourceOperation) *radappiov1alpha3.RecipeStatus { - ctx := t.Context() - - logger := t - status := &radappiov1alpha3.RecipeStatus{} - require.EventuallyWithT(t, func(t *assert.CollectT) { - logger.Logf("Fetching Recipe: %+v", name) - current := &radappiov1alpha3.Recipe{} - err := client.Get(ctx, name, current) - require.NoError(t, err) - - status = ¤t.Status - logger.Logf("Recipe.Status: %+v", current.Status) - assert.Equal(t, status.ObservedGeneration, current.Generation, "Status is not updated") - - if assert.Equal(t, radappiov1alpha3.PhraseUpdating, current.Status.Phrase) { - assert.NotEmpty(t, current.Status.Operation) - assert.NotEqual(t, oldOperation, current.Status.Operation) - } - - }, recipeTestWaitDuration, recipeTestWaitInterval, "failed to enter updating state") - - return status -} - -func waitForRecipeStateReady(t *testing.T, client client.Client, name types.NamespacedName) *radappiov1alpha3.RecipeStatus { - ctx := t.Context() - - logger := t - status := &radappiov1alpha3.RecipeStatus{} - require.EventuallyWithTf(t, func(t *assert.CollectT) { - logger.Logf("Fetching Recipe: %+v", name) - current := &radappiov1alpha3.Recipe{} - err := client.Get(ctx, name, current) - require.NoError(t, err) - - status = ¤t.Status - logger.Logf("Recipe.Status: %+v", current.Status) - assert.Equal(t, status.ObservedGeneration, current.Generation, "Status is not updated") - - if assert.Equal(t, radappiov1alpha3.PhraseReady, current.Status.Phrase) { - assert.Empty(t, current.Status.Operation) - } - }, recipeTestWaitDuration, recipeTestWaitInterval, "failed to enter updating state") - - return status -} - -func waitForRecipeStateDeleting(t *testing.T, client client.Client, name types.NamespacedName, oldOperation *radappiov1alpha3.ResourceOperation) *radappiov1alpha3.RecipeStatus { - ctx := t.Context() - - logger := t - status := &radappiov1alpha3.RecipeStatus{} - require.EventuallyWithTf(t, func(t *assert.CollectT) { - logger.Logf("Fetching Recipe: %+v", name) - current := &radappiov1alpha3.Recipe{} - err := client.Get(ctx, name, current) - assert.NoError(t, err) - - status = ¤t.Status - logger.Logf("Recipe.Status: %+v", current.Status) - assert.Equal(t, status.ObservedGeneration, current.Generation, "Status is not updated") - - if assert.Equal(t, radappiov1alpha3.PhraseDeleting, current.Status.Phrase) { - assert.NotEmpty(t, current.Status.Operation) - assert.NotEqual(t, oldOperation, current.Status.Operation) - } - }, recipeTestWaitDuration, recipeTestWaitInterval, "failed to enter deleting state") - - return status -} - -func waitForRecipeDeleted(t *testing.T, client client.Client, name types.NamespacedName) { - ctx := t.Context() - - logger := t - require.Eventuallyf(t, func() bool { - logger.Logf("Fetching Recipe: %+v", name) - current := &radappiov1alpha3.Recipe{} - err := client.Get(ctx, name, current) - if apierrors.IsNotFound(err) { - return true - } - - logger.Logf("Recipe.Status: %+v", current.Status) - return false - - }, recipeTestWaitDuration, recipeTestWaitInterval, "recipe still exists") -} - -// waitForSecretData waits until a secret contains the expected data. The test client reads from the -// informer cache, and the recipe status and its secret are delivered by independent informers, so -// observing the recipe as ready does not guarantee the secret's data is visible yet. -func waitForSecretData(t *testing.T, client client.Client, name types.NamespacedName, expected map[string][]byte) { - ctx := t.Context() - - logger := t - require.EventuallyWithTf(t, func(t *assert.CollectT) { - logger.Logf("Fetching Secret: %+v", name) - secret := corev1.Secret{} - if !assert.NoError(t, client.Get(ctx, name, &secret)) { - return - } - - assert.Equal(t, expected, secret.Data) - }, recipeTestWaitDuration, recipeTestWaitInterval, "secret data does not match") -} - -func makeDeployment(name types.NamespacedName) *appsv1.Deployment { - return &appsv1.Deployment{ - Name: name.Name, - Namespace: name.Namespace, - Annotations: map[string]string{}, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": name.Name, - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": name.Name, - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: name.Name, - Image: "nginx:latest", - }, - }, - }, - }, - }, - } -} - func makeDeploymentTemplate(name types.NamespacedName, template, providerConfig string, parameters map[string]string) *radappiov1alpha3.DeploymentTemplate { return &radappiov1alpha3.DeploymentTemplate{ Namespace: name.Namespace, diff --git a/pkg/controller/reconciler/util.go b/pkg/controller/reconciler/util.go index 82fd53d0e56..adbadf6b57a 100644 --- a/pkg/controller/reconciler/util.go +++ b/pkg/controller/reconciler/util.go @@ -18,15 +18,10 @@ package reconciler import ( "context" - "fmt" "os" - "strings" v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" "github.com/radius-project/radius/pkg/cli/clients" - "github.com/radius-project/radius/pkg/cli/clients_new/generated" - corerpv20231001preview "github.com/radius-project/radius/pkg/corerp/api/v20231001preview" - sdkclients "github.com/radius-project/radius/pkg/sdk/clients" "github.com/radius-project/radius/pkg/to" ucpv20231001preview "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" "github.com/radius-project/radius/pkg/ucp/resources" @@ -34,51 +29,6 @@ import ( "go.yaml.in/yaml/v3" ) -func resolveDependencies(ctx context.Context, radius RadiusClient, scope string, environmentName string, applicationName string) (resourceGroupID string, environmentID string, applicationID string, err error) { - found, err := findEnvironment(ctx, radius, scope, environmentName) - if found == nil { - return "", "", "", fmt.Errorf("could not find an environment named %q", environmentName) - } else if err != nil { - return "", "", "", err - } - - environmentID = *found - - // NOTE: using resource groups with lowercase here is a workaround for a casing bug in `rad app graph`. - // When https://github.com/radius-project/radius/issues/6422 is fixed we can use the more correct casing. - resourceGroupID = fmt.Sprintf("/planes/radius/local/resourcegroups/%s-%s", environmentName, applicationName) - err = createResourceGroupIfNotExists(ctx, radius, resourceGroupID) - if err != nil { - return "", "", "", fmt.Errorf("failed to create resource group: %w", err) - } - - applicationID = resourceGroupID + "/providers/Applications.Core/applications/" + applicationName - err = createApplicationIfNotExists(ctx, radius, environmentID, applicationID) - if err != nil { - return "", "", "", fmt.Errorf("failed to get or create application: %w", err) - } - - return resourceGroupID, environmentID, applicationID, nil -} - -func findEnvironment(ctx context.Context, radius RadiusClient, scope string, environmentName string) (*string, error) { - logger := ucplog.FromContextOrDiscard(ctx).WithValues("scope", scope) - logger.Info("Listing environments.") - - response, err := radius.Environments(scope).List(ctx, nil) - if err != nil { - return nil, err - } - - for _, env := range response.Value { - if strings.EqualFold(*env.Name, environmentName) { - return env.ID, nil - } - } - - return nil, nil -} - func createResourceGroupIfNotExists(ctx context.Context, radius RadiusClient, resourceGroupID string) error { id, err := resources.Parse(resourceGroupID) if err != nil { @@ -113,226 +63,6 @@ func createResourceGroupIfNotExists(ctx context.Context, radius RadiusClient, re return nil } -func createApplicationIfNotExists(ctx context.Context, radius RadiusClient, environmentID string, applicationID string) error { - id, err := resources.Parse(applicationID) - if err != nil { - return err - } - - logger := ucplog.FromContextOrDiscard(ctx).WithValues("scope", id.RootScope(), "application", applicationID, "environment", environmentID) - logger.Info("Fetching application.") - - _, err = radius.Applications(id.RootScope()).Get(context.Background(), id.Name(), nil) - if clients.Is404Error(err) { - // Need to create application. Keep going. - } else if err != nil { - return err - } else { - // Application already created. - logger.Info("Application already exists.") - return nil - } - - app := corerpv20231001preview.ApplicationResource{ - Location: to.Ptr(v1.LocationGlobal), - Name: new(id.Name()), - Properties: &corerpv20231001preview.ApplicationProperties{ - Environment: new(environmentID), - Extensions: []corerpv20231001preview.ExtensionClassification{ - &corerpv20231001preview.KubernetesNamespaceExtension{ - Kind: new("kubernetesNamespace"), - Namespace: new(id.Name()), - }, - }, - }, - } - _, err = radius.Applications(id.RootScope()).CreateOrUpdate(ctx, id.Name(), app, nil) - if err != nil { - return err - } - - return nil -} - -func deleteResource(ctx context.Context, radius RadiusClient, resourceID string) (sdkclients.Poller[generated.GenericResourcesClientDeleteResponse], error) { - id, err := resources.Parse(resourceID) - if err != nil { - return nil, err - } - - logger := ucplog.FromContextOrDiscard(ctx).WithValues("scope", id.RootScope(), "resourceType", id.Type()) - logger.Info("Deleting resource.") - - poller, err := radius.Resources(id.RootScope(), id.Type()).BeginDelete(ctx, id.Name(), nil) - if err != nil { - return nil, err - } - - if !poller.Done() { - return poller, nil - } - - // Handle synchronous completion - _, err = poller.Result(ctx) - if err != nil { - return nil, err - } - - return nil, nil -} - -func createOrUpdateResource(ctx context.Context, radius RadiusClient, resourceID string, properties map[string]any) (sdkclients.Poller[generated.GenericResourcesClientCreateOrUpdateResponse], error) { - id, err := resources.Parse(resourceID) - if err != nil { - return nil, err - } - - logger := ucplog.FromContextOrDiscard(ctx).WithValues("scope", id.RootScope(), "resourceType", id.Type()) - logger.Info("Creating or updating resource.") - - body := generated.GenericResource{ - Location: to.Ptr(v1.LocationGlobal), - Name: new(id.Name()), - Properties: properties, - } - poller, err := radius.Resources(id.RootScope(), id.Type()).BeginCreateOrUpdate(ctx, id.Name(), body, nil) - if err != nil { - return nil, err - } - - if !poller.Done() { - return poller, nil - } - - // Handle synchronous completion - _, err = poller.Result(ctx) - if err != nil { - return nil, err - } - - return nil, nil -} - -func fetchResource(ctx context.Context, radius RadiusClient, resourceID string) (generated.GenericResourcesClientGetResponse, error) { - id, err := resources.Parse(resourceID) - if err != nil { - return generated.GenericResourcesClientGetResponse{}, err - } - - logger := ucplog.FromContextOrDiscard(ctx).WithValues("scope", id.RootScope(), "resourceType", id.Type()) - logger.Info("Fetching resource.") - - return radius.Resources(id.RootScope(), id.Type()).Get(ctx, id.Name()) -} - -func deleteContainer(ctx context.Context, radius RadiusClient, containerID string) (sdkclients.Poller[corerpv20231001preview.ContainersClientDeleteResponse], error) { - id, err := parseContainerResourceID(containerID) - if err != nil { - return nil, err - } - - logger := ucplog.FromContextOrDiscard(ctx).WithValues("scope", id.RootScope(), "resourceType", id.Type()) - logger.Info("Deleting container.") - - poller, err := radius.Containers(id.RootScope()).BeginDelete(ctx, id.Name(), nil) - if err != nil { - return nil, err - } - - if !poller.Done() { - return poller, nil - } - - // Handle synchronous completion - _, err = poller.Result(ctx) - if err != nil { - return nil, err - } - - return nil, nil -} - -func makeKubernetesDeploymentResourceID(namespace string, name string) string { - return "/planes/kubernetes/local/namespaces/" + namespace + "/providers/apps/Deployment/" + name -} - -func containerHasResourceReference(container *corerpv20231001preview.ContainerResource, expectedResourceID string) bool { - if container == nil || container.Properties == nil { - return false - } - - for _, resource := range container.Properties.Resources { - if resource == nil || resource.ID == nil { - continue - } - - if strings.EqualFold(*resource.ID, expectedResourceID) { - return true - } - } - - return false -} - -func fetchContainerResource(ctx context.Context, radius RadiusClient, containerID string) (corerpv20231001preview.ContainerResource, error) { - id, err := parseContainerResourceID(containerID) - if err != nil { - return corerpv20231001preview.ContainerResource{}, err - } - - response, err := radius.Containers(id.RootScope()).Get(ctx, id.Name(), nil) - if err != nil { - return corerpv20231001preview.ContainerResource{}, err - } - - return response.ContainerResource, nil -} - -func createOrUpdateContainer(ctx context.Context, radius RadiusClient, containerID string, properties *corerpv20231001preview.ContainerProperties) (sdkclients.Poller[corerpv20231001preview.ContainersClientCreateOrUpdateResponse], error) { - id, err := parseContainerResourceID(containerID) - if err != nil { - return nil, err - } - - logger := ucplog.FromContextOrDiscard(ctx).WithValues("scope", id.RootScope(), "resourceType", id.Type()) - logger.Info("Creating or updating container.") - - body := corerpv20231001preview.ContainerResource{ - Location: to.Ptr(v1.LocationGlobal), - Name: new(id.Name()), - Properties: properties, - } - poller, err := radius.Containers(id.RootScope()).BeginCreateOrUpdate(ctx, id.Name(), body, nil) - if err != nil { - return nil, err - } - - if !poller.Done() { - return poller, nil - } - - // Handle synchronous completion - _, err = poller.Result(ctx) - if err != nil { - return nil, err - } - - return nil, nil -} - -func parseContainerResourceID(containerID string) (resources.ID, error) { - id, err := resources.ParseResource(containerID) - if err != nil { - return resources.ID{}, err - } - - if !strings.EqualFold(id.Type(), applicationsCoreContainersResourceType) { - return resources.ID{}, fmt.Errorf("resource type %q is not %q", id.Type(), applicationsCoreContainersResourceType) - } - - return id, nil -} - func generateDeploymentResourceName(resourceId string) (string, error) { id, err := resources.ParseResource(resourceId) if err != nil { diff --git a/pkg/controller/reconciler/util_test.go b/pkg/controller/reconciler/util_test.go index fc9fc4ab223..546d27c4164 100644 --- a/pkg/controller/reconciler/util_test.go +++ b/pkg/controller/reconciler/util_test.go @@ -3,8 +3,6 @@ package reconciler import ( "testing" - v20231001preview "github.com/radius-project/radius/pkg/corerp/api/v20231001preview" - "github.com/radius-project/radius/pkg/to" "github.com/stretchr/testify/require" ) @@ -88,55 +86,3 @@ func TestConvertToARMJSONParameters(t *testing.T) { }) } } - -func TestMakeKubernetesDeploymentResourceID(t *testing.T) { - got := makeKubernetesDeploymentResourceID("current-namespace", "current-app") - require.Equal(t, "/planes/kubernetes/local/namespaces/current-namespace/providers/apps/Deployment/current-app", got) -} - -func TestContainerHasResourceReference(t *testing.T) { - tests := []struct { - name string - container *v20231001preview.ContainerResource - expectedResource string - want bool - }{ - { - name: "has matching resource reference", - container: &v20231001preview.ContainerResource{ - Properties: &v20231001preview.ContainerProperties{ - Resources: []*v20231001preview.ResourceReference{{ - ID: to.Ptr("/planes/kubernetes/local/namespaces/current-namespace/providers/apps/Deployment/current-app"), - }}, - }, - }, - expectedResource: "/planes/kubernetes/local/namespaces/current-namespace/providers/apps/Deployment/current-app", - want: true, - }, - { - name: "missing matching reference", - container: &v20231001preview.ContainerResource{ - Properties: &v20231001preview.ContainerProperties{ - Resources: []*v20231001preview.ResourceReference{{ - ID: to.Ptr("/planes/kubernetes/local/namespaces/other-namespace/providers/apps/Deployment/other-app"), - }}, - }, - }, - expectedResource: "/planes/kubernetes/local/namespaces/current-namespace/providers/apps/Deployment/current-app", - want: false, - }, - { - name: "nil container", - container: nil, - expectedResource: "/planes/kubernetes/local/namespaces/current-namespace/providers/apps/Deployment/current-app", - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := containerHasResourceReference(tt.container, tt.expectedResource) - require.Equal(t, tt.want, got) - }) - } -} diff --git a/pkg/controller/service.go b/pkg/controller/service.go index 13d79c8396c..35c1ca7d1e8 100644 --- a/pkg/controller/service.go +++ b/pkg/controller/service.go @@ -38,7 +38,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "sigs.k8s.io/controller-runtime/pkg/webhook" ) var ( @@ -58,9 +57,6 @@ var _ hosting.Service = (*Service)(nil) type Service struct { // Options is the options for the controller. Options hostoptions.HostOptions - - // TLSConfigDir is the directory containing the TLS configuration. - TLSCertDir string } // Name returns the name of the service. @@ -77,7 +73,6 @@ func (s *Service) Run(ctx context.Context) error { metricsAddr = fmt.Sprintf(":%d", s.Options.Config.MetricsProvider.Prometheus.Port) } - port := s.Options.Config.Server.Port healthProbePort := *s.Options.Config.WorkerServer.Port mgr, err := ctrl.NewManager(s.Options.K8sConfig, ctrl.Options{ Logger: logger, @@ -88,36 +83,12 @@ func (s *Service) Run(ctx context.Context) error { HealthProbeBindAddress: fmt.Sprintf(":%d", healthProbePort), LeaderElection: false, LeaderElectionID: "c85b2113.radapp.io", - WebhookServer: webhook.NewServer(webhook.Options{ - Port: port, - CertDir: s.TLSCertDir, - })}) + }) if err != nil { return fmt.Errorf("failed to create controller manager: %w", err) } logger.Info("Registering controllers.") - //nolint:staticcheck // SA1019: GetEventRecorderFor is deprecated but migration to new events API requires significant refactoring - err = (&reconciler.RecipeReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - EventRecorder: mgr.GetEventRecorderFor("recipe-controller"), - Radius: reconciler.NewRadiusClient(s.Options.UCPConnection), - }).SetupWithManager(mgr) - if err != nil { - return fmt.Errorf("failed to setup %s controller: %w", "Recipe", err) - } - //nolint:staticcheck // SA1019: GetEventRecorderFor is deprecated but migration to new events API requires significant refactoring - err = (&reconciler.DeploymentReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - EventRecorder: mgr.GetEventRecorderFor("radius-deployment-controller"), - Radius: reconciler.NewRadiusClient(s.Options.UCPConnection), - }).SetupWithManager(mgr) - if err != nil { - return fmt.Errorf("failed to setup %s controller: %w", "Deployment", err) - } - resourceDeploymentsClient, err := sdkclients.NewResourceDeploymentsClient(&sdkclients.Options{ Cred: &aztoken.AnonymousCredential{}, BaseURI: s.Options.UCPConnection.Endpoint(), @@ -161,15 +132,6 @@ func (s *Service) Run(ctx context.Context) error { return fmt.Errorf("failed to setup %s controller: %w", "FluxController", err) } - if s.TLSCertDir == "" { - logger.Info("Webhooks will be skipped. TLS certificates not present.") - } else { - logger.Info("Registering validating webhook.") - if err = (&reconciler.RecipeWebhook{}).SetupWebhookWithManager(mgr); err != nil { - return fmt.Errorf("failed to create recipe-webhook: %w", err) - } - } - logger.Info("Registering health checks.") err = mgr.AddHealthzCheck("healthz", healthz.Ping) if err != nil { diff --git a/test/functional-portable/kubernetes/noncloud/kubernetes_test.go b/test/functional-portable/kubernetes/noncloud/kubernetes_test.go deleted file mode 100644 index 07b8d0ccc08..00000000000 --- a/test/functional-portable/kubernetes/noncloud/kubernetes_test.go +++ /dev/null @@ -1,284 +0,0 @@ -/* -Copyright 2023 The Radius Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubernetes_noncloud_test - -import ( - "context" - "encoding/json" - "fmt" - "testing" - "time" - - aztoken "github.com/radius-project/radius/pkg/azure/tokencredentials" - "github.com/radius-project/radius/pkg/cli/clients_new/generated" - radappiov1alpha3 "github.com/radius-project/radius/pkg/controller/api/radapp.io/v1alpha3" - "github.com/radius-project/radius/pkg/controller/reconciler" - "github.com/radius-project/radius/pkg/sdk" - "github.com/radius-project/radius/test/radcli" - "github.com/radius-project/radius/test/rp" - "github.com/radius-project/radius/test/testutil" - "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - watchtools "k8s.io/client-go/tools/watch" - controller_runtime "sigs.k8s.io/controller-runtime/pkg/client" -) - -func Test_TutorialApplication_KubernetesManifests(t *testing.T) { - ctx := t.Context() - opts := rp.NewRPTestOptions(t) - - namespace := "kubernetes-interop-tutorial" - envNamespace := "kubernetes-interop-tutorial-env" - environmentName := namespace + "-env" - applicationName := namespace - - // Create the namespace, if it already exists we can ignore the error. - _, err := opts.K8sClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{Name: namespace}, metav1.CreateOptions{}) - require.NoError(t, controller_runtime.IgnoreAlreadyExists(err)) - - cli := radcli.NewCLI(t, "") - - params := []string{ - testutil.GetBicepRecipeRegistry(), - testutil.GetBicepRecipeVersion(), - - // Avoid a conflict between app namespace and env namespace. - fmt.Sprintf("name=%s", environmentName), - fmt.Sprintf("namespace=%s", environmentName), - } - - err = cli.Deploy(ctx, "testdata/tutorial-environment.bicep", "", "", params...) - require.NoError(t, err) - - deployment := makeDeployment(types.NamespacedName{Name: "demo", Namespace: namespace}, environmentName, applicationName) - recipe := makeRecipe(types.NamespacedName{Name: "db", Namespace: namespace}, environmentName, applicationName) - - t.Run("Deploy", func(t *testing.T) { - t.Log("Creating recipe") - err = opts.Client.Create(ctx, recipe) - require.NoError(t, err) - - t.Log("Creating deployment") - err = opts.Client.Create(ctx, deployment) - require.NoError(t, err) - }) - - t.Run("Check Recipe status", func(t *testing.T) { - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - - // Get resource version - err = opts.Client.Get(ctx, types.NamespacedName{Name: "db", Namespace: namespace}, recipe) - require.NoError(t, err) - - t.Log("Waiting for recipe ready") - recipe, err = waitForRecipeReady(t, ctx, types.NamespacedName{Name: "db", Namespace: namespace}, opts.Client, recipe.ResourceVersion) - require.NoError(t, err) - - // Doing a basic check that the recipe has a resource provisioned. - require.NotEmpty(t, recipe.Status.Resource) - - client, err := generated.NewGenericResourcesClient(recipe.Spec.Type, recipe.Status.Scope, &aztoken.AnonymousCredential{}, sdk.NewClientOptions(opts.Connection)) - require.NoError(t, err) - - _, err = client.Get(ctx, recipe.Name, nil) - require.NoError(t, err) - }) - - t.Run("Check Deployment status", func(t *testing.T) { - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - - // Get resource version - err = opts.Client.Get(ctx, types.NamespacedName{Name: "demo", Namespace: namespace}, deployment) - require.NoError(t, err) - - t.Log("Waiting for deployment ready") - deployment, err = waitForDeploymentReady(t, ctx, types.NamespacedName{Name: "demo", Namespace: namespace}, opts.K8sClient, deployment.ResourceVersion) - require.NoError(t, err) - - // Doing a basic check that the Deployment has environment variables set. - require.NotEmpty(t, deployment.Spec.Template.Spec.Containers[0].EnvFrom) - - // Doing a basic check that the deployment has a resource provisioned. - client, err := generated.NewGenericResourcesClient("Applications.Core/containers", recipe.Status.Scope, &aztoken.AnonymousCredential{}, sdk.NewClientOptions(opts.Connection)) - require.NoError(t, err) - - _, err = client.Get(ctx, deployment.Name, nil) - require.NoError(t, err) - }) - - t.Run("Delete", func(t *testing.T) { - t.Log("Deleting recipe") - err = opts.Client.Delete(ctx, recipe) - require.NoError(t, err) - - require.Eventually(t, func() bool { - err = opts.Client.Get(ctx, types.NamespacedName{Name: "db", Namespace: namespace}, recipe) - return apierrors.IsNotFound(err) - }, time.Second*60, time.Second*5, "waiting for recipe to be deleted") - - t.Log("Deleting deployment") - err = opts.Client.Delete(ctx, deployment) - require.NoError(t, err) - - require.Eventually(t, func() bool { - err = opts.Client.Get(ctx, types.NamespacedName{Name: "demo", Namespace: namespace}, deployment) - return apierrors.IsNotFound(err) - }, time.Second*60, time.Second*5, "waiting for deployment to be deleted") - }) - - t.Run("Cleanup", func(t *testing.T) { - t.Log("Deleting namespace") - deleteNamespace(ctx, t, namespace, opts) - deleteNamespace(ctx, t, envNamespace, opts) - }) -} - -func makeDeployment(name types.NamespacedName, environmentName string, applicationName string) *appsv1.Deployment { - return &appsv1.Deployment{ - Name: name.Name, - Namespace: name.Namespace, - Annotations: map[string]string{ - "radapp.io/enabled": "true", - "radapp.io/connection-redis": "db", - "radapp.io/environment": environmentName, - "radapp.io/application": applicationName, - }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": "demo"}, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": "demo"}, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "demo", - Image: "ghcr.io/radius-project/tutorial/webapp:edge", - Ports: []corev1.ContainerPort{ - { - ContainerPort: 3000, - }, - }, - }, - }, - }, - }, - }, - } -} - -func makeRecipe(name types.NamespacedName, environmentName string, applicationName string) *radappiov1alpha3.Recipe { - return &radappiov1alpha3.Recipe{ - Name: name.Name, - Namespace: name.Namespace, - Annotations: map[string]string{ - "radapp.io/enabled": "true", - "radapp.io/connection-redis": "db", - }, - Spec: radappiov1alpha3.RecipeSpec{ - Type: "Applications.Datastores/redisCaches", - Environment: environmentName, - Application: applicationName, - }, - } -} - -func waitForRecipeReady(t *testing.T, ctx context.Context, name types.NamespacedName, client controller_runtime.WithWatch, initialVersion string) (*radappiov1alpha3.Recipe, error) { - // Based on https://gist.github.com/PrasadG193/52faed6499d2ec739f9630b9d044ffdc - lister := &cache.ListWatch{ - ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { - listOptions := &controller_runtime.ListOptions{Raw: &options, Namespace: name.Namespace, FieldSelector: fields.ParseSelectorOrDie("metadata.name=" + name.Name)} - recipes := &radappiov1alpha3.RecipeList{} - err := client.List(ctx, recipes, listOptions) - if err != nil { - return nil, err - } - - return recipes, nil - }, - WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { - listOptions := &controller_runtime.ListOptions{Raw: &options, Namespace: name.Namespace, FieldSelector: fields.ParseSelectorOrDie("metadata.name=" + name.Name)} - recipes := &radappiov1alpha3.RecipeList{} - return client.Watch(ctx, recipes, listOptions) - }, - } - watcher, err := watchtools.NewRetryWatcherWithContext(ctx, initialVersion, lister) - require.NoError(t, err) - defer watcher.Stop() - - for { - event := <-watcher.ResultChan() - r, ok := event.Object.(*radappiov1alpha3.Recipe) - if !ok { - // Not a recipe, likely an event. - t.Logf("Received event: %+v", event) - continue - } - - t.Logf("Received recipe. Status: %+v", r.Status) - if r.Status.Phrase == radappiov1alpha3.PhraseReady { - return r, nil - } - } -} - -func waitForDeploymentReady(t *testing.T, ctx context.Context, name types.NamespacedName, client *kubernetes.Clientset, initialVersion string) (*appsv1.Deployment, error) { - // Based on https://gist.github.com/PrasadG193/52faed6499d2ec739f9630b9d044ffdc - watcher, err := watchtools.NewRetryWatcherWithContext(ctx, initialVersion, cache.NewFilteredListWatchFromClient(client.AppsV1().RESTClient(), "deployments", name.Namespace, func(options *metav1.ListOptions) { - options.FieldSelector = "metadata.name=" + name.Name - })) - require.NoError(t, err) - defer watcher.Stop() - - for { - event := <-watcher.ResultChan() - d, ok := event.Object.(*appsv1.Deployment) - if !ok { - // Not a deployment, likely an event. - t.Logf("Received event: %+v", event) - continue - } - - t.Logf("Received deployment. Annotations: %+v", d.Annotations) - - data, ok := d.Annotations[reconciler.AnnotationRadiusStatus] - if !ok || data == "" { - continue - } - - status := map[string]any{} - err := json.Unmarshal([]byte(data), &status) - require.NoError(t, err) - - if d.Status.ObservedGeneration == d.Generation && d.Status.ReadyReplicas == 1 && status["phrase"] == "Ready" { - return d, nil - } - } -} diff --git a/test/functional-portable/kubernetes/noncloud/testdata/tutorial-environment.bicep b/test/functional-portable/kubernetes/noncloud/testdata/tutorial-environment.bicep deleted file mode 100644 index 10976922742..00000000000 --- a/test/functional-portable/kubernetes/noncloud/testdata/tutorial-environment.bicep +++ /dev/null @@ -1,25 +0,0 @@ -extension radius - -param name string -param namespace string -param registry string -param version string - -resource env 'Applications.Core/environments@2023-10-01-preview' = { - name: name - properties: { - compute: { - kind: 'kubernetes' - resourceId: 'self' - namespace: namespace - } - recipes: { - 'Applications.Datastores/redisCaches': { - default: { - templateKind: 'bicep' - templatePath: '${registry}/test/testrecipes/test-bicep-recipes/redis-recipe-value-backed:${version}' - } - } - } - } -}