diff --git a/charts/kserve-resources/templates/clusterrole.yaml b/charts/kserve-resources/templates/clusterrole.yaml index c35a7583171..45dcb8bca16 100644 --- a/charts/kserve-resources/templates/clusterrole.yaml +++ b/charts/kserve-resources/templates/clusterrole.yaml @@ -144,6 +144,14 @@ rules: - patch - update - watch +- apiGroups: + - operator.knative.dev + resources: + - knativeservings + verbs: + - get + - list + - watch - apiGroups: - rbac.authorization.k8s.io resourceNames: @@ -161,6 +169,7 @@ rules: - routes verbs: - create + - delete - get - list - patch diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 9e5843edc00..4cf1484df72 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -33,6 +33,7 @@ import ( typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "k8s.io/client-go/tools/record" + operatorv1beta1 "knative.dev/operator/pkg/apis/operator/v1beta1" knservingv1 "knative.dev/serving/pkg/apis/serving/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/config" @@ -176,6 +177,20 @@ func main() { os.Exit(1) } } + + knServingFound, knServingCheckErr := utils.IsCrdAvailable(cfg, operatorv1beta1.SchemeGroupVersion.String(), constants.KnativeServingKind) + if knServingCheckErr != nil { + setupLog.Error(knServingCheckErr, "error when checking if Knative KnativeServing kind is available") + os.Exit(1) + } + if knServingFound { + setupLog.Info("Setting up Knative Operator scheme") + if err := operatorv1beta1.AddToScheme(mgr.GetScheme()); err != nil { + setupLog.Error(err, "unable to add Knative Operator APIs to scheme") + os.Exit(1) + } + } + if !ingressConfig.DisableIstioVirtualHost { vsFound, vsCheckErr := utils.IsCrdAvailable(cfg, istioclientv1beta1.SchemeGroupVersion.String(), constants.IstioVirtualServiceKind) if vsCheckErr != nil { diff --git a/config/configmap/inferenceservice.yaml b/config/configmap/inferenceservice.yaml index 0feb63db983..a683b0ba05b 100644 --- a/config/configmap/inferenceservice.yaml +++ b/config/configmap/inferenceservice.yaml @@ -59,6 +59,7 @@ data: # revisions, which prevents the reconciliation loop to be triggered if the annotations is # configured here are used. # Default values are: + # "autoscaling.knative.dev/initial-scale", # "autoscaling.knative.dev/min-scale", # "autoscaling.knative.dev/max-scale", # "internal.serving.kserve.io/storage-initializer-sourceuri", diff --git a/config/overlays/odh/inferenceservice-config-patch.yaml b/config/overlays/odh/inferenceservice-config-patch.yaml index 75a47feb240..71f07abddb9 100644 --- a/config/overlays/odh/inferenceservice-config-patch.yaml +++ b/config/overlays/odh/inferenceservice-config-patch.yaml @@ -88,6 +88,7 @@ data: inferenceService: |- { "serviceAnnotationDisallowedList": [ + "autoscaling.knative.dev/initial-scale", "autoscaling.knative.dev/min-scale", "autoscaling.knative.dev/max-scale", "internal.serving.kserve.io/storage-initializer-sourceuri", diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index e48ccf560af..edbc9ecc53a 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -131,6 +131,14 @@ rules: - patch - update - watch +- apiGroups: + - operator.knative.dev + resources: + - knativeservings + verbs: + - get + - list + - watch - apiGroups: - rbac.authorization.k8s.io resourceNames: diff --git a/go.mod b/go.mod index 3f2cd5c87f7..808983abc9e 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2 k8s.io/utils v0.0.0-20240821151609-f90d01438635 knative.dev/networking v0.0.0-20240815142417-37fdbdd0854b + knative.dev/operator v0.42.2 knative.dev/pkg v0.0.0-20240815051656-89743d9bbf7c knative.dev/serving v0.42.2 sigs.k8s.io/controller-runtime v0.19.2 diff --git a/go.sum b/go.sum index 4882bfe6e6f..7624ecb6914 100644 --- a/go.sum +++ b/go.sum @@ -285,6 +285,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/manifestival/manifestival v0.7.2 h1:l4uFdWX/xQK4QcRfqGoMtBvaZeWPEuwD6hVsCwUqZY4= +github.com/manifestival/manifestival v0.7.2/go.mod h1:nl3T6HlfHCeidooWVTMI9vYNTBkQ1GdhLNb+smozbdk= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -764,8 +766,12 @@ k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2 h1:GKE9U8BH16uynoxQii0auT k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA= k8s.io/utils v0.0.0-20240821151609-f90d01438635 h1:2wThSvJoW/Ncn9TmQEYXRnevZXi2duqHWf5OX9S3zjI= k8s.io/utils v0.0.0-20240821151609-f90d01438635/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +knative.dev/caching v0.0.0-20240716132144-989f54c83776 h1:2nINnWuXtb9e2nG/EJxSCeghcmu6qmvmomJ7woiP5Is= +knative.dev/caching v0.0.0-20240716132144-989f54c83776/go.mod h1:Uj74eO9rLiK1eb8wmDBED1hJBZQ7MJ9cvq/d8Ktsm3c= knative.dev/networking v0.0.0-20240815142417-37fdbdd0854b h1:ws/Jeho6on84+5tfNKLAKriVVGIwivHbgPEtZjBfcs0= knative.dev/networking v0.0.0-20240815142417-37fdbdd0854b/go.mod h1:2eMQVGLBZ5Kj1C4kKPuPhO7BsUeF6fkmhZFDQPIP+88= +knative.dev/operator v0.42.2 h1:wgAWYHwoSFmV+wPHCt5dZahHTHLy2VCM4G82PEo9iSc= +knative.dev/operator v0.42.2/go.mod h1:cfSpJMgvwmuZ7USaxC+zgEuizMFc/xweREW5DG6J1DA= knative.dev/pkg v0.0.0-20240815051656-89743d9bbf7c h1:2crXVk4FG0dSG6WHaIT+WKbUzn7qG2wn0AfYmvA22zs= knative.dev/pkg v0.0.0-20240815051656-89743d9bbf7c/go.mod h1:cI2RPEEHZk+/dBpfHobs0aBdPA1mMZVUVWnGAc8NSzM= knative.dev/serving v0.42.2 h1:yKieg3MeNvpVz+4JJPbvmpee3v3LK3zO5h5HJBtzaNk= diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index a7220a199e7..d1f3e8d482d 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -93,6 +93,7 @@ var ( AutoscalerClass = KServeAPIGroupName + "/autoscalerClass" AutoscalerMetrics = KServeAPIGroupName + "/metrics" TargetUtilizationPercentage = KServeAPIGroupName + "/targetUtilizationPercentage" + InitialScaleAnnotationKey = KnativeAutoscalingAPIGroupName + "/initial-scale" MinScaleAnnotationKey = KnativeAutoscalingAPIGroupName + "/min-scale" MaxScaleAnnotationKey = KnativeAutoscalingAPIGroupName + "/max-scale" RollOutDurationAnnotationKey = KnativeServingAPIGroupName + "/rollout-duration" @@ -263,9 +264,14 @@ type InferenceServiceProtocol string // Knative constants const ( - KnativeLocalGateway = "knative-serving/knative-local-gateway" - KnativeIngressGateway = "knative-serving/knative-ingress-gateway" - VisibilityLabel = "networking.knative.dev/visibility" + AutoscalerKey = "autoscaler" + AutoscalerInitialScaleKey = "initial-scale" + AutoscalerAllowZeroScaleKey = "allow-zero-initial-scale" + DefaultKnServingName = "knative-serving" + DefaultKnServingNamespace = "knative-serving" + KnativeLocalGateway = "knative-serving/knative-local-gateway" + KnativeIngressGateway = "knative-serving/knative-ingress-gateway" + VisibilityLabel = "networking.knative.dev/visibility" ) var ( @@ -379,6 +385,7 @@ var ( // revisions, which prevents the reconciliation loop to be triggered if the annotations is // configured here are used. ServiceAnnotationDisallowedList = []string{ + autoscaling.InitialScaleAnnotationKey, autoscaling.MinScaleAnnotationKey, autoscaling.MaxScaleAnnotationKey, StorageInitializerSourceUriInternalAnnotationKey, @@ -504,6 +511,7 @@ const ( const ( IstioVirtualServiceKind = "VirtualService" KnativeServiceKind = "Service" + KnativeServingKind = "KnativeServing" HTTPRouteKind = "HTTPRoute" GatewayKind = "Gateway" ServiceKind = "Service" diff --git a/pkg/controller/v1alpha1/inferencegraph/controller.go b/pkg/controller/v1alpha1/inferencegraph/controller.go index c1bc0c3f490..a9788056c83 100644 --- a/pkg/controller/v1alpha1/inferencegraph/controller.go +++ b/pkg/controller/v1alpha1/inferencegraph/controller.go @@ -23,6 +23,7 @@ limitations under the License. // +kubebuilder:rbac:groups=serving.knative.dev,resources=services/status,verbs=get;update;patch // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=create;get;update;patch;watch;delete // +kubebuilder:rbac:groups=route.openshift.io,resources=routes/status,verbs=get +// +kubebuilder:rbac:groups=operator.knative.dev,resources=knativeservings,verbs=get;list;watch package inferencegraph import ( @@ -45,6 +46,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" + operatorv1beta1 "knative.dev/operator/pkg/apis/operator/v1beta1" "knative.dev/pkg/apis" knservingv1 "knative.dev/serving/pkg/apis/serving/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -266,10 +268,26 @@ func (r *InferenceGraphReconciler) Reconcile(ctx context.Context, req ctrl.Reque if !ksvcAvailable { r.Recorder.Event(graph, v1.EventTypeWarning, "ServerlessModeRejected", "It is not possible to use Serverless deployment mode when Knative Services are not available") - return reconcile.Result{Requeue: false}, reconcile.TerminalError(fmt.Errorf("the resolved deployment mode of InferenceGraph '%s' is Serverless, but Knative Serving is not available", graph.Name)) + return reconcile.Result{Requeue: false}, reconcile.TerminalError(fmt.Errorf("the resolved deployment mode of InferenceGraph '%s' is Serverless, but Knative Services are not available", graph.Name)) + } + + // Abort if Knative KnativeServings are not available + knServingFound, knServingCheckErr := utils.IsCrdAvailable(r.ClientConfig, operatorv1beta1.SchemeGroupVersion.String(), constants.KnativeServingKind) + if knServingCheckErr != nil { + return reconcile.Result{}, knServingCheckErr + } + + if !knServingFound { + r.Recorder.Event(graph, v1.EventTypeWarning, "ServerlessModeRejected", + "It is not possible to use Serverless deployment mode when Knative KnativeServings are not available") + return reconcile.Result{Requeue: false}, reconcile.TerminalError(fmt.Errorf("the resolved deployment mode of InferenceGraph '%s' is Serverless, but Knative KnativeServings are not available", graph.Name)) + } + + desired, err := createKnativeService(r.Client, graph.ObjectMeta, graph, routerConfig) + if err != nil { + return ctrl.Result{}, errors.Wrapf(err, "fails to create new knative service") } - desired := createKnativeService(graph.ObjectMeta, graph, routerConfig) err = controllerutil.SetControllerReference(graph, desired, r.Scheme) if err != nil { return reconcile.Result{}, err diff --git a/pkg/controller/v1alpha1/inferencegraph/controller_test.go b/pkg/controller/v1alpha1/inferencegraph/controller_test.go index 6dd12c9855f..41c9a833c6a 100644 --- a/pkg/controller/v1alpha1/inferencegraph/controller_test.go +++ b/pkg/controller/v1alpha1/inferencegraph/controller_test.go @@ -33,6 +33,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" + operatorv1beta1 "knative.dev/operator/pkg/apis/operator/v1beta1" "knative.dev/pkg/kmp" knservingv1 "knative.dev/serving/pkg/apis/serving/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -80,6 +81,235 @@ var _ = Describe("Inference Graph controller test", func() { var expectedReadinessProbe = constants.GetRouterReadinessProbe() + Context("with knative configured to allow zero initial scale", func() { + When("an InferenceGraph with min-scale:0 annotation is created", func() { + It("should create a knative service with initial-scale:0 and min-scale:0 annotations", func() { + var configMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.InferenceServiceConfigMapName, + Namespace: constants.KServeNamespace, + }, + Data: configs, + } + Expect(k8sClient.Create(context.TODO(), configMap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), configMap) + graphName := "initialscale1" + var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: graphName, Namespace: "default"}} + var serviceKey = expectedRequest.NamespacedName + ctx := context.Background() + minScale := 1 + ig := &v1alpha1.InferenceGraph{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceKey.Name, + Namespace: serviceKey.Namespace, + Annotations: map[string]string{ + constants.MinScaleAnnotationKey: "0", + }, + }, + Spec: v1alpha1.InferenceGraphSpec{ + MinReplicas: &minScale, + Nodes: map[string]v1alpha1.InferenceRouter{ + v1alpha1.GraphRootNodeName: { + RouterType: v1alpha1.Sequence, + Steps: []v1alpha1.InferenceStep{ + { + InferenceTarget: v1alpha1.InferenceTarget{ + ServiceURL: "http://someservice.exmaple.com", + }, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, ig)).Should(Succeed()) + defer k8sClient.Delete(ctx, ig) + + actualKnServiceCreated := &knservingv1.Service{} + Eventually(func() error { + return k8sClient.Get(context.TODO(), serviceKey, actualKnServiceCreated) + }, timeout). + Should(Succeed()) + + Expect(actualKnServiceCreated.Spec.Template.Annotations[constants.InitialScaleAnnotationKey]).To(Equal("0")) + Expect(actualKnServiceCreated.Spec.Template.Annotations[constants.MinScaleAnnotationKey]).To(Equal("0")) + }) + }) + When("an InferenceGraph without min-scale annotation is created", func() { + It("should create a knative service with no initial-scale annotation and min-scale annotation equal to the default min replicas value", func() { + var configMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.InferenceServiceConfigMapName, + Namespace: constants.KServeNamespace, + }, + Data: configs, + } + Expect(k8sClient.Create(context.TODO(), configMap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), configMap) + graphName := "initialscale2" + var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: graphName, Namespace: "default"}} + var serviceKey = expectedRequest.NamespacedName + ctx := context.Background() + minScale := 1 + ig := &v1alpha1.InferenceGraph{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceKey.Name, + Namespace: serviceKey.Namespace, + }, + Spec: v1alpha1.InferenceGraphSpec{ + MinReplicas: &minScale, + Nodes: map[string]v1alpha1.InferenceRouter{ + v1alpha1.GraphRootNodeName: { + RouterType: v1alpha1.Sequence, + Steps: []v1alpha1.InferenceStep{ + { + InferenceTarget: v1alpha1.InferenceTarget{ + ServiceURL: "http://someservice.exmaple.com", + }, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, ig)).Should(Succeed()) + defer k8sClient.Delete(ctx, ig) + + actualKnServiceCreated := &knservingv1.Service{} + Eventually(func() error { + return k8sClient.Get(context.TODO(), serviceKey, actualKnServiceCreated) + }, timeout). + Should(Succeed()) + + Expect(actualKnServiceCreated.Spec.Template.Annotations[constants.MinScaleAnnotationKey]).To(Equal(fmt.Sprint(constants.DefaultMinReplicas))) + Expect(constants.InitialScaleAnnotationKey).ShouldNot(BeKeyOf(actualKnServiceCreated.Spec.Template.Annotations)) + }) + }) + }) + Context("with knative configured to not allow zero initial scale", func() { + BeforeEach(func() { + // Update the existing knativeserving custom resource to set allow-zero-intial-scale to false + knativeService := &operatorv1beta1.KnativeServing{} + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: constants.DefaultKnServingName, Namespace: constants.DefaultKnServingNamespace}, knativeService)).ToNot(HaveOccurred()) + knativeService.Spec.CommonSpec.Config["autoscaler"]["allow-zero-initial-scale"] = "false" + Eventually(func() error { + return k8sClient.Update(context.TODO(), knativeService) + }, timeout).Should(Succeed()) + }) + AfterEach(func() { + // Restore the original knativeserving custom resource configuration + knativeServiceRestored := &operatorv1beta1.KnativeServing{} + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: constants.DefaultKnServingName, Namespace: constants.DefaultKnServingNamespace}, knativeServiceRestored)).ToNot(HaveOccurred()) + knativeServiceRestored.Spec.CommonSpec.Config["autoscaler"]["allow-zero-initial-scale"] = "true" + Eventually(func() error { + return k8sClient.Update(context.TODO(), knativeServiceRestored) + }, timeout).Should(Succeed()) + }) + When("an InferenceGraph with min-scale:0 annotation is created", func() { + It("should create a knative service with min-scale:0 annotation and no initial-scale annotation", func() { + var configMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.InferenceServiceConfigMapName, + Namespace: constants.KServeNamespace, + }, + Data: configs, + } + Expect(k8sClient.Create(context.TODO(), configMap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), configMap) + graphName := "initialscale3" + var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: graphName, Namespace: "default"}} + var serviceKey = expectedRequest.NamespacedName + ctx := context.Background() + minScale := 1 + ig := &v1alpha1.InferenceGraph{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceKey.Name, + Namespace: serviceKey.Namespace, + Annotations: map[string]string{ + constants.MinScaleAnnotationKey: "0", + }, + }, + Spec: v1alpha1.InferenceGraphSpec{ + MinReplicas: &minScale, + Nodes: map[string]v1alpha1.InferenceRouter{ + v1alpha1.GraphRootNodeName: { + RouterType: v1alpha1.Sequence, + Steps: []v1alpha1.InferenceStep{ + { + InferenceTarget: v1alpha1.InferenceTarget{ + ServiceURL: "http://someservice.exmaple.com", + }, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, ig)).Should(Succeed()) + defer k8sClient.Delete(ctx, ig) + + actualKnServiceCreated := &knservingv1.Service{} + Eventually(func() error { + return k8sClient.Get(context.TODO(), serviceKey, actualKnServiceCreated) + }, timeout). + Should(Succeed()) + + Expect(actualKnServiceCreated.Spec.Template.Annotations[constants.MinScaleAnnotationKey]).To(Equal("0")) + Expect(constants.InitialScaleAnnotationKey).ShouldNot(BeKeyOf(actualKnServiceCreated.Spec.Template.Annotations)) + }) + }) + When("an InferenceGraph without min-scale annotation is created", func() { + It("should create a knative service with no initial-scale annotation and min-scale annotation equal to the default minReplicas value", func() { + var configMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.InferenceServiceConfigMapName, + Namespace: constants.KServeNamespace, + }, + Data: configs, + } + Expect(k8sClient.Create(context.TODO(), configMap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), configMap) + graphName := "initialscale4" + var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: graphName, Namespace: "default"}} + var serviceKey = expectedRequest.NamespacedName + ctx := context.Background() + minScale := 1 + ig := &v1alpha1.InferenceGraph{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceKey.Name, + Namespace: serviceKey.Namespace, + }, + Spec: v1alpha1.InferenceGraphSpec{ + MinReplicas: &minScale, + Nodes: map[string]v1alpha1.InferenceRouter{ + v1alpha1.GraphRootNodeName: { + RouterType: v1alpha1.Sequence, + Steps: []v1alpha1.InferenceStep{ + { + InferenceTarget: v1alpha1.InferenceTarget{ + ServiceURL: "http://someservice.exmaple.com", + }, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, ig)).Should(Succeed()) + defer k8sClient.Delete(ctx, ig) + + actualKnServiceCreated := &knservingv1.Service{} + Eventually(func() error { + return k8sClient.Get(context.TODO(), serviceKey, actualKnServiceCreated) + }, timeout). + Should(Succeed()) + + Expect(actualKnServiceCreated.Spec.Template.Annotations[constants.MinScaleAnnotationKey]).To(Equal(fmt.Sprint(constants.DefaultMinReplicas))) + Expect(constants.InitialScaleAnnotationKey).ShouldNot(BeKeyOf(actualKnServiceCreated.Spec.Template.Annotations)) + }) + }) + }) + Context("When creating an inferencegraph with headers in global config", func() { It("Should create a knative service with headers as env var of podspec", func() { By("By creating a new InferenceGraph") diff --git a/pkg/controller/v1alpha1/inferencegraph/knative_reconciler.go b/pkg/controller/v1alpha1/inferencegraph/knative_reconciler.go index 9b74b26436f..8bab699cb13 100644 --- a/pkg/controller/v1alpha1/inferencegraph/knative_reconciler.go +++ b/pkg/controller/v1alpha1/inferencegraph/knative_reconciler.go @@ -21,10 +21,12 @@ import ( "encoding/json" "fmt" "reflect" + "strconv" "strings" v1alpha1api "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" "github.com/kserve/kserve/pkg/constants" + knutils "github.com/kserve/kserve/pkg/controller/v1alpha1/utils" "github.com/kserve/kserve/pkg/utils" "github.com/pkg/errors" "google.golang.org/protobuf/proto" @@ -135,10 +137,10 @@ func semanticEquals(desiredService, service *knservingv1.Service) bool { equality.Semantic.DeepEqual(desiredService.Spec.RouteSpec, service.Spec.RouteSpec) } -func createKnativeService(componentMeta metav1.ObjectMeta, graph *v1alpha1api.InferenceGraph, config *RouterConfig) *knservingv1.Service { +func createKnativeService(client client.Client, componentMeta metav1.ObjectMeta, graph *v1alpha1api.InferenceGraph, config *RouterConfig) (*knservingv1.Service, error) { bytes, err := json.Marshal(graph.Spec) if err != nil { - return nil + return nil, errors.Wrapf(err, "fails to marshal inference graph spec to json") } annotations := componentMeta.GetAnnotations() if annotations == nil { @@ -148,13 +150,10 @@ func createKnativeService(componentMeta metav1.ObjectMeta, graph *v1alpha1api.In if labels == nil { labels = make(map[string]string) //nolint:ineffassign, staticcheck } - // User can pass down scaling class annotation to overwrite the default scaling KPA - if _, ok := annotations[autoscaling.ClassAnnotationKey]; !ok { - annotations[autoscaling.ClassAnnotationKey] = autoscaling.KPA - } - if _, ok := annotations[autoscaling.MinScaleAnnotationKey]; !ok { - annotations[autoscaling.MinScaleAnnotationKey] = fmt.Sprint(constants.DefaultMinReplicas) + err = setAutoScalingAnnotations(client, annotations) + if err != nil { + return nil, errors.Wrapf(err, "fails to set autoscaling annotations for knative service") } // ksvc metadata.annotations @@ -249,7 +248,7 @@ func createKnativeService(componentMeta metav1.ObjectMeta, graph *v1alpha1api.In service.Spec.ConfigurationSpec.Template.Spec.PodSpec.Containers[0].Env = append(service.Spec.ConfigurationSpec.Template.Spec.PodSpec.Containers[0].Env, propagateEnv) } - return service + return service, nil } func constructResourceRequirements(graph v1alpha1api.InferenceGraph, config RouterConfig) v1.ResourceRequirements { @@ -274,3 +273,65 @@ func constructResourceRequirements(graph v1alpha1api.InferenceGraph, config Rout } return specResources } + +// setAutoScalingAnnotations checks the knative autoscaler configuration defined in the knativeserving custom resource +// and compares the values to the autoscaling configuration requested for the inference service. +// It then sets the necessary annotations for the desired autoscaling configuration. +func setAutoScalingAnnotations(client client.Client, + annotations map[string]string) error { + // User can pass down scaling class annotation to overwrite the default scaling KPA + if _, ok := annotations[autoscaling.ClassAnnotationKey]; !ok { + annotations[autoscaling.ClassAnnotationKey] = autoscaling.KPA + } + + // If a min-scale annotation is not set for the inference graph, then use the default min-scale value of 1. + var revisionMinScale int + if _, ok := annotations[autoscaling.MinScaleAnnotationKey]; !ok { + annotations[autoscaling.MinScaleAnnotationKey] = fmt.Sprint(constants.DefaultMinReplicas) + revisionMinScale = constants.DefaultMinReplicas + } else { + minScaleInt, err := strconv.Atoi(annotations[autoscaling.MinScaleAnnotationKey]) + if err != nil { + return errors.Wrapf(err, "fails to convert existing min-scale annotation to an integer") + } + revisionMinScale = minScaleInt + } + + // Retrieve the allow-zero-initial-scale and initial-scale values from the knative autoscaler configuration. + allowZeroInitialScale, globalInitialScale, err := knutils.GetAutoscalerConfiguration(client) + if err != nil { + return errors.Wrapf(err, "failed to retrieve the knative autoscaler configuration") + } + + initialScaleInt, err := strconv.Atoi(globalInitialScale) + if err != nil { + return errors.Wrapf(err, fmt.Sprintf("fails to convert configured knative serving global initial-scale value to an integer. initial-scale: %s", globalInitialScale)) + } + + // Provide transparency to users while aligning with knative serving's expected behavior, log a warning when the + // knative autoscaler's gloabal initial-scale value exceeds the requested minScale value for the inference graph. + if initialScaleInt > revisionMinScale { + log.Info("knative autoscaler is globally configured with an initial-scale value that is greater than the requested min-scale for the inference graph.", + "initial-scale", initialScaleInt, + "min-scale", revisionMinScale) + } + + // knative will choose the larger of min-scale and initial-scale as the initial target scale for a knative revision. + // When min-scale is 0, if allow-zero-initial scale is true, set initial-scale to 0 for the created knative revision. + // This will prevent any pods from being created to initialize a knative revision when an inference graph has minReplicas set to 0. + // Configuring scaling for knative: https://knative.dev/docs/serving/autoscaling/scale-bounds/#initial-scale + if revisionMinScale == 0 { + if allowZeroInitialScale == "true" { + log.Info("kserve will override the global knative autoscaler configuration for initial-scale on a per revision basis with 0 when an inference graph is requested with min-scale 0", + "allow-zero-initial-scale", allowZeroInitialScale, + "initial-scale", globalInitialScale) + annotations[constants.InitialScaleAnnotationKey] = "0" + } else { + log.Info("The current knative autoscaler global configuration does not allow zero initial scale", + "allow-zero-initial-scale", allowZeroInitialScale, + "initial-scale", globalInitialScale) + } + } + + return nil +} diff --git a/pkg/controller/v1alpha1/inferencegraph/suite_test.go b/pkg/controller/v1alpha1/inferencegraph/suite_test.go index 261bd38983e..6364b3f28da 100644 --- a/pkg/controller/v1alpha1/inferencegraph/suite_test.go +++ b/pkg/controller/v1alpha1/inferencegraph/suite_test.go @@ -30,6 +30,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "knative.dev/operator/pkg/apis/operator/base" + operatorv1beta1 "knative.dev/operator/pkg/apis/operator/v1beta1" knservingv1 "knative.dev/serving/pkg/apis/serving/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -92,13 +94,37 @@ var _ = BeforeSuite(func() { Expect(err).ToNot(HaveOccurred()) Expect(clientset).ToNot(BeNil()) - //Create namespace + // Create namespaces kfservingNamespaceObj := &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: constants.KServeNamespace, }, } + knativeServingNamespace := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.DefaultKnServingNamespace, + }, + } Expect(k8sClient.Create(context.Background(), kfservingNamespaceObj)).Should(Succeed()) + Expect(k8sClient.Create(context.Background(), knativeServingNamespace)).Should(Succeed()) + + // Create knativeserving custom resource + knativeCr := &operatorv1beta1.KnativeServing{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.DefaultKnServingName, + Namespace: constants.DefaultKnServingNamespace, + }, + Spec: operatorv1beta1.KnativeServingSpec{ + CommonSpec: base.CommonSpec{ + Config: base.ConfigMapData{ + "autoscaler": map[string]string{ + "allow-zero-initial-scale": "true", + }, + }, + }, + }, + } + Expect(k8sClient.Create(context.Background(), knativeCr)).Should(Succeed()) k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme.Scheme, diff --git a/pkg/controller/v1alpha1/utils/utils.go b/pkg/controller/v1alpha1/utils/utils.go new file mode 100644 index 00000000000..6909f9d6f9b --- /dev/null +++ b/pkg/controller/v1alpha1/utils/utils.go @@ -0,0 +1,68 @@ +/* +Copyright 2021 The KServe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + + "github.com/kserve/kserve/pkg/constants" + "github.com/pkg/errors" + operatorv1beta1 "knative.dev/operator/pkg/apis/operator/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// GetAutoscalerConfiguration reads the global knative serving configuration and retrieves values related to the autoscaler. +// This configuration is defined in the knativeserving custom resource. +func GetAutoscalerConfiguration(client client.Client) (string, string, error) { + // Set allow-zero-initial-scale and intitial-scale to their default values to start. + // If autoscaling values are not set in the configuration, then the defaults are used. + allowZeroInitialScale := "false" + globalInitialScale := "1" + + // List all knativeserving custom resources to handle scenarios where the custom resource is not created in the default knative-serving namespace. + knservingList := &operatorv1beta1.KnativeServingList{} + err := client.List(context.TODO(), knservingList) + if err != nil { + return allowZeroInitialScale, globalInitialScale, errors.Wrapf( + err, + "fails to retrieve the knativeserving custom resource.", + ) + } else if len(knservingList.Items) == 0 { + return allowZeroInitialScale, globalInitialScale, errors.New("no knativeserving resources found in cluster.") + } + + // Always use the first knativeserving resource returned. + // We are operating under the assumption that there should be a single knativeserving custom resource created on the cluster. + knserving := knservingList.Items[0] + + // Check for both the autoscaler key with and without the 'config-' prefix. Both are supported by knative. + var knservingAutoscalerConfig map[string]string + if _, ok := knserving.Spec.Config[constants.AutoscalerKey]; ok { + knservingAutoscalerConfig = knserving.Spec.Config[constants.AutoscalerKey] + } else if _, ok := knserving.Spec.Config["config-"+constants.AutoscalerKey]; ok { + knservingAutoscalerConfig = knserving.Spec.Config["config-"+constants.AutoscalerKey] + } + + if configuredAllowZeroInitialScale, ok := knservingAutoscalerConfig[constants.AutoscalerAllowZeroScaleKey]; ok { + allowZeroInitialScale = configuredAllowZeroInitialScale + } + if configuredInitialScale, ok := knservingAutoscalerConfig[constants.AutoscalerInitialScaleKey]; ok { + globalInitialScale = configuredInitialScale + } + + return allowZeroInitialScale, globalInitialScale, nil +} diff --git a/pkg/controller/v1beta1/inferenceservice/components/explainer.go b/pkg/controller/v1beta1/inferenceservice/components/explainer.go index cde182ca6aa..bd3e91419c8 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/explainer.go +++ b/pkg/controller/v1beta1/inferenceservice/components/explainer.go @@ -188,8 +188,11 @@ func (e *Explainer) Reconcile(isvc *v1beta1.InferenceService) (ctrl.Result, erro } isvc.Status.PropagateRawStatus(v1beta1.ExplainerComponent, deployment, r.URL) } else { - r := knative.NewKsvcReconciler(e.client, e.scheme, objectMeta, &isvc.Spec.Explainer.ComponentExtensionSpec, + r, err := knative.NewKsvcReconciler(e.client, e.scheme, objectMeta, &isvc.Spec.Explainer.ComponentExtensionSpec, &podSpec, isvc.Status.Components[v1beta1.ExplainerComponent], e.inferenceServiceConfig.ServiceLabelDisallowedList) + if err != nil { + return ctrl.Result{}, errors.Wrapf(err, "fails to create new knative service reconciler for explainer") + } if err := controllerutil.SetControllerReference(isvc, r.Service, e.scheme); err != nil { return ctrl.Result{}, errors.Wrapf(err, "fails to set owner reference for explainer") diff --git a/pkg/controller/v1beta1/inferenceservice/components/predictor.go b/pkg/controller/v1beta1/inferenceservice/components/predictor.go index f5a0a59798c..3d4b02bafe4 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/predictor.go +++ b/pkg/controller/v1beta1/inferenceservice/components/predictor.go @@ -395,13 +395,16 @@ func (p *Predictor) Reconcile(isvc *v1beta1.InferenceService) (ctrl.Result, erro isvc.Status.PropagateRawStatus(v1beta1.PredictorComponent, deploymentList, r.URL) } else { podLabelKey = constants.RevisionLabel - r := knative.NewKsvcReconciler(p.client, p.scheme, objectMeta, &isvc.Spec.Predictor.ComponentExtensionSpec, + r, err := knative.NewKsvcReconciler(p.client, p.scheme, objectMeta, &isvc.Spec.Predictor.ComponentExtensionSpec, &podSpec, isvc.Status.Components[v1beta1.PredictorComponent], p.inferenceServiceConfig.ServiceLabelDisallowedList) + if err != nil { + return ctrl.Result{}, errors.Wrapf(err, "fails to create new knative service reconciler for predictor") + } + if err := controllerutil.SetControllerReference(isvc, r.Service, p.scheme); err != nil { return ctrl.Result{}, errors.Wrapf(err, "fails to set owner reference for predictor") } - var err error kstatus, err = r.Reconcile() if err != nil { return ctrl.Result{}, errors.Wrapf(err, "fails to reconcile predictor") diff --git a/pkg/controller/v1beta1/inferenceservice/components/transformer.go b/pkg/controller/v1beta1/inferenceservice/components/transformer.go index 6687bc477dd..73e71dfca5e 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/transformer.go +++ b/pkg/controller/v1beta1/inferenceservice/components/transformer.go @@ -216,8 +216,12 @@ func (p *Transformer) Reconcile(isvc *v1beta1.InferenceService) (ctrl.Result, er } isvc.Status.PropagateRawStatus(v1beta1.TransformerComponent, deployment, r.URL) } else { - r := knative.NewKsvcReconciler(p.client, p.scheme, objectMeta, &isvc.Spec.Transformer.ComponentExtensionSpec, + r, err := knative.NewKsvcReconciler(p.client, p.scheme, objectMeta, &isvc.Spec.Transformer.ComponentExtensionSpec, &podSpec, isvc.Status.Components[v1beta1.TransformerComponent], p.inferenceServiceConfig.ServiceLabelDisallowedList) + if err != nil { + return ctrl.Result{}, errors.Wrapf(err, "fails to create new knative service reconciler") + } + if err := controllerutil.SetControllerReference(isvc, r.Service, p.scheme); err != nil { return ctrl.Result{}, errors.Wrapf(err, "fails to set owner reference for transformer") } diff --git a/pkg/controller/v1beta1/inferenceservice/controller.go b/pkg/controller/v1beta1/inferenceservice/controller.go index fbc219fd49e..1beb2bd3097 100644 --- a/pkg/controller/v1beta1/inferenceservice/controller.go +++ b/pkg/controller/v1beta1/inferenceservice/controller.go @@ -36,6 +36,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" + operatorv1beta1 "knative.dev/operator/pkg/apis/operator/v1beta1" "knative.dev/pkg/apis" knservingv1 "knative.dev/serving/pkg/apis/serving/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -83,6 +84,7 @@ import ( // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch // +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=operator.knative.dev,resources=knativeservings,verbs=get;list;watch // InferenceServiceState describes the Readiness of the InferenceService type InferenceServiceState string @@ -183,7 +185,7 @@ func (r *InferenceServiceReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, nil } - // Abort early if the resolved deployment mode is Serverless, but Knative Services are not available + // Abort early if the resolved deployment mode is Serverless, but Knative Services or KnativeServings are not available if deploymentMode == constants.Serverless { ksvcAvailable, checkKsvcErr := utils.IsCrdAvailable(r.ClientConfig, knservingv1.SchemeGroupVersion.String(), constants.KnativeServiceKind) if checkKsvcErr != nil { @@ -193,7 +195,19 @@ func (r *InferenceServiceReconciler) Reconcile(ctx context.Context, req ctrl.Req if !ksvcAvailable { r.Recorder.Event(isvc, v1.EventTypeWarning, "ServerlessModeRejected", "It is not possible to use Serverless deployment mode when Knative Services are not available") - return reconcile.Result{Requeue: false}, reconcile.TerminalError(fmt.Errorf("the resolved deployment mode of InferenceService '%s' is Serverless, but Knative Serving is not available", isvc.Name)) + return reconcile.Result{Requeue: false}, reconcile.TerminalError(fmt.Errorf("the resolved deployment mode of InferenceService '%s' is Serverless, but Knative Services are not available", isvc.Name)) + } + + // Abort if Knative KnativeServings are not available + knServingFound, knServingCheckErr := utils.IsCrdAvailable(r.ClientConfig, operatorv1beta1.SchemeGroupVersion.String(), constants.KnativeServingKind) + if knServingCheckErr != nil { + return reconcile.Result{}, knServingCheckErr + } + + if !knServingFound { + r.Recorder.Event(isvc, v1.EventTypeWarning, "ServerlessModeRejected", + "It is not possible to use Serverless deployment mode when Knative KnativeServings are not available") + return reconcile.Result{Requeue: false}, reconcile.TerminalError(fmt.Errorf("the resolved deployment mode of InferenceService '%s' is Serverless, but Knative KnativeServings are not available", isvc.Name)) } } diff --git a/pkg/controller/v1beta1/inferenceservice/controller_test.go b/pkg/controller/v1beta1/inferenceservice/controller_test.go index 9d8fae62b04..b7dc42feab7 100644 --- a/pkg/controller/v1beta1/inferenceservice/controller_test.go +++ b/pkg/controller/v1beta1/inferenceservice/controller_test.go @@ -41,6 +41,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" + operatorv1beta1 "knative.dev/operator/pkg/apis/operator/v1beta1" "knative.dev/pkg/apis" duckv1 "knative.dev/pkg/apis/duck/v1" "knative.dev/pkg/network" @@ -92,6 +93,249 @@ var _ = Describe("v1beta1 inference service controller", func() { }`, } ) + Context("with knative configured to allow zero initial scale", func() { + When("an InferenceService with minReplicas=0 is created", func() { + It("should create a knative service with initial-scale:0 and min-scale:0 annotations", func() { + // Create configmap + var configMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.InferenceServiceConfigMapName, + Namespace: constants.KServeNamespace, + }, + Data: configs, + } + Expect(k8sClient.Create(context.TODO(), configMap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), configMap) + + // Create InferenceService + serviceName := "initialscale1" + var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: serviceName, Namespace: "default"}} + var serviceKey = expectedRequest.NamespacedName + var storageUri = "s3://test/mnist/export" + ctx := context.Background() + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceKey.Name, + Namespace: serviceKey.Namespace, + }, + Spec: v1beta1.InferenceServiceSpec{ + Predictor: v1beta1.PredictorSpec{ + ComponentExtensionSpec: v1beta1.ComponentExtensionSpec{ + MinReplicas: v1beta1.GetIntReference(0), + }, + Tensorflow: &v1beta1.TFServingSpec{ + PredictorExtensionSpec: v1beta1.PredictorExtensionSpec{ + StorageURI: &storageUri, + RuntimeVersion: proto.String("1.14.0"), + Container: v1.Container{ + Name: constants.InferenceServiceContainerName, + Resources: defaultResource, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, isvc)).Should(Succeed()) + defer k8sClient.Delete(ctx, isvc) + + actualService := &knservingv1.Service{} + predictorServiceKey := types.NamespacedName{Name: constants.PredictorServiceName(serviceKey.Name), + Namespace: serviceKey.Namespace} + Eventually(func() error { return k8sClient.Get(context.TODO(), predictorServiceKey, actualService) }, timeout). + Should(Succeed()) + + Expect(actualService.Spec.Template.Annotations[constants.InitialScaleAnnotationKey]).To(Equal("0")) + Expect(actualService.Spec.Template.Annotations[constants.MinScaleAnnotationKey]).To(Equal("0")) + }) + }) + When("an InferenceService with minReplicas!=0 is created", func() { + It("should create a knative service with no initial-scale annotation and min-scale: annotation", func() { + // Create configmap + var configMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.InferenceServiceConfigMapName, + Namespace: constants.KServeNamespace, + }, + Data: configs, + } + Expect(k8sClient.Create(context.TODO(), configMap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), configMap) + + // Create InferenceService + serviceName := "initialscale2" + var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: serviceName, Namespace: "default"}} + var serviceKey = expectedRequest.NamespacedName + var storageUri = "s3://test/mnist/export" + ctx := context.Background() + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceKey.Name, + Namespace: serviceKey.Namespace, + }, + Spec: v1beta1.InferenceServiceSpec{ + Predictor: v1beta1.PredictorSpec{ + ComponentExtensionSpec: v1beta1.ComponentExtensionSpec{ + MinReplicas: v1beta1.GetIntReference(1), + }, + Tensorflow: &v1beta1.TFServingSpec{ + PredictorExtensionSpec: v1beta1.PredictorExtensionSpec{ + StorageURI: &storageUri, + RuntimeVersion: proto.String("1.14.0"), + Container: v1.Container{ + Name: constants.InferenceServiceContainerName, + Resources: defaultResource, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, isvc)).Should(Succeed()) + defer k8sClient.Delete(ctx, isvc) + + actualService := &knservingv1.Service{} + predictorServiceKey := types.NamespacedName{Name: constants.PredictorServiceName(serviceKey.Name), + Namespace: serviceKey.Namespace} + Eventually(func() error { return k8sClient.Get(context.TODO(), predictorServiceKey, actualService) }, timeout). + Should(Succeed()) + + Expect(actualService.Spec.Template.Annotations[constants.MinScaleAnnotationKey]).To(Equal("1")) + Expect(constants.InitialScaleAnnotationKey).ShouldNot(BeKeyOf(actualService.Spec.Template.Annotations)) + }) + }) + }) + Context("with knative configured to not allow zero initial scale", func() { + BeforeEach(func() { + // Update the existing knativeserving custom resource to set allow-zero-intial-scale to false + knativeService := &operatorv1beta1.KnativeServing{} + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: constants.DefaultKnServingName, Namespace: constants.DefaultKnServingNamespace}, knativeService)).ToNot(HaveOccurred()) + knativeService.Spec.CommonSpec.Config["autoscaler"]["allow-zero-initial-scale"] = "false" + Eventually(func() error { + return k8sClient.Update(context.TODO(), knativeService) + }, timeout).Should(Succeed()) + }) + AfterEach(func() { + // Restore the original knativeserving custom resource configuration + knativeServiceRestored := &operatorv1beta1.KnativeServing{} + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: constants.DefaultKnServingName, Namespace: constants.DefaultKnServingNamespace}, knativeServiceRestored)).ToNot(HaveOccurred()) + knativeServiceRestored.Spec.CommonSpec.Config["autoscaler"]["allow-zero-initial-scale"] = "true" + Eventually(func() error { + return k8sClient.Update(context.TODO(), knativeServiceRestored) + }, timeout).Should(Succeed()) + }) + When("an InferenceService with minReplicas=0 is created", func() { + It("should create a knative service with no initial-scale annotation and min-scale:0 annotation", func() { + // Create configmap + var configMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.InferenceServiceConfigMapName, + Namespace: constants.KServeNamespace, + }, + Data: configs, + } + Expect(k8sClient.Create(context.TODO(), configMap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), configMap) + + // Create InferenceService + serviceName := "initialscale3" + var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: serviceName, Namespace: "default"}} + var serviceKey = expectedRequest.NamespacedName + var storageUri = "s3://test/mnist/export" + ctx := context.Background() + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceKey.Name, + Namespace: serviceKey.Namespace, + }, + Spec: v1beta1.InferenceServiceSpec{ + Predictor: v1beta1.PredictorSpec{ + ComponentExtensionSpec: v1beta1.ComponentExtensionSpec{ + MinReplicas: v1beta1.GetIntReference(0), + }, + Tensorflow: &v1beta1.TFServingSpec{ + PredictorExtensionSpec: v1beta1.PredictorExtensionSpec{ + StorageURI: &storageUri, + RuntimeVersion: proto.String("1.14.0"), + Container: v1.Container{ + Name: constants.InferenceServiceContainerName, + Resources: defaultResource, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, isvc)).Should(Succeed()) + defer k8sClient.Delete(ctx, isvc) + + actualService := &knservingv1.Service{} + predictorServiceKey := types.NamespacedName{Name: constants.PredictorServiceName(serviceKey.Name), + Namespace: serviceKey.Namespace} + Eventually(func() error { return k8sClient.Get(context.TODO(), predictorServiceKey, actualService) }, timeout). + Should(Succeed()) + + Expect(actualService.Spec.Template.Annotations[constants.MinScaleAnnotationKey]).To(Equal("0")) + Expect(constants.InitialScaleAnnotationKey).ShouldNot(BeKeyOf(actualService.Spec.Template.Annotations)) + }) + }) + When("an InferenceService with minReplicas!=0 is created", func() { + It("should create a knative service with no initial-scale annotation and min-scale: annotation", func() { + // Create configmap + var configMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.InferenceServiceConfigMapName, + Namespace: constants.KServeNamespace, + }, + Data: configs, + } + Expect(k8sClient.Create(context.TODO(), configMap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), configMap) + + // Create InferenceService + serviceName := "initialscale4" + var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: serviceName, Namespace: "default"}} + var serviceKey = expectedRequest.NamespacedName + var storageUri = "s3://test/mnist/export" + ctx := context.Background() + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceKey.Name, + Namespace: serviceKey.Namespace, + }, + Spec: v1beta1.InferenceServiceSpec{ + Predictor: v1beta1.PredictorSpec{ + ComponentExtensionSpec: v1beta1.ComponentExtensionSpec{ + MinReplicas: v1beta1.GetIntReference(1), + }, + Tensorflow: &v1beta1.TFServingSpec{ + PredictorExtensionSpec: v1beta1.PredictorExtensionSpec{ + StorageURI: &storageUri, + RuntimeVersion: proto.String("1.14.0"), + Container: v1.Container{ + Name: constants.InferenceServiceContainerName, + Resources: defaultResource, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, isvc)).Should(Succeed()) + defer k8sClient.Delete(ctx, isvc) + + actualService := &knservingv1.Service{} + predictorServiceKey := types.NamespacedName{Name: constants.PredictorServiceName(serviceKey.Name), + Namespace: serviceKey.Namespace} + Eventually(func() error { return k8sClient.Get(context.TODO(), predictorServiceKey, actualService) }, timeout). + Should(Succeed()) + + Expect(actualService.Spec.Template.Annotations[constants.MinScaleAnnotationKey]).To(Equal("1")) + Expect(constants.InitialScaleAnnotationKey).ShouldNot(BeKeyOf(actualService.Spec.Template.Annotations)) + }) + }) + }) + Context("an annotation is applied to an InferenceService resource", func() { When("the annotation is a member of the serviceAnnotationDisallowedList in the inferenceservice-config configmap", func() { It("should not be propagated to any knative revisions", func() { diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/knative/ksvc_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/knative/ksvc_reconciler.go index fe929e61ad1..455fa8b8eab 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/knative/ksvc_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/knative/ksvc_reconciler.go @@ -19,9 +19,11 @@ package knative import ( "context" "fmt" + "strconv" "github.com/kserve/kserve/pkg/apis/serving/v1beta1" "github.com/kserve/kserve/pkg/constants" + knutils "github.com/kserve/kserve/pkg/controller/v1alpha1/utils" "github.com/kserve/kserve/pkg/utils" "github.com/pkg/errors" "google.golang.org/protobuf/proto" @@ -62,44 +64,31 @@ func NewKsvcReconciler(client client.Client, componentExt *v1beta1.ComponentExtensionSpec, podSpec *corev1.PodSpec, componentStatus v1beta1.ComponentStatusSpec, - disallowedLabelList []string) *KsvcReconciler { + disallowedLabelList []string) (*KsvcReconciler, error) { + ksvc, err := createKnativeService(client, componentMeta, componentExt, podSpec, componentStatus, disallowedLabelList) + if err != nil { + return nil, errors.Wrapf(err, fmt.Sprintf("fails to create knative service for inference service %s", componentMeta.Name)) + } return &KsvcReconciler{ client: client, scheme: scheme, - Service: createKnativeService(componentMeta, componentExt, podSpec, componentStatus, disallowedLabelList), + Service: ksvc, componentExt: componentExt, componentStatus: componentStatus, - } + }, nil } -func createKnativeService(componentMeta metav1.ObjectMeta, +func createKnativeService(client client.Client, + componentMeta metav1.ObjectMeta, componentExtension *v1beta1.ComponentExtensionSpec, podSpec *corev1.PodSpec, componentStatus v1beta1.ComponentStatusSpec, - disallowedLabelList []string) *knservingv1.Service { + disallowedLabelList []string) (*knservingv1.Service, error) { annotations := componentMeta.GetAnnotations() - if componentExtension.MinReplicas == nil { - annotations[constants.MinScaleAnnotationKey] = fmt.Sprint(constants.DefaultMinReplicas) - } else { - annotations[constants.MinScaleAnnotationKey] = fmt.Sprint(*componentExtension.MinReplicas) - } - - if componentExtension.MaxReplicas != 0 { - annotations[constants.MaxScaleAnnotationKey] = fmt.Sprint(componentExtension.MaxReplicas) - } - - // User can pass down scaling class annotation to overwrite the default scaling KPA - if _, ok := annotations[autoscaling.ClassAnnotationKey]; !ok { - annotations[autoscaling.ClassAnnotationKey] = autoscaling.KPA - } - - if componentExtension.ScaleTarget != nil { - annotations[autoscaling.TargetAnnotationKey] = fmt.Sprint(*componentExtension.ScaleTarget) - } - - if componentExtension.ScaleMetric != nil { - annotations[autoscaling.MetricAnnotationKey] = fmt.Sprint(*componentExtension.ScaleMetric) + err := setAutoScalingAnnotations(client, annotations, componentExtension) + if err != nil { + return nil, errors.Wrapf(err, "fails to set autoscaling annotations for knative service") } // ksvc metadata.annotations @@ -190,7 +179,7 @@ func createKnativeService(componentMeta metav1.ObjectMeta, }, }, } - return service + return service, nil } func reconcileKsvc(desired *knservingv1.Service, existing *knservingv1.Service) error { @@ -279,3 +268,75 @@ func semanticEquals(desiredService, service *knservingv1.Service) bool { equality.Semantic.DeepEqual(desiredService.ObjectMeta.Labels, service.ObjectMeta.Labels) && equality.Semantic.DeepEqual(desiredService.Spec.RouteSpec, service.Spec.RouteSpec) } + +// setAutoScalingAnnotations checks the knative autoscaler configuration defined in the knativeserving custom resource +// and compares the values to the autoscaling configuration requested for the inference service. +// It then sets the necessary annotations for the desired autoscaling configuration. +func setAutoScalingAnnotations(client client.Client, + annotations map[string]string, + componentExtension *v1beta1.ComponentExtensionSpec) error { + // User can pass down scaling class annotation to overwrite the default scaling KPA + if _, ok := annotations[autoscaling.ClassAnnotationKey]; !ok { + annotations[autoscaling.ClassAnnotationKey] = autoscaling.KPA + } + + if componentExtension.ScaleTarget != nil { + annotations[autoscaling.TargetAnnotationKey] = fmt.Sprint(*componentExtension.ScaleTarget) + } + + if componentExtension.ScaleMetric != nil { + annotations[autoscaling.MetricAnnotationKey] = fmt.Sprint(*componentExtension.ScaleMetric) + } + + // If a minReplicas value is not set for the inference service, then use the default min-scale value of 1. + var revisionMinScale int + if componentExtension.MinReplicas == nil { + annotations[autoscaling.MinScaleAnnotationKey] = fmt.Sprint(constants.DefaultMinReplicas) + revisionMinScale = constants.DefaultMinReplicas + } else { + annotations[autoscaling.MinScaleAnnotationKey] = fmt.Sprint(*componentExtension.MinReplicas) + revisionMinScale = *componentExtension.MinReplicas + } + + if componentExtension.MaxReplicas != 0 { + annotations[autoscaling.MaxScaleAnnotationKey] = fmt.Sprint(componentExtension.MaxReplicas) + } + + // Retrieve the allow-zero-initial-scale and initial-scale values from the knative autoscaler configuration. + allowZeroInitialScale, globalInitialScale, err := knutils.GetAutoscalerConfiguration(client) + if err != nil { + return errors.Wrapf(err, "failed to retrieve the knative autoscaler configuration") + } + + initialScaleInt, err := strconv.Atoi(globalInitialScale) + if err != nil { + return errors.Wrapf(err, fmt.Sprintf("failed to convert configured knative autoscaler global initial-scale value (%s) to an integer", globalInitialScale)) + } + + // Provide transparency to users while aligning with knative serving's expected behavior, log a warning when the + // knative autoscaler global initial-scale value exceeds the requested minScale value for the inference service. + if initialScaleInt > revisionMinScale { + log.Info("knative autoscaler is globally configured with an initial-scale value that is greater than the requested min-scale for the inference service", + "initial-scale", initialScaleInt, + "min-scale", revisionMinScale) + } + + // knative will choose the larger of min-scale and initial-scale as the initial target scale for a knative revision. + // When min-scale is 0, if allow-zero-initial scale is true, set initial-scale to 0 for the created knative revision. + // This will prevent any pods from being created to initialize a knative revision when an inference service has minReplicas set to 0. + // Configuring scaling for knative: https://knative.dev/docs/serving/autoscaling/scale-bounds/#initial-scale + if revisionMinScale == 0 { + if allowZeroInitialScale == "true" { + log.Info("kserve will override the global knative autoscaler configuration for initial-scale on a per revision basis with 0 when an inference service is requested with min-scale 0", + "allow-zero-initial-scale", allowZeroInitialScale, + "initial-scale", globalInitialScale) + annotations[constants.InitialScaleAnnotationKey] = "0" + } else { + log.Info("The current knative autoscaler global configuration does not allow zero initial scale.", + "allow-zero-initial-scale", allowZeroInitialScale, + "initial-scale", globalInitialScale) + } + } + + return nil +} diff --git a/pkg/controller/v1beta1/inferenceservice/suite_test.go b/pkg/controller/v1beta1/inferenceservice/suite_test.go index 35aab6f0435..16b858b2df5 100644 --- a/pkg/controller/v1beta1/inferenceservice/suite_test.go +++ b/pkg/controller/v1beta1/inferenceservice/suite_test.go @@ -29,6 +29,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "knative.dev/operator/pkg/apis/operator/base" + operatorv1beta1 "knative.dev/operator/pkg/apis/operator/v1beta1" knservingv1 "knative.dev/serving/pkg/apis/serving/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -95,13 +97,37 @@ var _ = BeforeSuite(func() { Expect(err).ToNot(HaveOccurred()) Expect(clientset).ToNot(BeNil()) - //Create namespace + // Create namespaces kfservingNamespaceObj := &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: constants.KServeNamespace, }, } + knativeServingNamespace := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.DefaultKnServingNamespace, + }, + } Expect(k8sClient.Create(context.Background(), kfservingNamespaceObj)).Should(Succeed()) + Expect(k8sClient.Create(context.Background(), knativeServingNamespace)).Should(Succeed()) + + // Create knativeserving custom resource + knativeCr := &operatorv1beta1.KnativeServing{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.DefaultKnServingName, + Namespace: constants.DefaultKnServingNamespace, + }, + Spec: operatorv1beta1.KnativeServingSpec{ + CommonSpec: base.CommonSpec{ + Config: base.ConfigMapData{ + "autoscaler": map[string]string{ + "allow-zero-initial-scale": "true", + }, + }, + }, + }, + } + Expect(k8sClient.Create(context.Background(), knativeCr)).Should(Succeed()) k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme.Scheme, diff --git a/pkg/testing/envtest_setup.go b/pkg/testing/envtest_setup.go index 47f9c1b307f..09d2cafbd1d 100644 --- a/pkg/testing/envtest_setup.go +++ b/pkg/testing/envtest_setup.go @@ -26,6 +26,7 @@ import ( "github.com/onsi/gomega" "k8s.io/client-go/kubernetes/scheme" + operatorv1beta1 "knative.dev/operator/pkg/apis/operator/v1beta1" knservingv1 "knative.dev/serving/pkg/apis/serving/v1" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -56,6 +57,10 @@ func SetupEnvTest(crdDirectoryPaths []string) *envtest.Environment { log.Error(err, "Failed to add istio scheme") } + if err := operatorv1beta1.SchemeBuilder.AddToScheme(scheme.Scheme); err != nil { + log.Error(err, "Failed to add knative operator scheme") + } + if err := gatewayapiv1.Install(scheme.Scheme); err != nil { log.Error(err, "Failed to add gateway scheme") } diff --git a/test/crds/knative_knativeserving.yaml b/test/crds/knative_knativeserving.yaml new file mode 100644 index 00000000000..2fb7d773871 --- /dev/null +++ b/test/crds/knative_knativeserving.yaml @@ -0,0 +1,2510 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: knativeservings.operator.knative.dev +spec: + conversion: + strategy: None + group: operator.knative.dev + names: + kind: KnativeServing + listKind: KnativeServingList + plural: knativeservings + singular: knativeserving + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.version + name: Version + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: Schema for the knativeservings API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec defines the desired state of KnativeServing + properties: + config: + additionalProperties: + additionalProperties: + type: string + type: object + description: A means to override the corresponding entries in the + upstream configmaps + type: object + controller-custom-certs: + description: Enabling the controller to trust registries with self-signed + certificates + properties: + name: + description: The name of the ConfigMap or Secret + type: string + type: + description: One of ConfigMap or Secret + enum: + - ConfigMap + - Secret + - "" + type: string + type: object + deployments: + description: A mapping of deployment name to override + items: + properties: + affinity: + description: If specified, the pod's scheduling constraints. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for + the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most preferred. + items: + description: An empty preferred scheduling term matches + all objects with implicit weight 0 (i.e. it's a + no-op). A null preferred scheduling term matches + no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. If + the operator is Gt or Lt, the values + array must have a single element, + which will be interpreted as an integer. + This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. If + the operator is Gt or Lt, the values + array must have a single element, + which will be interpreted as an integer. + This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range + 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an + update), the system may or may not try to eventually + evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: A null or empty node selector term + matches no objects. The requirements of them + are ANDed. The TopologySelectorTerm type implements + a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. If + the operator is Gt or Lt, the values + array must have a single element, + which will be interpreted as an integer. + This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. If + the operator is Gt or Lt, the values + array must have a single element, + which will be interpreted as an integer. + This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node has pods which matches the + corresponding podAffinityTerm; the node(s) with the + highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose value + of the label with key topologyKey matches + that of any node on which any of the selected + pods is running. Empty topologyKey is not + allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range + 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a + pod label update), the system may or may not try to + eventually evict the pod from its node. When there + are multiple elements, the lists of nodes corresponding + to each podAffinityTerm are intersected, i.e. all + terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or + not co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any + node on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified + namespaces, where co-located is defined as running + on a node whose value of the label with key + topologyKey matches that of any node on which + any of the selected pods is running. Empty topologyKey + is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the anti-affinity expressions + specified by this field, but it may choose a node + that violates one or more of the expressions. The + node that is most preferred is the one with the greatest + sum of weights, i.e. for each node that meets all + of the scheduling requirements (resource request, + requiredDuringScheduling anti-affinity expressions, + etc.), compute a sum by iterating through the elements + of this field and adding "weight" to the sum if the + node has pods which matches the corresponding podAffinityTerm; + the node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose value + of the label with key topologyKey matches + that of any node on which any of the selected + pods is running. Empty topologyKey is not + allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range + 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the anti-affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a + pod label update), the system may or may not try to + eventually evict the pod from its node. When there + are multiple elements, the lists of nodes corresponding + to each podAffinityTerm are intersected, i.e. all + terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or + not co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any + node on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified + namespaces, where co-located is defined as running + on a node whose value of the label with key + topologyKey matches that of any node on which + any of the selected pods is running. Empty topologyKey + is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + description: Annotations overrides labels for the deployment + and its template. + type: object + env: + description: Env overrides env vars for the containers. + items: + properties: + container: + description: The container name + type: string + envVars: + description: The desired EnvVarRequirements + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are + expanded using the previously defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, the + reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults + to "".' + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + required: + - container + type: object + type: array + hostNetwork: + description: Use the host's network namespace if true. Make + sure to understand the security implications if you want to + enable it. When hostNetwork is enabled, this will set dnsPolicy + to ClusterFirstWithHostNet automatically. + type: boolean + labels: + additionalProperties: + type: string + description: Labels overrides labels for the deployment and + its template. + type: object + livenessProbes: + description: LivenessProbes overrides liveness probes for the + containers. + items: + description: ProbesRequirementsOverride enables the user to + override any container's env vars. + properties: + container: + description: The container name + type: string + failureThreshold: + description: Minimum consecutive failures for the probe + to be considered failed after having succeeded. Defaults + to 3. Minimum value is 1. + format: int32 + type: integer + initialDelaySeconds: + description: 'Number of seconds after the container has + started before liveness probes are initiated. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe + to be considered successful after having failed. Defaults + to 1. Must be 1 for liveness and startup. Minimum value + is 1. + format: int32 + type: integer + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and + the time when the processes are forcibly halted with + a kill signal. Set this value longer than the expected + cleanup time for your process. If this value is nil, + the pod's terminationGracePeriodSeconds will be used. + Otherwise, this value overrides the value provided by + the pod spec. Value must be non-negative integer. The + value zero indicates stop immediately via the kill signal + (no opportunity to shut down). This is a beta field + and requires enabling ProbeTerminationGracePeriod feature + gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + required: + - container + type: object + type: array + name: + description: The name of the deployment + type: string + nodeSelector: + additionalProperties: + type: string + description: NodeSelector overrides nodeSelector for the deployment. + type: object + readinessProbes: + description: ReadinessProbes overrides readiness probes for + the containers. + items: + description: ProbesRequirementsOverride enables the user to + override any container's env vars. + properties: + container: + description: The container name + type: string + failureThreshold: + description: Minimum consecutive failures for the probe + to be considered failed after having succeeded. Defaults + to 3. Minimum value is 1. + format: int32 + type: integer + initialDelaySeconds: + description: 'Number of seconds after the container has + started before liveness probes are initiated. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe + to be considered successful after having failed. Defaults + to 1. Must be 1 for liveness and startup. Minimum value + is 1. + format: int32 + type: integer + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and + the time when the processes are forcibly halted with + a kill signal. Set this value longer than the expected + cleanup time for your process. If this value is nil, + the pod's terminationGracePeriodSeconds will be used. + Otherwise, this value overrides the value provided by + the pod spec. Value must be non-negative integer. The + value zero indicates stop immediately via the kill signal + (no opportunity to shut down). This is a beta field + and requires enabling ProbeTerminationGracePeriod feature + gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + required: + - container + type: object + type: array + replicas: + description: The number of replicas that HA parts of the control + plane will be scaled to + minimum: 0 + type: integer + resources: + description: If specified, the container's resources. + items: + description: The pod this Resource is used to specify the + requests and limits for a certain container based on the + name. + properties: + container: + description: The name of the container + type: string + limits: + properties: + cpu: + pattern: ^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$ + type: string + memory: + pattern: ^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$ + type: string + type: object + requests: + properties: + cpu: + pattern: ^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$ + type: string + memory: + pattern: ^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$ + type: string + type: object + type: object + type: array + tolerations: + description: If specified, the pod's tolerations. + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple using + the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. When specified, + allowed values are NoSchedule, PreferNoSchedule and + NoExecute. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. If the + key is empty, operator must be Exists; this combination + means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship + to the value. Valid operators are Exists and Equal. + Defaults to Equal. Exists is equivalent to wildcard + for value, so that a pod can tolerate all taints of + a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of + time the toleration (which must be of effect NoExecute, + otherwise this field is ignored) tolerates the taint. + By default, it is not set, which means tolerate the + taint forever (do not evict). Zero and negative values + will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches + to. If the operator is Exists, the value should be empty, + otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + description: If specified, the pod's topology spread constraints. + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values + array must be non-empty. If the operator is + Exists or DoesNotExist, the values array must + be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + maxSkew: + description: 'MaxSkew describes the degree to which pods + may be unevenly distributed. It''s the maximum permitted + difference between the number of matching pods in any + two topology domains of a given topology type. For example, + in a 3-zone cluster, MaxSkew is set to 1, and pods with + the same labelSelector spread as 1/1/0: | zone1 | zone2 + | zone3 | | P | P | | - if MaxSkew is + 1, incoming pod can only be scheduled to zone3 to become + 1/1/1; scheduling it onto zone1(zone2) would make the + ActualSkew(2-0) on zone1(zone2) violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto + any zone. It''s a required field. Default value is 1 + and 0 is not allowed.' + format: int32 + type: integer + topologyKey: + description: TopologyKey is the key of node labels. Nodes + that have a label with this key and identical values + are considered to be in the same topology. We consider + each as a "bucket", and try to put balanced + number of pods into each bucket. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal + with a pod if it doesn''t satisfy the spread constraint. + - DoNotSchedule (default) tells the scheduler not to + schedule it - ScheduleAnyway tells the scheduler to + still schedule it It''s considered as "Unsatisfiable" + if and only if placing incoming pod on any topology + violates "MaxSkew". For example, in a 3-zone cluster, + MaxSkew is set to 1, and pods with the same labelSelector + spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming + pod can only be scheduled to zone2(zone3) to become + 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be + imbalanced, but scheduler won''t make it *more* imbalanced. + It''s a required field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + type: array + high-availability: + description: Allows specification of HA control plane + properties: + replicas: + description: The number of replicas that HA parts of the control + plane will be scaled to + minimum: 0 + type: integer + type: object + ingress: + description: The ingress configuration for Knative Serving + properties: + istio: + description: Istio settings + properties: + enabled: + type: boolean + knative-local-gateway: + description: A means to override the knative-local-gateway + properties: + selector: + additionalProperties: + type: string + description: The selector for the ingress-gateway. + type: object + servers: + description: A list of server specifications. + items: + properties: + hosts: + description: One or more hosts exposed by this gateway. + items: + format: string + type: string + type: array + port: + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port + number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + target_port: + description: A valid non-negative integer target + port number. + type: integer + type: object + tls: + properties: + credentialName: + description: TLS certificate name. + format: string + type: string + mode: + description: TLS mode can be SIMPLE, MUTUAL, + ISTIO_MUTUAL. + format: string + type: string + type: object + type: object + type: array + type: object + type: object + kourier: + description: Kourier settings + properties: + bootstrap-configmap: + type: string + enabled: + type: boolean + http-port: + type: integer + https-port: + type: integer + service-load-balancer-ip: + type: string + service-type: + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + podDisruptionBudgets: + description: A mapping of podDisruptionBudget name to override + items: + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + description: An eviction is allowed if at most "maxUnavailable" + pods selected by "selector" are unavailable after the eviction, + i.e. even in absence of the evicted pod. For example, one + can prevent all voluntary evictions by specifying 0. This + is a mutually exclusive setting with "minAvailable". + x-kubernetes-int-or-string: true + minAvailable: + anyOf: + - type: integer + - type: string + description: An eviction is allowed if at least "minAvailable" + pods selected by "selector" will still be available after + the eviction, i.e. even in the absence of the evicted pod. So + for example you can prevent all voluntary evictions by specifying + "100%". + x-kubernetes-int-or-string: true + name: + description: The name of the podDisruptionBudget + type: string + type: object + type: array + security: + description: The security configuration for Knative Serving + properties: + securityGuard: + description: Security Guard settings + properties: + enabled: + type: boolean + type: object + type: object + services: + description: A mapping of service name to override + items: + properties: + annotations: + additionalProperties: + type: string + description: Annotations overrides labels for the service + type: object + labels: + additionalProperties: + type: string + description: Labels overrides labels for the service + type: object + name: + description: The name of the service + type: string + selector: + additionalProperties: + type: string + description: Selector overrides selector for the service + type: object + type: object + type: array + workloads: + description: A mapping of deployment or statefulset name to override + items: + properties: + affinity: + description: If specified, the pod's scheduling constraints. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for + the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most preferred. + items: + description: An empty preferred scheduling term matches + all objects with implicit weight 0 (i.e. it's a + no-op). A null preferred scheduling term matches + no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. If + the operator is Gt or Lt, the values + array must have a single element, + which will be interpreted as an integer. + This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. If + the operator is Gt or Lt, the values + array must have a single element, + which will be interpreted as an integer. + This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range + 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an + update), the system may or may not try to eventually + evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: A null or empty node selector term + matches no objects. The requirements of them + are ANDed. The TopologySelectorTerm type implements + a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. If + the operator is Gt or Lt, the values + array must have a single element, + which will be interpreted as an integer. + This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. If + the operator is Gt or Lt, the values + array must have a single element, + which will be interpreted as an integer. + This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node has pods which matches the + corresponding podAffinityTerm; the node(s) with the + highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose value + of the label with key topologyKey matches + that of any node on which any of the selected + pods is running. Empty topologyKey is not + allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range + 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a + pod label update), the system may or may not try to + eventually evict the pod from its node. When there + are multiple elements, the lists of nodes corresponding + to each podAffinityTerm are intersected, i.e. all + terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or + not co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any + node on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified + namespaces, where co-located is defined as running + on a node whose value of the label with key + topologyKey matches that of any node on which + any of the selected pods is running. Empty topologyKey + is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the anti-affinity expressions + specified by this field, but it may choose a node + that violates one or more of the expressions. The + node that is most preferred is the one with the greatest + sum of weights, i.e. for each node that meets all + of the scheduling requirements (resource request, + requiredDuringScheduling anti-affinity expressions, + etc.), compute a sum by iterating through the elements + of this field and adding "weight" to the sum if the + node has pods which matches the corresponding podAffinityTerm; + the node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose value + of the label with key topologyKey matches + that of any node on which any of the selected + pods is running. Empty topologyKey is not + allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range + 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the anti-affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a + pod label update), the system may or may not try to + eventually evict the pod from its node. When there + are multiple elements, the lists of nodes corresponding + to each podAffinityTerm are intersected, i.e. all + terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or + not co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any + node on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified + namespaces, where co-located is defined as running + on a node whose value of the label with key + topologyKey matches that of any node on which + any of the selected pods is running. Empty topologyKey + is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + description: Annotations overrides labels for the deployment + and its template. + type: object + env: + description: Env overrides env vars for the containers. + items: + properties: + container: + description: The container name + type: string + envVars: + description: The desired EnvVarRequirements + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are + expanded using the previously defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, the + reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults + to "".' + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + required: + - container + type: object + type: array + hostNetwork: + description: Use the host's network namespace if true. Make + sure to understand the security implications if you want to + enable it. When hostNetwork is enabled, this will set dnsPolicy + to ClusterFirstWithHostNet automatically. + type: boolean + labels: + additionalProperties: + type: string + description: Labels overrides labels for the deployment and + its template. + type: object + livenessProbes: + description: LivenessProbes overrides liveness probes for the + containers. + items: + description: ProbesRequirementsOverride enables the user to + override any container's env vars. + properties: + container: + description: The container name + type: string + failureThreshold: + description: Minimum consecutive failures for the probe + to be considered failed after having succeeded. Defaults + to 3. Minimum value is 1. + format: int32 + type: integer + initialDelaySeconds: + description: 'Number of seconds after the container has + started before liveness probes are initiated. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe + to be considered successful after having failed. Defaults + to 1. Must be 1 for liveness and startup. Minimum value + is 1. + format: int32 + type: integer + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and + the time when the processes are forcibly halted with + a kill signal. Set this value longer than the expected + cleanup time for your process. If this value is nil, + the pod's terminationGracePeriodSeconds will be used. + Otherwise, this value overrides the value provided by + the pod spec. Value must be non-negative integer. The + value zero indicates stop immediately via the kill signal + (no opportunity to shut down). This is a beta field + and requires enabling ProbeTerminationGracePeriod feature + gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + required: + - container + type: object + type: array + name: + description: The name of the deployment + type: string + nodeSelector: + additionalProperties: + type: string + description: NodeSelector overrides nodeSelector for the deployment. + type: object + readinessProbes: + description: ReadinessProbes overrides readiness probes for + the containers. + items: + description: ProbesRequirementsOverride enables the user to + override any container's env vars. + properties: + container: + description: The container name + type: string + failureThreshold: + description: Minimum consecutive failures for the probe + to be considered failed after having succeeded. Defaults + to 3. Minimum value is 1. + format: int32 + type: integer + initialDelaySeconds: + description: 'Number of seconds after the container has + started before liveness probes are initiated. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe + to be considered successful after having failed. Defaults + to 1. Must be 1 for liveness and startup. Minimum value + is 1. + format: int32 + type: integer + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and + the time when the processes are forcibly halted with + a kill signal. Set this value longer than the expected + cleanup time for your process. If this value is nil, + the pod's terminationGracePeriodSeconds will be used. + Otherwise, this value overrides the value provided by + the pod spec. Value must be non-negative integer. The + value zero indicates stop immediately via the kill signal + (no opportunity to shut down). This is a beta field + and requires enabling ProbeTerminationGracePeriod feature + gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + required: + - container + type: object + type: array + replicas: + description: The number of replicas that HA parts of the control + plane will be scaled to + minimum: 0 + type: integer + resources: + description: If specified, the container's resources. + items: + description: The pod this Resource is used to specify the + requests and limits for a certain container based on the + name. + properties: + container: + description: The name of the container + type: string + limits: + properties: + cpu: + pattern: ^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$ + type: string + memory: + pattern: ^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$ + type: string + type: object + requests: + properties: + cpu: + pattern: ^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$ + type: string + memory: + pattern: ^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$ + type: string + type: object + type: object + type: array + tolerations: + description: If specified, the pod's tolerations. + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple using + the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. When specified, + allowed values are NoSchedule, PreferNoSchedule and + NoExecute. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. If the + key is empty, operator must be Exists; this combination + means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship + to the value. Valid operators are Exists and Equal. + Defaults to Equal. Exists is equivalent to wildcard + for value, so that a pod can tolerate all taints of + a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of + time the toleration (which must be of effect NoExecute, + otherwise this field is ignored) tolerates the taint. + By default, it is not set, which means tolerate the + taint forever (do not evict). Zero and negative values + will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches + to. If the operator is Exists, the value should be empty, + otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + description: If specified, the pod's topology spread constraints. + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values + array must be non-empty. If the operator is + Exists or DoesNotExist, the values array must + be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + maxSkew: + description: 'MaxSkew describes the degree to which pods + may be unevenly distributed. It''s the maximum permitted + difference between the number of matching pods in any + two topology domains of a given topology type. For example, + in a 3-zone cluster, MaxSkew is set to 1, and pods with + the same labelSelector spread as 1/1/0: | zone1 | zone2 + | zone3 | | P | P | | - if MaxSkew is + 1, incoming pod can only be scheduled to zone3 to become + 1/1/1; scheduling it onto zone1(zone2) would make the + ActualSkew(2-0) on zone1(zone2) violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto + any zone. It''s a required field. Default value is 1 + and 0 is not allowed.' + format: int32 + type: integer + topologyKey: + description: TopologyKey is the key of node labels. Nodes + that have a label with this key and identical values + are considered to be in the same topology. We consider + each as a "bucket", and try to put balanced + number of pods into each bucket. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal + with a pod if it doesn''t satisfy the spread constraint. + - DoNotSchedule (default) tells the scheduler not to + schedule it - ScheduleAnyway tells the scheduler to + still schedule it It''s considered as "Unsatisfiable" + if and only if placing incoming pod on any topology + violates "MaxSkew". For example, in a 3-zone cluster, + MaxSkew is set to 1, and pods with the same labelSelector + spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming + pod can only be scheduled to zone2(zone3) to become + 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be + imbalanced, but scheduler won''t make it *more* imbalanced. + It''s a required field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + version: + description: Version the cluster should be on. + type: string + volumeMounts: + description: VolumeMounts allows configuration of additional + VolumeMounts on the output StatefulSet definition. VolumeMounts + specified will be appended to other VolumeMounts in the alertmanager + container, that are generated as a result of StorageSpec objects. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: Path within the container at which the volume + should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts are + propagated from the host to container and the other + way around. When not set, MountPropagationNone is used. + This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from which + the container's volume should be mounted. Behaves similarly + to SubPath but environment variable references $(VAR_NAME) + are expanded using the container's environment. Defaults + to "" (volume's root). SubPathExpr and SubPath are mutually + exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + status: + description: Status defines the observed state of KnativeServing + properties: + conditions: + description: The latest available observations of a resource's current + state. + items: + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition + transitioned from one status to another. We use VolatileTime + in place of metav1.Time to exclude this from creating equality.Semantic + differences (all other things held constant). + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + severity: + description: Severity with which to treat failures of this type + of condition. When this is not specified, it defaults to Error. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - type + - status + type: object + type: array + manifests: + description: The list of serving manifests, which have been installed + by the operator + items: + type: string + type: array + observedGeneration: + description: The generation last processed by the controller + type: integer + version: + description: The version of the installed release + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: KnativeServing + listKind: KnativeServingList + plural: knativeservings + singular: knativeserving + conditions: + - lastTransitionTime: "2025-02-11T21:32:16Z" + message: no conflicts found + reason: NoConflicts + status: "True" + type: NamesAccepted + - lastTransitionTime: "2025-02-11T21:32:17Z" + message: the initial names have been accepted + reason: InitialNamesAccepted + status: "True" + type: Established + storedVersions: + - v1beta1