MON-4115: expose label metrics for jobs and cronjobs

Adds `AdditionalLabelsAllowList` to KSM config.

Signed-off-by: Pranshu Srivastava <rexagod@gmail.com>

Author: rexagod

Documentation/api.md

@@ -44,6 +44,7 @@ Configuring Cluster Monitoring is optional. If the config does not exist or is e
 * [PrometheusOperatorConfig](#prometheusoperatorconfig)
 * [PrometheusRestrictedConfig](#prometheusrestrictedconfig)
 * [RemoteWriteSpec](#remotewritespec)
+* [ResourceLabels](#resourcelabels)
 * [TLSConfig](#tlsconfig)
 * [TelemeterClientConfig](#telemeterclientconfig)
 * [ThanosQuerierConfig](#thanosquerierconfig)
@@ -175,6 +176,7 @@ The `KubeStateMetricsConfig` resource defines settings for the `kube-state-metri
 | resources | *[v1.ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#resourcerequirements-v1-core) | Defines resource requests and limits for the KubeStateMetrics container. |
 | tolerations | [][v1.Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#toleration-v1-core) | Defines tolerations for the pods. |
 | topologySpreadConstraints | []v1.TopologySpreadConstraint | Defines a pod's topology spread constraints. |
+| additionalResourceLabels | [][ResourceLabels](#resourcelabels) | Defines additional resource labels to expose as metrics in addition to the default labels. Currently, only `jobs` and `cronjobs` resources are supported due to cardinality concerns. Each entry specifies a resource name and a list of label names (use `*` to expose all labels). |
 
 [Back to TOC](#table-of-contents)
 
@@ -555,6 +557,25 @@ The `RemoteWriteSpec` resource defines the settings for remote write storage.
 
 [Back to TOC](#table-of-contents)
 
+## ResourceLabels
+
+#### Description
+
+The `ResourceLabels` resource defines which Kubernetes labels to expose as metrics for a given resource type.
+
+#### Required
+   - ` resource `
+   - ` labels `
+
+<em>appears in: [KubeStateMetricsConfig](#kubestatemetricsconfig)</em>
+
+| Property | Type | Description |
+| -------- | ---- | ----------- |
+| resource | string | Defines the Kubernetes resource name (for example, `jobs` or `cronjobs`). |
+| labels | []string | Defines the list of Kubernetes labels to expose as metrics for this resource. Use `*` to expose all labels. |
+
+[Back to TOC](#table-of-contents)
+
 ## TLSConfig
 
 #### Description

Documentation/openshiftdocs/index.adoc

@@ -64,6 +64,7 @@ The configuration file itself is always defined under the `config.yaml` key in t
 * link:modules/prometheusoperatorconfig.adoc[PrometheusOperatorConfig]
 * link:modules/prometheusrestrictedconfig.adoc[PrometheusRestrictedConfig]
 * link:modules/remotewritespec.adoc[RemoteWriteSpec]
+* link:modules/resourcelabels.adoc[ResourceLabels]
 * link:modules/tlsconfig.adoc[TLSConfig]
 * link:modules/telemeterclientconfig.adoc[TelemeterClientConfig]
 * link:modules/thanosquerierconfig.adoc[ThanosQuerierConfig]

Documentation/openshiftdocs/modules/kubestatemetricsconfig.adoc

@@ -26,6 +26,8 @@ Appears in: link:clustermonitoringconfiguration.adoc[ClusterMonitoringConfigurat
 
 |topologySpreadConstraints|[]v1.TopologySpreadConstraint|Defines a pod's topology spread constraints.
 
+|additionalResourceLabels|[]link:resourcelabels.adoc[ResourceLabels]|Defines additional resource labels to expose as metrics in addition to the default labels. Currently, only `jobs` and `cronjobs` resources are supported due to cardinality concerns. Each entry specifies a resource name and a list of label names (use `*` to expose all labels).
+
 |===
 
 link:../index.adoc[Back to TOC]

Documentation/openshiftdocs/modules/resourcelabels.adoc

@@ -0,0 +1,30 @@
+// DO NOT EDIT THE CONTENT IN THIS FILE. It is automatically generated from the 
+	// source code for the Cluster Monitoring Operator. Any changes made to this 
+	// file will be overwritten when the content is re-generated. If you wish to 
+	// make edits, read the docgen utility instructions in the source code for the 
+	// CMO.
+	:_content-type: ASSEMBLY
+
+== ResourceLabels
+
+=== Description
+
+The `ResourceLabels` resource defines which Kubernetes labels to expose as metrics for a given resource type.
+
+=== Required
+* `resource`
+* `labels`
+
+
+Appears in: link:kubestatemetricsconfig.adoc[KubeStateMetricsConfig]
+
+[options="header"]
+|===
+| Property | Type | Description 
+|resource|string|Defines the Kubernetes resource name (for example, `jobs` or `cronjobs`).
+
+|labels|[]string|Defines the list of Kubernetes labels to expose as metrics for this resource. Use `*` to expose all labels.
+
+|===
+
+link:../index.adoc[Back to TOC]

pkg/manifests/config.go

@@ -363,9 +363,40 @@ func NewConfigFromString(content string) (*Config, error) {
 		)
 	}
 
+	// Validate additional resource labels for KSM.
+	if err := validateAdditionalResourceLabels(c.ClusterMonitoringConfiguration.KubeStateMetricsConfig); err != nil {
+		return nil, err
+	}
+
 	return &c, nil
 }
 
+var supportedResourceLabelsResources = []string{"jobs", "cronjobs"}
+
+func validateAdditionalResourceLabels(ksm *KubeStateMetricsConfig) error {
+	if ksm == nil {
+		return nil
+	}
+
+	seen := map[string]bool{}
+	for _, rl := range ksm.AdditionalResourceLabels {
+		if rl.Resource == "" {
+			return fmt.Errorf("%w: additionalResourceLabels: resource name must not be empty", ErrConfigValidation)
+		}
+		if !slices.Contains(supportedResourceLabelsResources, rl.Resource) {
+			return fmt.Errorf("%w: additionalResourceLabels: unsupported resource %q, supported resources are: %v", ErrConfigValidation, rl.Resource, supportedResourceLabelsResources)
+		}
+		if seen[rl.Resource] {
+			return fmt.Errorf("%w: additionalResourceLabels: duplicate resource %q", ErrConfigValidation, rl.Resource)
+		}
+		seen[rl.Resource] = true
+		if len(rl.Labels) == 0 {
+			return fmt.Errorf("%w: additionalResourceLabels: resource %q must have at least one label", ErrConfigValidation, rl.Resource)
+		}
+	}
+	return nil
+}
+
 func (c *Config) applyDefaults() {
 	if c.Images == nil {
 		c.Images = &Images{}

pkg/manifests/config_test.go

@@ -753,6 +753,98 @@ func TestCollectionProfileValues(t *testing.T) {
 	}
 }
 
+func TestAdditionalResourceLabelsValidation(t *testing.T) {
+	for _, tc := range []struct {
+		name        string
+		config      string
+		expectError bool
+	}{
+		{
+			name:        "no additional resource labels",
+			config:      "",
+			expectError: false,
+		},
+		{
+			name: "valid jobs resource",
+			config: `kubeStateMetrics:
+  additionalResourceLabels:
+  - resource: jobs
+    labels:
+    - foo`,
+			expectError: false,
+		},
+		{
+			name: "valid cronjobs resource",
+			config: `kubeStateMetrics:
+  additionalResourceLabels:
+  - resource: cronjobs
+    labels:
+    - bar`,
+			expectError: false,
+		},
+		{
+			name: "valid multiple resources",
+			config: `kubeStateMetrics:
+  additionalResourceLabels:
+  - resource: jobs
+    labels:
+    - foo
+  - resource: cronjobs
+    labels:
+    - bar`,
+			expectError: false,
+		},
+		{
+			name: "unsupported resource",
+			config: `kubeStateMetrics:
+  additionalResourceLabels:
+  - resource: pods
+    labels:
+    - foo`,
+			expectError: true,
+		},
+		{
+			name: "empty resource name",
+			config: `kubeStateMetrics:
+  additionalResourceLabels:
+  - resource: ""
+    labels:
+    - foo`,
+			expectError: true,
+		},
+		{
+			name: "duplicate resource",
+			config: `kubeStateMetrics:
+  additionalResourceLabels:
+  - resource: jobs
+    labels:
+    - foo
+  - resource: jobs
+    labels:
+    - bar`,
+			expectError: true,
+		},
+		{
+			name: "no labels",
+			config: `kubeStateMetrics:
+  additionalResourceLabels:
+  - resource: jobs
+    labels: []`,
+			expectError: true,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := NewConfigFromString(tc.config)
+			if tc.expectError {
+				require.Error(t, err)
+				require.ErrorIs(t, err, ErrConfigValidation)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
 func TestDeprecatedConfig(t *testing.T) {
 	for _, tc := range []struct {
 		name                string

pkg/manifests/manifests.go

@@ -788,6 +788,18 @@ func (f *Factory) KubeStateMetricsDeployment() (*appsv1.Deployment, error) {
 			if f.config.ClusterMonitoringConfiguration.KubeStateMetricsConfig.Resources != nil {
 				d.Spec.Template.Spec.Containers[i].Resources = *f.config.ClusterMonitoringConfiguration.KubeStateMetricsConfig.Resources
 			}
+			additionalResourceLabels := f.config.ClusterMonitoringConfiguration.KubeStateMetricsConfig.AdditionalResourceLabels
+			if len(additionalResourceLabels) > 0 {
+				for j := range container.Args {
+					if strings.HasPrefix(container.Args[j], "--metric-labels-allowlist=") {
+						var parts []string
+						for _, rl := range additionalResourceLabels {
+							parts = append(parts, fmt.Sprintf("%s=[%s]", rl.Resource, strings.Join(rl.Labels, ",")))
+						}
+						container.Args[j] += "," + strings.Join(parts, ",")
+					}
+				}
+			}
 		}
 	}
 

pkg/manifests/manifests_test.go

@@ -3834,6 +3834,91 @@ func TestKubeStateMetrics(t *testing.T) {
 	}
 }
 
+func TestKubeStateMetricsAdditionalResourceLabels(t *testing.T) {
+	defaultAllowList := "--metric-labels-allowlist=pods=[*],nodes=[*],namespaces=[*],persistentvolumes=[*],persistentvolumeclaims=[*],poddisruptionbudgets=[*]"
+
+	tests := []struct {
+		name        string
+		config      string
+		expectedArg string
+	}{
+		{
+			name:        "no additional resource labels",
+			config:      "",
+			expectedArg: defaultAllowList,
+		},
+		{
+			name: "single resource with specific labels",
+			config: `kubeStateMetrics:
+  additionalResourceLabels:
+  - resource: jobs
+    labels:
+    - foo
+    - bar`,
+			expectedArg: defaultAllowList + ",jobs=[foo,bar]",
+		},
+		{
+			name: "multiple resources",
+			config: `kubeStateMetrics:
+  additionalResourceLabels:
+  - resource: jobs
+    labels:
+    - foo
+  - resource: cronjobs
+    labels:
+    - bar
+    - baz`,
+			expectedArg: defaultAllowList + ",jobs=[foo],cronjobs=[bar,baz]",
+		},
+		{
+			name: "wildcard labels",
+			config: `kubeStateMetrics:
+  additionalResourceLabels:
+  - resource: cronjobs
+    labels:
+    - "*"`,
+			expectedArg: defaultAllowList + ",cronjobs=[*]",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			c, err := NewConfigFromString(tc.config)
+			if err != nil {
+				t.Fatal(err)
+			}
+			c.SetImages(map[string]string{
+				"kube-state-metrics": "docker.io/openshift/origin-kube-state-metrics:latest",
+				"kube-rbac-proxy":    "docker.io/openshift/origin-kube-rbac-proxy:latest",
+			})
+
+			f := NewFactory("openshift-monitoring", "openshift-user-workload-monitoring", c, defaultInfrastructureReader(), &fakeProxyReader{}, NewAssets(assetsPath), &APIServerConfig{}, &configv1.Console{})
+
+			d, err := f.KubeStateMetricsDeployment()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			found := false
+			for _, container := range d.Spec.Template.Spec.Containers {
+				if container.Name == "kube-state-metrics" {
+					for _, arg := range container.Args {
+						if strings.HasPrefix(arg, "--metric-labels-allowlist=") {
+							found = true
+							if arg != tc.expectedArg {
+								t.Fatalf("expected arg %q, got %q", tc.expectedArg, arg)
+							}
+						}
+					}
+				}
+			}
+			if !found {
+				t.Fatal("--metric-labels-allowlist arg not found in kube-state-metrics container")
+			}
+		})
+	}
+}
+
 func TestOpenShiftStateMetrics(t *testing.T) {
 	config := `openshiftStateMetrics:
   resources:

pkg/manifests/types.go

@@ -182,6 +182,16 @@ type DedicatedServiceMonitors struct {
 	Enabled bool `json:"enabled,omitempty"`
 }
 
+// The `ResourceLabels` resource defines which Kubernetes labels to expose
+// as metrics for a given resource type.
+type ResourceLabels struct {
+	// Defines the Kubernetes resource name (for example, `jobs` or `cronjobs`).
+	Resource string `json:"resource"`
+	// Defines the list of Kubernetes labels to expose as metrics for this
+	// resource. Use `*` to expose all labels.
+	Labels []string `json:"labels"`
+}
+
 // The `KubeStateMetricsConfig` resource defines settings for the
 // `kube-state-metrics` agent.
 type KubeStateMetricsConfig struct {
@@ -193,6 +203,11 @@ type KubeStateMetricsConfig struct {
 	Tolerations []v1.Toleration `json:"tolerations,omitempty"`
 	// Defines a pod's topology spread constraints.
 	TopologySpreadConstraints []v1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"`
+	// Defines additional resource labels to expose as metrics in addition to
+	// the default labels. Currently, only `jobs` and `cronjobs` resources are
+	// supported due to cardinality concerns. Each entry specifies a resource
+	// name and a list of label names (use `*` to expose all labels).
+	AdditionalResourceLabels []ResourceLabels `json:"additionalResourceLabels,omitempty"`
 }
 
 // The `PrometheusK8sConfig` resource defines settings for the Prometheus