Replace MongoDB storage for scheduling strategies/intents with K8s CRDs

config/manager_config.go

@@ -79,6 +79,7 @@ type AccountConfig struct {
 type K8SConfig struct {
 	KubeConfigPath string `mapstructure:"kube_config_path"`
 	IsInCluster    bool   `mapstructure:"in_cluster"`
+	CRDNamespace   string `mapstructure:"crd_namespace"`
 }
 
 var managerCfg *ManageConfig

deployment/crds/schedulingintent-crd.yaml

@@ -0,0 +1,73 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: schedulingintents.gthulhu.io
+spec:
+  group: gthulhu.io
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                strategyID:
+                  type: string
+                podID:
+                  type: string
+                podName:
+                  type: string
+                nodeID:
+                  type: string
+                k8sNamespace:
+                  type: string
+                commandRegex:
+                  type: string
+                priority:
+                  type: integer
+                executionTime:
+                  type: integer
+                  format: int64
+                podLabels:
+                  type: object
+                  additionalProperties:
+                    type: string
+                state:
+                  type: integer
+                creatorID:
+                  type: string
+                updaterID:
+                  type: string
+                createdTime:
+                  type: integer
+                  format: int64
+                updatedTime:
+                  type: integer
+                  format: int64
+      additionalPrinterColumns:
+        - name: Strategy
+          type: string
+          jsonPath: .spec.strategyID
+        - name: Pod
+          type: string
+          jsonPath: .spec.podName
+        - name: Node
+          type: string
+          jsonPath: .spec.nodeID
+        - name: State
+          type: integer
+          jsonPath: .spec.state
+        - name: Age
+          type: date
+          jsonPath: .metadata.creationTimestamp
+  scope: Namespaced
+  names:
+    plural: schedulingintents
+    singular: schedulingintent
+    kind: SchedulingIntent
+    shortNames:
+      - si

deployment/crds/schedulingstrategy-crd.yaml

@@ -0,0 +1,66 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: schedulingstrategies.gthulhu.io
+spec:
+  group: gthulhu.io
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                strategyNamespace:
+                  type: string
+                labelSelectors:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      key:
+                        type: string
+                      value:
+                        type: string
+                k8sNamespaces:
+                  type: array
+                  items:
+                    type: string
+                commandRegex:
+                  type: string
+                priority:
+                  type: integer
+                executionTime:
+                  type: integer
+                  format: int64
+                creatorID:
+                  type: string
+                updaterID:
+                  type: string
+                createdTime:
+                  type: integer
+                  format: int64
+                updatedTime:
+                  type: integer
+                  format: int64
+      additionalPrinterColumns:
+        - name: Priority
+          type: integer
+          jsonPath: .spec.priority
+        - name: Creator
+          type: string
+          jsonPath: .spec.creatorID
+        - name: Age
+          type: date
+          jsonPath: .metadata.creationTimestamp
+  scope: Namespaced
+  names:
+    plural: schedulingstrategies
+    singular: schedulingstrategy
+    kind: SchedulingStrategy
+    shortNames:
+      - ss

deployment/rbac/manager-crd-rbac.yaml

@@ -0,0 +1,35 @@
+# ClusterRole granting the API Server (manager mode) full access to
+# SchedulingStrategy and SchedulingIntent custom resources.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: gthulhu-manager-crd
+rules:
+  - apiGroups: ["gthulhu.io"]
+    resources: ["schedulingstrategies", "schedulingintents"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+---
+# Bind the ClusterRole to the manager ServiceAccount so that only the
+# manager pods can manipulate scheduling CRs.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: gthulhu-manager-crd-binding
+subjects:
+  - kind: ServiceAccount
+    name: gthulhu-manager
+    namespace: gthulhu-system
+roleRef:
+  kind: ClusterRole
+  name: gthulhu-manager-crd
+  apiGroup: rbac.authorization.k8s.io
+---
+# Read-only ClusterRole for monitoring / dashboards.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: gthulhu-crd-viewer
+rules:
+  - apiGroups: ["gthulhu.io"]
+    resources: ["schedulingstrategies", "schedulingintents"]
+    verbs: ["get", "list", "watch"]

manager/app/module.go

@@ -10,6 +10,10 @@ import (
 	"github.com/Gthulhu/api/manager/service"
 	"github.com/Gthulhu/api/pkg/container"
 	"go.uber.org/fx"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	dynamicfake "k8s.io/client-go/dynamic/fake"
 )
 
 // ConfigModule creates an Fx module that provides configuration structs
@@ -48,6 +52,12 @@ func AdapterModule() (fx.Option, error) {
 				InCluster:      k8sConfig.IsInCluster,
 			})
 		}),
+		fx.Provide(func(k8sConfig config.K8SConfig) (dynamic.Interface, error) {
+			return k8sadapter.NewDynamicClient(k8sadapter.Options{
+				KubeConfigPath: k8sConfig.KubeConfigPath,
+				InCluster:      k8sConfig.IsInCluster,
+			})
+		}),
 		fx.Provide(client.NewDecisionMakerClient),
 	), nil
 }
@@ -100,6 +110,18 @@ func TestRepoModule(cfg config.ManageConfig, containerBuilder *container.Contain
 	}
 	return fx.Options(
 		configModule,
+		fx.Provide(NewFakeDynamicClient),
 		fx.Provide(repository.NewRepository),
 	), nil
 }
+
+// NewFakeDynamicClient returns a fake Kubernetes dynamic client for testing.
+func NewFakeDynamicClient() dynamic.Interface {
+	scheme := runtime.NewScheme()
+	return dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme,
+		map[schema.GroupVersionResource]string{
+			{Group: "gthulhu.io", Version: "v1alpha1", Resource: "schedulingstrategies"}: "SchedulingStrategyList",
+			{Group: "gthulhu.io", Version: "v1alpha1", Resource: "schedulingintents"}:    "SchedulingIntentList",
+		},
+	)
+}

manager/domain/interface.go

@@ -67,7 +67,7 @@ type Repository interface {
 	BatchUpdateIntentsState(ctx context.Context, intentIDs []bson.ObjectID, newState IntentState) error
 	QueryStrategies(ctx context.Context, opt *QueryStrategyOptions) error
 	QueryIntents(ctx context.Context, opt *QueryIntentOptions) error
-	UpdateStrategy(ctx context.Context, strategyID bson.ObjectID, update bson.M) error
+	UpdateStrategy(ctx context.Context, strategy *ScheduleStrategy) error
 	DeleteStrategy(ctx context.Context, strategyID bson.ObjectID) error
 	DeleteIntents(ctx context.Context, intentIDs []bson.ObjectID) error
 	DeleteIntentsByStrategyID(ctx context.Context, strategyID bson.ObjectID) error