diff --git a/.github/workflows/container-images-1.0.yaml b/.github/workflows/container-images-1.0.yaml new file mode 100644 index 00000000..f2f61499 --- /dev/null +++ b/.github/workflows/container-images-1.0.yaml @@ -0,0 +1,116 @@ +# Build and push slurm-operator container images to Docker Hub (1.0 line). +# +# Tagging: +# - push to slurm-1.0-together-changes: togethercomputer/slurm-operator: (from ./VERSION, no suffix) +# - pull_request (same-repo only): ...:-dev- +# +# GitHub only evaluates workflows that exist on the ref being pushed. This file must live on +# slurm-1.0-together-changes (not only on main), or pushes to that branch will not run any workflow. +# +# Required repository secrets: +# - DOCKERHUB_USERNAME +# - DOCKERHUB_TOKEN +# Optional: +# - ROBOT_GITHUB_TOKEN + +name: Container images v1.0 + +on: + pull_request: + branches: + - slurm-1.0-together-changes + push: + branches: + - slurm-1.0-together-changes + workflow_dispatch: + +env: + REGISTRY: togethercomputer + RELEASE_BRANCH: slurm-1.0-together-changes + +concurrency: + group: container-images-1.0-${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + + - name: Set image VERSION for docker-bake + id: image-version + run: | + set -euo pipefail + BASE="$(tr -d ' \n\r\t' < VERSION)" + SHORT="$(git rev-parse --short HEAD)" + if [ "${{ github.event_name }}" = "pull_request" ]; then + IMG="${BASE}-dev-${SHORT}" + else + IMG="${BASE}" + fi + echo "version=${IMG}" >> "$GITHUB_OUTPUT" + echo "Image VERSION (make/docker-bake): ${IMG}" + + - name: Whether to push to Docker Hub + id: should-push + env: + RELEASE_REF: refs/heads/${{ env.RELEASE_BRANCH }} + run: | + if [ "${{ github.event_name }}" = "push" ] && [ "${{ github.ref }}" = "${RELEASE_REF}" ]; then + echo "push=true" >> "$GITHUB_OUTPUT" + elif [ "${{ github.event_name }}" = "pull_request" ] && \ + [ "${{ github.event.pull_request.head.repo.full_name }}" = "${{ github.repository }}" ]; then + echo "push=true" >> "$GITHUB_OUTPUT" + elif [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ github.ref }}" = "${RELEASE_REF}" ]; then + echo "push=true" >> "$GITHUB_OUTPUT" + else + echo "push=false" >> "$GITHUB_OUTPUT" + fi + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Configure git for private Go modules + run: | + git config --global url."https://together-robot:${ROBOT_GITHUB_TOKEN}@github.com/togethercomputer/".insteadOf "https://github.com/togethercomputer/" + env: + ROBOT_GITHUB_TOKEN: ${{ secrets.ROBOT_GITHUB_TOKEN }} + + - name: Set up Helm + uses: azure/setup-helm@v4 + with: + version: v3.16.3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + if: steps.should-push.outputs.push == 'true' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build (images + charts, no registry push) + if: steps.should-push.outputs.push != 'true' + run: make build VERSION="${{ steps.image-version.outputs.version }}" + + - name: Package charts (push path) + if: steps.should-push.outputs.push == 'true' + run: make build-chart + + - name: Build and push images + if: steps.should-push.outputs.push == 'true' + run: make push-images VERSION="${{ steps.image-version.outputs.version }}" diff --git a/.gitignore b/.gitignore index 712805de..483a725b 100644 --- a/.gitignore +++ b/.gitignore @@ -137,3 +137,7 @@ values-*.yaml .helm_ls_cache docs/build/html *debug_bin* +test-values.yaml + +# codemogger db +.codemogger/ diff --git a/Makefile b/Makefile index 9d996252..10346f49 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ version-match: version ## Check if versions are consistent. .PHONY: all all: build ## Build slurm-operator. -REGISTRY ?= slinky.slurm.net +REGISTRY ?= togethercomputer BUILDER ?= project-v3-builder .PHONY: build @@ -81,7 +81,8 @@ build-chart: ## Build charts. push: push-images push-charts ## Push OCI packages. .PHONY: push-images -push-images: build-images ## Push container images. +push-images: ## Build and push container images (single buildx bake --push). + - $(CONTAINER_TOOL) buildx create --name $(BUILDER) REGISTRY=$(REGISTRY) VERSION=$(VERSION) $(CONTAINER_TOOL) buildx bake --builder=$(BUILDER) --push .PHONY: push-charts diff --git a/PROJECT b/PROJECT index c65f8485..599bcddd 100644 --- a/PROJECT +++ b/PROJECT @@ -9,7 +9,7 @@ plugins: manifests.sdk.operatorframework.io/v2: {} scorecard.sdk.operatorframework.io/v2: {} projectName: slurm-operator -repo: github.com/SlinkyProject/slurm-operator +repo: github.com/togethercomputer/slurm-operator resources: - api: crdVersion: v1beta1 @@ -18,7 +18,7 @@ resources: domain: slurm.net group: slinky kind: Controller - path: github.com/SlinkyProject/slurm-operator/api/v1beta1 + path: github.com/togethercomputer/slurm-operator/api/v1beta1 version: v1beta1 webhooks: validation: true @@ -30,7 +30,7 @@ resources: domain: slurm.net group: slinky kind: RestApi - path: github.com/SlinkyProject/slurm-operator/api/v1beta1 + path: github.com/togethercomputer/slurm-operator/api/v1beta1 version: v1beta1 webhooks: validation: true @@ -42,7 +42,7 @@ resources: domain: slurm.net group: slinky kind: Accounting - path: github.com/SlinkyProject/slurm-operator/api/v1beta1 + path: github.com/togethercomputer/slurm-operator/api/v1beta1 version: v1beta1 webhooks: validation: true @@ -54,7 +54,7 @@ resources: domain: slurm.net group: slinky kind: NodeSet - path: github.com/SlinkyProject/slurm-operator/api/v1beta1 + path: github.com/togethercomputer/slurm-operator/api/v1beta1 version: v1beta1 webhooks: validation: true @@ -66,7 +66,7 @@ resources: domain: slurm.net group: slinky kind: LoginSet - path: github.com/SlinkyProject/slurm-operator/api/v1beta1 + path: github.com/togethercomputer/slurm-operator/api/v1beta1 version: v1beta1 webhooks: validation: true @@ -78,7 +78,7 @@ resources: domain: slurm.net group: slinky kind: Token - path: github.com/SlinkyProject/slurm-operator/api/v1beta1 + path: github.com/togethercomputer/slurm-operator/api/v1beta1 version: v1beta1 webhooks: validation: true @@ -90,6 +90,6 @@ resources: domain: slurm.net group: slinky kind: Controller - path: github.com/SlinkyProject/slurm-operator/api/v1beta1 + path: github.com/togethercomputer/slurm-operator/api/v1beta1 version: v1beta1 version: "3" diff --git a/README.md b/README.md index b074ace9..c7d935cb 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,9 @@
[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg?style=for-the-badge)](./LICENSES/Apache-2.0.txt) -[![Tag](https://img.shields.io/github/v/tag/SlinkyProject/slurm-operator?style=for-the-badge)](https://github.com/SlinkyProject/slurm-operator/tags/) -[![Go-Version](https://img.shields.io/github/go-mod/go-version/SlinkyProject/slurm-operator?style=for-the-badge)](./go.mod) -[![Last-Commit](https://img.shields.io/github/last-commit/SlinkyProject/slurm-operator?style=for-the-badge)](https://github.com/SlinkyProject/slurm-operator/commits/) +[![Tag](https://img.shields.io/github/v/tag/togethercomputer/slurm-operator?style=for-the-badge)](https://github.com/togethercomputer/slurm-operator/tags/) +[![Go-Version](https://img.shields.io/github/go-mod/go-version/togethercomputer/slurm-operator?style=for-the-badge)](./go.mod) +[![Last-Commit](https://img.shields.io/github/last-commit/togethercomputer/slurm-operator?style=for-the-badge)](https://github.com/togethercomputer/slurm-operator/commits/)
diff --git a/VERSION b/VERSION index 3eefcb9d..7ee7020b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.0 +1.0.10 diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 5930a0ca..3dfeeab7 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -18,6 +18,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -55,6 +56,7 @@ type Flags struct { metricsAddr string secureMetrics bool enableHTTP2 bool + namespace string } func parseFlags(flags *Flags) { @@ -81,6 +83,9 @@ func parseFlags(flags *Flags) { "If set the metrics endpoint is served securely") flag.BoolVar(&flags.enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") + flag.StringVar(&flags.namespace, "namespace", "", + "If set, the operator only watches Slinky resources in this namespace. "+ + "Empty (the default) watches all namespaces.") flag.Parse() } @@ -107,7 +112,9 @@ func main() { tlsOpts = append(tlsOpts, disableHTTP2) } - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + leaderElectionID := "0033bda7.slinky.slurm.net" + + mgrOpts := ctrl.Options{ Scheme: scheme, Metrics: server.Options{ TLSOpts: tlsOpts, @@ -115,9 +122,19 @@ func main() { }, HealthProbeBindAddress: flags.probeAddr, LeaderElection: flags.enableLeaderElection, - LeaderElectionID: "0033bda7.slinky.slurm.net", + LeaderElectionID: leaderElectionID, LeaderElectionReleaseOnCancel: true, - }) + } + + // Restrict informers to a single namespace and give the leader-election + // lock a unique name so multiple operator instances can coexist on the + // same cluster without racing over the same CRs. + if flags.namespace != "" { + mgrOpts.Cache.DefaultNamespaces = map[string]cache.Config{flags.namespace: {}} + mgrOpts.LeaderElectionID = leaderElectionID + "." + flags.namespace + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), mgrOpts) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) diff --git a/docker-bake.hcl b/docker-bake.hcl index 016abb32..9b1a4ceb 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -4,7 +4,7 @@ ################################################################################ variable "REGISTRY" { - default = "ghcr.io/slinkyproject" + default = "togethercomputer" } variable "VERSION" { @@ -22,15 +22,15 @@ target "_common" { labels = { # Ref: https://github.com/opencontainers/image-spec/blob/v1.0/annotations.md "org.opencontainers.image.authors" = "slinky@schedmd.com" - "org.opencontainers.image.documentation" = "https://github.com/SlinkyProject/slurm-operator" + "org.opencontainers.image.documentation" = "https://github.com/togethercomputer/slurm-operator" "org.opencontainers.image.license" = "Apache-2.0" "org.opencontainers.image.vendor" = "SchedMD LLC." "org.opencontainers.image.version" = "${VERSION}" - "org.opencontainers.image.source" = "https://github.com/SlinkyProject/slurm-operator" + "org.opencontainers.image.source" = "https://github.com/togethercomputer/slurm-operator" # Ref: https://docs.redhat.com/en/documentation/red_hat_software_certification/2025/html/red_hat_openshift_software_certification_policy_guide/assembly-requirements-for-container-images_openshift-sw-cert-policy-introduction#con-image-metadata-requirements_openshift-sw-cert-policy-container-images "vendor" = "SchedMD LLC." "version" = "${VERSION}" - "release" = "https://github.com/SlinkyProject/slurm-operator" + "release" = "https://github.com/togethercomputer/slurm-operator" } } diff --git a/docs/index.rst b/docs/index.rst index ca81273b..352388ca 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -345,12 +345,12 @@ limitations under the License. .. |License| image:: https://img.shields.io/badge/License-Apache_2.0-blue.svg?style=for-the-badge :target: ./LICENSES/Apache-2.0.txt -.. |Tag| image:: https://img.shields.io/github/v/tag/SlinkyProject/slurm-operator?style=for-the-badge - :target: https://github.com/SlinkyProject/slurm-operator/tags/ -.. |Go-Version| image:: https://img.shields.io/github/go-mod/go-version/SlinkyProject/slurm-operator?style=for-the-badge +.. |Tag| image:: https://img.shields.io/github/v/tag/togethercomputer/slurm-operator?style=for-the-badge + :target: https://github.com/togethercomputer/slurm-operator/tags/ +.. |Go-Version| image:: https://img.shields.io/github/go-mod/go-version/togethercomputer/slurm-operator?style=for-the-badge :target: ./go.mod -.. |Last-Commit| image:: https://img.shields.io/github/last-commit/SlinkyProject/slurm-operator?style=for-the-badge - :target: https://github.com/SlinkyProject/slurm-operator/commits/ +.. |Last-Commit| image:: https://img.shields.io/github/last-commit/togethercomputer/slurm-operator?style=for-the-badge + :target: https://github.com/togethercomputer/slurm-operator/commits/ .. toctree:: :maxdepth: 2 diff --git a/docs/versioning.md b/docs/versioning.md index ded3faec..3bcdb2d7 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -58,4 +58,4 @@ any kind (e.g., component flag changes). [semver]: https://semver.org/ [slurm-bridge]: https://github.com/SlinkyProject/slurm-bridge [slurm-client]: https://github.com/SlinkyProject/slurm-client -[slurm-operator]: https://github.com/SlinkyProject/slurm-operator +[slurm-operator]: https://github.com/togethercomputer/slurm-operator diff --git a/go.mod b/go.mod index 352d27cb..68539a2d 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/SlinkyProject/slurm-operator +module github.com/togethercomputer/slurm-operator go 1.25.0 diff --git a/helm/slurm-operator-crds/Chart.yaml b/helm/slurm-operator-crds/Chart.yaml index e2ebc293..f15b9863 100644 --- a/helm/slurm-operator-crds/Chart.yaml +++ b/helm/slurm-operator-crds/Chart.yaml @@ -13,7 +13,7 @@ home: https://slinky.schedmd.com/ icon: https://github.com/SlinkyProject/docs/blob/main/docs/_static/images/slinky.svg sources: - - https://github.com/SlinkyProject/slurm-operator + - https://github.com/togethercomputer/slurm-operator maintainers: - name: SchedMD LLC. diff --git a/helm/slurm-operator-crds/README.md b/helm/slurm-operator-crds/README.md index 2041f67a..ea9625f1 100644 --- a/helm/slurm-operator-crds/README.md +++ b/helm/slurm-operator-crds/README.md @@ -14,5 +14,5 @@ Slurm Operator CRDs ## Source Code -* +* diff --git a/helm/slurm-operator/Chart.yaml b/helm/slurm-operator/Chart.yaml index 8a270134..4710de2f 100644 --- a/helm/slurm-operator/Chart.yaml +++ b/helm/slurm-operator/Chart.yaml @@ -15,7 +15,7 @@ home: https://slinky.schedmd.com/ icon: https://github.com/SlinkyProject/docs/blob/main/docs/_static/images/slinky.svg sources: - - https://github.com/SlinkyProject/slurm-operator + - https://github.com/togethercomputer/slurm-operator maintainers: - name: SchedMD LLC. diff --git a/helm/slurm-operator/README.md b/helm/slurm-operator/README.md index 4a61fa81..f522ff2f 100644 --- a/helm/slurm-operator/README.md +++ b/helm/slurm-operator/README.md @@ -14,7 +14,7 @@ Slurm Operator ## Source Code -* +* ## Requirements diff --git a/helm/slurm-operator/templates/operator/rbac.yaml b/helm/slurm-operator/templates/operator/rbac.yaml index 87f84dda..1206298e 100644 --- a/helm/slurm-operator/templates/operator/rbac.yaml +++ b/helm/slurm-operator/templates/operator/rbac.yaml @@ -45,6 +45,7 @@ rules: - list - watch - patch + - update - apiGroups: - "" resources: diff --git a/helm/slurm/Chart.yaml b/helm/slurm/Chart.yaml index b59061af..cb1cc151 100644 --- a/helm/slurm/Chart.yaml +++ b/helm/slurm/Chart.yaml @@ -17,7 +17,7 @@ icon: https://github.com/SlinkyProject/docs/blob/main/docs/_static/images/slurm- sources: - https://github.com/SchedMD/slurm - https://github.com/SlinkyProject/containers - - https://github.com/SlinkyProject/slurm-operator + - https://github.com/togethercomputer/slurm-operator maintainers: - name: SchedMD LLC. diff --git a/helm/slurm/README.md b/helm/slurm/README.md index c10fe3ce..3dbdb6ce 100644 --- a/helm/slurm/README.md +++ b/helm/slurm/README.md @@ -16,7 +16,7 @@ Slurm Cluster * * -* +* ## Requirements diff --git a/helm/slurm/templates/_helpers.tpl b/helm/slurm/templates/_helpers.tpl index 62b138ca..1e4920a2 100644 --- a/helm/slurm/templates/_helpers.tpl +++ b/helm/slurm/templates/_helpers.tpl @@ -161,3 +161,14 @@ Ref: https://github.com/helm/helm/issues/11376#issuecomment-1256831105 {{- end -}} {{- mulf (float64 $value) $unit -}} {{- end -}} + +{{/* +Default pod dnsConfig snippet. + +We keep this as a helper so templates can include it consistently. +*/}} +{{- define "slurm.dnsConfig" -}} +options: + - name: ndots + value: "1" +{{- end -}} diff --git a/helm/slurm/templates/_slurm.tpl b/helm/slurm/templates/_slurm.tpl index f93df720..2a3c9fc1 100644 --- a/helm/slurm/templates/_slurm.tpl +++ b/helm/slurm/templates/_slurm.tpl @@ -30,3 +30,27 @@ Define auth/jwt HS256 secret ref key {{- define "slurm.authJwtHs256Ref.key" -}} {{- print "jwt_hs256.key" -}} {{- end }} + +{{/* +Define login name +*/}} +{{- define "slurm.login.name" -}} +{{- printf "%s-login" (include "slurm.fullname" .) -}} +{{- end }} + +{{/* +Define login labels +*/}} +{{- define "slurm.login.labels" -}} +app.kubernetes.io/component: login +{{ include "slurm.login.selectorLabels" . }} +{{ include "slurm.labels" . }} +{{- end }} + +{{/* +Define login selectorLabels +*/}} +{{- define "slurm.login.selectorLabels" -}} +app.kubernetes.io/name: login +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/helm/slurm/templates/controller/_helpers.tpl b/helm/slurm/templates/controller/_helpers.tpl index 75631cda..e2c29b15 100644 --- a/helm/slurm/templates/controller/_helpers.tpl +++ b/helm/slurm/templates/controller/_helpers.tpl @@ -105,3 +105,17 @@ Worker epilog slurmctld scripts. {{- define "slurm.controller.epilogSlurmctldName" -}} {{- printf "%s-epilog-slurmctld-scripts" (include "slurm.fullname" .) -}} {{- end }} + +{{/* +Init script for login nodes. +*/}} +{{- define "slurm.initScriptLoginName" -}} +{{- printf "%s-init-script-login" (include "slurm.fullname" .) -}} +{{- end }} + +{{/* +Init script for compute nodes. +*/}} +{{- define "slurm.initScriptNodesName" -}} +{{- printf "%s-init-script-nodes" (include "slurm.fullname" .) -}} +{{- end }} diff --git a/helm/slurm/templates/controller/controller-cr.yaml b/helm/slurm/templates/controller/controller-cr.yaml index 2db2396a..5ad3bcf8 100644 --- a/helm/slurm/templates/controller/controller-cr.yaml +++ b/helm/slurm/templates/controller/controller-cr.yaml @@ -97,14 +97,14 @@ spec: {{- include "format-podTemplate" $podTemplate | nindent 2 }} persistence: {{- with .Values.controller.persistence.existingClaim }} - {{- toYaml . | nindent 4 }} + existingClaim: {{ toYaml . | nindent 6 | trim }} {{- end }}{{- /* with .Values.controller.persistence.existingClaim */}} {{- with .Values.controller.persistence.storageClassName }} - {{- toYaml . | nindent 4 }} + storageClassName: {{ . }} {{- end }}{{- /* with .Values.controller.persistence.storageClassName */}} - {{- with omit .Values.controller.persistence "existingClaim" "storageClassName"}} + {{- with omit .Values.controller.persistence "existingClaim" "storageClassName" "enabled"}} {{- toYaml . | nindent 4 }} - {{- end }}{{- /* with omit .Values.controller.persistence "existingClaim" "storageClassName" */}} + {{- end }}{{- /* with omit .Values.controller.persistence "existingClaim" "storageClassName" "enabled" */}} {{- if or (.Values.controller.service.spec) (.Values.controller.service.metadata) (omit .Values.controller.service "metadata" "spec") }} {{- include "format-service" .Values.controller.service | nindent 2 }} {{ else }} diff --git a/helm/slurm/templates/login/init-script-configmap.yaml b/helm/slurm/templates/login/init-script-configmap.yaml new file mode 100644 index 00000000..f6c3f500 --- /dev/null +++ b/helm/slurm/templates/login/init-script-configmap.yaml @@ -0,0 +1,17 @@ +{{- /* +SPDX-FileCopyrightText: Copyright (C) SchedMD LLC. +SPDX-License-Identifier: Apache-2.0 +*/}} + +{{- if .Values.initScriptLogin -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "slurm.initScriptLoginName" . }} + namespace: {{ include "slurm.namespace" . }} + labels: + {{- include "slurm.labels" . | nindent 4 }} +data: + init.sh: | + {{- .Values.initScriptLogin | nindent 4 }} +{{- end }}{{- /* if .Values.initScriptLogin */}} diff --git a/helm/slurm/templates/login/login-deployment.yaml b/helm/slurm/templates/login/login-deployment.yaml new file mode 100644 index 00000000..5addbb1c --- /dev/null +++ b/helm/slurm/templates/login/login-deployment.yaml @@ -0,0 +1,163 @@ +{{- /* +SPDX-FileCopyrightText: Copyright (C) SchedMD LLC. +SPDX-License-Identifier: Apache-2.0 +*/}} + +{{- if .Values.login.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "slurm.login.name" . }} + namespace: {{ include "slurm.namespace" . }} + labels: + {{- include "slurm.login.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.login.replicas }} + selector: + matchLabels: + {{- include "slurm.login.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "slurm.login.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.login.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + automountServiceAccountToken: false + enableServiceLinks: false + dnsConfig: + {{- include "slurm.dnsConfig" . | nindent 8 }} + {{- with .Values.login.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.login.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.login.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + {{- include "slurm.login.selectorLabels" . | nindent 12 }} + initContainers: + - name: fix-perms + image: "{{ .Values.login.image.repository }}:{{ .Values.login.image.tag }}" + command: ["sh", "-c", "set -e && cp /mnt/slurm/* /etc/slurm/ && chmod 644 /etc/slurm/*.conf && chmod 600 /etc/slurm/*.key && chown -R root:root /etc/slurm/"] + volumeMounts: + - name: slurm-config-projected + mountPath: /mnt/slurm + readOnly: true + - name: slurm-config + mountPath: /etc/slurm + containers: + - name: login + image: "{{ .Values.login.image.repository }}:{{ .Values.login.image.tag }}" + imagePullPolicy: {{ .Values.login.imagePullPolicy | default .Values.imagePullPolicy }} + {{- with .Values.login.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.login.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + env: + - name: SLURM_CLUSTER_NAME + value: {{ include "slurm.fullname" . }} + - name: SLURM_CONF_SERVER # TCL-4403: must use -controller service name + value: {{ include "slurm.fullname" . }}-controller.{{ include "slurm.namespace" . }}.svc.cluster.local + - name: SACKD_OPTIONS + value: "--conf-server {{ include "slurm.fullname" . }}-controller.{{ include "slurm.namespace" . }}:6817" + volumeMounts: + - name: slurm-config + mountPath: /etc/slurm + {{- if .Values.login.sharedMemorySize }} + - name: dshm + mountPath: /dev/shm + {{- end }} + {{- if .Values.initScriptLogin }} + - name: init-script + mountPath: /root/init.sh + subPath: init.sh + {{- end }} + {{- range .Values.login.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- if .subPath }} + subPath: {{ .subPath }} + {{- end }} + {{- if .readOnly }} + readOnly: {{ .readOnly }} + {{- end }} + {{- end }} + volumes: + - name: slurm-config + emptyDir: {} + - name: slurm-config-projected # TCL-4402: projected volume, copied by initContainer with split permissions + projected: + defaultMode: 0600 + sources: + - configMap: + name: {{ include "slurm.fullname" . }}-config + - secret: + name: {{ include "slurm.fullname" . }}-auth-slurm + - secret: + name: {{ include "slurm.fullname" . }}-auth-jwths256 + {{- if .Values.login.sharedMemorySize }} + - name: dshm + emptyDir: + medium: Memory + sizeLimit: {{ .Values.login.sharedMemorySize }} + {{- end }} + {{- if .Values.initScriptLogin }} + - name: init-script + configMap: + name: {{ include "slurm.initScriptLoginName" . }} + defaultMode: 0755 + {{- end }} + {{- range .Values.login.extraVolumes }} + - name: {{ .name }} + {{- if .configMap }} + configMap: + name: {{ .configMap.name }} + {{- with .configMap.items }} + items: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- else if .secret }} + secret: + secretName: {{ .secret.secretName }} + {{- with .secret.items }} + items: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- else if .persistentVolumeClaim }} + persistentVolumeClaim: + claimName: {{ .persistentVolumeClaim.claimName }} + {{- else if .nfs }} + nfs: + server: {{ .nfs.server }} + path: {{ .nfs.path }} + {{- else if .emptyDir }} + emptyDir: {} + {{- else if .hostPath }} + hostPath: + path: {{ .hostPath.path }} + {{- if .hostPath.type }} + type: {{ .hostPath.type }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/slurm/templates/login/login-service.yaml b/helm/slurm/templates/login/login-service.yaml new file mode 100644 index 00000000..2f9ad63a --- /dev/null +++ b/helm/slurm/templates/login/login-service.yaml @@ -0,0 +1,23 @@ +{{- /* +SPDX-FileCopyrightText: Copyright (C) SchedMD LLC. +SPDX-License-Identifier: Apache-2.0 +*/}} + +{{- if .Values.login.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "slurm.login.name" . }} + namespace: {{ include "slurm.namespace" . }} + labels: + {{- include "slurm.login.labels" . | nindent 4 }} +spec: + type: ClusterIP + selector: + {{- include "slurm.login.selectorLabels" . | nindent 4 }} + ports: + - name: ssh + port: 22 + targetPort: 22 + protocol: TCP +{{- end }} \ No newline at end of file diff --git a/helm/slurm/templates/loginset/_helpers.tpl b/helm/slurm/templates/loginset/_helpers.tpl deleted file mode 100644 index 5e705f3a..00000000 --- a/helm/slurm/templates/loginset/_helpers.tpl +++ /dev/null @@ -1,11 +0,0 @@ -{{- /* -SPDX-FileCopyrightText: Copyright (C) SchedMD LLC. -SPDX-License-Identifier: Apache-2.0 -*/}} - -{{/* -Define login name -*/}} -{{- define "slurm.login.name" -}} -{{- printf "%s-login" (include "slurm.fullname" .) -}} -{{- end }} diff --git a/helm/slurm/templates/loginset/loginset-cr.yaml b/helm/slurm/templates/loginset/loginset-cr.yaml deleted file mode 100644 index 754e03b3..00000000 --- a/helm/slurm/templates/loginset/loginset-cr.yaml +++ /dev/null @@ -1,49 +0,0 @@ -{{- /* -SPDX-FileCopyrightText: Copyright (C) SchedMD LLC. -SPDX-License-Identifier: Apache-2.0 -*/}} - -{{- range $key, $loginset := $.Values.loginsets -}} -{{- if $loginset.enabled }} -{{- $name := printf "%s-%s" (include "slurm.login.name" $) $key }} -{{- $podMetadata := $loginset.metadata | default dict -}} -{{- $podSpec := $loginset.podSpec | default dict -}} -{{- $imagePullSecrets := concat list ($podSpec.imagePullSecrets | default list) ($.Values.imagePullSecrets | default list) -}} -{{- $_ := set $podSpec "imagePullSecrets" $imagePullSecrets -}} -{{- $priorityClassName := $podSpec.priorityClassName | default $.Values.priorityClassName -}} -{{- $_ := set $podSpec "priorityClassName" $priorityClassName -}} -{{- $podTemplate := dict "metadata" $podMetadata "spec" $podSpec -}} ---- -apiVersion: slinky.slurm.net/v1beta1 -kind: LoginSet -metadata: - name: {{ $name }} - namespace: {{ include "slurm.namespace" $ }} - labels: - {{- include "slurm.labels" $ | nindent 4 }} -spec: - controllerRef: - name: {{ include "slurm.fullname" $ }} - namespace: {{ include "slurm.namespace" $ }} - {{- with $loginset.extraSshdConfig }} - extraSshdConfig: | - {{- . | nindent 4 }} - {{- end }}{{- /* with $loginset.extraSshdConfig */}} - sssdConfRef: - name: {{ $name }}-sssd-conf - key: sssd.conf - {{- with $loginset.rootSshAuthorizedKeys }} - rootSshAuthorizedKeys: | - {{- . | nindent 4 }} - {{- end }}{{- /* with $loginset.rootSshAuthorizedKeys */}} - replicas: {{ $loginset.replicas }} - login: - {{- $_ := set $loginset.login "imagePullPolicy" (default $.Values.imagePullPolicy $loginset.login.imagePullPolicy) -}} - {{- include "format-container" $loginset.login | nindent 4 }} - {{- include "format-podTemplate" $podTemplate | nindent 2 }} - {{- with $loginset.service }} - service: - {{- toYaml . | nindent 4 }} - {{- end }}{{- /* with $loginset.service */}} -{{- end }}{{- /* $loginset.enabled */}} -{{- end }}{{- /* range $loginset := $.Values.loginsets */}} diff --git a/helm/slurm/templates/loginset/loginset-secret.yaml b/helm/slurm/templates/loginset/loginset-secret.yaml deleted file mode 100644 index 3ed66459..00000000 --- a/helm/slurm/templates/loginset/loginset-secret.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- /* -SPDX-FileCopyrightText: Copyright (C) SchedMD LLC. -SPDX-License-Identifier: Apache-2.0 -*/}} - -{{- range $key, $loginset := $.Values.loginsets -}} -{{- if $loginset.enabled }} -{{- $name := printf "%s-%s" (include "slurm.login.name" $) $key }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ $name }}-sssd-conf - namespace: {{ include "slurm.namespace" $ }} - labels: - loginset.slinky.slurm.net/name: {{ $name }} - {{- include "slurm.labels" $ | nindent 4 }} -type: Opaque -stringData: - sssd.conf: | - {{- required "sssdConf is required" $loginset.sssdConf | nindent 4 }} -{{- end }}{{- /* $loginset.enabled */}} -{{- end }}{{- /* range $loginset := $.Values.loginsets */}} diff --git a/helm/slurm/templates/nodeset/init-script-configmap.yaml b/helm/slurm/templates/nodeset/init-script-configmap.yaml new file mode 100644 index 00000000..f1e5bee1 --- /dev/null +++ b/helm/slurm/templates/nodeset/init-script-configmap.yaml @@ -0,0 +1,17 @@ +{{- /* +SPDX-FileCopyrightText: Copyright (C) SchedMD LLC. +SPDX-License-Identifier: Apache-2.0 +*/}} + +{{- if .Values.initScriptNodes -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "slurm.initScriptNodesName" . }} + namespace: {{ include "slurm.namespace" . }} + labels: + {{- include "slurm.labels" . | nindent 4 }} +data: + init.sh: | + {{- .Values.initScriptNodes | nindent 4 }} +{{- end }}{{- /* if .Values.initScriptNodes */}} diff --git a/helm/slurm/templates/nodeset/nodeset-cr.yaml b/helm/slurm/templates/nodeset/nodeset-cr.yaml index 22842b8e..65692144 100644 --- a/helm/slurm/templates/nodeset/nodeset-cr.yaml +++ b/helm/slurm/templates/nodeset/nodeset-cr.yaml @@ -13,6 +13,50 @@ SPDX-License-Identifier: Apache-2.0 {{- $priorityClassName := $podSpec.priorityClassName | default $.Values.priorityClassName -}} {{- $_ := set $podSpec "priorityClassName" $priorityClassName -}} {{- $_ := set $podSpec "hostname" (printf "%s-" $key) -}} +{{- /* Add shared memory volume and mount if shmSize is specified (Together addition) */}} +{{- if $podSpec.shmSize }} + {{- $volumes := $podSpec.volumes | default list -}} + {{- $shmVolume := dict "name" "dshm" "emptyDir" (dict "medium" "Memory" "sizeLimit" $podSpec.shmSize) -}} + {{- $volumes = append $volumes $shmVolume -}} + {{- $_ := set $podSpec "volumes" $volumes -}} + {{- $volumeMounts := $nodeset.slurmd.volumeMounts | default list -}} + {{- $shmVolumeMount := dict "name" "dshm" "mountPath" "/dev/shm" -}} + {{- $volumeMounts = append $volumeMounts $shmVolumeMount -}} + {{- $_ := set $nodeset.slurmd "volumeMounts" $volumeMounts -}} +{{- end }} +{{- /* Add existing data claim volumes and mounts (Together addition) */}} +{{- if and $nodeset.persistence $nodeset.persistence.existingDataClaims }} + {{- $volumes := $podSpec.volumes | default list -}} + {{- $volumeMounts := $nodeset.slurmd.volumeMounts | default list -}} + {{- range $nodeset.persistence.existingDataClaims }} + {{- $pvcVolume := dict "name" .name "persistentVolumeClaim" (dict "claimName" .name) -}} + {{- $volumes = append $volumes $pvcVolume -}} + {{- $pvcVolumeMount := dict "name" .name "mountPath" .mountPath -}} + {{- $volumeMounts = append $volumeMounts $pvcVolumeMount -}} + {{- end }} + {{- $_ := set $podSpec "volumes" $volumes -}} + {{- $_ := set $nodeset.slurmd "volumeMounts" $volumeMounts -}} +{{- end }} +{{- /* Add volumeClaimTemplate mounts to slurmd (Together addition) */}} +{{- if and $nodeset.persistence $nodeset.persistence.volumeClaimTemplate }} + {{- $volumeMounts := $nodeset.slurmd.volumeMounts | default list -}} + {{- range $nodeset.persistence.volumeClaimTemplate }} + {{- $vctVolumeMount := dict "name" .metadata.name "mountPath" .spec.mountPath -}} + {{- $volumeMounts = append $volumeMounts $vctVolumeMount -}} + {{- end }} + {{- $_ := set $nodeset.slurmd "volumeMounts" $volumeMounts -}} +{{- end }} +{{- /* Add init script volume and mount for compute nodes (Together addition) */}} +{{- if $.Values.initScriptNodes }} + {{- $volumes := $podSpec.volumes | default list -}} + {{- $initScriptVolume := dict "name" "init-script" "configMap" (dict "name" (include "slurm.initScriptNodesName" $) "defaultMode" 0755) -}} + {{- $volumes = append $volumes $initScriptVolume -}} + {{- $_ := set $podSpec "volumes" $volumes -}} + {{- $volumeMounts := $nodeset.slurmd.volumeMounts | default list -}} + {{- $initScriptVolumeMount := dict "name" "init-script" "mountPath" "/root/init.sh" "subPath" "init.sh" -}} + {{- $volumeMounts = append $volumeMounts $initScriptVolumeMount -}} + {{- $_ := set $nodeset.slurmd "volumeMounts" $volumeMounts -}} +{{- end }} {{- $podTemplate := dict "metadata" $podMetadata "spec" $podSpec -}} {{- if $nodeset.useResourceLimits }} {{- $envLimts := list (dict "name" "POD_CPUS" "value" (include "slurm.worker.podCpus" $nodeset.slurmd | toString)) (dict "name" "POD_MEMORY" "value" (include "slurm.worker.podMemory" $nodeset.slurmd | toString)) -}} @@ -63,6 +107,19 @@ spec: updateStrategy: {{- toYaml . | nindent 4 }} {{- end }}{{- /* with $nodeset.updateStrategy */}} + {{- if and $nodeset.persistence $nodeset.persistence.volumeClaimTemplate }} + volumeClaimTemplates: + {{- range $nodeset.persistence.volumeClaimTemplate }} + - metadata: + name: {{ .metadata.name }} + spec: + accessModes: {{ .spec.accessModes | toYaml | nindent 10 }} + resources: + {{- toYaml .spec.resources | nindent 10 }} + storageClassName: {{ .spec.storageClassName }} + volumeMode: {{ .spec.volumeMode }} + {{- end }} + {{- end }} taintKubeNodes: {{ $nodeset.taintKubeNodes }} {{- end }}{{- /* $nodeset.enabled */}} {{- end }}{{- /* range $nodeset := $.Values.nodesets */}} diff --git a/helm/slurm/values.yaml b/helm/slurm/values.yaml index b27a1605..b3bb115d 100644 --- a/helm/slurm/values.yaml +++ b/helm/slurm/values.yaml @@ -118,6 +118,24 @@ epilogScripts: {} # set -euo pipefail # exit 0 +# -- (string) Init script mounted to /root/init.sh on all login nodes. +# WARNING: The script must include a shebang (!) so it can be executed correctly. +# Ref: https://en.wikipedia.org/wiki/Shebang_(Unix) +initScriptLogin: null + # | + # #!/usr/bin/env bash + # set -euo pipefail + # echo "Login node init" + +# -- (string) Init script mounted to /root/init.sh on all NodeSets (compute nodes). +# WARNING: The script must include a shebang (!) so it can be executed correctly. +# Ref: https://en.wikipedia.org/wiki/Shebang_(Unix) +initScriptNodes: null + # | + # #!/usr/bin/env bash + # set -euo pipefail + # echo "Compute node init" + # Slurm controller (slurmctld) configuration. controller: # -- Configures this component as external (not in Kubernetes). @@ -462,121 +480,6 @@ accounting: # port: 6819 # nodePort: 30819 -# Slurm LoginSet (sackd, sshd, sssd) configurations. -loginsets: - # Sample LoginSet. - slinky: - # -- Enable use of this LoginSet. - enabled: false - # -- Number of replicas to deploy. - replicas: 1 - # login container configurations. - login: - # -- The image to use, `${repository}:${tag}`. - # Ref: https://kubernetes.io/docs/concepts/containers/images/#image-names - image: - repository: ghcr.io/slinkyproject/login - tag: 25.11-ubuntu24.04 - # -- Environment passed to the image. - env: [] - # - name: SACKD_OPTIONS - # value: -vvv - # -- The container security context to use. - # Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container - securityContext: - privileged: false - # capabilities: - # add: - # - SYS_CHROOT - # -- The container resource limits and requests. - # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container - resources: {} - # requests: - # cpu: 1 - # memory: 1Gi - # -- List of volume mounts to use. - # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ - volumeMounts: [] - # - name: nfs-home - # mountPath: /home - # -- SSH public keys to write into `/root/.ssh/authorized_keys`. - rootSshAuthorizedKeys: null - # -- Extra configuration lines appended to `/etc/ssh/sshd_config`. - # Ref: https://manpages.ubuntu.com/manpages/noble/man5/sshd_config.5.html - extraSshdConfig: null - # -- The `sssd.conf` to use. - # Ref: https://man.archlinux.org/man/sssd.conf.5 - sssdConf: | - [sssd] - config_file_version = 2 - services = nss,pam - domains = DEFAULT - - [nss] - filter_groups = root,slurm - filter_users = root,slurm - - [pam] - - [domain/DEFAULT] - auth_provider = ldap - id_provider = ldap - ldap_uri = ldap://ldap.example.com - ldap_search_base = dc=example,dc=com - ldap_user_search_base = ou=Users,dc=example,dc=com - ldap_group_search_base = ou=Groups,dc=example,dc=com - # -- Labels and annotations. - # Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ - metadata: {} - # annotations: {} - # labels: {} - # -- (corev1.PodSpec) Extend the pod template, and/or override certain configurations. - # Ref: https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates - podSpec: - # -- The pod resource limits and requests. - # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container - resources: {} - # requests: - # cpu: 1 - # memory: 1Gi - # -- Additional initContainers for the pod. - # Ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ - # Ref: https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/ - initContainers: [] - # -- (map[string]string) Node label selector for pod assignment. - # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector - nodeSelector: - kubernetes.io/os: linux - # -- Affinity for pod assignment. - # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity - affinity: {} - # -- Tolerations for pod assignment. - # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - tolerations: [] - # - key: key1 - # operator: Exists - # effect: NoSchedule - # -- List of volumes to use. - # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ - volumes: [] - # - name: nfs-home - # nfs: - # server: nfs-server.example.com - # path: /exports/home - # -- The service configuration. - service: - # -- Labels and annotations. - # Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ - metadata: {} - # annotations: {} - # labels: {} - # -- (corev1.ServiceSpec) Extend the service template, and/or override certain configurations. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/ - spec: - type: LoadBalancer - # port: 22 - # nodePort: 32222 - # Slurm NodeSet (slurmd) configurations. nodesets: # Sample NodeSet. @@ -685,6 +588,8 @@ nodesets: tolerations: [] # - key: nvidia.com/gpu # effect: NoSchedule + # -- (string) Set the size of the shared memory for the nodeset. + shmSize: 16Gi # -- List of volumes to use. # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ volumes: [] @@ -694,6 +599,14 @@ nodesets: # path: /exports/home # -- Taint the Kubernetes nodes on which nodeset pods are scheduled with NoExecute taintKubeNodes: false + # -- Persistence configuration for the nodeset. + persistence: + # -- (list) Existing PersistentVolumeClaims to mount (Together addition) + existingDataClaims: [] + # - name: data-cpu-pv + # mountPath: /data + # - name: scratch-gpu-pv + # mountPath: /scratch # Slurm partition configurations. partitions: @@ -729,3 +642,93 @@ vendor: jobMappingDir: "/var/lib/dcgm-exporter/job-mapping" # -- Script execution priority (lower numbers run first) scriptPriority: "90" + +login: + # + # -- (bool) + # Enables login nodes. + enabled: true + # + # -- (integer) + # Set the number of replicas to deploy. + replicas: 1 + # + # -- (string) + # Set the image pull policy. + imagePullPolicy: IfNotPresent + # + # Set the image to use. + image: + # + # -- (string) + # Set the image repository to use. + repository: ghcr.io/slinkyproject/login + # + # -- (string) + # Set the image tag to use. + tag: 25.05-ubuntu24.04 + # + # -- (object) + # The security context given to the container. + # Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + securityContext: + privileged: false + # capabilities: + # add: + # - SYS_CHROOT + # + # --(list) + # List of volume mounts. + # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ + extraVolumeMounts: [] + # - name: nfs-home + # mountPath: /home + # - name: nfs-data + # mountPath: /mnt/data + # + # --(list) + # Define list of pod volumes. + # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ + extraVolumes: [] + # - name: nfs-home + # nfs: + # server: nfs-server.example.com + # path: /exports/home/ + # - name: nfs-data + # persistentVolumeClaim: + # claimName: nfs-data + # + # -- (string) + # Set the priority class to use. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass + priorityClassName: "" + # + # -- (map) + # Selector which must match a node's labels for the pod to be scheduled on that node. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + # + # -- (object) + # Set affinity for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + # + # -- (list) + # Configure pod tolerations. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + # + # -- (object) + # Set container resource requests and limits for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container + resources: {} + # requests: + # cpu: 1 + # memory: 1Gi + # limits: + # cpu: 2 + # memory: 4Gi + +# +# Slurm compute (slurmd) configurations. diff --git a/internal/annotations/node.go b/internal/annotations/node.go new file mode 100644 index 00000000..3493ebab --- /dev/null +++ b/internal/annotations/node.go @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: Copyright (C) SchedMD LLC. +// SPDX-License-Identifier: Apache-2.0 + +package annotations + +const ( + // NodeWeight can be used to set to an int32 that represents the weight of + // scheduling a NodeSet Pod compared to other Nodes. + // Note that this is honored on a best-effort basis, and so it does not + // offer guarantees on Node scheduling order. + NodeWeight = "slinky.slurm.net/node-weight" + // NodeCordon is used to mark a node as cordoned for Together's use case + NodeCordon = "slinky.slurm.net/node-cordon" +) \ No newline at end of file diff --git a/internal/builder/accounting_app.go b/internal/builder/accounting_app.go index d7bec44f..7bad5f72 100644 --- a/internal/builder/accounting_app.go +++ b/internal/builder/accounting_app.go @@ -27,6 +27,10 @@ const ( SlurmdbdPort = 6819 slurmdbdConfFile = "slurmdbd.conf" + + // Volume names for accounting initContainer fix + slurmSecretsTmpVolume = "slurm-secrets-tmp" + slurmSecretsTmpDir = "/tmp/slurm-secrets" ) func (b *Builder) BuildAccounting(accounting *slinkyv1beta1.Accounting) (*appsv1.StatefulSet, error) { @@ -95,14 +99,17 @@ func (b *Builder) accountingPodTemplate(accounting *slinkyv1beta1.Accounting) (c }, base: corev1.PodSpec{ AutomountServiceAccountToken: ptr.To(false), + // InitContainer copies secrets from temp location to /etc/slurm with correct permissions. + // This is necessary because Kubernetes projected volumes with FSGroup add group read + // permission, but slurmdbd.conf requires exactly 0600 (owner read/write only). + InitContainers: []corev1.Container{ + accountingInitContainer(), + }, Containers: []corev1.Container{ b.slurmdbdContainer(spec.Slurmdbd.Container), }, SecurityContext: &corev1.PodSecurityContext{ - RunAsNonRoot: ptr.To(true), - RunAsUser: ptr.To(slurmUserUid), - RunAsGroup: ptr.To(slurmUserGid), - FSGroup: ptr.To(slurmUserGid), + RunAsNonRoot: ptr.To(false), // initContainer needs to run as root }, Volumes: accountingVolumes(accounting), }, @@ -114,11 +121,19 @@ func (b *Builder) accountingPodTemplate(accounting *slinkyv1beta1.Accounting) (c func accountingVolumes(accounting *slinkyv1beta1.Accounting) []corev1.Volume { out := []corev1.Volume{ + // EmptyDir for /etc/slurm - initContainer copies secrets here with correct permissions { Name: slurmEtcVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + // Projected volume with secrets - mounted to temp location, copied by initContainer + { + Name: slurmSecretsTmpVolume, VolumeSource: corev1.VolumeSource{ Projected: &corev1.ProjectedVolumeSource{ - DefaultMode: ptr.To[int32](0o600), + DefaultMode: ptr.To[int32](0o644), // readable by initContainer Sources: []corev1.VolumeProjection{ { Secret: &corev1.SecretProjection{ @@ -159,6 +174,35 @@ func accountingVolumes(accounting *slinkyv1beta1.Accounting) []corev1.Volume { return out } +// accountingInitContainer creates an init container that copies secrets from the +// projected volume to an emptyDir with correct ownership (slurm:slurm) and +// permissions (0600 for slurmdbd.conf, 0400 for keys). +// This is necessary because Kubernetes projected volumes with FSGroup add group +// read permission, but slurmdbd.conf requires exactly 0600 (owner read/write only). +func accountingInitContainer() corev1.Container { + return corev1.Container{ + Name: "init-slurm-config", + Image: "busybox:1.36", + Command: []string{ + "sh", "-c", + `cp /tmp/slurm-secrets/* /etc/slurm/ && \ + chown -R 401:401 /etc/slurm && \ + chmod 600 /etc/slurm/slurmdbd.conf && \ + chmod 400 /etc/slurm/slurm.key && \ + chmod 400 /etc/slurm/jwt_hs256.key && \ + ls -la /etc/slurm/`, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To[int64](0), // Run as root to chown + RunAsGroup: ptr.To[int64](0), + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: slurmSecretsTmpVolume, MountPath: slurmSecretsTmpDir, ReadOnly: true}, + {Name: slurmEtcVolume, MountPath: slurmEtcDir}, + }, + } +} + func (b *Builder) slurmdbdContainer(merge corev1.Container) corev1.Container { opts := ContainerOpts{ base: corev1.Container{ @@ -183,7 +227,7 @@ func (b *Builder) slurmdbdContainer(merge corev1.Container) corev1.Container { RunAsGroup: ptr.To(slurmUserGid), }, VolumeMounts: []corev1.VolumeMount{ - {Name: slurmEtcVolume, MountPath: slurmEtcDir, ReadOnly: true}, + {Name: slurmEtcVolume, MountPath: slurmEtcDir}, // emptyDir populated by initContainer {Name: slurmPidFileVolume, MountPath: slurmPidFileDir}, }, }, diff --git a/internal/builder/controller_app.go b/internal/builder/controller_app.go index 394657cc..29e662ad 100644 --- a/internal/builder/controller_app.go +++ b/internal/builder/controller_app.go @@ -242,27 +242,24 @@ func (b *Builder) slurmctldContainer(merge corev1.Container, clusterName string) }, StartupProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/livez", - Port: intstr.FromString(labels.ControllerApp), + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(SlurmctldPort), }, }, - FailureThreshold: 6, + FailureThreshold: 30, PeriodSeconds: 10, }, ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/readyz", - Port: intstr.FromString(labels.ControllerApp), + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(SlurmctldPort), }, }, }, LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/livez", - Port: intstr.FromString(labels.ControllerApp), + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(SlurmctldPort), }, }, FailureThreshold: 6, diff --git a/internal/builder/controller_config.go b/internal/builder/controller_config.go index 8b7e64c0..8b4fc735 100644 --- a/internal/builder/controller_config.go +++ b/internal/builder/controller_config.go @@ -270,6 +270,14 @@ func buildSlurmConf( conf.AddProperty(config.NewPropertyRaw(partitionLineRendered)) } + conf.AddProperty(config.NewPropertyRaw("#")) + conf.AddProperty(config.NewPropertyRaw("### SYSTEM DEFAULTS ###")) + conf.AddProperty(config.NewProperty("UnkillableStepTimeout", 600)) + conf.AddProperty(config.NewProperty("HealthCheckInterval", 60)) + conf.AddProperty(config.NewProperty("HealthCheckNodeState", "ANY")) + conf.AddProperty(config.NewProperty("HealthCheckProgram", "/usr/bin/gpu_healthcheck.sh")) + conf.AddProperty(config.NewProperty("JobRequeue", 0)) + extraConf := controller.Spec.ExtraConf conf.AddProperty(config.NewPropertyRaw("#")) conf.AddProperty(config.NewPropertyRaw("### EXTRA CONFIG ###")) @@ -284,6 +292,7 @@ func buildCgroupConf() string { conf.AddProperty(config.NewProperty("CgroupPlugin", "cgroup/v2")) conf.AddProperty(config.NewProperty("IgnoreSystemd", "yes")) + conf.AddProperty(config.NewProperty("ConstrainRAMSpace", "yes")) return conf.Build() } diff --git a/internal/builder/controller_config_test.go b/internal/builder/controller_config_test.go index 7e85fb22..afbba3b7 100644 --- a/internal/builder/controller_config_test.go +++ b/internal/builder/controller_config_test.go @@ -159,6 +159,25 @@ func TestBuilder_BuildControllerConfig(t *testing.T) { case got.Data[slurmConfFile] == "" && got.BinaryData[slurmConfFile] == nil: t.Errorf("got.Data[%s] = %v", slurmConfFile, got.Data[slurmConfFile]) + + default: + slurmConf := got.Data[slurmConfFile] + for _, directive := range []string{ + "UnkillableStepTimeout=600", + "HealthCheckInterval=60", + "HealthCheckNodeState=ANY", + "HealthCheckProgram=/usr/bin/gpu_healthcheck.sh", + "JobRequeue=0", + } { + if !strings.Contains(slurmConf, directive) { + t.Errorf("slurm.conf missing system default %q", directive) + } + } + if cgroupConf, ok := got.Data[cgroupConfFile]; ok { + if !strings.Contains(cgroupConf, "ConstrainRAMSpace=yes") { + t.Errorf("cgroup.conf missing ConstrainRAMSpace=yes") + } + } } }) } diff --git a/internal/builder/scripts/reconfigure.sh b/internal/builder/scripts/reconfigure.sh index 921803e2..1aad2919 100755 --- a/internal/builder/scripts/reconfigure.sh +++ b/internal/builder/scripts/reconfigure.sh @@ -25,7 +25,12 @@ function main() { local lastHash="" local newHash="" - echo "[$(date)] Start '$SLURM_DIR' polling" + # Initialize lastHash to avoid a spurious reconfigure on startup. + # slurmctld may still be initializing and an early SIGHUP can deadlock it. + # TCL-4401 + lastHash="$(getHash)" + + echo "[$(date)] Start '$SLURM_DIR' polling (initial hash captured)" while true; do newHash="$(getHash)" if [ "$newHash" != "$lastHash" ]; then diff --git a/internal/builder/worker_app.go b/internal/builder/worker_app.go index c3e7c81e..02404d75 100644 --- a/internal/builder/worker_app.go +++ b/internal/builder/worker_app.go @@ -121,19 +121,17 @@ func (b *Builder) slurmdContainer(nodeset *slinkyv1beta1.NodeSet, controller *sl }, StartupProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/livez", - Port: intstr.FromString(labels.WorkerApp), + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(SlurmdPort), }, }, - FailureThreshold: 6, + FailureThreshold: 30, PeriodSeconds: 10, }, LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/livez", - Port: intstr.FromString(labels.WorkerApp), + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(SlurmdPort), }, }, FailureThreshold: 6, @@ -141,9 +139,8 @@ func (b *Builder) slurmdContainer(nodeset *slinkyv1beta1.NodeSet, controller *sl }, ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/readyz", - Port: intstr.FromString(labels.WorkerApp), + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(SlurmdPort), }, }, }, diff --git a/internal/controller/nodeset/nodeset_sync.go b/internal/controller/nodeset/nodeset_sync.go index 39a209ae..9a29504d 100644 --- a/internal/controller/nodeset/nodeset_sync.go +++ b/internal/controller/nodeset/nodeset_sync.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" slinkyv1beta1 "github.com/SlinkyProject/slurm-operator/api/v1beta1" + "github.com/SlinkyProject/slurm-operator/internal/annotations" "github.com/SlinkyProject/slurm-operator/internal/builder/labels" nodesetutils "github.com/SlinkyProject/slurm-operator/internal/controller/nodeset/utils" "github.com/SlinkyProject/slurm-operator/internal/utils" @@ -243,6 +244,10 @@ func (r *NodeSetReconciler) sync( return err } + if err := r.syncOrphanedSlurmNodes(ctx, nodeset, pods); err != nil { + return err + } + if err := r.syncNodeSet(ctx, nodeset, pods, hash); err != nil { return err } @@ -250,6 +255,16 @@ func (r *NodeSetReconciler) sync( return nil } +// syncOrphanedSlurmNodes removes Slurm node registrations that have no matching pod. +// This handles ghost entries left behind when pods terminate without running PreStop. +func (r *NodeSetReconciler) syncOrphanedSlurmNodes( + ctx context.Context, + nodeset *slinkyv1beta1.NodeSet, + pods []*corev1.Pod, +) error { + return r.slurmControl.DeleteOrphanedNodes(ctx, nodeset, pods) +} + // syncClusterWorkerService manages the cluster worker hostname service for the Slurm cluster. func (r *NodeSetReconciler) syncClusterWorkerService(ctx context.Context, nodeset *slinkyv1beta1.NodeSet) error { service, err := r.builder.BuildClusterWorkerService(nodeset) @@ -671,13 +686,22 @@ func (r *NodeSetReconciler) doPodScaleIn( return err } } - if isDrained, err := r.slurmControl.IsNodeDrained(ctx, nodeset, pod); !isDrained || err != nil { + isDrained, err := r.slurmControl.IsNodeDrained(ctx, nodeset, pod) + if err != nil { + r.expectations.DeletionObserved(logger, key, podKey) + return err + } + + // Update K8s node annotation to reflect drain state + if err := r.updateNodeCordonAnnotation(ctx, nodeset, pod, isDrained); err != nil { + logger.Error(err, "Failed to update node cordon annotation", "pod", klog.KObj(pod)) + // Don't fail the reconciliation, just log the error + } + + if !isDrained { // Decrement expectations and requeue reconcile because the Slurm node is not drained yet. // We must wait until fully drained to terminate the pod. r.expectations.DeletionObserved(logger, key, podKey) - if err != nil { - return err - } } return nil }) @@ -721,6 +745,12 @@ func (r *NodeSetReconciler) processCondemned( return err } + // Update K8s node annotation to reflect drain state + if err := r.updateNodeCordonAnnotation(ctx, nodeset, pod, isDrained); err != nil { + logger.Error(err, "Failed to update node cordon annotation", "pod", klog.KObj(pod)) + // Don't fail the reconciliation, just log the error + } + if podutils.IsRunning(pod) && !isDrained { logger.V(2).Info("NodeSet Pod is draining, pending termination for scale-in", "pod", klog.KObj(pod)) @@ -885,6 +915,8 @@ func (r *NodeSetReconciler) makePodUncordonAndUndrain( pod *corev1.Pod, reason string, ) error { + logger := log.FromContext(ctx) + if err := r.makePodUncordon(ctx, pod); err != nil { return err } @@ -893,6 +925,12 @@ func (r *NodeSetReconciler) makePodUncordonAndUndrain( return err } + // Update K8s node annotation to reflect undrained state + if err := r.updateNodeCordonAnnotation(ctx, nodeset, pod, false); err != nil { + logger.Error(err, "Failed to update node cordon annotation", "pod", klog.KObj(pod)) + // Don't fail the reconciliation, just log the error + } + return nil } @@ -1119,3 +1157,58 @@ func (r *NodeSetReconciler) syncClusterWorkerPDB( return nil } + +// updateNodeCordonAnnotation updates the K8s node annotation to reflect cordon state based on Slurm drain status +func (r *NodeSetReconciler) updateNodeCordonAnnotation( + ctx context.Context, + nodeset *slinkyv1beta1.NodeSet, + pod *corev1.Pod, + isDrained bool, +) error { + logger := klog.FromContext(ctx) + + // Get the K8s node that the pod is running on + if pod.Spec.NodeName == "" { + // Pod not scheduled yet, nothing to do + return nil + } + + node := &corev1.Node{} + nodeKey := types.NamespacedName{Name: pod.Spec.NodeName} + if err := r.Get(ctx, nodeKey, node); err != nil { + if apierrors.IsNotFound(err) { + // Node doesn't exist, nothing to do + return nil + } + return err + } + + // Initialize annotations if nil + if node.Annotations == nil { + node.Annotations = make(map[string]string) + } + + // Check current annotation value + currentValue := node.Annotations[annotations.NodeCordon] + expectedValue := fmt.Sprintf("%t", isDrained) + + // Only update if the value has changed + if currentValue != expectedValue { + node.Annotations[annotations.NodeCordon] = expectedValue + + if isDrained { + logger.Info("Node is drained, marking as cordoned", + "node", node.Name, "pod", klog.KObj(pod)) + } else { + logger.Info("Node is not drained, marking as uncordoned", + "node", node.Name, "pod", klog.KObj(pod)) + } + + // Update the node + if err := r.Update(ctx, node); err != nil { + return fmt.Errorf("failed to update node %s cordon annotation: %w", node.Name, err) + } + } + + return nil +} diff --git a/internal/controller/nodeset/slurmcontrol/slurmcontrol.go b/internal/controller/nodeset/slurmcontrol/slurmcontrol.go index 5c520be1..69698de0 100644 --- a/internal/controller/nodeset/slurmcontrol/slurmcontrol.go +++ b/internal/controller/nodeset/slurmcontrol/slurmcontrol.go @@ -7,6 +7,7 @@ import ( "context" "math" "net/http" + "strconv" "strings" "time" @@ -52,6 +53,10 @@ type SlurmControlInterface interface { CalculateNodeStatus(ctx context.Context, nodeset *slinkyv1beta1.NodeSet, pods []*corev1.Pod) (SlurmNodeStatus, error) // GetNodeDeadlines returns a map of node to its deadline time.Time calculated from running jobs. GetNodeDeadlines(ctx context.Context, nodeset *slinkyv1beta1.NodeSet, pods []*corev1.Pod) (*timestore.TimeStore, error) + // DeleteNode removes a Slurm node registration by name. + DeleteNode(ctx context.Context, nodeset *slinkyv1beta1.NodeSet, nodeName string) error + // DeleteOrphanedNodes removes Slurm node registrations that have no matching pod. + DeleteOrphanedNodes(ctx context.Context, nodeset *slinkyv1beta1.NodeSet, pods []*corev1.Pod) error } // realSlurmControl is the default implementation of SlurmControlInterface. @@ -72,6 +77,10 @@ func (r *realSlurmControl) RefreshNodeCache(ctx context.Context, nodeset *slinky nodeList := &slurmtypes.V0044NodeList{} opts := &slurmclient.ListOptions{RefreshCache: true} if err := slurmClient.List(ctx, nodeList, opts); err != nil { + // Tolerate "Not Found" and "No Content" errors when there are no nodes yet + if tolerateError(err) { + return nil + } return err } @@ -90,6 +99,10 @@ func (r *realSlurmControl) GetNodeNames(ctx context.Context, nodeset *slinkyv1be nodeList := &slurmtypes.V0044NodeList{} if err := slurmClient.List(ctx, nodeList); err != nil { + // Tolerate "Not Found" and "No Content" errors when there are no nodes yet + if tolerateError(err) { + return nil, nil + } return nil, err } @@ -541,6 +554,10 @@ func (r *realSlurmControl) GetNodeDeadlines(ctx context.Context, nodeset *slinky jobList := &slurmtypes.V0044JobInfoList{} if err := slurmClient.List(ctx, jobList); err != nil { + // Tolerate "Not Found" and "No Content" errors when there are no jobs yet + if tolerateError(err) { + return ts, nil + } return nil, err } @@ -577,6 +594,100 @@ func (r *realSlurmControl) GetNodeDeadlines(ctx context.Context, nodeset *slinky return ts, nil } +// DeleteNode implements SlurmControlInterface. +func (r *realSlurmControl) DeleteNode(ctx context.Context, nodeset *slinkyv1beta1.NodeSet, nodeName string) error { + logger := log.FromContext(ctx) + + slurmClient := r.lookupClient(nodeset) + if slurmClient == nil { + logger.V(2).Info("no client for nodeset, cannot do DeleteNode()", + "nodeName", nodeName) + return nil + } + + slurmNode := &slurmtypes.V0044Node{} + key := slurmobject.ObjectKey(nodeName) + if err := slurmClient.Get(ctx, key, slurmNode); err != nil { + if tolerateError(err) { + return nil + } + return err + } + + logger.V(1).Info("deleting slurm node", + "nodeName", nodeName) + if err := slurmClient.Delete(ctx, slurmNode); err != nil { + if tolerateError(err) { + return nil + } + return err + } + + return nil +} + +// DeleteOrphanedNodes implements SlurmControlInterface. +func (r *realSlurmControl) DeleteOrphanedNodes(ctx context.Context, nodeset *slinkyv1beta1.NodeSet, pods []*corev1.Pod) error { + logger := log.FromContext(ctx) + + slurmClient := r.lookupClient(nodeset) + if slurmClient == nil { + logger.V(2).Info("no client for nodeset, cannot do DeleteOrphanedNodes()") + return nil + } + + nodeList := &slurmtypes.V0044NodeList{} + if err := slurmClient.List(ctx, nodeList); err != nil { + if tolerateError(err) { + return nil + } + return err + } + + podNodeNameSet := set.New[string]() + for _, pod := range pods { + podNodeNameSet.Insert(nodesetutils.GetNodeName(pod)) + } + + nodeNamePrefix := nodeNamePrefixForNodeSet(nodeset) + + for _, node := range nodeList.Items { + nodeName := ptr.Deref(node.Name, "") + if nodeName == "" || !isNodeFromNodeSet(nodeName, nodeNamePrefix) { + continue + } + if podNodeNameSet.Has(nodeName) { + continue + } + logger.Info("deleting orphaned slurm node (no matching pod)", + "nodeName", nodeName) + if err := r.DeleteNode(ctx, nodeset, nodeName); err != nil { + return err + } + } + + return nil +} + +func nodeNamePrefixForNodeSet(nodeset *slinkyv1beta1.NodeSet) string { + if h := nodeset.Spec.Template.PodSpecWrapper.Hostname; h != "" { + return h + } + + return nodeset.Name + "-" +} + +func isNodeFromNodeSet(nodeName, prefix string) bool { + ordinal, found := strings.CutPrefix(nodeName, prefix) + if !found || ordinal == "" { + return false + } + + _, err := strconv.Atoi(ordinal) + + return err == nil +} + func (r *realSlurmControl) lookupClient(nodeset *slinkyv1beta1.NodeSet) slurmclient.Client { return r.clientMap.Get(nodeset.Spec.ControllerRef.NamespacedName()) } diff --git a/internal/controller/nodeset/slurmcontrol/slurmcontrol_test.go b/internal/controller/nodeset/slurmcontrol/slurmcontrol_test.go index 696b98d8..181ad71e 100644 --- a/internal/controller/nodeset/slurmcontrol/slurmcontrol_test.go +++ b/internal/controller/nodeset/slurmcontrol/slurmcontrol_test.go @@ -273,6 +273,246 @@ var _ = Describe("SlurmControlInterface", func() { }) }) + Context("DeleteNode()", func() { + It("Should delete the Slurm node", func() { + By("Setup initial system state") + nodeset = newNodeSet("foo", controller.Name, 1) + pod = nodesetutils.NewNodeSetPod(nodeset, controller, 0, "") + slurmNodename := nodesetutils.GetNodeName(pod) + node := &types.V0044Node{ + V0044Node: api.V0044Node{ + Name: ptr.To(slurmNodename), + State: ptr.To([]api.V0044NodeState{ + api.V0044NodeStateDOWN, + }), + }, + } + sclient = fake.NewClientBuilder().WithObjects(node).Build() + controllers := newSlurmClientMap(controller.Name, sclient) + slurmcontrol = NewSlurmControl(controllers) + + By("Deleting Slurm node") + err := slurmcontrol.DeleteNode(ctx, nodeset, slurmNodename) + Expect(err).ToNot(HaveOccurred()) + + By("Verify node no longer exists") + checkNode := &types.V0044Node{} + key := object.ObjectKey(slurmNodename) + err = sclient.Get(ctx, key, checkNode) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("Not Found")) + }) + + It("Should tolerate deleting a non-existent node", func() { + By("Setup initial system state with no nodes") + nodeset = newNodeSet("foo", controller.Name, 1) + sclient = fake.NewClientBuilder().Build() + controllers := newSlurmClientMap(controller.Name, sclient) + slurmcontrol = NewSlurmControl(controllers) + + By("Deleting non-existent Slurm node should not error") + err := slurmcontrol.DeleteNode(ctx, nodeset, "nonexistent-0") + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("DeleteOrphanedNodes()", func() { + It("Should delete Slurm nodes with no matching pod", func() { + By("Setup: 3 Slurm nodes but only 2 pods") + nodeset = newNodeSet("foo", controller.Name, 3) + pod0 := nodesetutils.NewNodeSetPod(nodeset, controller, 0, "") + pod1 := nodesetutils.NewNodeSetPod(nodeset, controller, 1, "") + pod2 := nodesetutils.NewNodeSetPod(nodeset, controller, 2, "") + node0 := &types.V0044Node{ + V0044Node: api.V0044Node{ + Name: ptr.To(nodesetutils.GetNodeName(pod0)), + State: ptr.To([]api.V0044NodeState{api.V0044NodeStateIDLE}), + }, + } + node1 := &types.V0044Node{ + V0044Node: api.V0044Node{ + Name: ptr.To(nodesetutils.GetNodeName(pod1)), + State: ptr.To([]api.V0044NodeState{api.V0044NodeStateIDLE}), + }, + } + orphanNode := &types.V0044Node{ + V0044Node: api.V0044Node{ + Name: ptr.To(nodesetutils.GetNodeName(pod2)), + State: ptr.To([]api.V0044NodeState{api.V0044NodeStateDOWN}), + }, + } + sclient = fake.NewClientBuilder().WithObjects(node0, node1, orphanNode).Build() + controllers := newSlurmClientMap(controller.Name, sclient) + slurmcontrol = NewSlurmControl(controllers) + + By("Delete orphaned nodes (pod2 has no matching pod)") + pods := []*corev1.Pod{pod0, pod1} + err := slurmcontrol.DeleteOrphanedNodes(ctx, nodeset, pods) + Expect(err).ToNot(HaveOccurred()) + + By("Verify orphan is deleted") + checkNode := &types.V0044Node{} + err = sclient.Get(ctx, object.ObjectKey(nodesetutils.GetNodeName(pod2)), checkNode) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("Not Found")) + + By("Verify non-orphans still exist") + err = sclient.Get(ctx, object.ObjectKey(nodesetutils.GetNodeName(pod0)), checkNode) + Expect(err).ToNot(HaveOccurred()) + err = sclient.Get(ctx, object.ObjectKey(nodesetutils.GetNodeName(pod1)), checkNode) + Expect(err).ToNot(HaveOccurred()) + }) + + It("Should do nothing when all nodes have matching pods", func() { + By("Setup: 2 Slurm nodes and 2 pods") + nodeset = newNodeSet("foo", controller.Name, 2) + pod0 := nodesetutils.NewNodeSetPod(nodeset, controller, 0, "") + pod1 := nodesetutils.NewNodeSetPod(nodeset, controller, 1, "") + node0 := &types.V0044Node{ + V0044Node: api.V0044Node{ + Name: ptr.To(nodesetutils.GetNodeName(pod0)), + State: ptr.To([]api.V0044NodeState{api.V0044NodeStateIDLE}), + }, + } + node1 := &types.V0044Node{ + V0044Node: api.V0044Node{ + Name: ptr.To(nodesetutils.GetNodeName(pod1)), + State: ptr.To([]api.V0044NodeState{api.V0044NodeStateIDLE}), + }, + } + sclient = fake.NewClientBuilder().WithObjects(node0, node1).Build() + controllers := newSlurmClientMap(controller.Name, sclient) + slurmcontrol = NewSlurmControl(controllers) + + By("No orphans to delete") + pods := []*corev1.Pod{pod0, pod1} + err := slurmcontrol.DeleteOrphanedNodes(ctx, nodeset, pods) + Expect(err).ToNot(HaveOccurred()) + + By("Both nodes still exist") + checkNode := &types.V0044Node{} + err = sclient.Get(ctx, object.ObjectKey(nodesetutils.GetNodeName(pod0)), checkNode) + Expect(err).ToNot(HaveOccurred()) + err = sclient.Get(ctx, object.ObjectKey(nodesetutils.GetNodeName(pod1)), checkNode) + Expect(err).ToNot(HaveOccurred()) + }) + + It("Should not delete nodes belonging to a different NodeSet", func() { + By("Setup: NodeSet 'foo' with 1 pod, plus a node from NodeSet 'bar'") + nodeset = newNodeSet("foo", controller.Name, 1) + pod0 := nodesetutils.NewNodeSetPod(nodeset, controller, 0, "") + fooNode := &types.V0044Node{ + V0044Node: api.V0044Node{ + Name: ptr.To(nodesetutils.GetNodeName(pod0)), + State: ptr.To([]api.V0044NodeState{api.V0044NodeStateIDLE}), + }, + } + otherNodeSet := newNodeSet("bar", controller.Name, 1) + otherPod := nodesetutils.NewNodeSetPod(otherNodeSet, controller, 0, "") + barNode := &types.V0044Node{ + V0044Node: api.V0044Node{ + Name: ptr.To(nodesetutils.GetNodeName(otherPod)), + State: ptr.To([]api.V0044NodeState{api.V0044NodeStateDOWN}), + }, + } + sclient = fake.NewClientBuilder().WithObjects(fooNode, barNode).Build() + controllers := newSlurmClientMap(controller.Name, sclient) + slurmcontrol = NewSlurmControl(controllers) + + By("Delete orphans for NodeSet 'foo' only") + pods := []*corev1.Pod{pod0} + err := slurmcontrol.DeleteOrphanedNodes(ctx, nodeset, pods) + Expect(err).ToNot(HaveOccurred()) + + By("Verify bar's node was NOT deleted") + checkNode := &types.V0044Node{} + err = sclient.Get(ctx, object.ObjectKey(nodesetutils.GetNodeName(otherPod)), checkNode) + Expect(err).ToNot(HaveOccurred()) + }) + + It("Should not delete nodes from a NodeSet whose name shares a prefix", func() { + By("Setup: NodeSet 'foo' with 1 pod, plus a node from NodeSet 'foo-extra'") + nodeset = newNodeSet("foo", controller.Name, 1) + pod0 := nodesetutils.NewNodeSetPod(nodeset, controller, 0, "") + fooNode := &types.V0044Node{ + V0044Node: api.V0044Node{ + Name: ptr.To(nodesetutils.GetNodeName(pod0)), + State: ptr.To([]api.V0044NodeState{api.V0044NodeStateIDLE}), + }, + } + otherNodeSet := newNodeSet("foo-extra", controller.Name, 1) + otherPod := nodesetutils.NewNodeSetPod(otherNodeSet, controller, 0, "") + otherNode := &types.V0044Node{ + V0044Node: api.V0044Node{ + Name: ptr.To(nodesetutils.GetNodeName(otherPod)), + State: ptr.To([]api.V0044NodeState{api.V0044NodeStateDOWN}), + }, + } + sclient = fake.NewClientBuilder().WithObjects(fooNode, otherNode).Build() + controllers := newSlurmClientMap(controller.Name, sclient) + slurmcontrol = NewSlurmControl(controllers) + + By("Delete orphans for NodeSet 'foo' only") + pods := []*corev1.Pod{pod0} + err := slurmcontrol.DeleteOrphanedNodes(ctx, nodeset, pods) + Expect(err).ToNot(HaveOccurred()) + + By("Verify foo-extra's node was NOT deleted") + checkNode := &types.V0044Node{} + err = sclient.Get(ctx, object.ObjectKey(nodesetutils.GetNodeName(otherPod)), checkNode) + Expect(err).ToNot(HaveOccurred()) + }) + + It("Should use hostname template prefix when set (matches real prod naming)", func() { + By("Setup: NodeSet with hostname template 'slinky-' (like prod)") + nodeset = newNodeSet("slurm-worker-slinky", controller.Name, 1) + nodeset.Spec.Template.PodSpecWrapper.Hostname = "slinky-" + pod0 := nodesetutils.NewNodeSetPod(nodeset, controller, 0, "") + Expect(nodesetutils.GetNodeName(pod0)).To(Equal("slinky-0")) + + activeNode := &types.V0044Node{ + V0044Node: api.V0044Node{ + Name: ptr.To("slinky-0"), + State: ptr.To([]api.V0044NodeState{api.V0044NodeStateIDLE}), + }, + } + orphanNode := &types.V0044Node{ + V0044Node: api.V0044Node{ + Name: ptr.To("slinky-99"), + State: ptr.To([]api.V0044NodeState{api.V0044NodeStateDOWN}), + }, + } + unrelatedNode := &types.V0044Node{ + V0044Node: api.V0044Node{ + Name: ptr.To("slinky-extra-0"), + State: ptr.To([]api.V0044NodeState{api.V0044NodeStateDOWN}), + }, + } + sclient = fake.NewClientBuilder().WithObjects(activeNode, orphanNode, unrelatedNode).Build() + controllers := newSlurmClientMap(controller.Name, sclient) + slurmcontrol = NewSlurmControl(controllers) + + By("Delete orphans — should delete slinky-99 but keep slinky-0 and slinky-extra-0") + pods := []*corev1.Pod{pod0} + err := slurmcontrol.DeleteOrphanedNodes(ctx, nodeset, pods) + Expect(err).ToNot(HaveOccurred()) + + By("Verify slinky-99 was deleted") + checkNode := &types.V0044Node{} + err = sclient.Get(ctx, object.ObjectKey("slinky-99"), checkNode) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("Not Found")) + + By("Verify slinky-0 still exists") + err = sclient.Get(ctx, object.ObjectKey("slinky-0"), checkNode) + Expect(err).ToNot(HaveOccurred()) + + By("Verify slinky-extra-0 still exists (not an integer ordinal)") + err = sclient.Get(ctx, object.ObjectKey("slinky-extra-0"), checkNode) + Expect(err).ToNot(HaveOccurred()) + }) + }) + Context("GetNodeDeadlines()", func() { now := time.Now()