From 89350084b3af2e89981aab4e18fd098b3fdf7939 Mon Sep 17 00:00:00 2001 From: Juno13340 <35788036+Juno13340@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:31:05 -0400 Subject: [PATCH 1/4] Add Kimi K2.5 generation vLLM runtime and K2.6/K2.7-Code models --- config/models/kustomization.yaml | 2 + config/models/moonshotai/Kimi-K2.6.yaml | 14 ++ config/models/moonshotai/Kimi-K2.7-Code.yaml | 14 ++ config/runtimes/kustomization.yaml | 1 + .../kimi-k25-single-node-8gpu-rt.yaml | 188 ++++++++++++++++++ 5 files changed, 219 insertions(+) create mode 100644 config/models/moonshotai/Kimi-K2.6.yaml create mode 100644 config/models/moonshotai/Kimi-K2.7-Code.yaml create mode 100644 config/runtimes/vllm/moonshotai/kimi-k25-single-node-8gpu-rt.yaml diff --git a/config/models/kustomization.yaml b/config/models/kustomization.yaml index e884fc7c0..c1b4550fd 100644 --- a/config/models/kustomization.yaml +++ b/config/models/kustomization.yaml @@ -116,6 +116,8 @@ resources: # moonshotai - moonshotai/Kimi-K2-Instruct.yaml + - moonshotai/Kimi-K2.6.yaml + - moonshotai/Kimi-K2.7-Code.yaml - moonshotai/Kimi-VL-A3B-Instruct.yaml # nvidia diff --git a/config/models/moonshotai/Kimi-K2.6.yaml b/config/models/moonshotai/Kimi-K2.6.yaml new file mode 100644 index 000000000..bfa87e651 --- /dev/null +++ b/config/models/moonshotai/Kimi-K2.6.yaml @@ -0,0 +1,14 @@ +apiVersion: ome.io/v1beta1 +kind: ClusterBaseModel +metadata: + name: kimi-k2-6 +spec: + modelCapabilities: + - IMAGE_TEXT_TO_TEXT + vendor: moonshotai + displayName: moonshotai.kimi-k2-6 + disabled: false + version: "1.0.0" + storage: + storageUri: hf://moonshotai/Kimi-K2.6 + path: /raid/models/moonshotai/Kimi-K2.6 diff --git a/config/models/moonshotai/Kimi-K2.7-Code.yaml b/config/models/moonshotai/Kimi-K2.7-Code.yaml new file mode 100644 index 000000000..5c27d95e3 --- /dev/null +++ b/config/models/moonshotai/Kimi-K2.7-Code.yaml @@ -0,0 +1,14 @@ +apiVersion: ome.io/v1beta1 +kind: ClusterBaseModel +metadata: + name: kimi-k2-7-code +spec: + modelCapabilities: + - IMAGE_TEXT_TO_TEXT + vendor: moonshotai + displayName: moonshotai.kimi-k2-7-code + disabled: false + version: "1.0.0" + storage: + storageUri: hf://moonshotai/Kimi-K2.7-Code + path: /raid/models/moonshotai/Kimi-K2.7-Code diff --git a/config/runtimes/kustomization.yaml b/config/runtimes/kustomization.yaml index 0b78b5905..c6147f18e 100644 --- a/config/runtimes/kustomization.yaml +++ b/config/runtimes/kustomization.yaml @@ -52,3 +52,4 @@ resources: - vllm/mixtral-8x7b-instruct-rt.yaml - vllm/deepseek-ai/deepseek-v4-flash-rt.yaml - vllm/deepseek-ai/deepseek-v4-pro-rt.yaml +- vllm/moonshotai/kimi-k25-single-node-8gpu-rt.yaml diff --git a/config/runtimes/vllm/moonshotai/kimi-k25-single-node-8gpu-rt.yaml b/config/runtimes/vllm/moonshotai/kimi-k25-single-node-8gpu-rt.yaml new file mode 100644 index 000000000..d40f467dc --- /dev/null +++ b/config/runtimes/vllm/moonshotai/kimi-k25-single-node-8gpu-rt.yaml @@ -0,0 +1,188 @@ +apiVersion: ome.io/v1beta1 +kind: ClusterServingRuntime +metadata: + name: vllm-kimi-k25-single-node-8gpu + annotations: + ome.io/fit-quantization: int4 + ome.io/engine-ready-timeout-sec: "3600" +spec: + acceleratorRequirements: + acceleratorClasses: + - nvidia-b200-8 + - nvidia-h200-8 + # - nvidia-b300-8 + # - nvidia-h100-8 + # - nvidia-a100-80gb-8 + disabled: false + routerConfig: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '29000' + prometheus.io/scrape: 'true' + labels: + logging-forward: enabled + runner: + name: router + image: docker.io/lightseekorg/smg:1.5.0 + ports: + - containerPort: 8080 + name: http + resources: + limits: + cpu: "1" + memory: 2Gi + args: + - --host + - 0.0.0.0 + - --port + - "8080" + - --service-discovery + - --service-discovery-namespace + - $(NAMESPACE) + - --service-discovery-port + - "8080" + - --selector + - component=engine ome.io/inferenceservice=$(INFERENCESERVICE_NAME) + - --enable-igw + - --request-id-headers + - opc-request-id + - --log-json + - --disable-retries + - --disable-circuit-breaker + - --disable-tokenizer-autoload + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: INFERENCESERVICE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ome.io/inferenceservice'] + readinessProbe: + httpGet: + path: /readiness + port: 8080 + failureThreshold: 5 + periodSeconds: 30 + timeoutSeconds: 10 + livenessProbe: + httpGet: + path: /liveness + port: 8080 + failureThreshold: 5 + periodSeconds: 30 + timeoutSeconds: 10 + startupProbe: + httpGet: + path: /readiness + port: 8080 + failureThreshold: 177 + periodSeconds: 20 + timeoutSeconds: 10 + initialDelaySeconds: 60 + supportedModelFormats: + - modelFramework: + name: transformers + version: "4.57.1" + modelFormat: + name: safetensors + version: "1.0.0" + modelArchitecture: KimiK25ForConditionalGeneration + autoSelect: true + priority: 1 + acceleratorConfig: + nvidia-b200-8: + tensorParallelismOverride: + tensorParallelSize: 8 + nvidia-h200-8: + tensorParallelismOverride: + tensorParallelSize: 8 + # nvidia-b300-8: + # tensorParallelismOverride: + # tensorParallelSize: 8 + # nvidia-h100-8: + # tensorParallelismOverride: + # tensorParallelSize: 8 + # nvidia-a100-80gb-8: + # tensorParallelismOverride: + # tensorParallelSize: 8 + modelSizeRange: + min: 150B + max: 300B + protocolVersions: + - openAI + engineConfig: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + prometheus.io/path: "/metrics" + labels: + logging-forward: enabled + volumes: + - name: dshm + emptyDir: + medium: Memory + runner: + name: ome-container + image: docker.io/vllm/vllm-openai:v0.23.0-cu129 + ports: + - containerPort: 8080 + name: http1 + protocol: TCP + args: + - $(MODEL_PATH) + - --port=8080 + - --max-log-len=0 + - --served-model-name=vllm-model + - --trust-remote-code + - --enable-expert-parallel + - --tensor-parallel-size=8 + - --max-model-len=262144 + - '--limit-mm-per-prompt={"image":5,"video":1}' + - --tool-call-parser=kimi_k2 + - --enable-auto-tool-choice + - --reasoning-parser=kimi_k2 + - --mm-encoder-tp-mode=data + env: + - name: VLLM_ENGINE_READY_TIMEOUT_S + value: '3600' + - name: VLLM_RPC_TIMEOUT + value: '600000' + volumeMounts: + - mountPath: /dev/shm + name: dshm + resources: + requests: + cpu: 64 + memory: 512Gi + nvidia.com/gpu: 8 + limits: + cpu: 64 + memory: 512Gi + nvidia.com/gpu: 8 + readinessProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 3 + successThreshold: 1 + periodSeconds: 90 + timeoutSeconds: 60 + livenessProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 5 + successThreshold: 1 + periodSeconds: 60 + timeoutSeconds: 60 + startupProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 348 + successThreshold: 1 + periodSeconds: 10 + initialDelaySeconds: 120 + timeoutSeconds: 30 From 7edbe696b306fbd51207177e571d496f5d68a151 Mon Sep 17 00:00:00 2001 From: Juno13340 <35788036+Juno13340@users.noreply.github.com> Date: Tue, 23 Jun 2026 13:06:17 -0400 Subject: [PATCH 2/4] Address review: drop unused fit-quantization annotation, add router requests, use /health startup probe, remove dead comments --- .../kimi-k25-single-node-8gpu-rt.yaml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/config/runtimes/vllm/moonshotai/kimi-k25-single-node-8gpu-rt.yaml b/config/runtimes/vllm/moonshotai/kimi-k25-single-node-8gpu-rt.yaml index d40f467dc..5ad5518f2 100644 --- a/config/runtimes/vllm/moonshotai/kimi-k25-single-node-8gpu-rt.yaml +++ b/config/runtimes/vllm/moonshotai/kimi-k25-single-node-8gpu-rt.yaml @@ -3,16 +3,12 @@ kind: ClusterServingRuntime metadata: name: vllm-kimi-k25-single-node-8gpu annotations: - ome.io/fit-quantization: int4 ome.io/engine-ready-timeout-sec: "3600" spec: acceleratorRequirements: acceleratorClasses: - nvidia-b200-8 - nvidia-h200-8 - # - nvidia-b300-8 - # - nvidia-h100-8 - # - nvidia-a100-80gb-8 disabled: false routerConfig: annotations: @@ -28,6 +24,9 @@ spec: - containerPort: 8080 name: http resources: + requests: + cpu: "1" + memory: 2Gi limits: cpu: "1" memory: 2Gi @@ -75,7 +74,7 @@ spec: timeoutSeconds: 10 startupProbe: httpGet: - path: /readiness + path: /health port: 8080 failureThreshold: 177 periodSeconds: 20 @@ -98,15 +97,6 @@ spec: nvidia-h200-8: tensorParallelismOverride: tensorParallelSize: 8 - # nvidia-b300-8: - # tensorParallelismOverride: - # tensorParallelSize: 8 - # nvidia-h100-8: - # tensorParallelismOverride: - # tensorParallelSize: 8 - # nvidia-a100-80gb-8: - # tensorParallelismOverride: - # tensorParallelSize: 8 modelSizeRange: min: 150B max: 300B From 421852cf8ecf9c03a4fac1b46bcb4c6d8a086e36 Mon Sep 17 00:00:00 2001 From: Juno13340 <35788036+Juno13340@users.noreply.github.com> Date: Tue, 23 Jun 2026 13:15:11 -0400 Subject: [PATCH 3/4] Add TEXT_TO_TEXT capability to Kimi K2.6/K2.7-Code, matching general-purpose multimodal model convention --- config/models/moonshotai/Kimi-K2.6.yaml | 1 + config/models/moonshotai/Kimi-K2.7-Code.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/config/models/moonshotai/Kimi-K2.6.yaml b/config/models/moonshotai/Kimi-K2.6.yaml index bfa87e651..a1d11b194 100644 --- a/config/models/moonshotai/Kimi-K2.6.yaml +++ b/config/models/moonshotai/Kimi-K2.6.yaml @@ -4,6 +4,7 @@ metadata: name: kimi-k2-6 spec: modelCapabilities: + - TEXT_TO_TEXT - IMAGE_TEXT_TO_TEXT vendor: moonshotai displayName: moonshotai.kimi-k2-6 diff --git a/config/models/moonshotai/Kimi-K2.7-Code.yaml b/config/models/moonshotai/Kimi-K2.7-Code.yaml index 5c27d95e3..cbd2ea9da 100644 --- a/config/models/moonshotai/Kimi-K2.7-Code.yaml +++ b/config/models/moonshotai/Kimi-K2.7-Code.yaml @@ -4,6 +4,7 @@ metadata: name: kimi-k2-7-code spec: modelCapabilities: + - TEXT_TO_TEXT - IMAGE_TEXT_TO_TEXT vendor: moonshotai displayName: moonshotai.kimi-k2-7-code From b4a2ad6c9047776b122c4885a7dc41ed406bc75f Mon Sep 17 00:00:00 2001 From: Juno13340 <35788036+Juno13340@users.noreply.github.com> Date: Tue, 23 Jun 2026 13:21:44 -0400 Subject: [PATCH 4/4] Rename Kimi K2.5 runtime to vllm-kimi-k25-tp8 --- config/runtimes/kustomization.yaml | 2 +- .../{kimi-k25-single-node-8gpu-rt.yaml => kimi-k25-tp8-rt.yaml} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename config/runtimes/vllm/moonshotai/{kimi-k25-single-node-8gpu-rt.yaml => kimi-k25-tp8-rt.yaml} (99%) diff --git a/config/runtimes/kustomization.yaml b/config/runtimes/kustomization.yaml index c6147f18e..76091ebeb 100644 --- a/config/runtimes/kustomization.yaml +++ b/config/runtimes/kustomization.yaml @@ -52,4 +52,4 @@ resources: - vllm/mixtral-8x7b-instruct-rt.yaml - vllm/deepseek-ai/deepseek-v4-flash-rt.yaml - vllm/deepseek-ai/deepseek-v4-pro-rt.yaml -- vllm/moonshotai/kimi-k25-single-node-8gpu-rt.yaml +- vllm/moonshotai/kimi-k25-tp8-rt.yaml diff --git a/config/runtimes/vllm/moonshotai/kimi-k25-single-node-8gpu-rt.yaml b/config/runtimes/vllm/moonshotai/kimi-k25-tp8-rt.yaml similarity index 99% rename from config/runtimes/vllm/moonshotai/kimi-k25-single-node-8gpu-rt.yaml rename to config/runtimes/vllm/moonshotai/kimi-k25-tp8-rt.yaml index 5ad5518f2..3e7604d13 100644 --- a/config/runtimes/vllm/moonshotai/kimi-k25-single-node-8gpu-rt.yaml +++ b/config/runtimes/vllm/moonshotai/kimi-k25-tp8-rt.yaml @@ -1,7 +1,7 @@ apiVersion: ome.io/v1beta1 kind: ClusterServingRuntime metadata: - name: vllm-kimi-k25-single-node-8gpu + name: vllm-kimi-k25-tp8 annotations: ome.io/engine-ready-timeout-sec: "3600" spec: