From 02f35c7b48fffc817723cc3c66c1f9d0db0d1d23 Mon Sep 17 00:00:00 2001 From: Aniruddh Krovvidi Date: Fri, 17 Jul 2026 13:13:43 -0700 Subject: [PATCH 1/4] feat: add SEA-LION v4 32B runtime --- .../aisingapore/Qwen-SEA-LION-v4-32B-IT.yaml | 22 +++ config/models/kustomization.yaml | 3 + config/runtimes/kustomization.yaml | 1 + config/runtimes/vllm/Qwen/qwen-3-32b-rt.yaml | 168 ++++++++++++++++++ .../aisingapore/qwen-sea-lion-v4-32b-it.yaml | 16 ++ 5 files changed, 210 insertions(+) create mode 100644 config/models/aisingapore/Qwen-SEA-LION-v4-32B-IT.yaml create mode 100644 config/runtimes/vllm/Qwen/qwen-3-32b-rt.yaml create mode 100644 config/samples/isvc/aisingapore/qwen-sea-lion-v4-32b-it.yaml diff --git a/config/models/aisingapore/Qwen-SEA-LION-v4-32B-IT.yaml b/config/models/aisingapore/Qwen-SEA-LION-v4-32B-IT.yaml new file mode 100644 index 000000000..621e70b56 --- /dev/null +++ b/config/models/aisingapore/Qwen-SEA-LION-v4-32B-IT.yaml @@ -0,0 +1,22 @@ +apiVersion: ome.io/v1beta1 +kind: ClusterBaseModel +metadata: + name: qwen-sea-lion-v4-32b-it +spec: + modelCapabilities: + - TEXT_TO_TEXT + vendor: aisingapore + displayName: aisingapore.qwen-sea-lion-v4-32b-it + disabled: false + version: "1.0.0" + modelArchitecture: Qwen3ForCausalLM + modelFormat: + name: safetensors + version: "1.0.0" + modelFramework: + name: transformers + version: "4.51.0" + modelParameterSize: 32.762B + storage: + storageUri: hf://aisingapore/Qwen-SEA-LION-v4-32B-IT + path: /raid/models/aisingapore/Qwen-SEA-LION-v4-32B-IT diff --git a/config/models/kustomization.yaml b/config/models/kustomization.yaml index e884fc7c0..0df095d57 100644 --- a/config/models/kustomization.yaml +++ b/config/models/kustomization.yaml @@ -5,6 +5,9 @@ resources: # adept - adept/persimmon-8b-chat.yaml + # aisingapore + - aisingapore/Qwen-SEA-LION-v4-32B-IT.yaml + # Alibaba-NLP - Alibaba-NLP/gme-Qwen2-VL-2B-Instruct.yaml - Alibaba-NLP/gte-Qwen2-7B-instruct.yaml diff --git a/config/runtimes/kustomization.yaml b/config/runtimes/kustomization.yaml index 41b1a2df9..fdc22b5f5 100644 --- a/config/runtimes/kustomization.yaml +++ b/config/runtimes/kustomization.yaml @@ -33,6 +33,7 @@ resources: - srt/mixtral-8x7b-instruct-pd-rt.yaml - srt/mixtral-8x7b-instruct-rt.yaml # vLLM runtimes +- vllm/Qwen/qwen-3-32b-rt.yaml - vllm/e5-mistral-7b-instruct-rt.yaml - vllm/llama-3-1-405b-instruct-fp8-rt.yaml - vllm/llama-3-1-8b-instruct-rt.yaml diff --git a/config/runtimes/vllm/Qwen/qwen-3-32b-rt.yaml b/config/runtimes/vllm/Qwen/qwen-3-32b-rt.yaml new file mode 100644 index 000000000..7417ef967 --- /dev/null +++ b/config/runtimes/vllm/Qwen/qwen-3-32b-rt.yaml @@ -0,0 +1,168 @@ +apiVersion: ome.io/v1beta1 +kind: ClusterServingRuntime +metadata: + name: vllm-qwen-3-32b +spec: + disabled: false + routerConfig: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "29000" + prometheus.io/scrape: "true" + labels: + logging-forward: enabled + runner: + name: router + image: docker.io/lightseekorg/smg:1.4.1 + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "8" + memory: 16Gi + args: + - --host + - 0.0.0.0 + - --port + - "8080" + - --service-discovery + - --service-discovery-namespace + - $(NAMESPACE) + - --service-discovery-port + - "8080" + - --selector + - component=engine ome.io/inferenceservice=$(INFERENCESERVICE_NAME) + - --enable-igw + - --request-id-headers + - opc-request-id + - --log-json + - --disable-retries + - --disable-circuit-breaker + - --disable-tokenizer-autoload + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: INFERENCESERVICE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ome.io/inferenceservice'] + readinessProbe: + httpGet: + path: /readiness + port: 8080 + failureThreshold: 5 + periodSeconds: 30 + timeoutSeconds: 10 + livenessProbe: + httpGet: + path: /liveness + port: 8080 + failureThreshold: 5 + periodSeconds: 30 + timeoutSeconds: 10 + startupProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 5 + periodSeconds: 20 + timeoutSeconds: 10 + initialDelaySeconds: 30 + supportedModelFormats: + - modelFramework: + name: transformers + version: "4.51.0" + modelFormat: + name: safetensors + version: "1.0.0" + modelArchitecture: Qwen3ForCausalLM + autoSelect: false + priority: 3 + version: "1.0.0" + modelSizeRange: + min: 30B + max: 34B + protocolVersions: + - openAI + engineConfig: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + prometheus.io/path: /metrics + labels: + logging-forward: enabled + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + volumes: + - name: dshm + emptyDir: + medium: Memory + runner: + name: ome-container + image: docker.io/vllm/vllm-openai:v0.23.0-cu129 + ports: + - containerPort: 8080 + name: http1 + protocol: TCP + command: + - vllm + - serve + args: + - $(MODEL_PATH) + - --port=8080 + - --enable-log-requests + - --served-model-name=vllm-model + - --tensor-parallel-size=2 + - --gpu-memory-utilization=0.9 + - --max-model-len=32768 + - --reasoning-parser=qwen3 + - --enable-auto-tool-choice + - --tool-call-parser=hermes + env: + - name: VLLM_LOGGING_LEVEL + value: INFO + volumeMounts: + - mountPath: /dev/shm + name: dshm + resources: + requests: + cpu: 20 + memory: 120Gi + nvidia.com/gpu: 2 + limits: + cpu: 20 + memory: 120Gi + nvidia.com/gpu: 2 + readinessProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 3 + successThreshold: 1 + periodSeconds: 90 + timeoutSeconds: 60 + livenessProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 5 + successThreshold: 1 + periodSeconds: 60 + timeoutSeconds: 60 + startupProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 100 + successThreshold: 1 + periodSeconds: 6 + initialDelaySeconds: 60 + timeoutSeconds: 30 diff --git a/config/samples/isvc/aisingapore/qwen-sea-lion-v4-32b-it.yaml b/config/samples/isvc/aisingapore/qwen-sea-lion-v4-32b-it.yaml new file mode 100644 index 000000000..c554e4d3c --- /dev/null +++ b/config/samples/isvc/aisingapore/qwen-sea-lion-v4-32b-it.yaml @@ -0,0 +1,16 @@ +apiVersion: ome.io/v1beta1 +kind: InferenceService +metadata: + name: qwen-sea-lion-v4-32b-it + namespace: qwen-sea-lion-v4-32b-it +spec: + model: + name: qwen-sea-lion-v4-32b-it + runtime: + name: vllm-qwen-3-32b + engine: + minReplicas: 1 + maxReplicas: 1 + router: + minReplicas: 1 + maxReplicas: 1 From 030b04b5295cd1847f2af765dbe353d32ec3eb36 Mon Sep 17 00:00:00 2001 From: Aniruddh Krovvidi Date: Fri, 17 Jul 2026 13:39:08 -0700 Subject: [PATCH 2/4] fix(runtime): isolate SEA-LION serving defaults --- config/runtimes/kustomization.yaml | 2 +- .../qwen-sea-lion-v4-32b-it-rt.yaml} | 6 ++++-- .../samples/isvc/aisingapore/qwen-sea-lion-v4-32b-it.yaml | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) rename config/runtimes/vllm/{Qwen/qwen-3-32b-rt.yaml => aisingapore/qwen-sea-lion-v4-32b-it-rt.yaml} (95%) diff --git a/config/runtimes/kustomization.yaml b/config/runtimes/kustomization.yaml index fdc22b5f5..775c19025 100644 --- a/config/runtimes/kustomization.yaml +++ b/config/runtimes/kustomization.yaml @@ -33,7 +33,7 @@ resources: - srt/mixtral-8x7b-instruct-pd-rt.yaml - srt/mixtral-8x7b-instruct-rt.yaml # vLLM runtimes -- vllm/Qwen/qwen-3-32b-rt.yaml +- vllm/aisingapore/qwen-sea-lion-v4-32b-it-rt.yaml - vllm/e5-mistral-7b-instruct-rt.yaml - vllm/llama-3-1-405b-instruct-fp8-rt.yaml - vllm/llama-3-1-8b-instruct-rt.yaml diff --git a/config/runtimes/vllm/Qwen/qwen-3-32b-rt.yaml b/config/runtimes/vllm/aisingapore/qwen-sea-lion-v4-32b-it-rt.yaml similarity index 95% rename from config/runtimes/vllm/Qwen/qwen-3-32b-rt.yaml rename to config/runtimes/vllm/aisingapore/qwen-sea-lion-v4-32b-it-rt.yaml index 7417ef967..00214d296 100644 --- a/config/runtimes/vllm/Qwen/qwen-3-32b-rt.yaml +++ b/config/runtimes/vllm/aisingapore/qwen-sea-lion-v4-32b-it-rt.yaml @@ -1,7 +1,7 @@ apiVersion: ome.io/v1beta1 kind: ClusterServingRuntime metadata: - name: vllm-qwen-3-32b + name: vllm-qwen-sea-lion-v4-32b-it spec: disabled: false routerConfig: @@ -122,9 +122,11 @@ spec: - --served-model-name=vllm-model - --tensor-parallel-size=2 - --gpu-memory-utilization=0.9 - - --max-model-len=32768 + - --max-model-len=40960 - --reasoning-parser=qwen3 + - '--default-chat-template-kwargs={"enable_thinking":false}' - --enable-auto-tool-choice + - --exclude-tools-when-tool-choice-none - --tool-call-parser=hermes env: - name: VLLM_LOGGING_LEVEL diff --git a/config/samples/isvc/aisingapore/qwen-sea-lion-v4-32b-it.yaml b/config/samples/isvc/aisingapore/qwen-sea-lion-v4-32b-it.yaml index c554e4d3c..aaa0c6ca9 100644 --- a/config/samples/isvc/aisingapore/qwen-sea-lion-v4-32b-it.yaml +++ b/config/samples/isvc/aisingapore/qwen-sea-lion-v4-32b-it.yaml @@ -7,7 +7,7 @@ spec: model: name: qwen-sea-lion-v4-32b-it runtime: - name: vllm-qwen-3-32b + name: vllm-qwen-sea-lion-v4-32b-it engine: minReplicas: 1 maxReplicas: 1 From e4f53ad8cf54efc85803d06ed3bc7f40af21c141 Mon Sep 17 00:00:00 2001 From: Aniruddh Krovvidi Date: Mon, 20 Jul 2026 08:24:03 -0700 Subject: [PATCH 3/4] chore(runtime): update SEA-LION serving images --- .../runtimes/vllm/aisingapore/qwen-sea-lion-v4-32b-it-rt.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/runtimes/vllm/aisingapore/qwen-sea-lion-v4-32b-it-rt.yaml b/config/runtimes/vllm/aisingapore/qwen-sea-lion-v4-32b-it-rt.yaml index 00214d296..d8fec003c 100644 --- a/config/runtimes/vllm/aisingapore/qwen-sea-lion-v4-32b-it-rt.yaml +++ b/config/runtimes/vllm/aisingapore/qwen-sea-lion-v4-32b-it-rt.yaml @@ -13,7 +13,7 @@ spec: logging-forward: enabled runner: name: router - image: docker.io/lightseekorg/smg:1.4.1 + image: docker.io/lightseekorg/smg:1.7.0 ports: - containerPort: 8080 name: http @@ -107,7 +107,7 @@ spec: medium: Memory runner: name: ome-container - image: docker.io/vllm/vllm-openai:v0.23.0-cu129 + image: docker.io/vllm/vllm-openai:v0.24.0-cu129 ports: - containerPort: 8080 name: http1 From 60434a99f33a1ec08573def15a79d14cc1dde6d1 Mon Sep 17 00:00:00 2001 From: Aniruddh Krovvidi Date: Mon, 20 Jul 2026 09:54:14 -0700 Subject: [PATCH 4/4] chore(runtime): target latest SEA-LION images --- .../runtimes/vllm/aisingapore/qwen-sea-lion-v4-32b-it-rt.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/runtimes/vllm/aisingapore/qwen-sea-lion-v4-32b-it-rt.yaml b/config/runtimes/vllm/aisingapore/qwen-sea-lion-v4-32b-it-rt.yaml index d8fec003c..a9b6a84a6 100644 --- a/config/runtimes/vllm/aisingapore/qwen-sea-lion-v4-32b-it-rt.yaml +++ b/config/runtimes/vllm/aisingapore/qwen-sea-lion-v4-32b-it-rt.yaml @@ -13,7 +13,7 @@ spec: logging-forward: enabled runner: name: router - image: docker.io/lightseekorg/smg:1.7.0 + image: docker.io/lightseekorg/smg:1.8.0 ports: - containerPort: 8080 name: http @@ -107,7 +107,7 @@ spec: medium: Memory runner: name: ome-container - image: docker.io/vllm/vllm-openai:v0.24.0-cu129 + image: docker.io/vllm/vllm-openai:v0.25.1-cu129 ports: - containerPort: 8080 name: http1