diff --git a/config/models/google/gemma-4-26B-A4B-it.yaml b/config/models/google/gemma-4-26B-A4B-it.yaml new file mode 100644 index 000000000..5c7900aea --- /dev/null +++ b/config/models/google/gemma-4-26B-A4B-it.yaml @@ -0,0 +1,16 @@ +apiVersion: ome.io/v1beta1 +kind: ClusterBaseModel +metadata: + name: gemma-4-26b-a4b-it +spec: + modelCapabilities: + - TEXT_TO_TEXT + - IMAGE_TEXT_TO_TEXT + - VIDEO_TEXT_TO_TEXT + vendor: google + disabled: false + version: "1.0.0" + displayName: google.gemma-4-26b-a4b-it + storage: + storageUri: hf://google/gemma-4-26B-A4B-it + path: /raid/models/google/gemma-4-26B-A4B-it diff --git a/config/models/google/gemma-4-31B-it.yaml b/config/models/google/gemma-4-31B-it.yaml new file mode 100644 index 000000000..e6848be49 --- /dev/null +++ b/config/models/google/gemma-4-31B-it.yaml @@ -0,0 +1,16 @@ +apiVersion: ome.io/v1beta1 +kind: ClusterBaseModel +metadata: + name: gemma-4-31b-it +spec: + modelCapabilities: + - TEXT_TO_TEXT + - IMAGE_TEXT_TO_TEXT + - VIDEO_TEXT_TO_TEXT + vendor: google + disabled: false + version: "1.0.0" + displayName: google.gemma-4-31b-it + storage: + storageUri: hf://google/gemma-4-31B-it + path: /raid/models/google/gemma-4-31B-it diff --git a/config/models/google/gemma-4-E2B-it.yaml b/config/models/google/gemma-4-E2B-it.yaml new file mode 100644 index 000000000..37cdcb1d4 --- /dev/null +++ b/config/models/google/gemma-4-E2B-it.yaml @@ -0,0 +1,17 @@ +apiVersion: ome.io/v1beta1 +kind: ClusterBaseModel +metadata: + name: gemma-4-e2b-it +spec: + modelCapabilities: + - TEXT_TO_TEXT + - IMAGE_TEXT_TO_TEXT + - VIDEO_TEXT_TO_TEXT + - AUDIO_TEXT_TO_TEXT + vendor: google + disabled: false + version: "1.0.0" + displayName: google.gemma-4-e2b-it + storage: + storageUri: hf://google/gemma-4-E2B-it + path: /raid/models/google/gemma-4-E2B-it diff --git a/config/models/google/gemma-4-E4B-it.yaml b/config/models/google/gemma-4-E4B-it.yaml new file mode 100644 index 000000000..3f4800d4d --- /dev/null +++ b/config/models/google/gemma-4-E4B-it.yaml @@ -0,0 +1,17 @@ +apiVersion: ome.io/v1beta1 +kind: ClusterBaseModel +metadata: + name: gemma-4-e4b-it +spec: + modelCapabilities: + - TEXT_TO_TEXT + - IMAGE_TEXT_TO_TEXT + - VIDEO_TEXT_TO_TEXT + - AUDIO_TEXT_TO_TEXT + vendor: google + disabled: false + version: "1.0.0" + displayName: google.gemma-4-e4b-it + storage: + storageUri: hf://google/gemma-4-E4B-it + path: /raid/models/google/gemma-4-E4B-it diff --git a/config/models/kustomization.yaml b/config/models/kustomization.yaml index e884fc7c0..fe492f29e 100644 --- a/config/models/kustomization.yaml +++ b/config/models/kustomization.yaml @@ -51,6 +51,10 @@ resources: # google - google/gemma-3-1b-it.yaml - google/gemma-3-4b-it.yaml + - google/gemma-4-E2B-it.yaml + - google/gemma-4-E4B-it.yaml + - google/gemma-4-26B-A4B-it.yaml + - google/gemma-4-31B-it.yaml # HuggingFaceTB - HuggingFaceTB/SmolLM-1.7B.yaml diff --git a/config/runtimes/kustomization.yaml b/config/runtimes/kustomization.yaml index 0b78b5905..b9728d46a 100644 --- a/config/runtimes/kustomization.yaml +++ b/config/runtimes/kustomization.yaml @@ -52,3 +52,5 @@ resources: - vllm/mixtral-8x7b-instruct-rt.yaml - vllm/deepseek-ai/deepseek-v4-flash-rt.yaml - vllm/deepseek-ai/deepseek-v4-pro-rt.yaml +- vllm/gemma-4-tp1-rt.yaml +- vllm/gemma-4-tp2-rt.yaml diff --git a/config/runtimes/vllm/gemma-4-tp1-rt.yaml b/config/runtimes/vllm/gemma-4-tp1-rt.yaml new file mode 100644 index 000000000..2161937ef --- /dev/null +++ b/config/runtimes/vllm/gemma-4-tp1-rt.yaml @@ -0,0 +1,256 @@ +apiVersion: ome.io/v1beta1 +kind: ClusterServingRuntime +metadata: + name: vllm-gemma-4-tp1 +spec: + disabled: false + acceleratorRequirements: + acceleratorClasses: + - nvidia-h100-1 + - nvidia-h100-2 + - nvidia-h100-4 + - nvidia-h100-8 + - nvidia-a100-80gb-1 + - nvidia-a100-80gb-2 + - nvidia-a100-80gb-4 + - nvidia-a100-80gb-8 + - nvidia-h200-1 + - nvidia-h200-2 + - nvidia-h200-4 + - nvidia-h200-8 + - nvidia-b200-1 + - nvidia-b200-2 + - nvidia-b200-4 + - nvidia-b200-8 + supportedModelFormats: + - modelFramework: + name: transformers + version: "5.5.0.dev0" + modelFormat: + name: safetensors + version: "1.0.0" + modelArchitecture: Gemma4ForConditionalGeneration + autoSelect: true + priority: 1 + version: "1.0.0" + acceleratorConfig: + nvidia-h100-1: + tensorParallelismOverride: + tensorParallelSize: 1 + nvidia-h100-2: + tensorParallelismOverride: + tensorParallelSize: 2 + nvidia-h100-4: + tensorParallelismOverride: + tensorParallelSize: 4 + nvidia-h100-8: + tensorParallelismOverride: + tensorParallelSize: 8 + nvidia-a100-80gb-1: + tensorParallelismOverride: + tensorParallelSize: 1 + nvidia-a100-80gb-2: + tensorParallelismOverride: + tensorParallelSize: 2 + nvidia-a100-80gb-4: + tensorParallelismOverride: + tensorParallelSize: 4 + nvidia-a100-80gb-8: + tensorParallelismOverride: + tensorParallelSize: 8 + nvidia-h200-1: + tensorParallelismOverride: + tensorParallelSize: 1 + nvidia-h200-2: + tensorParallelismOverride: + tensorParallelSize: 2 + nvidia-h200-4: + tensorParallelismOverride: + tensorParallelSize: 4 + nvidia-h200-8: + tensorParallelismOverride: + tensorParallelSize: 8 + nvidia-b200-1: + tensorParallelismOverride: + tensorParallelSize: 1 + nvidia-b200-2: + tensorParallelismOverride: + tensorParallelSize: 2 + nvidia-b200-4: + tensorParallelismOverride: + tensorParallelSize: 4 + nvidia-b200-8: + tensorParallelismOverride: + tensorParallelSize: 8 + modelSizeRange: + min: 4.6B + max: 27.7B + protocolVersions: + - openAI + routerConfig: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "29000" + prometheus.io/path: "/metrics" + labels: + logging-forward: enabled + runner: + name: router + image: docker.io/lightseekorg/smg:1.4.1 + ports: + - containerPort: 8080 + name: http + resources: + limits: + cpu: "1" + memory: 2Gi + args: + - --host + - 0.0.0.0 + - --port + - "8080" + - --service-discovery + - --service-discovery-namespace + - $(NAMESPACE) + - --service-discovery-port + - "8080" + - --selector + - component=engine ome.io/inferenceservice=$(INFERENCESERVICE_NAME) + - --enable-igw + - --request-id-headers + - opc-request-id + - --log-json + - --disable-retries + - --disable-circuit-breaker + - --disable-tokenizer-autoload + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: INFERENCESERVICE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ome.io/inferenceservice'] + readinessProbe: + httpGet: + path: /readiness + port: 8080 + failureThreshold: 5 + periodSeconds: 30 + timeoutSeconds: 10 + livenessProbe: + httpGet: + path: /liveness + port: 8080 + failureThreshold: 5 + periodSeconds: 30 + timeoutSeconds: 10 + startupProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 10 + periodSeconds: 20 + timeoutSeconds: 10 + engineConfig: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + prometheus.io/path: "/metrics" + labels: + logging-forward: enabled + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + volumes: + - name: dshm + emptyDir: + medium: Memory + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node.kubernetes.io/instance-type + operator: In + values: + - BM.GPU.A100-v2.8 + - BM.GPU.H100.8 + - BM.GPU.H200-NC.8 + - BM.GPU.H200.8 + runner: + name: ome-container + image: fra.ocir.io/idqj093njucb/official-vllm-openai:v0.19.1-nightly-gemma4 + ports: + - containerPort: 8080 + name: http1 + protocol: TCP + command: + - /bin/bash + - '-lc' + - -- + args: + - | + vllm serve \ + --port=8080 \ + --model="$MODEL_PATH" \ + --max-log-len=0 \ + --served-model-name=vllm-model \ + --tensor-parallel-size=1 \ + --max-model-len=-1 \ + --gpu-memory-utilization=0.9 \ + --enable-auto-tool-choice \ + --tool-call-parser=gemma4 \ + --reasoning-parser=gemma4 \ + --async-scheduling \ + --no-scheduler-reserve-full-isl \ + --limit-mm-per-prompt '{"image": 10, "audio": 1, "video": 1}' + env: + - name: VLLM_LOGGING_LEVEL + value: "INFO" + - name: VLLM_RPC_TIMEOUT + value: '30000' + - name: VLLM_ENGINE_ITERATION_TIMEOUT_S + value: '120' + volumeMounts: + - mountPath: /dev/shm + name: dshm + resources: + requests: + cpu: 10 + memory: 80Gi + nvidia.com/gpu: 1 + limits: + cpu: 10 + memory: 80Gi + nvidia.com/gpu: 1 + + readinessProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 3 + successThreshold: 1 + periodSeconds: 60 + timeoutSeconds: 200 + + livenessProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 5 + successThreshold: 1 + periodSeconds: 60 + timeoutSeconds: 60 + + startupProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 150 + successThreshold: 1 + periodSeconds: 6 + initialDelaySeconds: 60 + timeoutSeconds: 30 diff --git a/config/runtimes/vllm/gemma-4-tp2-rt.yaml b/config/runtimes/vllm/gemma-4-tp2-rt.yaml new file mode 100644 index 000000000..43ed100d2 --- /dev/null +++ b/config/runtimes/vllm/gemma-4-tp2-rt.yaml @@ -0,0 +1,237 @@ +apiVersion: ome.io/v1beta1 +kind: ClusterServingRuntime +metadata: + name: vllm-gemma-4-tp2 +spec: + disabled: false + acceleratorRequirements: + acceleratorClasses: + - nvidia-h100-2 + - nvidia-h100-4 + - nvidia-h100-8 + - nvidia-a100-80gb-2 + - nvidia-a100-80gb-4 + - nvidia-a100-80gb-8 + - nvidia-h200-2 + - nvidia-h200-4 + - nvidia-h200-8 + - nvidia-b200-2 + - nvidia-b200-4 + - nvidia-b200-8 + supportedModelFormats: + - modelFramework: + name: transformers + version: "5.5.0.dev0" + modelFormat: + name: safetensors + version: "1.0.0" + modelArchitecture: Gemma4ForConditionalGeneration + autoSelect: true + priority: 2 + version: "1.0.0" + acceleratorConfig: + nvidia-h100-2: + tensorParallelismOverride: + tensorParallelSize: 2 + nvidia-h100-4: + tensorParallelismOverride: + tensorParallelSize: 4 + nvidia-h100-8: + tensorParallelismOverride: + tensorParallelSize: 8 + nvidia-a100-80gb-2: + tensorParallelismOverride: + tensorParallelSize: 2 + nvidia-a100-80gb-4: + tensorParallelismOverride: + tensorParallelSize: 4 + nvidia-a100-80gb-8: + tensorParallelismOverride: + tensorParallelSize: 8 + nvidia-h200-2: + tensorParallelismOverride: + tensorParallelSize: 2 + nvidia-h200-4: + tensorParallelismOverride: + tensorParallelSize: 4 + nvidia-h200-8: + tensorParallelismOverride: + tensorParallelSize: 8 + nvidia-b200-2: + tensorParallelismOverride: + tensorParallelSize: 2 + nvidia-b200-4: + tensorParallelismOverride: + tensorParallelSize: 4 + nvidia-b200-8: + tensorParallelismOverride: + tensorParallelSize: 8 + modelSizeRange: + min: 28.2B + max: 34.4B + protocolVersions: + - openAI + routerConfig: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "29000" + prometheus.io/path: "/metrics" + labels: + logging-forward: enabled + runner: + name: router + image: docker.io/lightseekorg/smg:1.4.1 + ports: + - containerPort: 8080 + name: http + resources: + limits: + cpu: "1" + memory: 2Gi + args: + - --host + - 0.0.0.0 + - --port + - "8080" + - --service-discovery + - --service-discovery-namespace + - $(NAMESPACE) + - --service-discovery-port + - "8080" + - --selector + - component=engine ome.io/inferenceservice=$(INFERENCESERVICE_NAME) + - --enable-igw + - --request-id-headers + - opc-request-id + - --log-json + - --disable-retries + - --disable-circuit-breaker + - --disable-tokenizer-autoload + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: INFERENCESERVICE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ome.io/inferenceservice'] + readinessProbe: + httpGet: + path: /readiness + port: 8080 + failureThreshold: 5 + periodSeconds: 30 + timeoutSeconds: 10 + livenessProbe: + httpGet: + path: /liveness + port: 8080 + failureThreshold: 5 + periodSeconds: 30 + timeoutSeconds: 10 + startupProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 10 + periodSeconds: 20 + timeoutSeconds: 10 + engineConfig: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + prometheus.io/path: "/metrics" + labels: + logging-forward: enabled + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + volumes: + - name: dshm + emptyDir: + medium: Memory + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node.kubernetes.io/instance-type + operator: In + values: + - BM.GPU.A100-v2.8 + - BM.GPU.H100.8 + - BM.GPU.H200-NC.8 + - BM.GPU.H200.8 + runner: + name: ome-container + image: fra.ocir.io/idqj093njucb/official-vllm-openai:v0.19.1-nightly-gemma4 + ports: + - containerPort: 8080 + name: http1 + protocol: TCP + command: + - /bin/bash + - '-lc' + - -- + args: + - | + vllm serve \ + --port=8080 \ + --model="$MODEL_PATH" \ + --max-log-len=0 \ + --served-model-name=vllm-model \ + --tensor-parallel-size=2 \ + --max-model-len=-1 \ + --gpu-memory-utilization=0.9 \ + --enable-auto-tool-choice \ + --tool-call-parser=gemma4 \ + --reasoning-parser=gemma4 \ + --async-scheduling \ + --no-scheduler-reserve-full-isl \ + --limit-mm-per-prompt '{"image": 10, "audio": 0, "video": 1}' + env: + - name: VLLM_LOGGING_LEVEL + value: "INFO" + - name: VLLM_RPC_TIMEOUT + value: '30000' + - name: VLLM_ENGINE_ITERATION_TIMEOUT_S + value: '120' + volumeMounts: + - mountPath: /dev/shm + name: dshm + resources: + requests: + cpu: 10 + memory: 80Gi + nvidia.com/gpu: 2 + limits: + cpu: 10 + memory: 80Gi + nvidia.com/gpu: 2 + readinessProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 3 + successThreshold: 1 + periodSeconds: 90 + timeoutSeconds: 60 + livenessProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 5 + successThreshold: 1 + periodSeconds: 60 + timeoutSeconds: 60 + startupProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 150 + successThreshold: 1 + periodSeconds: 6 + initialDelaySeconds: 60 + timeoutSeconds: 30 diff --git a/config/samples/isvc/google/gemma-4-26B-A4B-it.yaml b/config/samples/isvc/google/gemma-4-26B-A4B-it.yaml new file mode 100644 index 000000000..a3b0ff575 --- /dev/null +++ b/config/samples/isvc/google/gemma-4-26B-A4B-it.yaml @@ -0,0 +1,19 @@ +# apiVersion: v1 +# kind: Namespace +# metadata: +# name: gemma-4-26b-a4b-it +--- +apiVersion: ome.io/v1beta1 +kind: InferenceService +metadata: + name: gemma-4-26b-a4b-it + namespace: gemma-4-26b-a4b-it +spec: + model: + name: gemma-4-26b-a4b-it + engine: + minReplicas: 1 + maxReplicas: 1 + router: + minReplicas: 1 + maxReplicas: 1 diff --git a/config/samples/isvc/google/gemma-4-31B-it.yaml b/config/samples/isvc/google/gemma-4-31B-it.yaml new file mode 100644 index 000000000..66bd1ccc0 --- /dev/null +++ b/config/samples/isvc/google/gemma-4-31B-it.yaml @@ -0,0 +1,19 @@ +# apiVersion: v1 +# kind: Namespace +# metadata: +# name: gemma-4-31b-it +--- +apiVersion: ome.io/v1beta1 +kind: InferenceService +metadata: + name: gemma-4-31b-it + namespace: gemma-4-31b-it +spec: + model: + name: gemma-4-31b-it + engine: + minReplicas: 1 + maxReplicas: 1 + router: + minReplicas: 1 + maxReplicas: 1 diff --git a/config/samples/isvc/google/gemma-4-E2B-it.yaml b/config/samples/isvc/google/gemma-4-E2B-it.yaml new file mode 100644 index 000000000..510df027c --- /dev/null +++ b/config/samples/isvc/google/gemma-4-E2B-it.yaml @@ -0,0 +1,19 @@ +# apiVersion: v1 +# kind: Namespace +# metadata: +# name: gemma-4-e2b-it +--- +apiVersion: ome.io/v1beta1 +kind: InferenceService +metadata: + name: gemma-4-e2b-it + namespace: gemma-4-e2b-it +spec: + model: + name: gemma-4-e2b-it + engine: + minReplicas: 1 + maxReplicas: 1 + router: + minReplicas: 1 + maxReplicas: 1 diff --git a/config/samples/isvc/google/gemma-4-E4B-it.yaml b/config/samples/isvc/google/gemma-4-E4B-it.yaml new file mode 100644 index 000000000..5fb3117cb --- /dev/null +++ b/config/samples/isvc/google/gemma-4-E4B-it.yaml @@ -0,0 +1,19 @@ +# apiVersion: v1 +# kind: Namespace +# metadata: +# name: gemma-4-e4b-it +--- +apiVersion: ome.io/v1beta1 +kind: InferenceService +metadata: + name: gemma-4-e4b-it + namespace: gemma-4-e4b-it +spec: + model: + name: gemma-4-e4b-it + engine: + minReplicas: 1 + maxReplicas: 1 + router: + minReplicas: 1 + maxReplicas: 1