Moonshot AI/Kimi-K2

# InferenceClass for the H200 shape, validated serving Kimi-K2 prefill/decode
# disaggregated on EKS. 8x NVIDIA H200 on an EKS p5en.48xlarge, with EFA.
#
# Both the GPU and the EFA fabric are claim: DRA devices. The two P/D engines
# each request 8 GPUs + 16 EFA interfaces, and the scheduler places one on each
# H200 node; NIXL ships KV cache between them over EFA.
apiVersion: modelplane.ai/v1alpha1
kind: InferenceClass
metadata:
  name: eks-h200-8x-p5en
spec:
  description: "EKS p5en.48xlarge, 8x NVIDIA H200, EFA"
  provisioning:
    provider: EKS
    eks:
      instanceType: p5en.48xlarge
      diskSizeGb: 1024
      accelerator:
        type: nvidia-h200
        count: 8
  devices:
  - name: gpu
    claim: DRA
    driver: gpu.nvidia.com
    deviceClassName: gpu.nvidia.com
    count: 8
    attributes:
      architecture: { string: Hopper }
    capacity:
      memory: { value: "140Gi" }   # advertised below the ~141 GiB the driver reports
  - name: efa
    claim: DRA
    driver: dra.net
    deviceClassName: efa.networking.k8s.aws
    count: 16

# An EKS InferenceCluster with a two-node H200 pool over EFA, validated serving
# Kimi-K2 as a prefill/decode pair (one engine per node). The H200 nodes come
# from an EC2 Capacity Block reserved for ML.
apiVersion: modelplane.ai/v1alpha1
kind: InferenceCluster
metadata:
  name: eks-kimi
  labels:
    modelplane.ai/region: us
spec:
  cluster:
    source: EKS
    eks:
      region: us-east-2
  nodePools:
  - name: gpu-h200
    className: eks-h200-8x-p5en
    nodeCount: 2
    minNodeCount: 2
    maxNodeCount: 2
    zones:
    - us-east-2b
    fabric:
      type: EFA
    capacityBlock:
      capacityReservationId: cr-0123456789abcdef0  # replace with your reservation ID

# The shared, read-write-many cache both P/D engines serve from. Hydrated once
# per matched cluster; both phases mount the same RWX volume at /mnt/models.
#
# This validated run served an INT4 quantization of Kimi K2 rather than the
# native 1T-parameter FP8 model, which would need four 8x H200 nodes. The quant
# repo is public, so no authSecret is needed here.
apiVersion: modelplane.ai/v1alpha1
kind: ModelCache
metadata:
  name: kimi-k2
  namespace: ml-team
spec:
  source: HuggingFace
  huggingFace:
    repo: RedHatAI/Kimi-K2-Instruct-quantized.w4a16
    sizeGiB: 600

# Kimi-K2 served prefill/decode disaggregated across two H200 nodes. The
# engines ran end to end on EKS; the routing written here hasn't run live yet.
#
# serving.mode: PrefillDecode makes the two engines below one P/D pair. The
# recipe writes the routing: the llm-d endpoint picker's command line, which
# passes on the pool, ports and config path Modelplane sets in its environment;
# its config, whose prefix-based decider skips prefill for short or cached
# prompts; and the llm-d sidecar beside the decode engine. Modelplane composes
# the InferencePool and runs the picker. Each engine is a single-node 8-GPU Standalone pod; the
# scheduler places prefill on one node and decode on the other, and NIXL moves
# the KV cache between them over EFA.
#
# Notes on the engine flags (the routing is this recipe's, and the engine config
# has real sharp edges):
#   EP, not TP8. --tensor-parallel-size=1 --data-parallel-size=8
#     --enable-expert-parallel is vLLM's DeepSeek-V3 / Kimi single-node recipe
#     (Kimi-K2 is DeepSeek-V3 arch). It keeps each expert whole on a GPU and
#     dodges the Marlin %128 alignment trap that a plain TP8 layout would hit.
#   --load-format=runai_streamer cold-reads ~509 GiB per engine off the shared
#     RWX cache in ~6 minutes (vs ~45 with the default loader).
#   --tokenizer / --override-generation-config are workarounds for two bugs in
#     this specific quant repo (an off-by-2 in its bundled tokenizer, and a
#     wrong eos_token_id), not normal flags. The override pulls a gated repo at
#     startup, so HF_TOKEN is set on both engines.
#   The picker config's blockSizeTokens: 64 matches --block-size=64 on the
#     engines; prefix-cache routing silently degrades if they differ.
#   The decode engine runs on :8001 and declares it as its http port; the
#     sidecar owns :8000. The sidecar's --kv-connector=nixlv2 matches the
#     engines' NixlConnector.
#   VLLM_NIXL_SIDE_CHANNEL_HOST is the pod IP on both engines. Each engine then
#     advertises an address its peer connects to. Without it KV transfer fails
#     and requests return 500.
apiVersion: modelplane.ai/v1alpha1
kind: ModelDeployment
metadata:
  name: kimi-k2
  namespace: ml-team
spec:
  replicas: 1
  template:
    spec:
      modelCacheRef:
        name: kimi-k2
      serving:
        mode: PrefillDecode        # the two engines below are one P/D pair
        router: EndpointPicker     # P/D writes its own routing
        endpointPicker:
          template:
            spec:
              containers:
              - name: epp
                image: ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0
                args:
                - --pool-name=$(MODELPLANE_POOL_NAME)
                - --pool-namespace=$(MODELPLANE_POOL_NAMESPACE)
                - --pool-group=inference.networking.k8s.io
                - --config-file=$(MODELPLANE_PICKER_CONFIG)
                - --grpc-port=$(MODELPLANE_GRPC_PORT)
                - --grpc-health-port=$(MODELPLANE_GRPC_HEALTH_PORT)
                - --metrics-port=$(MODELPLANE_METRICS_PORT)
                - --metrics-endpoint-auth=false
          config:
            apiVersion: llm-d.ai/v1alpha1
            kind: EndpointPickerConfig
            plugins:
            - type: approx-prefix-cache-producer
              parameters:
                autoTune: false
                blockSizeTokens: 64
                maxPrefixBlocksToMatch: 256
                lruCapacityPerServer: 31250
            - type: prefix-cache-scorer
            - type: queue-scorer
            - type: prefill-filter
            - type: decode-filter
            - type: max-score-picker
            - type: prefix-based-pd-decider
              parameters:
                nonCachedTokens: 16
            - type: disagg-profile-handler
              parameters:
                deciders:
                  prefill: prefix-based-pd-decider
            schedulingProfiles:
            - name: prefill
              plugins:
              - pluginRef: prefill-filter
              - pluginRef: max-score-picker
              - pluginRef: prefix-cache-scorer
                weight: 2
              - pluginRef: queue-scorer
                weight: 1
            - name: decode
              plugins:
              - pluginRef: decode-filter
              - pluginRef: max-score-picker
              - pluginRef: prefix-cache-scorer
                weight: 2
              - pluginRef: queue-scorer
                weight: 1
      engines:
      - name: kimi-prefill
        phase: Prefill
        members:
        - role: Standalone
          nodeSelector:
            devices:
            - name: gpu
              count: 8
              selectors:
              - cel: |
                  device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("130Gi")) >= 0
            - name: efa
              count: 16
              selectors:
              - cel: |
                  device.driver == "dra.net"
          template:
            spec:
              containers:
              - name: engine
                image: vllm/vllm-openai:v0.23.0
                command: ["vllm", "serve", "RedHatAI/Kimi-K2-Instruct-quantized.w4a16"]
                args:
                - --served-model-name=$(MODELPLANE_SERVED_MODEL_NAME)
                - --quantization=compressed-tensors
                - --tensor-parallel-size=1
                - --data-parallel-size=8
                - --enable-expert-parallel
                - --block-size=64
                - --max-model-len=131072
                - --trust-remote-code
                - --tool-call-parser=kimi_k2
                - --enable-auto-tool-choice
                - --load-format=runai_streamer
                - --model-loader-extra-config={"concurrency":16,"distributed":true}
                - --tokenizer=moonshotai/Kimi-K2-Instruct
                - --override-generation-config={"eos_token_id":163586}
                - --port=8000
                - --kv-transfer-config={"kv_connector":"NixlConnector","kv_role":"kv_producer"}
                env:
                - name: HF_TOKEN
                  valueFrom:
                    secretKeyRef: {name: hf-token, key: HF_TOKEN}
                - name: VLLM_NIXL_SIDE_CHANNEL_HOST
                  valueFrom:
                    fieldRef: {fieldPath: status.podIP}
                - name: VLLM_NIXL_SIDE_CHANNEL_PORT
                  value: "5557"
      - name: kimi-decode
        phase: Decode
        members:
        - role: Standalone
          nodeSelector:
            devices:
            - name: gpu
              count: 8
              selectors:
              - cel: |
                  device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("130Gi")) >= 0
            - name: efa
              count: 16
              selectors:
              - cel: |
                  device.driver == "dra.net"
          template:
            spec:
              containers:
              - name: sidecar
                image: ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0
                args:
                - --secure-proxy=false
                - --kv-connector=nixlv2
                - --vllm-port=8001
                ports:
                - containerPort: 8000
                readinessProbe:
                  httpGet:
                    path: /health
                    port: 8000
              - name: engine
                image: vllm/vllm-openai:v0.23.0
                command: ["vllm", "serve", "RedHatAI/Kimi-K2-Instruct-quantized.w4a16"]
                args:
                - --served-model-name=$(MODELPLANE_SERVED_MODEL_NAME)
                - --quantization=compressed-tensors
                - --tensor-parallel-size=1
                - --data-parallel-size=8
                - --enable-expert-parallel
                - --block-size=64
                - --max-model-len=131072
                - --trust-remote-code
                - --tool-call-parser=kimi_k2
                - --enable-auto-tool-choice
                - --load-format=runai_streamer
                - --model-loader-extra-config={"concurrency":16,"distributed":true}
                - --tokenizer=moonshotai/Kimi-K2-Instruct
                - --override-generation-config={"eos_token_id":163586}
                - --port=8001                                  # the sidecar owns 8000
                - --kv-transfer-config={"kv_connector":"NixlConnector","kv_role":"kv_consumer"}
                ports:
                - name: http
                  containerPort: 8001
                env:
                - name: HF_TOKEN
                  valueFrom:
                    secretKeyRef: {name: hf-token, key: HF_TOKEN}
                - name: VLLM_NIXL_SIDE_CHANNEL_HOST
                  valueFrom:
                    fieldRef: {fieldPath: status.podIP}
                - name: VLLM_NIXL_SIDE_CHANNEL_PORT
                  value: "5557"

# Exposes the kimi-k2 deployment as one model a caller can name.
# Name the model, ml-team/kimi-k2, in a request to an
# InferenceGateway that serves it.
apiVersion: modelplane.ai/v1alpha1
kind: ModelService
metadata:
  name: kimi-k2
  namespace: ml-team
spec:
  endpoints:
  - name: kimi-k2
    selector:
      matchLabels:
        modelplane.ai/deployment: kimi-k2