k3s-cluster/llamacpp/deployment-qwen36-36b-a3b.yaml

# Qwen3.6-35B-A3B (MoE: 35B total / 3B active, Q4_K_XL) — the "flash" variant.
#
# Despite having more total parameters than the 27B dense model, only 3B are
# active per token (Mixture-of-Experts), so inference is much faster. The full
# ~20 GiB of Q4 weights is still loaded into VRAM but only a small fraction is
# computed per token.
#
# Hardware: AMD Ryzen AI Max 395 (Strix Halo) — integrated Radeon 8060S,
# 128 GiB unified memory (32 GiB RAM / 96 GiB VRAM via firmware). Full GPU
# offload via the Vulkan backend. Shares the 96 GiB VRAM pool with the 27B
# model — see llamacpp/README.md for the combined VRAM budget.
#
# Image: ghcr.io/ggml-org/llama.cpp:server-vulkan (Mesa/RADV Vulkan driver,
# supports the Radeon 8060S / RDNA 3.5). Pin to a build tag for production.
#
# GPU access: mounts /dev/dri + privileged (simplest reliable path on k3s).
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: llamacpp-qwen36-36b-a3b
  namespace: llamacpp
  labels:
    app: llamacpp
    model: qwen3.6-36b-a3b
spec:
  replicas: 1
  strategy:
    type: Recreate   # never run two pods loading the same model into VRAM
  selector:
    matchLabels:
      app: llamacpp
      model: qwen3.6-36b-a3b
  template:
    metadata:
      labels:
        app: llamacpp
        model: qwen3.6-36b-a3b
    spec:
      nodeSelector:
        kubernetes.io/arch: amd64
        hardware: high-memory
      initContainers:
      # Idempotently download the GGUF into the shared models PVC on first boot.
      # Exits immediately if the file is already present (pod restart / recreate).
      - name: fetch-model
        image: alpine:3.20
        command: ["/bin/sh", "-c"]
        args:
        - |
          set -e
          if [ -f "/models/$MODEL_FILE" ]; then
            echo "Model $MODEL_FILE already present — skipping download."
            exit 0
          fi
          echo "Installing curl..."
          apk add --no-cache curl
          echo "Downloading $MODEL_FILE from $MODEL_URL ..."
          curl -fL --retry 5 --retry-delay 5 -o "/models/$MODEL_FILE" "$MODEL_URL"
          echo "Download complete: $(ls -lh /models/$MODEL_FILE)"
        env:
        - name: MODEL_URL
          value: "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/resolve/main/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf"
        - name: MODEL_FILE
          value: "Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf"
        volumeMounts:
        - name: models
          mountPath: /models
      containers:
      - name: llama-server
        image: ghcr.io/ggml-org/llama.cpp:server-vulkan
        imagePullPolicy: IfNotPresent
        # llama.cpp's CLI parser does NOT split on '=' — every value flag must be a
        # separate argv element (flag, then value). See common/arg.cpp in the repo.
        args:
        - -m                       # model file
        - /models/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf
        - --alias                 # /v1/models reports this name; matches the litellm alias
        - qwen3.6-36b-a3b
        - --host
        - 0.0.0.0
        - --port
        - "8080"
        - --jinja                  # use the GGUF's chat template (Qwen3 thinking format)
        - -ngl                     # offload ALL layers to the GPU (fits in 96 GiB VRAM)
        - "999"
        - -c                       # total KV-cache context, split across parallel slots
        - "32768"
        - -np                      # 4 parallel slots => 8192 tokens per concurrent request
        - "4"
        - --cont-batching          # continuous batching across slots
        - --threads                # CPU threads for sampling/overhead (GPU does the heavy lifting)
        - "8"
        ports:
        - name: http
          containerPort: 8080
        resources:
          # The model weights + KV cache live in GPU VRAM (96 GiB pool) and are
          # NOT counted against the cgroup memory limit. This limit only covers
          # CPU-side overhead + the mmap'd GGUF file pages during load (~20 GiB,
          # reclaimable). k8s sees ~32 GiB as the node's allocatable system RAM.
          # If the pod OOM-kills during load, raise the limit.
          requests:
            cpu: "1000m"
            memory: "2Gi"
          limits:
            cpu: "4000m"
            memory: "24Gi"
        readinessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 10
          failureThreshold: 6
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 180   # model load + Vulkan init can take a few minutes
          periodSeconds: 30
          failureThreshold: 5
        securityContext:
          # Vulkan on the AMD APU needs /dev/dri + the driver. Privileged is the
          # simplest reliable path on k3s without a device plugin.
          privileged: true
        volumeMounts:
        - name: models
          mountPath: /models
          readOnly: true
        - name: dri
          mountPath: /dev/dri
      volumes:
      - name: models
        persistentVolumeClaim:
          claimName: llamacpp-models
      - name: dri
        hostPath:
          path: /dev/dri
          type: Directory
---
apiVersion: v1
kind: Service
metadata:
  name: llamacpp-qwen36-36b-a3b
  namespace: llamacpp
  labels:
    app: llamacpp
    model: qwen3.6-36b-a3b
spec:
  type: ClusterIP
  selector:
    app: llamacpp
    model: qwen3.6-36b-a3b
  ports:
  - name: http
    port: 80
    targetPort: 8080