k3s-cluster/llamacpp/deployment-qwen36-27b.yaml

# Qwen3.6-27B (dense, Q4_K_XL) served by llama.cpp's llama-server on the NUCBox APU.
#
# Hardware: AMD Ryzen AI Max 395 (Strix Halo) — integrated Radeon 8060S,
# 128 GiB unified memory (32 GiB RAM / 96 GiB VRAM via firmware). The Q4 model
# (~16 GiB) is fully offloaded to the GPU via the Vulkan backend.
#
# Image: ghcr.io/ggml-org/llama.cpp:server-vulkan bundles the Mesa/RADV Vulkan
# driver, which supports the Radeon 8060S (RDNA 3.5). The project moved from the
# legacy `ggerganov/llama.cpp` namespace (which only has light/full tags) to
# `ggml-org/llama.cpp` (server-vulkan + pinned build tags like server-vulkan-bXXXX).
# Pin to a build tag (e.g. server-vulkan-b4738) for production reproducibility.
#
# GPU access: the container mounts /dev/dri (the DRM render nodes) and runs
# privileged. This is the simplest reliable option on k3s without a Vulkan
# device plugin; tighten later with supplementalGroups if desired.
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: llamacpp-qwen36-27b
  namespace: llamacpp
  labels:
    app: llamacpp
    model: qwen3.6-27b
spec:
  replicas: 1
  strategy:
    type: Recreate   # never run two pods loading the same model into VRAM
  selector:
    matchLabels:
      app: llamacpp
      model: qwen3.6-27b
  template:
    metadata:
      labels:
        app: llamacpp
        model: qwen3.6-27b
    spec:
      nodeSelector:
        kubernetes.io/arch: amd64
        hardware: high-memory
      initContainers:
      # Idempotently download the GGUF into the shared models PVC on first boot.
      # Exits immediately if the file is already present (pod restart / recreate).
      - name: fetch-model
        image: alpine:3.20
        command: ["/bin/sh", "-c"]
        args:
        - |
          set -e
          if [ -f "/models/$MODEL_FILE" ]; then
            echo "Model $MODEL_FILE already present — skipping download."
            exit 0
          fi
          echo "Installing curl..."
          apk add --no-cache curl
          echo "Downloading $MODEL_FILE from $MODEL_URL ..."
          curl -fL --retry 5 --retry-delay 5 -o "/models/$MODEL_FILE" "$MODEL_URL"
          echo "Download complete: $(ls -lh /models/$MODEL_FILE)"
        env:
        - name: MODEL_URL
          value: "https://huggingface.co/unsloth/Qwen3.6-27B-MTP-GGUF/resolve/main/Qwen3.6-27B-UD-Q4_K_XL.gguf"
        - name: MODEL_FILE
          value: "Qwen3.6-27B-UD-Q4_K_XL.gguf"
        volumeMounts:
        - name: models
          mountPath: /models
      containers:
      - name: llama-server
        image: ghcr.io/ggml-org/llama.cpp:server-vulkan
        imagePullPolicy: IfNotPresent
        # llama.cpp's CLI parser does NOT split on '=' — every value flag must be a
        # separate argv element (flag, then value). See common/arg.cpp in the repo.
        args:
        - -m                       # model file
        - /models/Qwen3.6-27B-UD-Q4_K_XL.gguf
        - --alias                 # /v1/models reports this name; matches the litellm alias
        - qwen3.6-27b
        - --host
        - 0.0.0.0
        - --port
        - "8080"
        - --jinja                  # use the GGUF's chat template (Qwen3 thinking format)
        - -ngl                     # offload ALL layers to the GPU (fits in 96 GiB VRAM)
        - "999"
        - -c                       # total KV-cache context, split across parallel slots
        - "131072"
        - -np                      # 2 parallel slots => 65536 tokens per concurrent request
        - "2"
        - --cont-batching          # continuous batching across slots
        - --cache-type-k            # quantize KV cache to q8_0 — halves KV VRAM (~32 GiB → ~16 GiB
        - q8_0                     # at 131k ctx); ~negligible quality loss, frees headroom for large -c
        - --cache-type-v
        - q8_0
        - --threads                # CPU threads for sampling/overhead (GPU does the heavy lifting)
        - "8"
        ports:
        - name: http
          containerPort: 8080
        resources:
          # The model weights + KV cache live in GPU VRAM (96 GiB pool) and are
          # NOT counted against the cgroup memory limit. This limit only covers
          # CPU-side overhead + the mmap'd GGUF file pages during load (~16 GiB,
          # reclaimable). k8s sees ~32 GiB as the node's allocatable system RAM,
          # so the request is kept low to stay schedulable alongside other pods.
          # If the pod OOM-kills during load, the amdgpu driver may be counting
          # some VRAM against the cgroup — raise the limit.
          requests:
            cpu: "1000m"
            memory: "2Gi"
          limits:
            cpu: "4000m"
            memory: "24Gi"
        readinessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 10
          failureThreshold: 6
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 180   # model load + Vulkan init can take a few minutes
          periodSeconds: 30
          failureThreshold: 5
        securityContext:
          # Vulkan on the AMD APU needs /dev/dri + the driver. Privileged is the
          # simplest reliable path on k3s without a device plugin.
          privileged: true
        volumeMounts:
        - name: models
          mountPath: /models
          readOnly: true
        - name: dri
          mountPath: /dev/dri
      volumes:
      - name: models
        persistentVolumeClaim:
          claimName: llamacpp-models
      - name: dri
        hostPath:
          path: /dev/dri
          type: Directory
---
apiVersion: v1
kind: Service
metadata:
  name: llamacpp-qwen36-27b
  namespace: llamacpp
  labels:
    app: llamacpp
    model: qwen3.6-27b
spec:
  type: ClusterIP
  selector:
    app: llamacpp
    model: qwen3.6-27b
  ports:
  - name: http
    port: 80
    targetPort: 8080