# Qwen3.6 (27B, Q4_K_XL) served by llama.cpp's llama-server on the NUCBox APU.
#
# Hardware: AMD Ryzen AI Max 395 (Strix Halo) — integrated Radeon 8060S,
# 128 GiB unified memory (32 GiB RAM / 96 GiB VRAM via firmware). The Q4 model
# (~16 GiB) is fully offloaded to the GPU via the Vulkan backend, leaving plenty
# of VRAM for the KV cache.
#
# Image: ghcr.io/ggerganov/llama.cpp:server-vulkan bundles the Mesa/RADV Vulkan
# driver, which supports the Radeon 8060S (RDNA 3.5). Pin to a digest for
# production once the rolling tag is confirmed working.
#
# GPU access: the container mounts /dev/dri (the DRM render nodes) and runs
# privileged. This is the simplest reliable option on k3s without a Vulkan
# device plugin; tighten later with supplementalGroups if desired.
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: llamacpp-qwen36
  namespace: llamacpp
  labels:
    app: llamacpp
    model: qwen3.6
spec:
  replicas: 1
  strategy:
    type: Recreate   # never run two pods loading the same model into VRAM
  selector:
    matchLabels:
      app: llamacpp
      model: qwen3.6
  template:
    metadata:
      labels:
        app: llamacpp
        model: qwen3.6
    spec:
      nodeSelector:
        kubernetes.io/arch: amd64
        hardware: high-memory
      initContainers:
      # Idempotently download the GGUF into the shared models PVC on first boot.
      # Exits immediately if the file is already present (pod restart / recreate).
      - name: fetch-model
        image: alpine:3.20
        command: ["/bin/sh", "-c"]
        args:
        - |
          set -e
          if [ -f "/models/$MODEL_FILE" ]; then
            echo "Model $MODEL_FILE already present — skipping download."
            exit 0
          fi
          echo "Installing curl..."
          apk add --no-cache curl
          echo "Downloading $MODEL_FILE from $MODEL_URL ..."
          curl -fL --retry 5 --retry-delay 5 -o "/models/$MODEL_FILE" "$MODEL_URL"
          echo "Download complete: $(ls -lh /models/$MODEL_FILE)"
        env:
        - name: MODEL_URL
          value: "https://huggingface.co/unsloth/Qwen3.6-27B-MTP-GGUF/resolve/main/Qwen3.6-27B-UD-Q4_K_XL.gguf"
        - name: MODEL_FILE
          value: "Qwen3.6-27B-UD-Q4_K_XL.gguf"
        volumeMounts:
        - name: models
          mountPath: /models
      containers:
      - name: llama-server
        image: ghcr.io/ggerganov/llama.cpp:server-vulkan
        imagePullPolicy: IfNotPresent
        args:
        - -m=/models/Qwen3.6-27B-UD-Q4_K_XL.gguf
        - --alias=qwen3.6          # /v1/models reports this name; matches the litellm alias
        - --host=0.0.0.0
        - --port=8080
        - --jinja                  # use the GGUF's chat template (Qwen3 thinking format)
        - -ngl=999                 # offload ALL layers to the GPU (fits in 96 GiB VRAM)
        - -c=32768                 # total KV-cache context, split across parallel slots
        - -np=4                    # 4 parallel slots => 8192 tokens per concurrent request
        - --cont-batching          # continuous batching across slots
        - --threads=8              # CPU threads for sampling/overhead (GPU does the heavy lifting)
        ports:
        - name: http
          containerPort: 8080
        resources:
          # The model weights + KV cache live in GPU VRAM (96 GiB pool) and are
          # NOT counted against the cgroup memory limit. This limit only covers
          # CPU-side overhead + the mmap'd GGUF file pages during load (~16 GiB,
          # reclaimable). k8s sees ~32 GiB as the node's allocatable system RAM,
          # so the request is kept low to stay schedulable alongside other pods.
          # If the pod OOM-kills during load, the amdgpu driver may be counting
          # some VRAM against the cgroup — raise the limit.
          requests:
            cpu: "1000m"
            memory: "2Gi"
          limits:
            cpu: "4000m"
            memory: "24Gi"
        readinessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 10
          failureThreshold: 6
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 180   # model load + Vulkan init can take a few minutes
          periodSeconds: 30
          failureThreshold: 5
        securityContext:
          # Vulkan on the AMD APU needs /dev/dri + the driver. Privileged is the
          # simplest reliable path on k3s without a device plugin.
          privileged: true
        volumeMounts:
        - name: models
          mountPath: /models
          readOnly: true
        - name: dri
          mountPath: /dev/dri
      volumes:
      - name: models
        persistentVolumeClaim:
          claimName: llamacpp-models
      - name: dri
        hostPath:
          path: /dev/dri
          type: Directory
---
apiVersion: v1
kind: Service
metadata:
  name: llamacpp-qwen36
  namespace: llamacpp
  labels:
    app: llamacpp
    model: qwen3.6
spec:
  type: ClusterIP
  selector:
    app: llamacpp
    model: qwen3.6
  ports:
  - name: http
    port: 80
    targetPort: 8080