# DeepSeek-V4-Flash-0731 (MoE: 256 experts / 6 active, UD-IQ1_M ≈ 87 GiB)
# served by llama.cpp's llama-server on the NUCBox APU.
#
# Hardware: AMD Ryzen AI Max 395 (Strix Halo) — integrated Radeon 8060S,
# ~120 GiB unified memory (≈90 GiB VRAM / 30 GiB CPU RAM via firmware). The
# IQ1_M model (~87 GiB) is *almost* the size of the whole VRAM pool, so it
# CANNOT be fully offloaded to the GPU: offloading all 43 layers + the KV
# cache + Vulkan compute buffers would overflow 90 GiB. Instead we offload
# 40 of 43 layers (-ngl 40) and keep the last 3 (~6 GiB) on CPU RAM, leaving
# ~8 GiB of VRAM headroom for the KV cache, compute buffers, and co-resident
# pods. This is the only model served on the NUCBox — the two Qwen3.6 models
# were removed to make room (their GGUF files should be deleted from the PVC,
# which the initContainer below does on first boot).
#
# KV cache is tiny thanks to DeepSeek-V4's MLA attention (num_kv_heads=1,
# head_dim=512 + 64 decoupled RoPE ⇒ ~576 elements/token/layer). At 64k
# context, q8_0 KV is only ~1.6 GiB, so context is cheap — but we cap -c at
# 65536 (the required minimum) to maximise VRAM headroom, not because KV is
# the constraint.
#
# Image: ghcr.io/ggml-org/llama.cpp:server-vulkan bundles the Mesa/RADV Vulkan
# driver, which supports the Radeon 8060S (RDNA 3.5). The Vulkan backend
# supports the IQ1_M matmul (incl. the MoE matmul_id variant), so the whole
# model runs on the GPU. deepseek4 is a brand-new arch (2026-07) so the
# floating `server-vulkan` tag is used to pull a recent enough build; pin to a
# specific server-vulkan-bXXXX tag once a known-good one is verified.
#
# GPU access: the container mounts /dev/dri (the DRM render nodes) and runs
# privileged — the simplest reliable option on k3s without a Vulkan device
# plugin.
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: llamacpp-deepseek-v4-flash-0731
  namespace: llamacpp
  labels:
    app: llamacpp
    model: deepseek-v4-flash-0731
spec:
  replicas: 1
  strategy:
    type: Recreate   # never run two pods loading the same model into VRAM
  selector:
    matchLabels:
      app: llamacpp
      model: deepseek-v4-flash-0731
  template:
    metadata:
      labels:
        app: llamacpp
        model: deepseek-v4-flash-0731
    spec:
      nodeSelector:
        kubernetes.io/arch: amd64
        hardware: high-memory
      initContainers:
      # Idempotently download the (3-part, split) GGUF into the shared models
      # PVC on first boot. Also removes the retired Qwen3.6 GGUFs so the new
      # 87 GiB model fits on the PVC alongside any other data. Exits
      # immediately if the first shard is already present (pod restart).
      - name: fetch-model
        image: alpine:3.20
        command: ["/bin/sh", "-c"]
        args:
        - |
          set -e
          # Reclaim space from the retired Qwen3.6 models (their Deployments
          # are gone; the GGUFs are dead weight on the shared PVC).
          for old in Qwen3.6-27B-UD-Q4_K_XL.gguf Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf; do
            if [ -f "/models/$old" ]; then
              echo "Removing retired model $old ..."
              rm -f "/models/$old"
            fi
          done
          # Download any missing shards of the split UD-IQ1_M GGUF.
          if [ -f "/models/$SHARD1" ]; then
            echo "First shard $SHARD1 already present — skipping download."
            exit 0
          fi
          echo "Installing curl..."
          apk add --no-cache curl
          for s in "$SHARD1" "$SHARD2" "$SHARD3"; do
            echo "Downloading $s from $HF_REPO ..."
            curl -fL --retry 5 --retry-delay 5 -o "/models/$s" "$HF_REPO/$s"
          done
          echo "Download complete:"
          ls -lh /models/DeepSeek-V4-Flash-0731-UD-IQ1_M-*.gguf
        env:
        - name: HF_REPO
          value: "https://huggingface.co/unsloth/DeepSeek-V4-Flash-0731-GGUF/resolve/main/UD-IQ1_M"
        - name: SHARD1
          value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00001-of-00003.gguf"
        - name: SHARD2
          value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00002-of-00003.gguf"
        - name: SHARD3
          value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00003-of-00003.gguf"
        volumeMounts:
        - name: models
          mountPath: /models
      containers:
      - name: llama-server
        image: ghcr.io/ggml-org/llama.cpp:server-vulkan
        imagePullPolicy: IfNotPresent
        # llama.cpp's CLI parser does NOT split on '=' — every value flag must
        # be a separate argv element (flag, then value). See common/arg.cpp.
        args:
        - -m                       # model file (first shard; llama.cpp auto-loads the rest)
        - /models/DeepSeek-V4-Flash-0731-UD-IQ1_M-00001-of-00003.gguf
        - --alias                 # /v1/models reports this name; matches the litellm alias
        - deepseek-v4-flash-0731
        - --host
        - 0.0.0.0
        - --port
        - "8080"
        - --jinja                  # use the GGUF's DeepSeek-V4 chat template (DSML / thinking)
        - -ngl                     # offload 40 of 43 layers to the GPU. The model (~87 GiB) is
        - "40"                     # nearly the whole 90 GiB VRAM pool, so full offload (-ngl 999)
                                   # would overflow once KV cache + Vulkan compute buffers are
                                   # added. Keeping 3 layers (~6 GiB) on CPU leaves ~8 GiB of
                                   # VRAM headroom for the KV cache, compute buffers, and
                                   # co-resident pods. Raise toward 43 if VRAM allows; lower
                                   # (e.g. 38) if the pod OOMs / Vulkan runs out of device mem.
        - -c                       # total KV-cache context (single slot gets the full window).
        - "65536"                  # 64k — the required minimum. MLA KV is tiny (~1.6 GiB at
                                   # q8_0), so context is cheap; -c is capped at the minimum to
                                   # maximise VRAM headroom, not because KV is the constraint.
                                   # Raise if VRAM headroom allows.
        - -np                      # 1 slot => the full 64k goes to a single concurrent request
        - "1"                      # (extra slots would multiply KV VRAM, which is fine here, but
                                   # 1 slot keeps it simple and headroom maximal).
        - --cont-batching          # continuous batching across slots
        - --cache-type-k            # quantize KV cache to q8_0 — MLA KV is already small (~576
        - q8_0                     # elem/token/layer); q8_0 halves it to ~1.6 GiB at 64k and
        - --cache-type-v            # maximises VRAM headroom with ~negligible quality loss.
        - q8_0
        - --temp                   # default sampling temperature (DeepSeek-V4 recommendation)
        - "1.0"
        - --top-p                  # default nucleus sampling threshold (DeepSeek-V4 recommendation)
        - "0.95"
        - --threads                # CPU threads for sampling + the 3 CPU-resident layers
        - "8"
        ports:
        - name: http
          containerPort: 8080
        resources:
          # The model weights + KV cache live in GPU VRAM (~90 GiB pool) and
          # are NOT counted against the cgroup memory limit. This limit only
          # covers CPU-side overhead + the mmap'd GGUF pages for the 3
          # CPU-resident layers (~6 GiB) plus reclaimable page cache during
          # load. k8s sees ~30 GiB as the node's allocatable system RAM, so the
          # limit is sized to cover the CPU layers + overhead while leaving
          # RAM for co-resident pods (litellm, the agents, etc.). If the pod is
          # OOM-killed during model load, raise the limit.
          requests:
            cpu: "1000m"
            memory: "4Gi"
          limits:
            cpu: "4000m"
            memory: "20Gi"
        readinessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 10
          failureThreshold: 6
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 300   # 87 GiB load + Vulkan init takes several minutes
          periodSeconds: 30
          failureThreshold: 5
        securityContext:
          # Vulkan on the AMD APU needs /dev/dri + the driver. Privileged is
          # the simplest reliable path on k3s without a device plugin.
          privileged: true
        volumeMounts:
        - name: models
          mountPath: /models
          readOnly: true
        - name: dri
          mountPath: /dev/dri
      volumes:
      - name: models
        persistentVolumeClaim:
          claimName: llamacpp-models
      - name: dri
        hostPath:
          path: /dev/dri
          type: Directory
---
apiVersion: v1
kind: Service
metadata:
  name: llamacpp-deepseek-v4-flash-0731
  namespace: llamacpp
  labels:
    app: llamacpp
    model: deepseek-v4-flash-0731
spec:
  type: ClusterIP
  selector:
    app: llamacpp
    model: deepseek-v4-flash-0731
  ports:
  - name: http
    port: 80
    targetPort: 8080