# DeepSeek-V4-Flash-0731 (MoE: 256 experts / 6 active, UD-IQ1_M ≈ 87 GiB)
# served by llama.cpp's llama-server on the NUCBox APU.
#
# Hardware: AMD Ryzen AI Max 395 (Strix Halo) — integrated Radeon 8060S,
# ~120 GiB unified memory (≈90 GiB VRAM / 30 GiB CPU RAM via firmware). The
# IQ1_M model (~87 GiB) is *almost* the size of the whole VRAM pool, so it
# CANNOT be fully offloaded to the GPU: offloading all 43 layers + the KV
# cache + Vulkan compute buffers would overflow 90 GiB. Instead we offload
# 38 of 43 layers (-ngl 38) and keep 5 layers (~10 GiB) on CPU RAM, leaving
# ~5 GiB of VRAM headroom for the KV cache + Vulkan compute buffers.
#
# VRAM is exclusive to this model (no other pod uses it); the other pods on
# the NUCBox only compete for the 30 GiB CPU RAM, so the headroom that
# matters here is VRAM headroom for compute buffers / fragmentation.
#
# KV CACHE MUST BE f16 (NOT quantized). The Vulkan backend has no Flash
# Attention for the deepseek4 arch, and quantized V cache requires Flash
# Attention (llama.cpp hard-errors otherwise: "quantized V cache was
# requested, but this requires Flash Attention"). Additionally, deepseek4 /
# MLA models require K and V cache types to be *identical*, so K cannot be
# quantized either. f16 KV at 64k is ~5.7 GiB (MLA KV: 576 K + 512 V
# elements/token/layer × 43 layers × 65536 tokens × 2 bytes). This is why
# -ngl is 38 rather than 40 — the larger f16 KV cache needs the extra VRAM.
#
# Image: ghcr.io/ggml-org/llama.cpp:server-vulkan bundles the Mesa/RADV Vulkan
# driver, which supports the Radeon 8060S (RDNA 3.5). The Vulkan backend
# supports the IQ1_M matmul (incl. the MoE matmul_id variant), so the whole
# model runs on the GPU. deepseek4 is a brand-new arch (2026-07); several
# deepseek4-specific fused ops (Lightning Indexer, HC pre/comb/post) are not
# yet implemented in Vulkan and fall back to CPU (logged as warnings, not
# fatal). The floating `server-vulkan` tag is used to pull a recent enough
# build; pin to a specific server-vulkan-bXXXX tag once a known-good one is
# verified.
#
# GPU access: the container mounts /dev/dri (the DRM render nodes) and runs
# privileged — the simplest reliable option on k3s without a Vulkan device
# plugin.
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: llamacpp-deepseek-v4-flash-0731
  namespace: llamacpp
  labels:
    app: llamacpp
    model: deepseek-v4-flash-0731
spec:
  replicas: 1
  strategy:
    type: Recreate   # never run two pods loading the same model into VRAM
  selector:
    matchLabels:
      app: llamacpp
      model: deepseek-v4-flash-0731
  template:
    metadata:
      labels:
        app: llamacpp
        model: deepseek-v4-flash-0731
    spec:
      nodeSelector:
        kubernetes.io/arch: amd64
        hardware: high-memory
      initContainers:
      # Idempotently download the (3-part, split) GGUF into the shared models
      # PVC on first boot. Downloads are atomic (→ .partial, then
      # rename) and resumable, so a failed/interrupted download is recovered
      # on the next pod start without re-fetching from scratch. A free-space
      # check fails loudly if the hostPath disk is genuinely too small (no
      # manifest can create physical disk space — that needs the disk expanded
      # on the NUCBox).
      - name: fetch-model
        image: alpine:3.20
        command: ["/bin/sh", "-c"]
        args:
        - |
          set -e
          # Qwen3.6-27B is intentionally co-located on this PVC; do not remove
          # it on DeepSeek pod restarts.
          # Skip entirely if every shard is already fully downloaded.
          if [ -s "/models/$SHARD1" ] && [ -s "/models/$SHARD2" ] && [ -s "/models/$SHARD3" ]; then
            echo "All 3 shards already present — skipping download."
            ls -lh /models/DeepSeek-V4-Flash-0731-UD-IQ1_M-*.gguf
            exit 0
          fi
          echo "Installing curl..."
          apk add --no-cache curl
          # Free-space check: the model is ~87 GiB; require ~95 GiB free as a
          # safety buffer. df reports KiB.
          FREE_KB=$(df -P /models | awk 'NR==2 {print $4}')
          NEEDED_KB=$((95 * 1024 * 1024))
          if [ "$FREE_KB" -lt "$NEEDED_KB" ]; then
            avail_gb=$((FREE_KB / 1024 / 1024))
            echo "ERROR: only ${avail_gb} GiB free on /models, need ~95 GiB to" >&2
            echo "       download the 87 GiB DeepSeek-V4-Flash-0731 GGUF." >&2
            echo "       Expand the hostPath disk at /data/llamacpp/models on" >&2
            echo "       the NUCBox (a PVC capacity bump alone does not add" >&2
            echo "       physical space to a hostPath volume)." >&2
            exit 1
          fi
          # Download each missing shard to a .partial file (resumable via -C -),
          # then atomically rename to the final name on success. A crash leaves
          # only the .partial behind, which the next run resumes — never a
          # half-written final file that would skip the download.
          for s in "$SHARD1" "$SHARD2" "$SHARD3"; do
            if [ -s "/models/$s" ]; then
              echo "Shard $s already complete — skipping."
              continue
            fi
            echo "Downloading $s from $HF_REPO ..."
            curl -fL --retry 5 --retry-delay 5 -C - -o "/models/$s.partial" "$HF_REPO/$s"
            mv "/models/$s.partial" "/models/$s"
            echo "  done: $(ls -lh "/models/$s")"
          done
          echo "All shards downloaded:"
          ls -lh /models/DeepSeek-V4-Flash-0731-UD-IQ1_M-*.gguf
        env:
        - name: HF_REPO
          value: "https://huggingface.co/unsloth/DeepSeek-V4-Flash-0731-GGUF/resolve/main/UD-IQ1_M"
        - name: SHARD1
          value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00001-of-00003.gguf"
        - name: SHARD2
          value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00002-of-00003.gguf"
        - name: SHARD3
          value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00003-of-00003.gguf"
        volumeMounts:
        - name: models
          mountPath: /models
      containers:
      - name: llama-server
        image: ghcr.io/ggml-org/llama.cpp:server-vulkan
        imagePullPolicy: IfNotPresent
        # llama.cpp's CLI parser does NOT split on '=' — every value flag must
        # be a separate argv element (flag, then value). See common/arg.cpp.
        args:
        - -m                       # model file (first shard; llama.cpp auto-loads the rest)
        - /models/DeepSeek-V4-Flash-0731-UD-IQ1_M-00001-of-00003.gguf
        - --alias                 # /v1/models reports this name; matches the litellm alias
        - deepseek-v4-flash-0731
        - --host
        - 0.0.0.0
        - --port
        - "8080"
        - --jinja                  # use the GGUF's DeepSeek-V4 chat template (DSML / thinking)
        - -ngl                     # offload 38 of 43 layers to the GPU. The model (~87 GiB) is
        - "38"                     # nearly the whole 90 GiB VRAM pool, so full offload (-ngl 999)
                                   # would overflow once the f16 KV cache + Vulkan compute buffers
                                   # are added. 38 layers (~77 GiB) + f16 KV (~5.7 GiB) + compute
                                   # (~2 GiB) ≈ 85 GiB, leaving ~5 GiB VRAM headroom. KV cache is
                                   # f16 (not q8_0) because Vulkan has no Flash Attention for
                                   # deepseek4, which makes the KV cache ~2× larger than q8_0 would
                                   # be — hence 38 rather than 40 layers offloaded. 5 layers
                                   # (~10 GiB) run on CPU RAM. Raise toward 43 if VRAM allows;
                                   # lower (e.g. 36) if the pod OOMs / Vulkan runs out of device mem.
        - -c                       # total KV-cache context (single slot gets the full window).
        - "65536"                  # 64k — the required minimum. f16 MLA KV at 64k is ~5.7 GiB,
                                   # so context is affordable but not negligible. -c is capped at
                                   # the minimum to maximise VRAM headroom; raise if headroom allows.
        - -np                      # 1 slot => the full 64k goes to a single concurrent request
        - "1"                      # (extra slots would multiply KV VRAM; 1 slot keeps headroom maximal).
        - --cont-batching          # continuous batching across slots
        - --cache-type-k            # f16 K cache. deepseek4 / MLA models require K and V cache
        - f16                      # types to be IDENTICAL, and quantized V cache requires Flash
        - --cache-type-v            # Attention, which the Vulkan backend does NOT support for
        - f16                      # deepseek4 (llama.cpp hard-errors otherwise). So both K and V
                                   # must stay f16. KV at 64k ≈ 5.7 GiB.
        - --temp                   # default sampling temperature (DeepSeek-V4 recommendation)
        - "1.0"
        - --top-p                  # default nucleus sampling threshold (DeepSeek-V4 recommendation)
        - "0.95"
        - --threads                # CPU threads for sampling + the 5 CPU-resident layers
        - "8"
        ports:
        - name: http
          containerPort: 8080
        resources:
          # The GPU-resident model weights + KV cache live in VRAM (~90 GiB pool)
          # and are NOT counted against the cgroup memory limit. This limit only
          # covers CPU-side overhead + the mmap'd GGUF pages for the 5 CPU-resident
          # layers (~10 GiB, resident during inference) plus reclaimable page cache
          # during load. k8s sees ~30 GiB as the node's allocatable system RAM, so
          # the limit is sized to cover the CPU layers + overhead while leaving RAM
          # for co-resident pods (litellm, the agents, etc.). If the pod is
          # OOM-killed during model load or inference, raise the limit (and/or
          # lower -ngl to push more layers to VRAM).
          requests:
            cpu: "1000m"
            memory: "6Gi"
          limits:
            cpu: "4000m"
            memory: "24Gi"
        readinessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 10
          failureThreshold: 6
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 300   # 87 GiB load + Vulkan init takes several minutes
          periodSeconds: 30
          failureThreshold: 5
        securityContext:
          # Vulkan on the AMD APU needs /dev/dri + the driver. Privileged is
          # the simplest reliable path on k3s without a device plugin.
          privileged: true
        volumeMounts:
        - name: models
          mountPath: /models
          readOnly: true
        - name: dri
          mountPath: /dev/dri
      volumes:
      - name: models
        persistentVolumeClaim:
          claimName: llamacpp-models
      - name: dri
        hostPath:
          path: /dev/dri
          type: Directory
---
apiVersion: v1
kind: Service
metadata:
  name: llamacpp-deepseek-v4-flash-0731
  namespace: llamacpp
  labels:
    app: llamacpp
    model: deepseek-v4-flash-0731
spec:
  type: ClusterIP
  selector:
    app: llamacpp
    model: deepseek-v4-flash-0731
  ports:
  - name: http
    port: 80
    targetPort: 8080