249 lines
12 KiB
YAML
249 lines
12 KiB
YAML
# DeepSeek-V4-Flash-0731 (MoE: 256 experts / 6 active, UD-IQ1_M ≈ 87 GiB)
|
||
# served by llama.cpp's llama-server on the NUCBox APU.
|
||
#
|
||
# Hardware: AMD Ryzen AI Max 395 (Strix Halo) — integrated Radeon 8060S,
|
||
# ~120 GiB unified memory (≈90 GiB VRAM / 30 GiB CPU RAM via firmware). The
|
||
# IQ1_M model (~87 GiB) is *almost* the size of the whole VRAM pool, so it
|
||
# CANNOT be fully offloaded to the GPU: offloading all 43 layers + the KV
|
||
# cache + Vulkan compute buffers would overflow 90 GiB. Instead we offload
|
||
# 38 of 43 layers (-ngl 38) and keep 5 layers (~10 GiB) on CPU RAM, leaving
|
||
# ~5 GiB of VRAM headroom for the KV cache + Vulkan compute buffers.
|
||
#
|
||
# VRAM is exclusive to this model (no other pod uses it); the other pods on
|
||
# the NUCBox only compete for the 30 GiB CPU RAM, so the headroom that
|
||
# matters here is VRAM headroom for compute buffers / fragmentation.
|
||
#
|
||
# KV CACHE MUST BE f16 (NOT quantized). The Vulkan backend has no Flash
|
||
# Attention for the deepseek4 arch, and quantized V cache requires Flash
|
||
# Attention (llama.cpp hard-errors otherwise: "quantized V cache was
|
||
# requested, but this requires Flash Attention"). Additionally, deepseek4 /
|
||
# MLA models require K and V cache types to be *identical*, so K cannot be
|
||
# quantized either. f16 KV at 64k is ~5.7 GiB (MLA KV: 576 K + 512 V
|
||
# elements/token/layer × 43 layers × 65536 tokens × 2 bytes). This is why
|
||
# -ngl is 38 rather than 40 — the larger f16 KV cache needs the extra VRAM.
|
||
#
|
||
# Image: ghcr.io/ggml-org/llama.cpp:server-vulkan bundles the Mesa/RADV Vulkan
|
||
# driver, which supports the Radeon 8060S (RDNA 3.5). The Vulkan backend
|
||
# supports the IQ1_M matmul (incl. the MoE matmul_id variant), so the whole
|
||
# model runs on the GPU. deepseek4 is a brand-new arch (2026-07); several
|
||
# deepseek4-specific fused ops (Lightning Indexer, HC pre/comb/post) are not
|
||
# yet implemented in Vulkan and fall back to CPU (logged as warnings, not
|
||
# fatal). The floating `server-vulkan` tag is used to pull a recent enough
|
||
# build; pin to a specific server-vulkan-bXXXX tag once a known-good one is
|
||
# verified.
|
||
#
|
||
# GPU access: the container mounts /dev/dri (the DRM render nodes) and runs
|
||
# privileged — the simplest reliable option on k3s without a Vulkan device
|
||
# plugin.
|
||
---
|
||
apiVersion: apps/v1
|
||
kind: Deployment
|
||
metadata:
|
||
name: llamacpp-deepseek-v4-flash-0731
|
||
namespace: llamacpp
|
||
labels:
|
||
app: llamacpp
|
||
model: deepseek-v4-flash-0731
|
||
spec:
|
||
replicas: 1
|
||
strategy:
|
||
type: Recreate # never run two pods loading the same model into VRAM
|
||
selector:
|
||
matchLabels:
|
||
app: llamacpp
|
||
model: deepseek-v4-flash-0731
|
||
template:
|
||
metadata:
|
||
labels:
|
||
app: llamacpp
|
||
model: deepseek-v4-flash-0731
|
||
spec:
|
||
nodeSelector:
|
||
kubernetes.io/arch: amd64
|
||
hardware: high-memory
|
||
initContainers:
|
||
# Idempotently download the (3-part, split) GGUF into the shared models
|
||
# PVC on first boot. Also removes the retired Qwen3.6 GGUFs so the new
|
||
# 87 GiB model fits on the PVC. Downloads are atomic (→ .partial, then
|
||
# rename) and resumable, so a failed/interrupted download is recovered
|
||
# on the next pod start without re-fetching from scratch. A free-space
|
||
# check fails loudly if the hostPath disk is genuinely too small (no
|
||
# manifest can create physical disk space — that needs the disk expanded
|
||
# on the NUCBox).
|
||
- name: fetch-model
|
||
image: alpine:3.20
|
||
command: ["/bin/sh", "-c"]
|
||
args:
|
||
- |
|
||
set -e
|
||
# Reclaim space from the retired Qwen3.6 models (their Deployments
|
||
# are gone; the GGUFs are dead weight on the shared PVC).
|
||
for old in Qwen3.6-27B-UD-Q4_K_XL.gguf Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf; do
|
||
if [ -f "/models/$old" ]; then
|
||
echo "Removing retired model $old ..."
|
||
rm -f "/models/$old"
|
||
fi
|
||
done
|
||
# Skip entirely if every shard is already fully downloaded.
|
||
if [ -s "/models/$SHARD1" ] && [ -s "/models/$SHARD2" ] && [ -s "/models/$SHARD3" ]; then
|
||
echo "All 3 shards already present — skipping download."
|
||
ls -lh /models/DeepSeek-V4-Flash-0731-UD-IQ1_M-*.gguf
|
||
exit 0
|
||
fi
|
||
echo "Installing curl..."
|
||
apk add --no-cache curl
|
||
# Free-space check: the model is ~87 GiB; require ~95 GiB free as a
|
||
# safety buffer. df reports KiB.
|
||
FREE_KB=$(df -P /models | awk 'NR==2 {print $4}')
|
||
NEEDED_KB=$((95 * 1024 * 1024))
|
||
if [ "$FREE_KB" -lt "$NEEDED_KB" ]; then
|
||
avail_gb=$((FREE_KB / 1024 / 1024))
|
||
echo "ERROR: only ${avail_gb} GiB free on /models, need ~95 GiB to" >&2
|
||
echo " download the 87 GiB DeepSeek-V4-Flash-0731 GGUF." >&2
|
||
echo " Expand the hostPath disk at /data/llamacpp/models on" >&2
|
||
echo " the NUCBox (a PVC capacity bump alone does not add" >&2
|
||
echo " physical space to a hostPath volume)." >&2
|
||
exit 1
|
||
fi
|
||
# Download each missing shard to a .partial file (resumable via -C -),
|
||
# then atomically rename to the final name on success. A crash leaves
|
||
# only the .partial behind, which the next run resumes — never a
|
||
# half-written final file that would skip the download.
|
||
for s in "$SHARD1" "$SHARD2" "$SHARD3"; do
|
||
if [ -s "/models/$s" ]; then
|
||
echo "Shard $s already complete — skipping."
|
||
continue
|
||
fi
|
||
echo "Downloading $s from $HF_REPO ..."
|
||
curl -fL --retry 5 --retry-delay 5 -C - -o "/models/$s.partial" "$HF_REPO/$s"
|
||
mv "/models/$s.partial" "/models/$s"
|
||
echo " done: $(ls -lh "/models/$s")"
|
||
done
|
||
echo "All shards downloaded:"
|
||
ls -lh /models/DeepSeek-V4-Flash-0731-UD-IQ1_M-*.gguf
|
||
env:
|
||
- name: HF_REPO
|
||
value: "https://huggingface.co/unsloth/DeepSeek-V4-Flash-0731-GGUF/resolve/main/UD-IQ1_M"
|
||
- name: SHARD1
|
||
value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00001-of-00003.gguf"
|
||
- name: SHARD2
|
||
value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00002-of-00003.gguf"
|
||
- name: SHARD3
|
||
value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00003-of-00003.gguf"
|
||
volumeMounts:
|
||
- name: models
|
||
mountPath: /models
|
||
containers:
|
||
- name: llama-server
|
||
image: ghcr.io/ggml-org/llama.cpp:server-vulkan
|
||
imagePullPolicy: IfNotPresent
|
||
# llama.cpp's CLI parser does NOT split on '=' — every value flag must
|
||
# be a separate argv element (flag, then value). See common/arg.cpp.
|
||
args:
|
||
- -m # model file (first shard; llama.cpp auto-loads the rest)
|
||
- /models/DeepSeek-V4-Flash-0731-UD-IQ1_M-00001-of-00003.gguf
|
||
- --alias # /v1/models reports this name; matches the litellm alias
|
||
- deepseek-v4-flash-0731
|
||
- --host
|
||
- 0.0.0.0
|
||
- --port
|
||
- "8080"
|
||
- --jinja # use the GGUF's DeepSeek-V4 chat template (DSML / thinking)
|
||
- -ngl # offload 38 of 43 layers to the GPU. The model (~87 GiB) is
|
||
- "38" # nearly the whole 90 GiB VRAM pool, so full offload (-ngl 999)
|
||
# would overflow once the f16 KV cache + Vulkan compute buffers
|
||
# are added. 38 layers (~77 GiB) + f16 KV (~5.7 GiB) + compute
|
||
# (~2 GiB) ≈ 85 GiB, leaving ~5 GiB VRAM headroom. KV cache is
|
||
# f16 (not q8_0) because Vulkan has no Flash Attention for
|
||
# deepseek4, which makes the KV cache ~2× larger than q8_0 would
|
||
# be — hence 38 rather than 40 layers offloaded. 5 layers
|
||
# (~10 GiB) run on CPU RAM. Raise toward 43 if VRAM allows;
|
||
# lower (e.g. 36) if the pod OOMs / Vulkan runs out of device mem.
|
||
- -c # total KV-cache context (single slot gets the full window).
|
||
- "65536" # 64k — the required minimum. f16 MLA KV at 64k is ~5.7 GiB,
|
||
# so context is affordable but not negligible. -c is capped at
|
||
# the minimum to maximise VRAM headroom; raise if headroom allows.
|
||
- -np # 1 slot => the full 64k goes to a single concurrent request
|
||
- "1" # (extra slots would multiply KV VRAM; 1 slot keeps headroom maximal).
|
||
- --cont-batching # continuous batching across slots
|
||
- --cache-type-k # f16 K cache. deepseek4 / MLA models require K and V cache
|
||
- f16 # types to be IDENTICAL, and quantized V cache requires Flash
|
||
- --cache-type-v # Attention, which the Vulkan backend does NOT support for
|
||
- f16 # deepseek4 (llama.cpp hard-errors otherwise). So both K and V
|
||
# must stay f16. KV at 64k ≈ 5.7 GiB.
|
||
- --temp # default sampling temperature (DeepSeek-V4 recommendation)
|
||
- "1.0"
|
||
- --top-p # default nucleus sampling threshold (DeepSeek-V4 recommendation)
|
||
- "0.95"
|
||
- --threads # CPU threads for sampling + the 5 CPU-resident layers
|
||
- "8"
|
||
ports:
|
||
- name: http
|
||
containerPort: 8080
|
||
resources:
|
||
# The GPU-resident model weights + KV cache live in VRAM (~90 GiB pool)
|
||
# and are NOT counted against the cgroup memory limit. This limit only
|
||
# covers CPU-side overhead + the mmap'd GGUF pages for the 5 CPU-resident
|
||
# layers (~10 GiB, resident during inference) plus reclaimable page cache
|
||
# during load. k8s sees ~30 GiB as the node's allocatable system RAM, so
|
||
# the limit is sized to cover the CPU layers + overhead while leaving RAM
|
||
# for co-resident pods (litellm, the agents, etc.). If the pod is
|
||
# OOM-killed during model load or inference, raise the limit (and/or
|
||
# lower -ngl to push more layers to VRAM).
|
||
requests:
|
||
cpu: "1000m"
|
||
memory: "6Gi"
|
||
limits:
|
||
cpu: "4000m"
|
||
memory: "24Gi"
|
||
readinessProbe:
|
||
httpGet:
|
||
path: /health
|
||
port: 8080
|
||
initialDelaySeconds: 30
|
||
periodSeconds: 10
|
||
failureThreshold: 6
|
||
livenessProbe:
|
||
httpGet:
|
||
path: /health
|
||
port: 8080
|
||
initialDelaySeconds: 300 # 87 GiB load + Vulkan init takes several minutes
|
||
periodSeconds: 30
|
||
failureThreshold: 5
|
||
securityContext:
|
||
# Vulkan on the AMD APU needs /dev/dri + the driver. Privileged is
|
||
# the simplest reliable path on k3s without a device plugin.
|
||
privileged: true
|
||
volumeMounts:
|
||
- name: models
|
||
mountPath: /models
|
||
readOnly: true
|
||
- name: dri
|
||
mountPath: /dev/dri
|
||
volumes:
|
||
- name: models
|
||
persistentVolumeClaim:
|
||
claimName: llamacpp-models
|
||
- name: dri
|
||
hostPath:
|
||
path: /dev/dri
|
||
type: Directory
|
||
---
|
||
apiVersion: v1
|
||
kind: Service
|
||
metadata:
|
||
name: llamacpp-deepseek-v4-flash-0731
|
||
namespace: llamacpp
|
||
labels:
|
||
app: llamacpp
|
||
model: deepseek-v4-flash-0731
|
||
spec:
|
||
type: ClusterIP
|
||
selector:
|
||
app: llamacpp
|
||
model: deepseek-v4-flash-0731
|
||
ports:
|
||
- name: http
|
||
port: 80
|
||
targetPort: 8080
|