211 lines
9.0 KiB
YAML
211 lines
9.0 KiB
YAML
# DeepSeek-V4-Flash-0731 (MoE: 256 experts / 6 active, UD-IQ1_M ≈ 87 GiB)
|
|
# served by llama.cpp's llama-server on the NUCBox APU.
|
|
#
|
|
# Hardware: AMD Ryzen AI Max 395 (Strix Halo) — integrated Radeon 8060S,
|
|
# ~120 GiB unified memory (≈90 GiB VRAM / 30 GiB CPU RAM via firmware). The
|
|
# IQ1_M model (~87 GiB) is *almost* the size of the whole VRAM pool, so it
|
|
# CANNOT be fully offloaded to the GPU: offloading all 43 layers + the KV
|
|
# cache + Vulkan compute buffers would overflow 90 GiB. Instead we offload
|
|
# 40 of 43 layers (-ngl 40) and keep the last 3 (~6 GiB) on CPU RAM, leaving
|
|
# ~8 GiB of VRAM headroom for the KV cache, compute buffers, and co-resident
|
|
# pods. This is the only model served on the NUCBox — the two Qwen3.6 models
|
|
# were removed to make room (their GGUF files should be deleted from the PVC,
|
|
# which the initContainer below does on first boot).
|
|
#
|
|
# KV cache is tiny thanks to DeepSeek-V4's MLA attention (num_kv_heads=1,
|
|
# head_dim=512 + 64 decoupled RoPE ⇒ ~576 elements/token/layer). At 64k
|
|
# context, q8_0 KV is only ~1.6 GiB, so context is cheap — but we cap -c at
|
|
# 65536 (the required minimum) to maximise VRAM headroom, not because KV is
|
|
# the constraint.
|
|
#
|
|
# Image: ghcr.io/ggml-org/llama.cpp:server-vulkan bundles the Mesa/RADV Vulkan
|
|
# driver, which supports the Radeon 8060S (RDNA 3.5). The Vulkan backend
|
|
# supports the IQ1_M matmul (incl. the MoE matmul_id variant), so the whole
|
|
# model runs on the GPU. deepseek4 is a brand-new arch (2026-07) so the
|
|
# floating `server-vulkan` tag is used to pull a recent enough build; pin to a
|
|
# specific server-vulkan-bXXXX tag once a known-good one is verified.
|
|
#
|
|
# GPU access: the container mounts /dev/dri (the DRM render nodes) and runs
|
|
# privileged — the simplest reliable option on k3s without a Vulkan device
|
|
# plugin.
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: llamacpp-deepseek-v4-flash-0731
|
|
namespace: llamacpp
|
|
labels:
|
|
app: llamacpp
|
|
model: deepseek-v4-flash-0731
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate # never run two pods loading the same model into VRAM
|
|
selector:
|
|
matchLabels:
|
|
app: llamacpp
|
|
model: deepseek-v4-flash-0731
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: llamacpp
|
|
model: deepseek-v4-flash-0731
|
|
spec:
|
|
nodeSelector:
|
|
kubernetes.io/arch: amd64
|
|
hardware: high-memory
|
|
initContainers:
|
|
# Idempotently download the (3-part, split) GGUF into the shared models
|
|
# PVC on first boot. Also removes the retired Qwen3.6 GGUFs so the new
|
|
# 87 GiB model fits on the PVC alongside any other data. Exits
|
|
# immediately if the first shard is already present (pod restart).
|
|
- name: fetch-model
|
|
image: alpine:3.20
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
set -e
|
|
# Reclaim space from the retired Qwen3.6 models (their Deployments
|
|
# are gone; the GGUFs are dead weight on the shared PVC).
|
|
for old in Qwen3.6-27B-UD-Q4_K_XL.gguf Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf; do
|
|
if [ -f "/models/$old" ]; then
|
|
echo "Removing retired model $old ..."
|
|
rm -f "/models/$old"
|
|
fi
|
|
done
|
|
# Download any missing shards of the split UD-IQ1_M GGUF.
|
|
if [ -f "/models/$SHARD1" ]; then
|
|
echo "First shard $SHARD1 already present — skipping download."
|
|
exit 0
|
|
fi
|
|
echo "Installing curl..."
|
|
apk add --no-cache curl
|
|
for s in "$SHARD1" "$SHARD2" "$SHARD3"; do
|
|
echo "Downloading $s from $HF_REPO ..."
|
|
curl -fL --retry 5 --retry-delay 5 -o "/models/$s" "$HF_REPO/$s"
|
|
done
|
|
echo "Download complete:"
|
|
ls -lh /models/DeepSeek-V4-Flash-0731-UD-IQ1_M-*.gguf
|
|
env:
|
|
- name: HF_REPO
|
|
value: "https://huggingface.co/unsloth/DeepSeek-V4-Flash-0731-GGUF/resolve/main/UD-IQ1_M"
|
|
- name: SHARD1
|
|
value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00001-of-00003.gguf"
|
|
- name: SHARD2
|
|
value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00002-of-00003.gguf"
|
|
- name: SHARD3
|
|
value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00003-of-00003.gguf"
|
|
volumeMounts:
|
|
- name: models
|
|
mountPath: /models
|
|
containers:
|
|
- name: llama-server
|
|
image: ghcr.io/ggml-org/llama.cpp:server-vulkan
|
|
imagePullPolicy: IfNotPresent
|
|
# llama.cpp's CLI parser does NOT split on '=' — every value flag must
|
|
# be a separate argv element (flag, then value). See common/arg.cpp.
|
|
args:
|
|
- -m # model file (first shard; llama.cpp auto-loads the rest)
|
|
- /models/DeepSeek-V4-Flash-0731-UD-IQ1_M-00001-of-00003.gguf
|
|
- --alias # /v1/models reports this name; matches the litellm alias
|
|
- deepseek-v4-flash-0731
|
|
- --host
|
|
- 0.0.0.0
|
|
- --port
|
|
- "8080"
|
|
- --jinja # use the GGUF's DeepSeek-V4 chat template (DSML / thinking)
|
|
- -ngl # offload 40 of 43 layers to the GPU. The model (~87 GiB) is
|
|
- "40" # nearly the whole 90 GiB VRAM pool, so full offload (-ngl 999)
|
|
# would overflow once KV cache + Vulkan compute buffers are
|
|
# added. Keeping 3 layers (~6 GiB) on CPU leaves ~8 GiB of
|
|
# VRAM headroom for the KV cache, compute buffers, and
|
|
# co-resident pods. Raise toward 43 if VRAM allows; lower
|
|
# (e.g. 38) if the pod OOMs / Vulkan runs out of device mem.
|
|
- -c # total KV-cache context (single slot gets the full window).
|
|
- "65536" # 64k — the required minimum. MLA KV is tiny (~1.6 GiB at
|
|
# q8_0), so context is cheap; -c is capped at the minimum to
|
|
# maximise VRAM headroom, not because KV is the constraint.
|
|
# Raise if VRAM headroom allows.
|
|
- -np # 1 slot => the full 64k goes to a single concurrent request
|
|
- "1" # (extra slots would multiply KV VRAM, which is fine here, but
|
|
# 1 slot keeps it simple and headroom maximal).
|
|
- --cont-batching # continuous batching across slots
|
|
- --cache-type-k # quantize KV cache to q8_0 — MLA KV is already small (~576
|
|
- q8_0 # elem/token/layer); q8_0 halves it to ~1.6 GiB at 64k and
|
|
- --cache-type-v # maximises VRAM headroom with ~negligible quality loss.
|
|
- q8_0
|
|
- --temp # default sampling temperature (DeepSeek-V4 recommendation)
|
|
- "1.0"
|
|
- --top-p # default nucleus sampling threshold (DeepSeek-V4 recommendation)
|
|
- "0.95"
|
|
- --threads # CPU threads for sampling + the 3 CPU-resident layers
|
|
- "8"
|
|
ports:
|
|
- name: http
|
|
containerPort: 8080
|
|
resources:
|
|
# The model weights + KV cache live in GPU VRAM (~90 GiB pool) and
|
|
# are NOT counted against the cgroup memory limit. This limit only
|
|
# covers CPU-side overhead + the mmap'd GGUF pages for the 3
|
|
# CPU-resident layers (~6 GiB) plus reclaimable page cache during
|
|
# load. k8s sees ~30 GiB as the node's allocatable system RAM, so the
|
|
# limit is sized to cover the CPU layers + overhead while leaving
|
|
# RAM for co-resident pods (litellm, the agents, etc.). If the pod is
|
|
# OOM-killed during model load, raise the limit.
|
|
requests:
|
|
cpu: "1000m"
|
|
memory: "4Gi"
|
|
limits:
|
|
cpu: "4000m"
|
|
memory: "20Gi"
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8080
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 10
|
|
failureThreshold: 6
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8080
|
|
initialDelaySeconds: 300 # 87 GiB load + Vulkan init takes several minutes
|
|
periodSeconds: 30
|
|
failureThreshold: 5
|
|
securityContext:
|
|
# Vulkan on the AMD APU needs /dev/dri + the driver. Privileged is
|
|
# the simplest reliable path on k3s without a device plugin.
|
|
privileged: true
|
|
volumeMounts:
|
|
- name: models
|
|
mountPath: /models
|
|
readOnly: true
|
|
- name: dri
|
|
mountPath: /dev/dri
|
|
volumes:
|
|
- name: models
|
|
persistentVolumeClaim:
|
|
claimName: llamacpp-models
|
|
- name: dri
|
|
hostPath:
|
|
path: /dev/dri
|
|
type: Directory
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: llamacpp-deepseek-v4-flash-0731
|
|
namespace: llamacpp
|
|
labels:
|
|
app: llamacpp
|
|
model: deepseek-v4-flash-0731
|
|
spec:
|
|
type: ClusterIP
|
|
selector:
|
|
app: llamacpp
|
|
model: deepseek-v4-flash-0731
|
|
ports:
|
|
- name: http
|
|
port: 80
|
|
targetPort: 8080
|