deploy deepseek v4
This commit is contained in:
Roger Oriol
210
llamacpp/deployment-deepseek-v4-flash-0731.yaml
Normal file
210
llamacpp/deployment-deepseek-v4-flash-0731.yaml
Normal file
@@ -0,0 +1,210 @@
|
||||
# DeepSeek-V4-Flash-0731 (MoE: 256 experts / 6 active, UD-IQ1_M ≈ 87 GiB)
|
||||
# served by llama.cpp's llama-server on the NUCBox APU.
|
||||
#
|
||||
# Hardware: AMD Ryzen AI Max 395 (Strix Halo) — integrated Radeon 8060S,
|
||||
# ~120 GiB unified memory (≈90 GiB VRAM / 30 GiB CPU RAM via firmware). The
|
||||
# IQ1_M model (~87 GiB) is *almost* the size of the whole VRAM pool, so it
|
||||
# CANNOT be fully offloaded to the GPU: offloading all 43 layers + the KV
|
||||
# cache + Vulkan compute buffers would overflow 90 GiB. Instead we offload
|
||||
# 40 of 43 layers (-ngl 40) and keep the last 3 (~6 GiB) on CPU RAM, leaving
|
||||
# ~8 GiB of VRAM headroom for the KV cache, compute buffers, and co-resident
|
||||
# pods. This is the only model served on the NUCBox — the two Qwen3.6 models
|
||||
# were removed to make room (their GGUF files should be deleted from the PVC,
|
||||
# which the initContainer below does on first boot).
|
||||
#
|
||||
# KV cache is tiny thanks to DeepSeek-V4's MLA attention (num_kv_heads=1,
|
||||
# head_dim=512 + 64 decoupled RoPE ⇒ ~576 elements/token/layer). At 64k
|
||||
# context, q8_0 KV is only ~1.6 GiB, so context is cheap — but we cap -c at
|
||||
# 65536 (the required minimum) to maximise VRAM headroom, not because KV is
|
||||
# the constraint.
|
||||
#
|
||||
# Image: ghcr.io/ggml-org/llama.cpp:server-vulkan bundles the Mesa/RADV Vulkan
|
||||
# driver, which supports the Radeon 8060S (RDNA 3.5). The Vulkan backend
|
||||
# supports the IQ1_M matmul (incl. the MoE matmul_id variant), so the whole
|
||||
# model runs on the GPU. deepseek4 is a brand-new arch (2026-07) so the
|
||||
# floating `server-vulkan` tag is used to pull a recent enough build; pin to a
|
||||
# specific server-vulkan-bXXXX tag once a known-good one is verified.
|
||||
#
|
||||
# GPU access: the container mounts /dev/dri (the DRM render nodes) and runs
|
||||
# privileged — the simplest reliable option on k3s without a Vulkan device
|
||||
# plugin.
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: llamacpp-deepseek-v4-flash-0731
|
||||
namespace: llamacpp
|
||||
labels:
|
||||
app: llamacpp
|
||||
model: deepseek-v4-flash-0731
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate # never run two pods loading the same model into VRAM
|
||||
selector:
|
||||
matchLabels:
|
||||
app: llamacpp
|
||||
model: deepseek-v4-flash-0731
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: llamacpp
|
||||
model: deepseek-v4-flash-0731
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: amd64
|
||||
hardware: high-memory
|
||||
initContainers:
|
||||
# Idempotently download the (3-part, split) GGUF into the shared models
|
||||
# PVC on first boot. Also removes the retired Qwen3.6 GGUFs so the new
|
||||
# 87 GiB model fits on the PVC alongside any other data. Exits
|
||||
# immediately if the first shard is already present (pod restart).
|
||||
- name: fetch-model
|
||||
image: alpine:3.20
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
# Reclaim space from the retired Qwen3.6 models (their Deployments
|
||||
# are gone; the GGUFs are dead weight on the shared PVC).
|
||||
for old in Qwen3.6-27B-UD-Q4_K_XL.gguf Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf; do
|
||||
if [ -f "/models/$old" ]; then
|
||||
echo "Removing retired model $old ..."
|
||||
rm -f "/models/$old"
|
||||
fi
|
||||
done
|
||||
# Download any missing shards of the split UD-IQ1_M GGUF.
|
||||
if [ -f "/models/$SHARD1" ]; then
|
||||
echo "First shard $SHARD1 already present — skipping download."
|
||||
exit 0
|
||||
fi
|
||||
echo "Installing curl..."
|
||||
apk add --no-cache curl
|
||||
for s in "$SHARD1" "$SHARD2" "$SHARD3"; do
|
||||
echo "Downloading $s from $HF_REPO ..."
|
||||
curl -fL --retry 5 --retry-delay 5 -o "/models/$s" "$HF_REPO/$s"
|
||||
done
|
||||
echo "Download complete:"
|
||||
ls -lh /models/DeepSeek-V4-Flash-0731-UD-IQ1_M-*.gguf
|
||||
env:
|
||||
- name: HF_REPO
|
||||
value: "https://huggingface.co/unsloth/DeepSeek-V4-Flash-0731-GGUF/resolve/main/UD-IQ1_M"
|
||||
- name: SHARD1
|
||||
value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00001-of-00003.gguf"
|
||||
- name: SHARD2
|
||||
value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00002-of-00003.gguf"
|
||||
- name: SHARD3
|
||||
value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00003-of-00003.gguf"
|
||||
volumeMounts:
|
||||
- name: models
|
||||
mountPath: /models
|
||||
containers:
|
||||
- name: llama-server
|
||||
image: ghcr.io/ggml-org/llama.cpp:server-vulkan
|
||||
imagePullPolicy: IfNotPresent
|
||||
# llama.cpp's CLI parser does NOT split on '=' — every value flag must
|
||||
# be a separate argv element (flag, then value). See common/arg.cpp.
|
||||
args:
|
||||
- -m # model file (first shard; llama.cpp auto-loads the rest)
|
||||
- /models/DeepSeek-V4-Flash-0731-UD-IQ1_M-00001-of-00003.gguf
|
||||
- --alias # /v1/models reports this name; matches the litellm alias
|
||||
- deepseek-v4-flash-0731
|
||||
- --host
|
||||
- 0.0.0.0
|
||||
- --port
|
||||
- "8080"
|
||||
- --jinja # use the GGUF's DeepSeek-V4 chat template (DSML / thinking)
|
||||
- -ngl # offload 40 of 43 layers to the GPU. The model (~87 GiB) is
|
||||
- "40" # nearly the whole 90 GiB VRAM pool, so full offload (-ngl 999)
|
||||
# would overflow once KV cache + Vulkan compute buffers are
|
||||
# added. Keeping 3 layers (~6 GiB) on CPU leaves ~8 GiB of
|
||||
# VRAM headroom for the KV cache, compute buffers, and
|
||||
# co-resident pods. Raise toward 43 if VRAM allows; lower
|
||||
# (e.g. 38) if the pod OOMs / Vulkan runs out of device mem.
|
||||
- -c # total KV-cache context (single slot gets the full window).
|
||||
- "65536" # 64k — the required minimum. MLA KV is tiny (~1.6 GiB at
|
||||
# q8_0), so context is cheap; -c is capped at the minimum to
|
||||
# maximise VRAM headroom, not because KV is the constraint.
|
||||
# Raise if VRAM headroom allows.
|
||||
- -np # 1 slot => the full 64k goes to a single concurrent request
|
||||
- "1" # (extra slots would multiply KV VRAM, which is fine here, but
|
||||
# 1 slot keeps it simple and headroom maximal).
|
||||
- --cont-batching # continuous batching across slots
|
||||
- --cache-type-k # quantize KV cache to q8_0 — MLA KV is already small (~576
|
||||
- q8_0 # elem/token/layer); q8_0 halves it to ~1.6 GiB at 64k and
|
||||
- --cache-type-v # maximises VRAM headroom with ~negligible quality loss.
|
||||
- q8_0
|
||||
- --temp # default sampling temperature (DeepSeek-V4 recommendation)
|
||||
- "1.0"
|
||||
- --top-p # default nucleus sampling threshold (DeepSeek-V4 recommendation)
|
||||
- "0.95"
|
||||
- --threads # CPU threads for sampling + the 3 CPU-resident layers
|
||||
- "8"
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
resources:
|
||||
# The model weights + KV cache live in GPU VRAM (~90 GiB pool) and
|
||||
# are NOT counted against the cgroup memory limit. This limit only
|
||||
# covers CPU-side overhead + the mmap'd GGUF pages for the 3
|
||||
# CPU-resident layers (~6 GiB) plus reclaimable page cache during
|
||||
# load. k8s sees ~30 GiB as the node's allocatable system RAM, so the
|
||||
# limit is sized to cover the CPU layers + overhead while leaving
|
||||
# RAM for co-resident pods (litellm, the agents, etc.). If the pod is
|
||||
# OOM-killed during model load, raise the limit.
|
||||
requests:
|
||||
cpu: "1000m"
|
||||
memory: "4Gi"
|
||||
limits:
|
||||
cpu: "4000m"
|
||||
memory: "20Gi"
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
failureThreshold: 6
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
initialDelaySeconds: 300 # 87 GiB load + Vulkan init takes several minutes
|
||||
periodSeconds: 30
|
||||
failureThreshold: 5
|
||||
securityContext:
|
||||
# Vulkan on the AMD APU needs /dev/dri + the driver. Privileged is
|
||||
# the simplest reliable path on k3s without a device plugin.
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: models
|
||||
mountPath: /models
|
||||
readOnly: true
|
||||
- name: dri
|
||||
mountPath: /dev/dri
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: llamacpp-models
|
||||
- name: dri
|
||||
hostPath:
|
||||
path: /dev/dri
|
||||
type: Directory
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: llamacpp-deepseek-v4-flash-0731
|
||||
namespace: llamacpp
|
||||
labels:
|
||||
app: llamacpp
|
||||
model: deepseek-v4-flash-0731
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: llamacpp
|
||||
model: deepseek-v4-flash-0731
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: 8080
|
||||
Reference in New Issue
Block a user