set platform engineer url to platform-engineer.rogi.casa
This commit is contained in:
Roger Oriol
158
llamacpp/deployment-qwen36-35b-a3b.yaml
Normal file
158
llamacpp/deployment-qwen36-35b-a3b.yaml
Normal file
@@ -0,0 +1,158 @@
|
||||
# Qwen3.6-35B-A3B (MoE: 35B total / 3B active, Q4_K_XL) — the "flash" variant.
|
||||
#
|
||||
# Despite having more total parameters than the 27B dense model, only 3B are
|
||||
# active per token (Mixture-of-Experts), so inference is much faster. The full
|
||||
# ~20 GiB of Q4 weights is still loaded into VRAM but only a small fraction is
|
||||
# computed per token.
|
||||
#
|
||||
# Hardware: AMD Ryzen AI Max 395 (Strix Halo) — integrated Radeon 8060S,
|
||||
# 128 GiB unified memory (32 GiB RAM / 96 GiB VRAM via firmware). Full GPU
|
||||
# offload via the Vulkan backend. Shares the 96 GiB VRAM pool with the 27B
|
||||
# model — see llamacpp/README.md for the combined VRAM budget.
|
||||
#
|
||||
# Image: ghcr.io/ggml-org/llama.cpp:server-vulkan (Mesa/RADV Vulkan driver,
|
||||
# supports the Radeon 8060S / RDNA 3.5). Pin to a build tag for production.
|
||||
#
|
||||
# GPU access: mounts /dev/dri + privileged (simplest reliable path on k3s).
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: llamacpp-qwen36-35b-a3b
|
||||
namespace: llamacpp
|
||||
labels:
|
||||
app: llamacpp
|
||||
model: qwen3.6-35b-a3b
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate # never run two pods loading the same model into VRAM
|
||||
selector:
|
||||
matchLabels:
|
||||
app: llamacpp
|
||||
model: qwen3.6-35b-a3b
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: llamacpp
|
||||
model: qwen3.6-35b-a3b
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: amd64
|
||||
hardware: high-memory
|
||||
initContainers:
|
||||
# Idempotently download the GGUF into the shared models PVC on first boot.
|
||||
# Exits immediately if the file is already present (pod restart / recreate).
|
||||
- name: fetch-model
|
||||
image: alpine:3.20
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
if [ -f "/models/$MODEL_FILE" ]; then
|
||||
echo "Model $MODEL_FILE already present — skipping download."
|
||||
exit 0
|
||||
fi
|
||||
echo "Installing curl..."
|
||||
apk add --no-cache curl
|
||||
echo "Downloading $MODEL_FILE from $MODEL_URL ..."
|
||||
curl -fL --retry 5 --retry-delay 5 -o "/models/$MODEL_FILE" "$MODEL_URL"
|
||||
echo "Download complete: $(ls -lh /models/$MODEL_FILE)"
|
||||
env:
|
||||
- name: MODEL_URL
|
||||
value: "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/resolve/main/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf"
|
||||
- name: MODEL_FILE
|
||||
value: "Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf"
|
||||
volumeMounts:
|
||||
- name: models
|
||||
mountPath: /models
|
||||
containers:
|
||||
- name: llama-server
|
||||
image: ghcr.io/ggml-org/llama.cpp:server-vulkan
|
||||
imagePullPolicy: IfNotPresent
|
||||
# llama.cpp's CLI parser does NOT split on '=' — every value flag must be a
|
||||
# separate argv element (flag, then value). See common/arg.cpp in the repo.
|
||||
args:
|
||||
- -m # model file
|
||||
- /models/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf
|
||||
- --alias # /v1/models reports this name; matches the litellm alias
|
||||
- qwen3.6-35b-a3b
|
||||
- --host
|
||||
- 0.0.0.0
|
||||
- --port
|
||||
- "8080"
|
||||
- --jinja # use the GGUF's chat template (Qwen3 thinking format)
|
||||
- -ngl # offload ALL layers to the GPU (fits in 96 GiB VRAM)
|
||||
- "999"
|
||||
- -c # total KV-cache context, split across parallel slots
|
||||
- "32768"
|
||||
- -np # 4 parallel slots => 8192 tokens per concurrent request
|
||||
- "4"
|
||||
- --cont-batching # continuous batching across slots
|
||||
- --threads # CPU threads for sampling/overhead (GPU does the heavy lifting)
|
||||
- "8"
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
resources:
|
||||
# The model weights + KV cache live in GPU VRAM (96 GiB pool) and are
|
||||
# NOT counted against the cgroup memory limit. This limit only covers
|
||||
# CPU-side overhead + the mmap'd GGUF file pages during load (~20 GiB,
|
||||
# reclaimable). k8s sees ~32 GiB as the node's allocatable system RAM.
|
||||
# If the pod OOM-kills during load, raise the limit.
|
||||
requests:
|
||||
cpu: "1000m"
|
||||
memory: "2Gi"
|
||||
limits:
|
||||
cpu: "4000m"
|
||||
memory: "24Gi"
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
failureThreshold: 6
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
initialDelaySeconds: 180 # model load + Vulkan init can take a few minutes
|
||||
periodSeconds: 30
|
||||
failureThreshold: 5
|
||||
securityContext:
|
||||
# Vulkan on the AMD APU needs /dev/dri + the driver. Privileged is the
|
||||
# simplest reliable path on k3s without a device plugin.
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: models
|
||||
mountPath: /models
|
||||
readOnly: true
|
||||
- name: dri
|
||||
mountPath: /dev/dri
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: llamacpp-models
|
||||
- name: dri
|
||||
hostPath:
|
||||
path: /dev/dri
|
||||
type: Directory
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: llamacpp-qwen36-35b-a3b
|
||||
namespace: llamacpp
|
||||
labels:
|
||||
app: llamacpp
|
||||
model: qwen3.6-35b-a3b
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: llamacpp
|
||||
model: qwen3.6-35b-a3b
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: 8080
|
||||
Reference in New Issue
Block a user