new llamacpp service
This commit is contained in:
Roger Oriol
148
llamacpp/deployment-qwen36.yaml
Normal file
148
llamacpp/deployment-qwen36.yaml
Normal file
@@ -0,0 +1,148 @@
|
||||
# Qwen3.6 (27B, Q4_K_XL) served by llama.cpp's llama-server on the NUCBox APU.
|
||||
#
|
||||
# Hardware: AMD Ryzen AI Max 395 (Strix Halo) — integrated Radeon 8060S,
|
||||
# 128 GiB unified memory (32 GiB RAM / 96 GiB VRAM via firmware). The Q4 model
|
||||
# (~16 GiB) is fully offloaded to the GPU via the Vulkan backend, leaving plenty
|
||||
# of VRAM for the KV cache.
|
||||
#
|
||||
# Image: ghcr.io/ggerganov/llama.cpp:server-vulkan bundles the Mesa/RADV Vulkan
|
||||
# driver, which supports the Radeon 8060S (RDNA 3.5). Pin to a digest for
|
||||
# production once the rolling tag is confirmed working.
|
||||
#
|
||||
# GPU access: the container mounts /dev/dri (the DRM render nodes) and runs
|
||||
# privileged. This is the simplest reliable option on k3s without a Vulkan
|
||||
# device plugin; tighten later with supplementalGroups if desired.
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: llamacpp-qwen36
|
||||
namespace: llamacpp
|
||||
labels:
|
||||
app: llamacpp
|
||||
model: qwen3.6
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate # never run two pods loading the same model into VRAM
|
||||
selector:
|
||||
matchLabels:
|
||||
app: llamacpp
|
||||
model: qwen3.6
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: llamacpp
|
||||
model: qwen3.6
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: amd64
|
||||
hardware: high-memory
|
||||
initContainers:
|
||||
# Idempotently download the GGUF into the shared models PVC on first boot.
|
||||
# Exits immediately if the file is already present (pod restart / recreate).
|
||||
- name: fetch-model
|
||||
image: alpine:3.20
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
if [ -f "/models/$MODEL_FILE" ]; then
|
||||
echo "Model $MODEL_FILE already present — skipping download."
|
||||
exit 0
|
||||
fi
|
||||
echo "Installing curl..."
|
||||
apk add --no-cache curl
|
||||
echo "Downloading $MODEL_FILE from $MODEL_URL ..."
|
||||
curl -fL --retry 5 --retry-delay 5 -o "/models/$MODEL_FILE" "$MODEL_URL"
|
||||
echo "Download complete: $(ls -lh /models/$MODEL_FILE)"
|
||||
env:
|
||||
- name: MODEL_URL
|
||||
value: "https://huggingface.co/unsloth/Qwen3.6-27B-MTP-GGUF/resolve/main/Qwen3.6-27B-UD-Q4_K_XL.gguf"
|
||||
- name: MODEL_FILE
|
||||
value: "Qwen3.6-27B-UD-Q4_K_XL.gguf"
|
||||
volumeMounts:
|
||||
- name: models
|
||||
mountPath: /models
|
||||
containers:
|
||||
- name: llama-server
|
||||
image: ghcr.io/ggerganov/llama.cpp:server-vulkan
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- -m=/models/Qwen3.6-27B-UD-Q4_K_XL.gguf
|
||||
- --alias=qwen3.6 # /v1/models reports this name; matches the litellm alias
|
||||
- --host=0.0.0.0
|
||||
- --port=8080
|
||||
- --jinja # use the GGUF's chat template (Qwen3 thinking format)
|
||||
- -ngl=999 # offload ALL layers to the GPU (fits in 96 GiB VRAM)
|
||||
- -c=32768 # total KV-cache context, split across parallel slots
|
||||
- -np=4 # 4 parallel slots => 8192 tokens per concurrent request
|
||||
- --cont-batching # continuous batching across slots
|
||||
- --threads=8 # CPU threads for sampling/overhead (GPU does the heavy lifting)
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
resources:
|
||||
# The model weights + KV cache live in GPU VRAM (96 GiB pool) and are
|
||||
# NOT counted against the cgroup memory limit. This limit only covers
|
||||
# CPU-side overhead + the mmap'd GGUF file pages during load (~16 GiB,
|
||||
# reclaimable). k8s sees ~32 GiB as the node's allocatable system RAM,
|
||||
# so the request is kept low to stay schedulable alongside other pods.
|
||||
# If the pod OOM-kills during load, the amdgpu driver may be counting
|
||||
# some VRAM against the cgroup — raise the limit.
|
||||
requests:
|
||||
cpu: "1000m"
|
||||
memory: "2Gi"
|
||||
limits:
|
||||
cpu: "4000m"
|
||||
memory: "24Gi"
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
failureThreshold: 6
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
initialDelaySeconds: 180 # model load + Vulkan init can take a few minutes
|
||||
periodSeconds: 30
|
||||
failureThreshold: 5
|
||||
securityContext:
|
||||
# Vulkan on the AMD APU needs /dev/dri + the driver. Privileged is the
|
||||
# simplest reliable path on k3s without a device plugin.
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: models
|
||||
mountPath: /models
|
||||
readOnly: true
|
||||
- name: dri
|
||||
mountPath: /dev/dri
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: llamacpp-models
|
||||
- name: dri
|
||||
hostPath:
|
||||
path: /dev/dri
|
||||
type: Directory
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: llamacpp-qwen36
|
||||
namespace: llamacpp
|
||||
labels:
|
||||
app: llamacpp
|
||||
model: qwen3.6
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: llamacpp
|
||||
model: qwen3.6
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: 8080
|
||||
Reference in New Issue
Block a user