forked from roger/k3s-cluster
140 lines
3.4 KiB
YAML
140 lines
3.4 KiB
YAML
# Qwen3.6-27B (dense, Q4_K_XL) served by llama.cpp alongside DeepSeek.
|
|
# The model is fully offloaded to the NUCBox Radeon 8060S; Qwen weights and
|
|
# DeepSeek weights share the llamacpp-models hostPath PVC.
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: llamacpp-qwen36-27b
|
|
namespace: llamacpp
|
|
labels:
|
|
app: llamacpp
|
|
model: qwen3.6-27b
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app: llamacpp
|
|
model: qwen3.6-27b
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: llamacpp
|
|
model: qwen3.6-27b
|
|
spec:
|
|
nodeSelector:
|
|
kubernetes.io/arch: amd64
|
|
hardware: high-memory
|
|
initContainers:
|
|
- name: fetch-model
|
|
image: alpine:3.20
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
set -e
|
|
if [ -s "/models/$MODEL_FILE" ]; then
|
|
echo "Model $MODEL_FILE already present — skipping download."
|
|
exit 0
|
|
fi
|
|
echo "Installing curl..."
|
|
apk add --no-cache curl
|
|
echo "Downloading $MODEL_FILE from $MODEL_URL ..."
|
|
curl -fL --retry 5 --retry-delay 5 -o "/models/$MODEL_FILE.partial" "$MODEL_URL"
|
|
mv "/models/$MODEL_FILE.partial" "/models/$MODEL_FILE"
|
|
echo "Download complete: $(ls -lh /models/$MODEL_FILE)"
|
|
env:
|
|
- name: MODEL_URL
|
|
value: "https://huggingface.co/unsloth/Qwen3.6-27B-MTP-GGUF/resolve/main/Qwen3.6-27B-UD-Q4_K_XL.gguf"
|
|
- name: MODEL_FILE
|
|
value: "Qwen3.6-27B-UD-Q4_K_XL.gguf"
|
|
volumeMounts:
|
|
- name: models
|
|
mountPath: /models
|
|
containers:
|
|
- name: llama-server
|
|
image: ghcr.io/ggml-org/llama.cpp:server-vulkan
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- -m
|
|
- /models/Qwen3.6-27B-UD-Q4_K_XL.gguf
|
|
- --alias
|
|
- qwen3.6-27b
|
|
- --host
|
|
- 0.0.0.0
|
|
- --port
|
|
- "8080"
|
|
- --jinja
|
|
- -ngl
|
|
- "999"
|
|
- -c
|
|
- "131072"
|
|
- -np
|
|
- "2"
|
|
- --cont-batching
|
|
- --cache-type-k
|
|
- q8_0
|
|
- --cache-type-v
|
|
- q8_0
|
|
- --threads
|
|
- "8"
|
|
ports:
|
|
- name: http
|
|
containerPort: 8080
|
|
resources:
|
|
requests:
|
|
cpu: "1000m"
|
|
memory: 2Gi
|
|
limits:
|
|
cpu: "4000m"
|
|
memory: 24Gi
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8080
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 10
|
|
failureThreshold: 6
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8080
|
|
initialDelaySeconds: 180
|
|
periodSeconds: 30
|
|
failureThreshold: 5
|
|
securityContext:
|
|
privileged: true
|
|
volumeMounts:
|
|
- name: models
|
|
mountPath: /models
|
|
readOnly: true
|
|
- name: dri
|
|
mountPath: /dev/dri
|
|
volumes:
|
|
- name: models
|
|
persistentVolumeClaim:
|
|
claimName: llamacpp-models
|
|
- name: dri
|
|
hostPath:
|
|
path: /dev/dri
|
|
type: Directory
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: llamacpp-qwen36-27b
|
|
namespace: llamacpp
|
|
labels:
|
|
app: llamacpp
|
|
model: qwen3.6-27b
|
|
spec:
|
|
type: ClusterIP
|
|
selector:
|
|
app: llamacpp
|
|
model: qwen3.6-27b
|
|
ports:
|
|
- name: http
|
|
port: 80
|
|
targetPort: 8080
|