diff --git a/litellm/litellm.yaml b/litellm/litellm.yaml index a4189c6..e3ca218 100644 --- a/litellm/litellm.yaml +++ b/litellm/litellm.yaml @@ -39,18 +39,19 @@ data: litellm_params: model: ollama/glm-4.7-flash api_base: http://10.88.20.12:11434 - # In-cluster LLM served by llama.cpp (llama-server, Vulkan on the - # NUCBox APU) — see llamacpp/deployment-deepseek-v4-flash-0731.yaml. - # DeepSeek-V4-Flash-0731 (UD-IQ1_M, ~87 GiB) is the sole model on the - # NUCBox; the two Qwen3.6 models were removed to make VRAM room. The - # chat template is applied via llama-server --jinja; reasoning (when - # thinking is enabled) comes back in `reasoning_content`. Default - # sampling (temp 1.0 / top-p 0.95) is set on the llama-server itself. + # In-cluster models served by llama.cpp (llama-server, Vulkan on the + # NUCBox APU). The chat template is applied via --jinja; reasoning (when + # enabled) comes back in `reasoning_content`. - model_name: deepseek-v4-flash-0731 litellm_params: model: openai/deepseek-v4-flash-0731 api_base: http://llamacpp-deepseek-v4-flash-0731.llamacpp/v1 api_key: "sk-no-auth" + - model_name: qwen3.6-27b + litellm_params: + model: openai/qwen3.6-27b + api_base: http://llamacpp-qwen36-27b.llamacpp/v1 + api_key: "sk-no-auth" litellm_settings: #set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production callbacks: ["arize_phoenix"] diff --git a/llamacpp/deployment-deepseek-v4-flash-0731.yaml b/llamacpp/deployment-deepseek-v4-flash-0731.yaml index a98bd26..14b07d4 100644 --- a/llamacpp/deployment-deepseek-v4-flash-0731.yaml +++ b/llamacpp/deployment-deepseek-v4-flash-0731.yaml @@ -63,8 +63,7 @@ spec: hardware: high-memory initContainers: # Idempotently download the (3-part, split) GGUF into the shared models - # PVC on first boot. Also removes the retired Qwen3.6 GGUFs so the new - # 87 GiB model fits on the PVC. Downloads are atomic (→ .partial, then + # PVC on first boot. Downloads are atomic (→ .partial, then # rename) and resumable, so a failed/interrupted download is recovered # on the next pod start without re-fetching from scratch. A free-space # check fails loudly if the hostPath disk is genuinely too small (no @@ -76,14 +75,8 @@ spec: args: - | set -e - # Reclaim space from the retired Qwen3.6 models (their Deployments - # are gone; the GGUFs are dead weight on the shared PVC). - for old in Qwen3.6-27B-UD-Q4_K_XL.gguf Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf; do - if [ -f "/models/$old" ]; then - echo "Removing retired model $old ..." - rm -f "/models/$old" - fi - done + # Qwen3.6-27B is intentionally co-located on this PVC; do not remove + # it on DeepSeek pod restarts. # Skip entirely if every shard is already fully downloaded. if [ -s "/models/$SHARD1" ] && [ -s "/models/$SHARD2" ] && [ -s "/models/$SHARD3" ]; then echo "All 3 shards already present — skipping download." diff --git a/llamacpp/deployment-qwen36-27b.yaml b/llamacpp/deployment-qwen36-27b.yaml new file mode 100644 index 0000000..ec9f8b3 --- /dev/null +++ b/llamacpp/deployment-qwen36-27b.yaml @@ -0,0 +1,139 @@ +# Qwen3.6-27B (dense, Q4_K_XL) served by llama.cpp alongside DeepSeek. +# The model is fully offloaded to the NUCBox Radeon 8060S; Qwen weights and +# DeepSeek weights share the llamacpp-models hostPath PVC. +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llamacpp-qwen36-27b + namespace: llamacpp + labels: + app: llamacpp + model: qwen3.6-27b +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: llamacpp + model: qwen3.6-27b + template: + metadata: + labels: + app: llamacpp + model: qwen3.6-27b + spec: + nodeSelector: + kubernetes.io/arch: amd64 + hardware: high-memory + initContainers: + - name: fetch-model + image: alpine:3.20 + command: ["/bin/sh", "-c"] + args: + - | + set -e + if [ -s "/models/$MODEL_FILE" ]; then + echo "Model $MODEL_FILE already present — skipping download." + exit 0 + fi + echo "Installing curl..." + apk add --no-cache curl + echo "Downloading $MODEL_FILE from $MODEL_URL ..." + curl -fL --retry 5 --retry-delay 5 -o "/models/$MODEL_FILE.partial" "$MODEL_URL" + mv "/models/$MODEL_FILE.partial" "/models/$MODEL_FILE" + echo "Download complete: $(ls -lh /models/$MODEL_FILE)" + env: + - name: MODEL_URL + value: "https://huggingface.co/unsloth/Qwen3.6-27B-MTP-GGUF/resolve/main/Qwen3.6-27B-UD-Q4_K_XL.gguf" + - name: MODEL_FILE + value: "Qwen3.6-27B-UD-Q4_K_XL.gguf" + volumeMounts: + - name: models + mountPath: /models + containers: + - name: llama-server + image: ghcr.io/ggml-org/llama.cpp:server-vulkan + imagePullPolicy: IfNotPresent + args: + - -m + - /models/Qwen3.6-27B-UD-Q4_K_XL.gguf + - --alias + - qwen3.6-27b + - --host + - 0.0.0.0 + - --port + - "8080" + - --jinja + - -ngl + - "999" + - -c + - "131072" + - -np + - "1" + - --cont-batching + - --cache-type-k + - q8_0 + - --cache-type-v + - q8_0 + - --threads + - "8" + ports: + - name: http + containerPort: 8080 + resources: + requests: + cpu: "1000m" + memory: 2Gi + limits: + cpu: "4000m" + memory: 24Gi + readinessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 180 + periodSeconds: 30 + failureThreshold: 5 + securityContext: + privileged: true + volumeMounts: + - name: models + mountPath: /models + readOnly: true + - name: dri + mountPath: /dev/dri + volumes: + - name: models + persistentVolumeClaim: + claimName: llamacpp-models + - name: dri + hostPath: + path: /dev/dri + type: Directory +--- +apiVersion: v1 +kind: Service +metadata: + name: llamacpp-qwen36-27b + namespace: llamacpp + labels: + app: llamacpp + model: qwen3.6-27b +spec: + type: ClusterIP + selector: + app: llamacpp + model: qwen3.6-27b + ports: + - name: http + port: 80 + targetPort: 8080