redeploy qwen3.6 27b

litellm/litellm.yaml

@@ -39,18 +39,19 @@ data:
         litellm_params:
           model: ollama/glm-4.7-flash
           api_base: http://10.88.20.12:11434
-      # In-cluster LLM served by llama.cpp (llama-server, Vulkan on the
+      # In-cluster models served by llama.cpp (llama-server, Vulkan on the
-      # NUCBox APU) — see llamacpp/deployment-deepseek-v4-flash-0731.yaml.
+      # NUCBox APU). The chat template is applied via --jinja; reasoning (when
-      # DeepSeek-V4-Flash-0731 (UD-IQ1_M, ~87 GiB) is the sole model on the
+      # enabled) comes back in `reasoning_content`.
-      # NUCBox; the two Qwen3.6 models were removed to make VRAM room. The
-      # chat template is applied via llama-server --jinja; reasoning (when
-      # thinking is enabled) comes back in `reasoning_content`. Default
-      # sampling (temp 1.0 / top-p 0.95) is set on the llama-server itself.
       - model_name: deepseek-v4-flash-0731
         litellm_params:
           model: openai/deepseek-v4-flash-0731
           api_base: http://llamacpp-deepseek-v4-flash-0731.llamacpp/v1
           api_key: "sk-no-auth"
+      - model_name: qwen3.6-27b
+        litellm_params:
+          model: openai/qwen3.6-27b
+          api_base: http://llamacpp-qwen36-27b.llamacpp/v1
+          api_key: "sk-no-auth"
     litellm_settings:
       #set_verbose: True  # Uncomment this if you want to see verbose logs; not recommended in production
       callbacks: ["arize_phoenix"]

llamacpp/deployment-deepseek-v4-flash-0731.yaml

@@ -63,8 +63,7 @@ spec:
         hardware: high-memory
       initContainers:
       # Idempotently download the (3-part, split) GGUF into the shared models
-      # PVC on first boot. Also removes the retired Qwen3.6 GGUFs so the new
+      # PVC on first boot. Downloads are atomic (→ .partial, then
-      # 87 GiB model fits on the PVC. Downloads are atomic (→ .partial, then
       # rename) and resumable, so a failed/interrupted download is recovered
       # on the next pod start without re-fetching from scratch. A free-space
       # check fails loudly if the hostPath disk is genuinely too small (no
@@ -76,14 +75,8 @@ spec:
         args:
         - |
           set -e
-          # Reclaim space from the retired Qwen3.6 models (their Deployments
+          # Qwen3.6-27B is intentionally co-located on this PVC; do not remove
-          # are gone; the GGUFs are dead weight on the shared PVC).
+          # it on DeepSeek pod restarts.
-          for old in Qwen3.6-27B-UD-Q4_K_XL.gguf Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf; do
-            if [ -f "/models/$old" ]; then
-              echo "Removing retired model $old ..."
-              rm -f "/models/$old"
-            fi
-          done
           # Skip entirely if every shard is already fully downloaded.
           if [ -s "/models/$SHARD1" ] && [ -s "/models/$SHARD2" ] && [ -s "/models/$SHARD3" ]; then
             echo "All 3 shards already present — skipping download."

llamacpp/deployment-qwen36-27b.yaml

@@ -0,0 +1,139 @@
+# Qwen3.6-27B (dense, Q4_K_XL) served by llama.cpp alongside DeepSeek.
+# The model is fully offloaded to the NUCBox Radeon 8060S; Qwen weights and
+# DeepSeek weights share the llamacpp-models hostPath PVC.
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: llamacpp-qwen36-27b
+  namespace: llamacpp
+  labels:
+    app: llamacpp
+    model: qwen3.6-27b
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: llamacpp
+      model: qwen3.6-27b
+  template:
+    metadata:
+      labels:
+        app: llamacpp
+        model: qwen3.6-27b
+    spec:
+      nodeSelector:
+        kubernetes.io/arch: amd64
+        hardware: high-memory
+      initContainers:
+      - name: fetch-model
+        image: alpine:3.20
+        command: ["/bin/sh", "-c"]
+        args:
+        - |
+          set -e
+          if [ -s "/models/$MODEL_FILE" ]; then
+            echo "Model $MODEL_FILE already present — skipping download."
+            exit 0
+          fi
+          echo "Installing curl..."
+          apk add --no-cache curl
+          echo "Downloading $MODEL_FILE from $MODEL_URL ..."
+          curl -fL --retry 5 --retry-delay 5 -o "/models/$MODEL_FILE.partial" "$MODEL_URL"
+          mv "/models/$MODEL_FILE.partial" "/models/$MODEL_FILE"
+          echo "Download complete: $(ls -lh /models/$MODEL_FILE)"
+        env:
+        - name: MODEL_URL
+          value: "https://huggingface.co/unsloth/Qwen3.6-27B-MTP-GGUF/resolve/main/Qwen3.6-27B-UD-Q4_K_XL.gguf"
+        - name: MODEL_FILE
+          value: "Qwen3.6-27B-UD-Q4_K_XL.gguf"
+        volumeMounts:
+        - name: models
+          mountPath: /models
+      containers:
+      - name: llama-server
+        image: ghcr.io/ggml-org/llama.cpp:server-vulkan
+        imagePullPolicy: IfNotPresent
+        args:
+        - -m
+        - /models/Qwen3.6-27B-UD-Q4_K_XL.gguf
+        - --alias
+        - qwen3.6-27b
+        - --host
+        - 0.0.0.0
+        - --port
+        - "8080"
+        - --jinja
+        - -ngl
+        - "999"
+        - -c
+        - "131072"
+        - -np
+        - "1"
+        - --cont-batching
+        - --cache-type-k
+        - q8_0
+        - --cache-type-v
+        - q8_0
+        - --threads
+        - "8"
+        ports:
+        - name: http
+          containerPort: 8080
+        resources:
+          requests:
+            cpu: "1000m"
+            memory: 2Gi
+          limits:
+            cpu: "4000m"
+            memory: 24Gi
+        readinessProbe:
+          httpGet:
+            path: /health
+            port: 8080
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          failureThreshold: 6
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: 8080
+          initialDelaySeconds: 180
+          periodSeconds: 30
+          failureThreshold: 5
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: models
+          mountPath: /models
+          readOnly: true
+        - name: dri
+          mountPath: /dev/dri
+      volumes:
+      - name: models
+        persistentVolumeClaim:
+          claimName: llamacpp-models
+      - name: dri
+        hostPath:
+          path: /dev/dri
+          type: Directory
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: llamacpp-qwen36-27b
+  namespace: llamacpp
+  labels:
+    app: llamacpp
+    model: qwen3.6-27b
+spec:
+  type: ClusterIP
+  selector:
+    app: llamacpp
+    model: qwen3.6-27b
+  ports:
+  - name: http
+    port: 80
+    targetPort: 8080