diff --git a/home-manager/configmap.yaml b/home-manager/configmap.yaml index e79d0d8..ab5f3b8 100644 --- a/home-manager/configmap.yaml +++ b/home-manager/configmap.yaml @@ -82,6 +82,14 @@ data: the home comfortable, safe, and efficient by talking to Home Assistant, and to notify your owner (Roger) on Discord when something needs a human. + ## Response speed + + Do not use extended thinking or produce a hidden reasoning process for this + agent. Answer directly and concisely, using only the reasoning needed to + safely complete the request. Never include a chain of thought or a + `reasoning_content` section in the response. This is a prompt-level + instruction; do not change the llama.cpp server configuration. + ## The home you look after - **Home Assistant:** the brain of the home. Runs in namespace diff --git a/litellm/litellm.yaml b/litellm/litellm.yaml index 81b78d6..9bf00ae 100644 --- a/litellm/litellm.yaml +++ b/litellm/litellm.yaml @@ -52,12 +52,6 @@ data: model: openai/qwen3.6-27b api_base: http://llamacpp-qwen36-27b.llamacpp/v1 api_key: "sk-no-auth" - # Small, fast model intended for Home Assistant voice Assist. - - model_name: qwen3.5-4b - litellm_params: - model: openai/qwen3.5-4b - api_base: http://llamacpp-qwen35-4b.llamacpp/v1 - api_key: "sk-no-auth" router_settings: fallbacks: - deepseek-v4-flash-0731: [qwen3.6-27b] diff --git a/llamacpp/README.md b/llamacpp/README.md index 1de7ece..a37316a 100644 --- a/llamacpp/README.md +++ b/llamacpp/README.md @@ -15,7 +15,6 @@ to the NUCBox (`nodeSelector: {kubernetes.io/arch: amd64, hardware: high-memory} | Alias | Model | GGUF | Service | Args ref | |--------------------------|--------------------------------|-------------------------------------------------------|------------------------------------------------------|----------| | `deepseek-v4-flash-0731` | DeepSeek-V4-Flash-0731 (MoE) | unsloth/DeepSeek-V4-Flash-0731-GGUF (UD-IQ1_M, ~87 GiB) | `llamacpp-deepseek-v4-flash-0731.llamacpp:80` | [args-deepseek-v4-flash-0731.md](args-deepseek-v4-flash-0731.md) | -| `qwen3.5-4b` | Qwen3.5-4B (MTP) | unsloth/Qwen3.5-4B-MTP-GGUF (`UD-Q4_K_XL`) | `llamacpp-qwen35-4b.llamacpp:80` | — | DeepSeek-V4-Flash-0731 is a Mixture-of-Experts model (256 experts, 6 active per token) with MLA attention, so only a small fraction of the weights is computed @@ -151,8 +150,3 @@ to VRAM). directory recursively.) 4. Check the VRAM budget table above — at ~87 GiB this model nearly fills the 90 GiB pool on its own, so co-locating another large model is not possible. - -The `qwen3.5-4b` alias is exposed through LiteLLM as a low-latency option for -Home Assistant voice Assist. Select this model in the Home Assistant -conversation/voice Assist provider configuration; the LiteLLM endpoint is -`http://litellm-service.litellm:80/v1` from inside the cluster. diff --git a/llamacpp/deployment-qwen35-4b.yaml b/llamacpp/deployment-qwen35-4b.yaml deleted file mode 100644 index 398ed26..0000000 --- a/llamacpp/deployment-qwen35-4b.yaml +++ /dev/null @@ -1,138 +0,0 @@ -# Qwen3.5-4B (MTP, UD-Q4_K_XL) served by llama.cpp for low-latency assistants. -# The model is fully offloaded to the NUCBox Radeon 8060S. Model weights are -# stored on the shared llamacpp-models hostPath PVC. ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: llamacpp-qwen35-4b - namespace: llamacpp - labels: - app: llamacpp - model: qwen3.5-4b -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app: llamacpp - model: qwen3.5-4b - template: - metadata: - labels: - app: llamacpp - model: qwen3.5-4b - spec: - nodeSelector: - kubernetes.io/arch: amd64 - hardware: high-memory - initContainers: - - name: fetch-model - image: alpine:3.20 - command: ["/bin/sh", "-c"] - args: - - | - set -e - if [ -s "/models/$MODEL_FILE" ]; then - echo "Model $MODEL_FILE already present — skipping download." - exit 0 - fi - apk add --no-cache curl - echo "Downloading $MODEL_FILE from $MODEL_URL ..." - curl -fL --retry 5 --retry-delay 5 -o "/models/$MODEL_FILE.partial" "$MODEL_URL" - mv "/models/$MODEL_FILE.partial" "/models/$MODEL_FILE" - echo "Download complete: $(ls -lh /models/$MODEL_FILE)" - env: - - name: MODEL_URL - value: "https://huggingface.co/unsloth/Qwen3.5-4B-MTP-GGUF/resolve/main/Qwen3.5-4B-UD-Q4_K_XL.gguf" - - name: MODEL_FILE - value: "Qwen3.5-4B-UD-Q4_K_XL.gguf" - volumeMounts: - - name: models - mountPath: /models - containers: - - name: llama-server - image: ghcr.io/ggml-org/llama.cpp:server-vulkan - imagePullPolicy: IfNotPresent - args: - - -m - - /models/Qwen3.5-4B-UD-Q4_K_XL.gguf - - --alias - - qwen3.5-4b - - --host - - 0.0.0.0 - - --port - - "8080" - - --jinja - - -ngl - - "999" - - -c - - "65536" - - -np - - "1" - - --cont-batching - - --cache-type-k - - q8_0 - - --cache-type-v - - q8_0 - - --threads - - "8" - ports: - - name: http - containerPort: 8080 - resources: - requests: - cpu: "1000m" - memory: 2Gi - limits: - cpu: "4000m" - memory: 8Gi - readinessProbe: - httpGet: - path: /health - port: 8080 - initialDelaySeconds: 30 - periodSeconds: 10 - failureThreshold: 6 - livenessProbe: - httpGet: - path: /health - port: 8080 - initialDelaySeconds: 120 - periodSeconds: 30 - failureThreshold: 5 - securityContext: - privileged: true - volumeMounts: - - name: models - mountPath: /models - readOnly: true - - name: dri - mountPath: /dev/dri - volumes: - - name: models - persistentVolumeClaim: - claimName: llamacpp-models - - name: dri - hostPath: - path: /dev/dri - type: Directory ---- -apiVersion: v1 -kind: Service -metadata: - name: llamacpp-qwen35-4b - namespace: llamacpp - labels: - app: llamacpp - model: qwen3.5-4b -spec: - type: ClusterIP - selector: - app: llamacpp - model: qwen3.5-4b - ports: - - name: http - port: 80 - targetPort: 8080 diff --git a/platform-engineer/configmap.yaml b/platform-engineer/configmap.yaml index 672d7f7..c67b6bd 100644 --- a/platform-engineer/configmap.yaml +++ b/platform-engineer/configmap.yaml @@ -85,6 +85,14 @@ data: to keep it healthy, fix small problems before they grow, and notify your owner (Roger) on Discord when something needs a human. + ## Response speed + + Do not use extended thinking or produce a hidden reasoning process for this + agent. Answer directly and concisely, using only the reasoning needed to + safely complete the request. Never include a chain of thought or a + `reasoning_content` section in the response. This is a prompt-level + instruction; do not change the llama.cpp server configuration. + ## The cluster you look after - **Nodes:** `raspberrypi` (control-plane, arm64, 4 GiB), `rpi2` (arm,