remove qwen 3.5 4b and disable thinking in agent crons
This commit is contained in:
@@ -82,6 +82,14 @@ data:
|
||||
the home comfortable, safe, and efficient by talking to Home Assistant, and
|
||||
to notify your owner (Roger) on Discord when something needs a human.
|
||||
|
||||
## Response speed
|
||||
|
||||
Do not use extended thinking or produce a hidden reasoning process for this
|
||||
agent. Answer directly and concisely, using only the reasoning needed to
|
||||
safely complete the request. Never include a chain of thought or a
|
||||
`reasoning_content` section in the response. This is a prompt-level
|
||||
instruction; do not change the llama.cpp server configuration.
|
||||
|
||||
## The home you look after
|
||||
|
||||
- **Home Assistant:** the brain of the home. Runs in namespace
|
||||
|
||||
@@ -52,12 +52,6 @@ data:
|
||||
model: openai/qwen3.6-27b
|
||||
api_base: http://llamacpp-qwen36-27b.llamacpp/v1
|
||||
api_key: "sk-no-auth"
|
||||
# Small, fast model intended for Home Assistant voice Assist.
|
||||
- model_name: qwen3.5-4b
|
||||
litellm_params:
|
||||
model: openai/qwen3.5-4b
|
||||
api_base: http://llamacpp-qwen35-4b.llamacpp/v1
|
||||
api_key: "sk-no-auth"
|
||||
router_settings:
|
||||
fallbacks:
|
||||
- deepseek-v4-flash-0731: [qwen3.6-27b]
|
||||
|
||||
@@ -15,7 +15,6 @@ to the NUCBox (`nodeSelector: {kubernetes.io/arch: amd64, hardware: high-memory}
|
||||
| Alias | Model | GGUF | Service | Args ref |
|
||||
|--------------------------|--------------------------------|-------------------------------------------------------|------------------------------------------------------|----------|
|
||||
| `deepseek-v4-flash-0731` | DeepSeek-V4-Flash-0731 (MoE) | unsloth/DeepSeek-V4-Flash-0731-GGUF (UD-IQ1_M, ~87 GiB) | `llamacpp-deepseek-v4-flash-0731.llamacpp:80` | [args-deepseek-v4-flash-0731.md](args-deepseek-v4-flash-0731.md) |
|
||||
| `qwen3.5-4b` | Qwen3.5-4B (MTP) | unsloth/Qwen3.5-4B-MTP-GGUF (`UD-Q4_K_XL`) | `llamacpp-qwen35-4b.llamacpp:80` | — |
|
||||
|
||||
DeepSeek-V4-Flash-0731 is a Mixture-of-Experts model (256 experts, 6 active per
|
||||
token) with MLA attention, so only a small fraction of the weights is computed
|
||||
@@ -151,8 +150,3 @@ to VRAM).
|
||||
directory recursively.)
|
||||
4. Check the VRAM budget table above — at ~87 GiB this model nearly fills the
|
||||
90 GiB pool on its own, so co-locating another large model is not possible.
|
||||
|
||||
The `qwen3.5-4b` alias is exposed through LiteLLM as a low-latency option for
|
||||
Home Assistant voice Assist. Select this model in the Home Assistant
|
||||
conversation/voice Assist provider configuration; the LiteLLM endpoint is
|
||||
`http://litellm-service.litellm:80/v1` from inside the cluster.
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
# Qwen3.5-4B (MTP, UD-Q4_K_XL) served by llama.cpp for low-latency assistants.
|
||||
# The model is fully offloaded to the NUCBox Radeon 8060S. Model weights are
|
||||
# stored on the shared llamacpp-models hostPath PVC.
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: llamacpp-qwen35-4b
|
||||
namespace: llamacpp
|
||||
labels:
|
||||
app: llamacpp
|
||||
model: qwen3.5-4b
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: llamacpp
|
||||
model: qwen3.5-4b
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: llamacpp
|
||||
model: qwen3.5-4b
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: amd64
|
||||
hardware: high-memory
|
||||
initContainers:
|
||||
- name: fetch-model
|
||||
image: alpine:3.20
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
if [ -s "/models/$MODEL_FILE" ]; then
|
||||
echo "Model $MODEL_FILE already present — skipping download."
|
||||
exit 0
|
||||
fi
|
||||
apk add --no-cache curl
|
||||
echo "Downloading $MODEL_FILE from $MODEL_URL ..."
|
||||
curl -fL --retry 5 --retry-delay 5 -o "/models/$MODEL_FILE.partial" "$MODEL_URL"
|
||||
mv "/models/$MODEL_FILE.partial" "/models/$MODEL_FILE"
|
||||
echo "Download complete: $(ls -lh /models/$MODEL_FILE)"
|
||||
env:
|
||||
- name: MODEL_URL
|
||||
value: "https://huggingface.co/unsloth/Qwen3.5-4B-MTP-GGUF/resolve/main/Qwen3.5-4B-UD-Q4_K_XL.gguf"
|
||||
- name: MODEL_FILE
|
||||
value: "Qwen3.5-4B-UD-Q4_K_XL.gguf"
|
||||
volumeMounts:
|
||||
- name: models
|
||||
mountPath: /models
|
||||
containers:
|
||||
- name: llama-server
|
||||
image: ghcr.io/ggml-org/llama.cpp:server-vulkan
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- -m
|
||||
- /models/Qwen3.5-4B-UD-Q4_K_XL.gguf
|
||||
- --alias
|
||||
- qwen3.5-4b
|
||||
- --host
|
||||
- 0.0.0.0
|
||||
- --port
|
||||
- "8080"
|
||||
- --jinja
|
||||
- -ngl
|
||||
- "999"
|
||||
- -c
|
||||
- "65536"
|
||||
- -np
|
||||
- "1"
|
||||
- --cont-batching
|
||||
- --cache-type-k
|
||||
- q8_0
|
||||
- --cache-type-v
|
||||
- q8_0
|
||||
- --threads
|
||||
- "8"
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1000m"
|
||||
memory: 2Gi
|
||||
limits:
|
||||
cpu: "4000m"
|
||||
memory: 8Gi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
failureThreshold: 6
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
initialDelaySeconds: 120
|
||||
periodSeconds: 30
|
||||
failureThreshold: 5
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: models
|
||||
mountPath: /models
|
||||
readOnly: true
|
||||
- name: dri
|
||||
mountPath: /dev/dri
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: llamacpp-models
|
||||
- name: dri
|
||||
hostPath:
|
||||
path: /dev/dri
|
||||
type: Directory
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: llamacpp-qwen35-4b
|
||||
namespace: llamacpp
|
||||
labels:
|
||||
app: llamacpp
|
||||
model: qwen3.5-4b
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: llamacpp
|
||||
model: qwen3.5-4b
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: 8080
|
||||
@@ -85,6 +85,14 @@ data:
|
||||
to keep it healthy, fix small problems before they grow, and notify your
|
||||
owner (Roger) on Discord when something needs a human.
|
||||
|
||||
## Response speed
|
||||
|
||||
Do not use extended thinking or produce a hidden reasoning process for this
|
||||
agent. Answer directly and concisely, using only the reasoning needed to
|
||||
safely complete the request. Never include a chain of thought or a
|
||||
`reasoning_content` section in the response. This is a prompt-level
|
||||
instruction; do not change the llama.cpp server configuration.
|
||||
|
||||
## The cluster you look after
|
||||
|
||||
- **Nodes:** `raspberrypi` (control-plane, arm64, 4 GiB), `rpi2` (arm,
|
||||
|
||||
Reference in New Issue
Block a user