roger/k3s-cluster
1
0
Fork 1
Code Issues Pull Requests 6 Actions Packages Projects Releases Wiki Activity

fix lfm 2.5

Browse Source
This commit is contained in:
Roger Oriol
2026-08-22 11:41:42 +02:00
parent b268b6cf99
commit 70bc473280
1 changed files with 9 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
llamacpp/deployment-lfm25-26b.yaml
View File

@@ -1,5 +1,5 @@
# LiquidAI LFM2.5-2.6B with speculative decoding served by llama.cpp. # LiquidAI LFM2.5-2.6B served by llama.cpp (Vulkan, no speculative decoding).
# The primary and draft GGUFs are downloaded into the shared model PVC before # The primary GGUF is downloaded into the shared model PVC before
# llama-server starts. # llama-server starts.
--- ---
apiVersion: apps/v1 apiVersion: apps/v1
@@ -36,8 +36,7 @@ spec:
set -eu set -eu
apk add --no-cache curl apk add --no-cache curl
for entry in \ for entry in \
"https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF/resolve/main/LFM2.5-2.6B-Q4_K_M.gguf|LFM2.5-2.6B-Q4_K_M.gguf" \ "https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF/resolve/main/LFM2.5-2.6B-Q4_K_M.gguf|LFM2.5-2.6B-Q4_K_M.gguf"; do
"https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF/resolve/main/LFM2.5-2.6B-Q4_0.gguf|LFM2.5-2.6B-Q4_0.gguf"; do
url=${entry%%|*} url=${entry%%|*}
file=${entry##*|} file=${entry##*|}
if [ -s "/models/$file" ]; then if [ -s "/models/$file" ]; then
@@ -60,13 +59,13 @@ spec:
args: args:
- -m - -m
- /models/LFM2.5-2.6B-Q4_K_M.gguf - /models/LFM2.5-2.6B-Q4_K_M.gguf
- --model-draft # Draft model + speculative decoding removed: the spec path crashes the
- /models/LFM2.5-2.6B-Q4_0.gguf # server (GGML_ASSERT slot.spec_i_batch...) when the KV cache is under
- --spec-default # pressure, taking all in-flight requests (incl. home-manager crons) down.
- --spec-type
- draft-simple
- --ctx-size - --ctx-size
- "121000" - "80000"
- --parallel
- "2"
- --temp - --temp
- "0.1" - "0.1"
- --cache-type-k - --cache-type-k