roger/k3s-cluster
1
0
Fork 1
Code Issues Pull Requests 6 Actions Packages Projects Releases Wiki Activity

fix llamacpp

Browse Source
This commit is contained in:
Roger Oriol
2026-07-23 00:26:58 +02:00
parent 70e4baf5e3
commit 8ef9db1e37
1 changed files with 18 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
llamacpp/deployment-qwen36.yaml
View File

@@ -70,17 +70,27 @@ spec:
- name: llama-server
image: ghcr.io/ggml-org/llama.cpp:server-vulkan
imagePullPolicy: IfNotPresent
# llama.cpp's CLI parser does NOT split on '=' — every value flag must be a
# separate argv element (flag, then value). See common/arg.cpp in the repo.
args:
- -m=/models/Qwen3.6-27B-UD-Q4_K_XL.gguf
- --alias=qwen3.6 # /v1/models reports this name; matches the litellm alias
- --host=0.0.0.0
- --port=8080
- -m # model file
- /models/Qwen3.6-27B-UD-Q4_K_XL.gguf
- --alias # /v1/models reports this name; matches the litellm alias
- qwen3.6
- --host
- 0.0.0.0
- --port
- "8080"
- --jinja # use the GGUF's chat template (Qwen3 thinking format)
- -ngl=999 # offload ALL layers to the GPU (fits in 96 GiB VRAM)
- -c=32768 # total KV-cache context, split across parallel slots
- -np=4 # 4 parallel slots => 8192 tokens per concurrent request
- -ngl # offload ALL layers to the GPU (fits in 96 GiB VRAM)
- "999"
- -c # total KV-cache context, split across parallel slots
- "32768"
- -np # 4 parallel slots => 8192 tokens per concurrent request
- "4"
- --cont-batching # continuous batching across slots
- --threads=8 # CPU threads for sampling/overhead (GPU does the heavy lifting)
- --threads # CPU threads for sampling/overhead (GPU does the heavy lifting)
- "8"
ports:
- name: http
containerPort: 8080