roger/k3s-cluster
1
0
Fork 1
Code Issues Pull Requests 6 Actions Packages Projects Releases Wiki Activity

fix llamacpp

Browse Source
This commit is contained in:
Roger Oriol
2026-07-23 00:26:58 +02:00
parent 70e4baf5e3
commit 8ef9db1e37
1 changed files with 18 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
llamacpp/deployment-qwen36.yaml
View File

@@ -70,17 +70,27 @@ spec:
- name: llama-server - name: llama-server
image: ghcr.io/ggml-org/llama.cpp:server-vulkan image: ghcr.io/ggml-org/llama.cpp:server-vulkan
imagePullPolicy: IfNotPresent imagePullPolicy: IfNotPresent
# llama.cpp's CLI parser does NOT split on '=' — every value flag must be a
# separate argv element (flag, then value). See common/arg.cpp in the repo.
args: args:
- -m=/models/Qwen3.6-27B-UD-Q4_K_XL.gguf - -m # model file
- --alias=qwen3.6 # /v1/models reports this name; matches the litellm alias - /models/Qwen3.6-27B-UD-Q4_K_XL.gguf
- --host=0.0.0.0 - --alias # /v1/models reports this name; matches the litellm alias
- --port=8080 - qwen3.6
- --host
- 0.0.0.0
- --port
- "8080"
- --jinja # use the GGUF's chat template (Qwen3 thinking format) - --jinja # use the GGUF's chat template (Qwen3 thinking format)
- -ngl=999 # offload ALL layers to the GPU (fits in 96 GiB VRAM) - -ngl # offload ALL layers to the GPU (fits in 96 GiB VRAM)
- -c=32768 # total KV-cache context, split across parallel slots - "999"
- -np=4 # 4 parallel slots => 8192 tokens per concurrent request - -c # total KV-cache context, split across parallel slots
- "32768"
- -np # 4 parallel slots => 8192 tokens per concurrent request
- "4"
- --cont-batching # continuous batching across slots - --cont-batching # continuous batching across slots
- --threads=8 # CPU threads for sampling/overhead (GPU does the heavy lifting) - --threads # CPU threads for sampling/overhead (GPU does the heavy lifting)
- "8"
ports: ports:
- name: http - name: http
containerPort: 8080 containerPort: 8080