diff --git a/llamacpp/deployment-qwen36.yaml b/llamacpp/deployment-qwen36.yaml index 29d40af..5e4c599 100644 --- a/llamacpp/deployment-qwen36.yaml +++ b/llamacpp/deployment-qwen36.yaml @@ -70,17 +70,27 @@ spec: - name: llama-server image: ghcr.io/ggml-org/llama.cpp:server-vulkan imagePullPolicy: IfNotPresent + # llama.cpp's CLI parser does NOT split on '=' — every value flag must be a + # separate argv element (flag, then value). See common/arg.cpp in the repo. args: - - -m=/models/Qwen3.6-27B-UD-Q4_K_XL.gguf - - --alias=qwen3.6 # /v1/models reports this name; matches the litellm alias - - --host=0.0.0.0 - - --port=8080 + - -m # model file + - /models/Qwen3.6-27B-UD-Q4_K_XL.gguf + - --alias # /v1/models reports this name; matches the litellm alias + - qwen3.6 + - --host + - 0.0.0.0 + - --port + - "8080" - --jinja # use the GGUF's chat template (Qwen3 thinking format) - - -ngl=999 # offload ALL layers to the GPU (fits in 96 GiB VRAM) - - -c=32768 # total KV-cache context, split across parallel slots - - -np=4 # 4 parallel slots => 8192 tokens per concurrent request + - -ngl # offload ALL layers to the GPU (fits in 96 GiB VRAM) + - "999" + - -c # total KV-cache context, split across parallel slots + - "32768" + - -np # 4 parallel slots => 8192 tokens per concurrent request + - "4" - --cont-batching # continuous batching across slots - - --threads=8 # CPU threads for sampling/overhead (GPU does the heavy lifting) + - --threads # CPU threads for sampling/overhead (GPU does the heavy lifting) + - "8" ports: - name: http containerPort: 8080