new llamacpp service

litellm/litellm.yaml

@@ -40,19 +40,15 @@ data:
           model: ollama/glm-4.7-flash
           api_base: http://10.88.20.12:11434
       # Used by the platform-engineer Hermes agent (deployed in ns platform-engineer).
-      # model_name is the alias Hermes requests; the underlying Ollama model is
-      # qwen3.6:latest (the fast non-27b tag). 27b is a slow reasoning model.
-      # `ollama_chat/` (not `ollama/`) uses Ollama's NATIVE /api/chat endpoint.
-      # `think: false` + `chat_template_kwargs.enable_thinking: false` disable
-      # Qwen3 thinking so the model emits content directly (otherwise the
-      # OpenAI-compat translation returns empty content with reasoning split off).
+      # Served by llama.cpp (llama-server, Vulkan on the NUCBox APU) — see
+      # llamacpp/deployment-qwen36.yaml. Thinking is left ON; reasoning comes
+      # back in `reasoning_content` (llama-server --jinja + default reasoning
+      # format). The alias `qwen3.6` is unchanged so Hermes keeps working.
       - model_name: qwen3.6
         litellm_params:
-          model: ollama_chat/qwen3.6:latest
-          api_base: http://10.88.20.12:11434
-          think: false
-          chat_template_kwargs:
-            enable_thinking: false
+          model: openai/qwen3.6
+          api_base: http://llamacpp-qwen36.llamacpp/v1
+          api_key: "sk-no-auth"
     litellm_settings:
       #set_verbose: True  # Uncomment this if you want to see verbose logs; not recommended in production
       callbacks: ["arize_phoenix"]