deploy deepseek v4

litellm/litellm.yaml

@@ -39,22 +39,17 @@ data:
         litellm_params:
           model: ollama/glm-4.7-flash
           api_base: http://10.88.20.12:11434
-      # Used by the platform-engineer Hermes agent (deployed in ns platform-engineer).
-      # Served by llama.cpp (llama-server, Vulkan on the NUCBox APU) — see
-      # llamacpp/deployment-qwen36-27b.yaml. Thinking is left ON; reasoning comes
-      # back in `reasoning_content` (llama-server --jinja + default reasoning format).
-      - model_name: qwen3.6-27b
+      # In-cluster LLM served by llama.cpp (llama-server, Vulkan on the
+      # NUCBox APU) — see llamacpp/deployment-deepseek-v4-flash-0731.yaml.
+      # DeepSeek-V4-Flash-0731 (UD-IQ1_M, ~87 GiB) is the sole model on the
+      # NUCBox; the two Qwen3.6 models were removed to make VRAM room. The
+      # chat template is applied via llama-server --jinja; reasoning (when
+      # thinking is enabled) comes back in `reasoning_content`. Default
+      # sampling (temp 1.0 / top-p 0.95) is set on the llama-server itself.
+      - model_name: deepseek-v4-flash-0731
         litellm_params:
-          model: openai/qwen3.6-27b
-          api_base: http://llamacpp-qwen36-27b.llamacpp/v1
-          api_key: "sk-no-auth"
-      # Faster "flash" variant: Qwen3.6-35B-A3B is a MoE (3B active params), so
-      # inference is much faster than the dense 27B despite more total weights.
-      # See llamacpp/deployment-qwen36-35b-a3b.yaml.
-      - model_name: qwen3.6-35b-a3b
-        litellm_params:
-          model: openai/qwen3.6-35b-a3b
-          api_base: http://llamacpp-qwen36-35b-a3b.llamacpp/v1
+          model: openai/deepseek-v4-flash-0731
+          api_base: http://llamacpp-deepseek-v4-flash-0731.llamacpp/v1
           api_key: "sk-no-auth"
     litellm_settings:
       #set_verbose: True  # Uncomment this if you want to see verbose logs; not recommended in production