From 7089763ffb4cabdb499542b941df16c11f18a0d0 Mon Sep 17 00:00:00 2001 From: Roger Oriol Date: Tue, 18 Aug 2026 18:56:07 +0200 Subject: [PATCH] deploy qwen 3.8 --- home-manager/configmap.yaml | 6 +- litellm/litellm.yaml | 15 +- llamacpp/README.md | 167 +++--------- llamacpp/args-deepseek-v4-flash-0731.md | 8 +- llamacpp/args-qwen36-27b.md | 4 +- .../deployment-deepseek-v4-flash-0731.yaml | 241 ------------------ ...36-27b.yaml => deployment-qwen38-27b.yaml} | 84 +++--- llamacpp/pv.yaml | 8 +- platform-engineer/configmap.yaml | 6 +- 9 files changed, 86 insertions(+), 453 deletions(-) delete mode 100644 llamacpp/deployment-deepseek-v4-flash-0731.yaml rename llamacpp/{deployment-qwen36-27b.yaml => deployment-qwen38-27b.yaml} (52%) diff --git a/home-manager/configmap.yaml b/home-manager/configmap.yaml index ab5f3b8..5fa8236 100644 --- a/home-manager/configmap.yaml +++ b/home-manager/configmap.yaml @@ -9,18 +9,18 @@ data: config.yaml: | model: provider: openai-api - default: deepseek-v4-flash-0731 + default: qwen3.8-27b base_url: "http://litellm-service.litellm:80/v1" api_mode: chat_completions auxiliary: compression: provider: openai-api - model: deepseek-v4-flash-0731 + model: qwen3.8-27b base_url: "http://litellm-service.litellm:80/v1" title_generation: provider: openai-api - model: deepseek-v4-flash-0731 + model: qwen3.8-27b base_url: "http://litellm-service.litellm:80/v1" terminal: diff --git a/litellm/litellm.yaml b/litellm/litellm.yaml index 9bf00ae..ab59818 100644 --- a/litellm/litellm.yaml +++ b/litellm/litellm.yaml @@ -42,19 +42,11 @@ data: # In-cluster models served by llama.cpp (llama-server, Vulkan on the # NUCBox APU). The chat template is applied via --jinja; reasoning (when # enabled) comes back in `reasoning_content`. - - model_name: deepseek-v4-flash-0731 + - model_name: qwen3.8-27b litellm_params: - model: openai/deepseek-v4-flash-0731 - api_base: http://llamacpp-deepseek-v4-flash-0731.llamacpp/v1 + model: openai/qwen3.8-27b + api_base: http://llamacpp-qwen38-27b.llamacpp/v1 api_key: "sk-no-auth" - - model_name: qwen3.6-27b - litellm_params: - model: openai/qwen3.6-27b - api_base: http://llamacpp-qwen36-27b.llamacpp/v1 - api_key: "sk-no-auth" - router_settings: - fallbacks: - - deepseek-v4-flash-0731: [qwen3.6-27b] litellm_settings: #set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production callbacks: ["arize_phoenix"] @@ -139,4 +131,3 @@ spec: - name: http port: 80 targetPort: 4000 - diff --git a/llamacpp/README.md b/llamacpp/README.md index a37316a..1bcf6fd 100644 --- a/llamacpp/README.md +++ b/llamacpp/README.md @@ -2,151 +2,58 @@ In-cluster LLM inference via llama.cpp's `llama-server`, serving a local model on the NUCBox APU (AMD Ryzen AI Max 395 / Strix Halo, Radeon 8060S, ~120 GiB -unified memory: ~90 GiB VRAM / 30 GiB CPU RAM). +unified memory). -LiteLLM (`litellm/`) points at these in-cluster Services instead of the -external `10.88.20.12:11434` Ollama endpoint. +LiteLLM (`litellm/`) points at the in-cluster Service instead of the external +Ollama endpoint. -## Layout +## Active Model -One Deployment + Service **per model**, all in namespace `llamacpp`, all pinned -to the NUCBox (`nodeSelector: {kubernetes.io/arch: amd64, hardware: high-memory}`): +| Alias | Model | Configuration | Service | +|---|---|---|---| +| `qwen3.8-27b` | Qwen3.8-27B with MTP | `Q4_K_M` primary, `Q4_0` draft, 196k context, q8_0 K/V cache | `llamacpp-qwen38-27b.llamacpp:80` | -| Alias | Model | GGUF | Service | Args ref | -|--------------------------|--------------------------------|-------------------------------------------------------|------------------------------------------------------|----------| -| `deepseek-v4-flash-0731` | DeepSeek-V4-Flash-0731 (MoE) | unsloth/DeepSeek-V4-Flash-0731-GGUF (UD-IQ1_M, ~87 GiB) | `llamacpp-deepseek-v4-flash-0731.llamacpp:80` | [args-deepseek-v4-flash-0731.md](args-deepseek-v4-flash-0731.md) | +The active Deployment uses llama.cpp's Hugging Face downloader for both model +repositories: -DeepSeek-V4-Flash-0731 is a Mixture-of-Experts model (256 experts, 6 active per -token) with MLA attention, so only a small fraction of the weights is computed -per token. The full ~87 GiB of IQ1_M weights is loaded into the unified memory -pool. +- Primary: `ggml-org/Qwen3.8-27B-GGUF:Q4_K_M` +- Draft: `ggml-org/Qwen3.8-27B-GGUF:Q4_0` -> **Previously** the NUCBox ran two co-resident Qwen3.6 models (a 27B dense and -> a 35B-A3B MoE "flash"). DeepSeek-V4-Flash-0731 (~87 GiB) nearly fills the -> whole 90 GiB VRAM pool on its own, so both Qwen models were removed to make -> room. Their GGUF files are deleted from the shared PVC by the new pod's -> `fetch-model` initContainer on first boot. Their deployment arguments are -> still documented for redeployment: -> - [args-qwen36-27b.md](args-qwen36-27b.md) — dense 27B (deeper reasoning) -> - [args-qwen36-35b-a3b.md](args-qwen36-35b-a3b.md) — MoE 35B-A3B "flash" (fast) +The model cache is stored on the shared hostPath PVC at +`/data/llamacpp/models` on the NUCBox. The server is configured with +`--spec-default --spec-type draft-mtp`, `--reasoning-preserve`, `--fit off`, +and `--agent`. -Model files are downloaded idempotently by an initContainer into a shared -hostPath PVC (`/data/llamacpp/models` on the NUCBox), so pods survive reboots -without re-downloading. The UD-IQ1_M GGUF is split across 3 shards -(`-00001-of-00003` … `-00003-of-00003`); llama.cpp auto-loads all shards when -pointed at the first one. +## Retired Models + +The following deployments are no longer active, but their argument references +are retained for future redeployment: + +- [DeepSeek-V4-Flash-0731](args-deepseek-v4-flash-0731.md) +- [Qwen3.6-27B](args-qwen36-27b.md) +- [Qwen3.6-35B-A3B](args-qwen36-35b-a3b.md) ## GPU / Vulkan The `server-vulkan` image (`ghcr.io/ggml-org/llama.cpp:server-vulkan`) bundles -the Mesa/RADV Vulkan driver, which supports the Radeon 8060S (RDNA 3.5). The -Vulkan backend supports the `IQ1_M` matmul (including the MoE `matmul_id` -variant), so the whole model runs on the GPU. `deepseek4` is a brand-new arch -(2026-07), so the floating `server-vulkan` tag is used to pull a recent enough -build — pin to a specific `server-vulkan-bXXXX` tag once a known-good one is -verified. +the Mesa/RADV Vulkan driver for the Radeon 8060S. The container mounts +`/dev/dri` and runs privileged, which is the current way to provide Vulkan +access on k3s without a device plugin. -The container mounts `/dev/dri` and runs `privileged: true` — the simplest -reliable way to give Vulkan access to the DRM render node on k3s without a -device plugin. Tighten later with `supplementalGroups` (the host's `render` -group GID) if desired. - -### Verify the GPU is actually used +Verify GPU use with: ```bash -kubectl logs -n llamacpp deploy/llamacpp-deepseek-v4-flash-0731 | grep -iE 'vulkan|gpu|offload|device' +kubectl logs -n llamacpp deploy/llamacpp-qwen38-27b | grep -iE 'vulkan|gpu|offload|device' ``` -If only a CPU device shows up, the container can't see the GPU — check that -`/dev/dri/renderD128` exists on the NUCBox and that the `amdgpu` module is loaded. +If only a CPU device appears, check that `/dev/dri/renderD128` exists on the +NUCBox and that the `amdgpu` module is loaded. -## VRAM budget (single model) +## Adding Or Replacing A Model -The model (~87 GiB IQ1_M) is almost the size of the entire 90 GiB VRAM pool, so -it **cannot be fully offloaded**: `-ngl 999` would try to put all 43 layers into -VRAM and overflow once the KV cache + Vulkan compute buffers are added. Instead -`-ngl 38` offloads 38 of 43 layers to the GPU and keeps 5 layers (~10 GiB) on -CPU RAM, leaving ~5 GiB of VRAM headroom for the KV cache, compute buffers, and -fragmentation. - -**KV cache is f16, not q8_0** — the Vulkan backend has no Flash Attention for -the `deepseek4` arch, and quantized V cache requires Flash Attention (llama.cpp -hard-errors: *"quantized V cache was requested, but this requires Flash -Attention"*). `deepseek4`/MLA models also require K and V cache types to be -*identical*, so K cannot be quantized either. f16 MLA KV at 64k is ~5.7 GiB -(576 K + 512 V elements/token/layer × 43 layers × 65536 tokens × 2 bytes) — -larger than q8_0 would be, which is why `-ngl` is 38 rather than 40. - -Approximate VRAM usage: - -| Component | VRAM | -|---------------------------------|-------------| -| Weights (38 GPU layers) | ~77 GiB | -| KV cache (f16, 64k, 1 slot) | ~5.7 GiB | -| Vulkan compute buffers | ~2 GiB | -| **Total in VRAM** | **~85 GiB** | -| **Headroom (of 90 GiB)** | **~5 GiB** | - -5 layers (~10 GiB) live in CPU RAM (counted against the pod's cgroup memory -limit, not VRAM). VRAM is exclusive to this model; the other NUCBox pods only -compete for the 30 GiB CPU RAM. - -Several `deepseek4`-specific fused ops (Lightning Indexer, HC pre/comb/post) -are not yet implemented in the Vulkan backend and fall back to CPU (logged as -warnings, not fatal). Inference still works; it will speed up once those ops -land in a future `server-vulkan` build. - -## Tuning - -The key knobs (in `deployment-deepseek-v4-flash-0731.yaml`): - -- `-ngl 38` — offload 38 of 43 layers to GPU. The model (~87 GiB) is nearly the - whole 90 GiB VRAM pool, so full offload would overflow once the f16 KV cache + - compute buffers are added. 5 layers (~10 GiB) on CPU leaves ~5 GiB VRAM - headroom. Raise toward 43 if VRAM allows; lower (e.g. 36) if the pod OOMs / - Vulkan runs out of device memory. -- `-c 65536` — total KV-cache context (64k, the required minimum). 1 slot gets - the full 64k. f16 MLA KV at 64k is ~5.7 GiB; capped at the minimum to maximise - VRAM headroom. Raise if headroom allows. -- `-np 1` — 1 parallel slot (the full 64k goes to a single concurrent request). - Extra slots multiply the f16 KV cost (~5.7 GiB/slot); 1 slot keeps headroom - maximal. -- `--cache-type-k f16 --cache-type-v f16` — **f16 KV cache (NOT quantized).** - The Vulkan backend has no Flash Attention for `deepseek4`, and quantized V - cache requires Flash Attention. `deepseek4`/MLA models also require K and V - cache types to be identical, so K cannot be quantized either. This is the - reason `-ngl` is 38 rather than 40. -- `--temp 1.0 --top-p 0.95` — default sampling parameters (DeepSeek-V4 - recommendation). These are server defaults; clients can override per request - via the OpenAI-compatible API. -- `--threads 8` — CPU threads for sampling + the 5 CPU-resident layers. - -## Memory accounting - -k8s sees only the ~30 GiB system RAM as allocatable (the ~90 GiB VRAM is -reserved by firmware and managed by `amdgpu`). The GPU-resident model weights -and KV cache live in VRAM and are **not** counted against the container's cgroup -memory limit — that limit only covers CPU-side overhead, the mmap'd GGUF pages -for the 5 CPU-resident layers (~10 GiB, resident during inference), and -reclaimable page cache during load. If the pod is OOM-killed during model load -or inference, raise the memory limit (and/or lower `-ngl` to push more layers -to VRAM). - -## Adding / replacing a model - -1. Copy `deployment-deepseek-v4-flash-0731.yaml` → `deployment-.yaml`; - change the `model:` label, GGUF URL/file(s), `--alias`, and Service name. - For split GGUFs, point `-m` at the first shard and download all shards in - the `fetch-model` initContainer. -2. Point LiteLLM at it in `litellm/litellm.yaml`: - ```yaml - - model_name: - litellm_params: - model: openai/ - api_base: http://.llamacpp/v1 - api_key: "sk-no-auth" - ``` -3. (No gen-apps.sh change needed — the `llamacpp` app already syncs the whole - directory recursively.) -4. Check the VRAM budget table above — at ~87 GiB this model nearly fills the - 90 GiB pool on its own, so co-locating another large model is not possible. +1. Copy `deployment-qwen38-27b.yaml` to `deployment-.yaml` and change the + model repositories, alias, labels, and Service name. +2. Add the matching alias and Service URL to `litellm/litellm.yaml`. +3. No `gen-apps.sh` change is needed because the `llamacpp` ArgoCD Application + syncs the directory recursively. +4. Check the model and KV-cache size against the NUCBox's available VRAM. diff --git a/llamacpp/args-deepseek-v4-flash-0731.md b/llamacpp/args-deepseek-v4-flash-0731.md index ab598e4..fc13b81 100644 --- a/llamacpp/args-deepseek-v4-flash-0731.md +++ b/llamacpp/args-deepseek-v4-flash-0731.md @@ -1,8 +1,8 @@ -# Deployment arguments — `deepseek-v4-flash-0731` +# Deployment arguments — `deepseek-v4-flash-0731` (RETIRED) -Reference for the llama-server flags used in -`deployment-deepseek-v4-flash-0731.yaml`. Keep this in sync if the -Deployment is edited. +Historical reference for the llama-server flags used by the retired +`deployment-deepseek-v4-flash-0731.yaml`. The Deployment can be recovered from +git history if this model is needed again. ## Model & source diff --git a/llamacpp/args-qwen36-27b.md b/llamacpp/args-qwen36-27b.md index ba78c47..7928850 100644 --- a/llamacpp/args-qwen36-27b.md +++ b/llamacpp/args-qwen36-27b.md @@ -6,8 +6,8 @@ > model was live, so it can be redeployed later if the DeepSeek model is taken > down or moved to different hardware. > -> To redeploy: restore `deployment-qwen36-27b.yaml` (the manifest is preserved -> in git history) and re-add the LiteLLM entry. Re-check the VRAM budget — +> To redeploy: restore `deployment-qwen36-27b.yaml` from git history and +> re-add the LiteLLM entry. Re-check the VRAM budget — > co-locating with the 87 GiB DeepSeek model is **not** possible on the current > 90 GiB pool. diff --git a/llamacpp/deployment-deepseek-v4-flash-0731.yaml b/llamacpp/deployment-deepseek-v4-flash-0731.yaml deleted file mode 100644 index 14b07d4..0000000 --- a/llamacpp/deployment-deepseek-v4-flash-0731.yaml +++ /dev/null @@ -1,241 +0,0 @@ -# DeepSeek-V4-Flash-0731 (MoE: 256 experts / 6 active, UD-IQ1_M ≈ 87 GiB) -# served by llama.cpp's llama-server on the NUCBox APU. -# -# Hardware: AMD Ryzen AI Max 395 (Strix Halo) — integrated Radeon 8060S, -# ~120 GiB unified memory (≈90 GiB VRAM / 30 GiB CPU RAM via firmware). The -# IQ1_M model (~87 GiB) is *almost* the size of the whole VRAM pool, so it -# CANNOT be fully offloaded to the GPU: offloading all 43 layers + the KV -# cache + Vulkan compute buffers would overflow 90 GiB. Instead we offload -# 38 of 43 layers (-ngl 38) and keep 5 layers (~10 GiB) on CPU RAM, leaving -# ~5 GiB of VRAM headroom for the KV cache + Vulkan compute buffers. -# -# VRAM is exclusive to this model (no other pod uses it); the other pods on -# the NUCBox only compete for the 30 GiB CPU RAM, so the headroom that -# matters here is VRAM headroom for compute buffers / fragmentation. -# -# KV CACHE MUST BE f16 (NOT quantized). The Vulkan backend has no Flash -# Attention for the deepseek4 arch, and quantized V cache requires Flash -# Attention (llama.cpp hard-errors otherwise: "quantized V cache was -# requested, but this requires Flash Attention"). Additionally, deepseek4 / -# MLA models require K and V cache types to be *identical*, so K cannot be -# quantized either. f16 KV at 64k is ~5.7 GiB (MLA KV: 576 K + 512 V -# elements/token/layer × 43 layers × 65536 tokens × 2 bytes). This is why -# -ngl is 38 rather than 40 — the larger f16 KV cache needs the extra VRAM. -# -# Image: ghcr.io/ggml-org/llama.cpp:server-vulkan bundles the Mesa/RADV Vulkan -# driver, which supports the Radeon 8060S (RDNA 3.5). The Vulkan backend -# supports the IQ1_M matmul (incl. the MoE matmul_id variant), so the whole -# model runs on the GPU. deepseek4 is a brand-new arch (2026-07); several -# deepseek4-specific fused ops (Lightning Indexer, HC pre/comb/post) are not -# yet implemented in Vulkan and fall back to CPU (logged as warnings, not -# fatal). The floating `server-vulkan` tag is used to pull a recent enough -# build; pin to a specific server-vulkan-bXXXX tag once a known-good one is -# verified. -# -# GPU access: the container mounts /dev/dri (the DRM render nodes) and runs -# privileged — the simplest reliable option on k3s without a Vulkan device -# plugin. ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: llamacpp-deepseek-v4-flash-0731 - namespace: llamacpp - labels: - app: llamacpp - model: deepseek-v4-flash-0731 -spec: - replicas: 1 - strategy: - type: Recreate # never run two pods loading the same model into VRAM - selector: - matchLabels: - app: llamacpp - model: deepseek-v4-flash-0731 - template: - metadata: - labels: - app: llamacpp - model: deepseek-v4-flash-0731 - spec: - nodeSelector: - kubernetes.io/arch: amd64 - hardware: high-memory - initContainers: - # Idempotently download the (3-part, split) GGUF into the shared models - # PVC on first boot. Downloads are atomic (→ .partial, then - # rename) and resumable, so a failed/interrupted download is recovered - # on the next pod start without re-fetching from scratch. A free-space - # check fails loudly if the hostPath disk is genuinely too small (no - # manifest can create physical disk space — that needs the disk expanded - # on the NUCBox). - - name: fetch-model - image: alpine:3.20 - command: ["/bin/sh", "-c"] - args: - - | - set -e - # Qwen3.6-27B is intentionally co-located on this PVC; do not remove - # it on DeepSeek pod restarts. - # Skip entirely if every shard is already fully downloaded. - if [ -s "/models/$SHARD1" ] && [ -s "/models/$SHARD2" ] && [ -s "/models/$SHARD3" ]; then - echo "All 3 shards already present — skipping download." - ls -lh /models/DeepSeek-V4-Flash-0731-UD-IQ1_M-*.gguf - exit 0 - fi - echo "Installing curl..." - apk add --no-cache curl - # Free-space check: the model is ~87 GiB; require ~95 GiB free as a - # safety buffer. df reports KiB. - FREE_KB=$(df -P /models | awk 'NR==2 {print $4}') - NEEDED_KB=$((95 * 1024 * 1024)) - if [ "$FREE_KB" -lt "$NEEDED_KB" ]; then - avail_gb=$((FREE_KB / 1024 / 1024)) - echo "ERROR: only ${avail_gb} GiB free on /models, need ~95 GiB to" >&2 - echo " download the 87 GiB DeepSeek-V4-Flash-0731 GGUF." >&2 - echo " Expand the hostPath disk at /data/llamacpp/models on" >&2 - echo " the NUCBox (a PVC capacity bump alone does not add" >&2 - echo " physical space to a hostPath volume)." >&2 - exit 1 - fi - # Download each missing shard to a .partial file (resumable via -C -), - # then atomically rename to the final name on success. A crash leaves - # only the .partial behind, which the next run resumes — never a - # half-written final file that would skip the download. - for s in "$SHARD1" "$SHARD2" "$SHARD3"; do - if [ -s "/models/$s" ]; then - echo "Shard $s already complete — skipping." - continue - fi - echo "Downloading $s from $HF_REPO ..." - curl -fL --retry 5 --retry-delay 5 -C - -o "/models/$s.partial" "$HF_REPO/$s" - mv "/models/$s.partial" "/models/$s" - echo " done: $(ls -lh "/models/$s")" - done - echo "All shards downloaded:" - ls -lh /models/DeepSeek-V4-Flash-0731-UD-IQ1_M-*.gguf - env: - - name: HF_REPO - value: "https://huggingface.co/unsloth/DeepSeek-V4-Flash-0731-GGUF/resolve/main/UD-IQ1_M" - - name: SHARD1 - value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00001-of-00003.gguf" - - name: SHARD2 - value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00002-of-00003.gguf" - - name: SHARD3 - value: "DeepSeek-V4-Flash-0731-UD-IQ1_M-00003-of-00003.gguf" - volumeMounts: - - name: models - mountPath: /models - containers: - - name: llama-server - image: ghcr.io/ggml-org/llama.cpp:server-vulkan - imagePullPolicy: IfNotPresent - # llama.cpp's CLI parser does NOT split on '=' — every value flag must - # be a separate argv element (flag, then value). See common/arg.cpp. - args: - - -m # model file (first shard; llama.cpp auto-loads the rest) - - /models/DeepSeek-V4-Flash-0731-UD-IQ1_M-00001-of-00003.gguf - - --alias # /v1/models reports this name; matches the litellm alias - - deepseek-v4-flash-0731 - - --host - - 0.0.0.0 - - --port - - "8080" - - --jinja # use the GGUF's DeepSeek-V4 chat template (DSML / thinking) - - -ngl # offload 38 of 43 layers to the GPU. The model (~87 GiB) is - - "38" # nearly the whole 90 GiB VRAM pool, so full offload (-ngl 999) - # would overflow once the f16 KV cache + Vulkan compute buffers - # are added. 38 layers (~77 GiB) + f16 KV (~5.7 GiB) + compute - # (~2 GiB) ≈ 85 GiB, leaving ~5 GiB VRAM headroom. KV cache is - # f16 (not q8_0) because Vulkan has no Flash Attention for - # deepseek4, which makes the KV cache ~2× larger than q8_0 would - # be — hence 38 rather than 40 layers offloaded. 5 layers - # (~10 GiB) run on CPU RAM. Raise toward 43 if VRAM allows; - # lower (e.g. 36) if the pod OOMs / Vulkan runs out of device mem. - - -c # total KV-cache context (single slot gets the full window). - - "65536" # 64k — the required minimum. f16 MLA KV at 64k is ~5.7 GiB, - # so context is affordable but not negligible. -c is capped at - # the minimum to maximise VRAM headroom; raise if headroom allows. - - -np # 1 slot => the full 64k goes to a single concurrent request - - "1" # (extra slots would multiply KV VRAM; 1 slot keeps headroom maximal). - - --cont-batching # continuous batching across slots - - --cache-type-k # f16 K cache. deepseek4 / MLA models require K and V cache - - f16 # types to be IDENTICAL, and quantized V cache requires Flash - - --cache-type-v # Attention, which the Vulkan backend does NOT support for - - f16 # deepseek4 (llama.cpp hard-errors otherwise). So both K and V - # must stay f16. KV at 64k ≈ 5.7 GiB. - - --temp # default sampling temperature (DeepSeek-V4 recommendation) - - "1.0" - - --top-p # default nucleus sampling threshold (DeepSeek-V4 recommendation) - - "0.95" - - --threads # CPU threads for sampling + the 5 CPU-resident layers - - "8" - ports: - - name: http - containerPort: 8080 - resources: - # The GPU-resident model weights + KV cache live in VRAM (~90 GiB pool) - # and are NOT counted against the cgroup memory limit. This limit only - # covers CPU-side overhead + the mmap'd GGUF pages for the 5 CPU-resident - # layers (~10 GiB, resident during inference) plus reclaimable page cache - # during load. k8s sees ~30 GiB as the node's allocatable system RAM, so - # the limit is sized to cover the CPU layers + overhead while leaving RAM - # for co-resident pods (litellm, the agents, etc.). If the pod is - # OOM-killed during model load or inference, raise the limit (and/or - # lower -ngl to push more layers to VRAM). - requests: - cpu: "1000m" - memory: "6Gi" - limits: - cpu: "4000m" - memory: "24Gi" - readinessProbe: - httpGet: - path: /health - port: 8080 - initialDelaySeconds: 30 - periodSeconds: 10 - failureThreshold: 6 - livenessProbe: - httpGet: - path: /health - port: 8080 - initialDelaySeconds: 300 # 87 GiB load + Vulkan init takes several minutes - periodSeconds: 30 - failureThreshold: 5 - securityContext: - # Vulkan on the AMD APU needs /dev/dri + the driver. Privileged is - # the simplest reliable path on k3s without a device plugin. - privileged: true - volumeMounts: - - name: models - mountPath: /models - readOnly: true - - name: dri - mountPath: /dev/dri - volumes: - - name: models - persistentVolumeClaim: - claimName: llamacpp-models - - name: dri - hostPath: - path: /dev/dri - type: Directory ---- -apiVersion: v1 -kind: Service -metadata: - name: llamacpp-deepseek-v4-flash-0731 - namespace: llamacpp - labels: - app: llamacpp - model: deepseek-v4-flash-0731 -spec: - type: ClusterIP - selector: - app: llamacpp - model: deepseek-v4-flash-0731 - ports: - - name: http - port: 80 - targetPort: 8080 diff --git a/llamacpp/deployment-qwen36-27b.yaml b/llamacpp/deployment-qwen38-27b.yaml similarity index 52% rename from llamacpp/deployment-qwen36-27b.yaml rename to llamacpp/deployment-qwen38-27b.yaml index ec9f8b3..3ea0f55 100644 --- a/llamacpp/deployment-qwen36-27b.yaml +++ b/llamacpp/deployment-qwen38-27b.yaml @@ -1,15 +1,15 @@ -# Qwen3.6-27B (dense, Q4_K_XL) served by llama.cpp alongside DeepSeek. -# The model is fully offloaded to the NUCBox Radeon 8060S; Qwen weights and -# DeepSeek weights share the llamacpp-models hostPath PVC. +# Qwen3.8-27B with MTP speculative decoding served by llama.cpp. +# The primary and draft GGUFs are downloaded by llama-server into the shared +# persistent llama.cpp cache on the NUCBox Radeon 8060S. --- apiVersion: apps/v1 kind: Deployment metadata: - name: llamacpp-qwen36-27b + name: llamacpp-qwen38-27b namespace: llamacpp labels: app: llamacpp - model: qwen3.6-27b + model: qwen3.8-27b spec: replicas: 1 strategy: @@ -17,68 +17,47 @@ spec: selector: matchLabels: app: llamacpp - model: qwen3.6-27b + model: qwen3.8-27b template: metadata: labels: app: llamacpp - model: qwen3.6-27b + model: qwen3.8-27b spec: nodeSelector: kubernetes.io/arch: amd64 hardware: high-memory - initContainers: - - name: fetch-model - image: alpine:3.20 - command: ["/bin/sh", "-c"] - args: - - | - set -e - if [ -s "/models/$MODEL_FILE" ]; then - echo "Model $MODEL_FILE already present — skipping download." - exit 0 - fi - echo "Installing curl..." - apk add --no-cache curl - echo "Downloading $MODEL_FILE from $MODEL_URL ..." - curl -fL --retry 5 --retry-delay 5 -o "/models/$MODEL_FILE.partial" "$MODEL_URL" - mv "/models/$MODEL_FILE.partial" "/models/$MODEL_FILE" - echo "Download complete: $(ls -lh /models/$MODEL_FILE)" - env: - - name: MODEL_URL - value: "https://huggingface.co/unsloth/Qwen3.6-27B-MTP-GGUF/resolve/main/Qwen3.6-27B-UD-Q4_K_XL.gguf" - - name: MODEL_FILE - value: "Qwen3.6-27B-UD-Q4_K_XL.gguf" - volumeMounts: - - name: models - mountPath: /models containers: - name: llama-server image: ghcr.io/ggml-org/llama.cpp:server-vulkan imagePullPolicy: IfNotPresent args: - - -m - - /models/Qwen3.6-27B-UD-Q4_K_XL.gguf + - -hf + - ggml-org/Qwen3.8-27B-GGUF:Q4_K_M + - -hfd + - ggml-org/Qwen3.8-27B-GGUF:Q4_0 + - --spec-default + - --spec-type + - draft-mtp + - --ctx-size + - "196608" + - --cache-type-k + - q8_0 + - --cache-type-v + - q8_0 + - --reasoning-preserve + - --fit + - off + - --agent + - --chat-template-kwargs + - '{"reasoning_effort":"medium"}' - --alias - - qwen3.6-27b + - qwen3.8-27b - --host - 0.0.0.0 - --port - "8080" - --jinja - - -ngl - - "999" - - -c - - "131072" - - -np - - "1" - - --cont-batching - - --cache-type-k - - q8_0 - - --cache-type-v - - q8_0 - - --threads - - "8" ports: - name: http containerPort: 8080 @@ -107,8 +86,7 @@ spec: privileged: true volumeMounts: - name: models - mountPath: /models - readOnly: true + mountPath: /root/.cache/llama.cpp - name: dri mountPath: /dev/dri volumes: @@ -123,16 +101,16 @@ spec: apiVersion: v1 kind: Service metadata: - name: llamacpp-qwen36-27b + name: llamacpp-qwen38-27b namespace: llamacpp labels: app: llamacpp - model: qwen3.6-27b + model: qwen3.8-27b spec: type: ClusterIP selector: app: llamacpp - model: qwen3.6-27b + model: qwen3.8-27b ports: - name: http port: 80 diff --git a/llamacpp/pv.yaml b/llamacpp/pv.yaml index aa1207a..b9e2961 100644 --- a/llamacpp/pv.yaml +++ b/llamacpp/pv.yaml @@ -9,11 +9,9 @@ # nodeAffinity keeps the PV bound to the NUCBox even if labels change later. # # IMPORTANT: capacity is only metadata for a hostPath volume — k8s does NOT -# enforce it and bumping it does NOT add physical disk space. The -# DeepSeek-V4-Flash-0731 UD-IQ1_M GGUF is ~87 GiB across 3 shards, so the -# hostPath filesystem (/data on the NUCBox) must physically have ~95 GiB free. -# The fetch-model initContainer checks free space and fails loudly if the disk -# is too small; expanding the disk is a host operation, not a manifest change. +# enforce it and bumping it does NOT add physical disk space. The active Qwen +# primary and draft GGUFs are downloaded into this cache, so the hostPath +# filesystem must have enough free space for both models. apiVersion: v1 kind: PersistentVolume metadata: diff --git a/platform-engineer/configmap.yaml b/platform-engineer/configmap.yaml index c67b6bd..f9b6c04 100644 --- a/platform-engineer/configmap.yaml +++ b/platform-engineer/configmap.yaml @@ -9,18 +9,18 @@ data: config.yaml: | model: provider: openai-api - default: deepseek-v4-flash-0731 + default: qwen3.8-27b base_url: "http://litellm-service.litellm:80/v1" api_mode: chat_completions auxiliary: compression: provider: openai-api - model: deepseek-v4-flash-0731 # in-cluster llama.cpp model (replaces the Qwen flash/dense pair) + model: qwen3.8-27b # in-cluster llama.cpp model base_url: "http://litellm-service.litellm:80/v1" title_generation: provider: openai-api - model: deepseek-v4-flash-0731 # in-cluster llama.cpp model (replaces the Qwen flash/dense pair) + model: qwen3.8-27b # in-cluster llama.cpp model base_url: "http://litellm-service.litellm:80/v1" terminal: