diff --git a/argocd/apps/llamacpp.yaml b/argocd/apps/llamacpp.yaml new file mode 100644 index 0000000..f2216e8 --- /dev/null +++ b/argocd/apps/llamacpp.yaml @@ -0,0 +1,24 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: llamacpp + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + project: k3s-cluster + source: + repoURL: https://git.rogi.casa/roger/k3s-cluster.git + targetRevision: main + path: llamacpp + directory: + recurse: true + destination: + server: https://kubernetes.default.svc + namespace: llamacpp + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=false diff --git a/argocd/gen-apps.sh b/argocd/gen-apps.sh index 31f9b13..c6e9631 100755 --- a/argocd/gen-apps.sh +++ b/argocd/gen-apps.sh @@ -29,6 +29,7 @@ APPS=( "gym-tracker|gym-tracker|gym-tracker|true|true" "homeassistant|home-assistant|homeassistant|true|true" "jellyfin|jellyfin|jellyfin|true|true" + "llamacpp|llamacpp|llamacpp|true|true" "litellm|litellm|litellm|true|true" "minecraft-server|minecraft|minecraft-server|true|true" "monitoring|monitoring|monitoring|true|true" diff --git a/litellm/litellm.yaml b/litellm/litellm.yaml index 4c1cf8f..1090ca8 100644 --- a/litellm/litellm.yaml +++ b/litellm/litellm.yaml @@ -40,19 +40,15 @@ data: model: ollama/glm-4.7-flash api_base: http://10.88.20.12:11434 # Used by the platform-engineer Hermes agent (deployed in ns platform-engineer). - # model_name is the alias Hermes requests; the underlying Ollama model is - # qwen3.6:latest (the fast non-27b tag). 27b is a slow reasoning model. - # `ollama_chat/` (not `ollama/`) uses Ollama's NATIVE /api/chat endpoint. - # `think: false` + `chat_template_kwargs.enable_thinking: false` disable - # Qwen3 thinking so the model emits content directly (otherwise the - # OpenAI-compat translation returns empty content with reasoning split off). + # Served by llama.cpp (llama-server, Vulkan on the NUCBox APU) — see + # llamacpp/deployment-qwen36.yaml. Thinking is left ON; reasoning comes + # back in `reasoning_content` (llama-server --jinja + default reasoning + # format). The alias `qwen3.6` is unchanged so Hermes keeps working. - model_name: qwen3.6 litellm_params: - model: ollama_chat/qwen3.6:latest - api_base: http://10.88.20.12:11434 - think: false - chat_template_kwargs: - enable_thinking: false + model: openai/qwen3.6 + api_base: http://llamacpp-qwen36.llamacpp/v1 + api_key: "sk-no-auth" litellm_settings: #set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production callbacks: ["arize_phoenix"] diff --git a/llamacpp/README.md b/llamacpp/README.md new file mode 100644 index 0000000..34bd127 --- /dev/null +++ b/llamacpp/README.md @@ -0,0 +1,88 @@ +# llama.cpp (llama-server) + +In-cluster LLM inference via llama.cpp's `llama-server`, serving local models on +the NUCBox APU (AMD Ryzen AI Max 395 / Strix Halo, Radeon 8060S, 128 GiB unified +memory: 32 GiB RAM / 96 GiB VRAM). + +This replaces the bare-metal Ollama setup for models that benefit from +always-loaded weights + tuned batching. LiteLLM (`litellm/`) points at these +in-cluster Services instead of the external `10.88.20.12:11434` Ollama endpoint. + +## Layout + +One Deployment + Service **per model**, all in namespace `llamacpp`, all pinned +to the NUCBox (`nodeSelector: {kubernetes.io/arch: amd64, hardware: high-memory}`): + +| Model | GGUF | Service | litellm alias | +|----------|---------------------------------------------|----------------------------------|---------------| +| qwen3.6 | unsloth/Qwen3.6-27B-MTP-GGUF (Q4_K_XL) | `llamacpp-qwen36.llamacpp:80` | `qwen3.6` | + +Model files are downloaded idempotently by an initContainer into a shared +hostPath PVC (`/data/llamacpp/models` on the NUCBox), so pods survive reboots +without re-downloading. + +## GPU / Vulkan + +The `server-vulkan` image bundles the Mesa/RADV Vulkan driver, which supports +the Radeon 8060S (RDNA 3.5). Full layer offload (`-ngl 999`) puts the ~16 GiB +Q4 model entirely in the 96 GiB VRAM pool. + +The container mounts `/dev/dri` and runs `privileged: true` — the simplest +reliable way to give Vulkan access to the DRM render node on k3s without a +device plugin. Tighten later with `supplementalGroups` (the host's `render` +group GID) if desired. + +### Verify the GPU is actually used + +```bash +kubectl exec -n llamacpp deploy/llamacpp-qwen36 -- \ + llama-server --list-devices -m /models/Qwen3.6-27B-UD-Q4_K_XL.gguf +# or check the startup logs for a "vulkan" device line + ngl offload count +kubectl logs -n llamacpp deploy/llamacpp-qwen36 | grep -iE 'vulkan|gpu|offload|device' +``` + +If only a CPU device shows up, the container can't see the GPU — check that +`/dev/dri/renderD128` exists on the NUCBox and that the `amdgpu` module is loaded. + +## Tuning + +The key knobs (in `deployment-qwen36.yaml`): + +- `-ngl 999` — offload all layers to GPU. Reduce only if VRAM is tight (it + isn't, with 96 GiB). +- `-c 32768` — total KV-cache context. With `-np 4` this is 8192 tokens per + concurrent request. For a 27B model the full 32k×4 KV cache is ~32 GiB of + VRAM; raise or lower `-c` to trade context length for VRAM headroom. +- `-np 4` — parallel slots (concurrent requests). Matches the requested + concurrency. Each extra slot multiplies KV-cache VRAM usage. +- `--threads 8` — CPU threads for sampling/overhead. Mostly irrelevant under + full GPU offload; tune if CPU-bound. + +## Memory accounting + +k8s sees only the ~32 GiB system RAM as allocatable (the 96 GiB VRAM is +reserved by firmware and managed by `amdgpu`). The model weights and KV cache +live in VRAM and are **not** counted against the container's cgroup memory +limit — that limit only covers CPU-side overhead and the mmap'd GGUF pages +during load. If the pod is OOM-killed during model load, raise the memory limit. + +## Adding a model + +1. Create `deployment-.yaml` + `service-.yaml` (copy the qwen3.6 + pair; change `model:` label, the GGUF URL/file, `--alias`, and Service name). +2. Point LiteLLM at it in `litellm/litellm.yaml`: + ```yaml + - model_name: # keep the alias so consumers don't change + litellm_params: + model: openai/ + api_base: http://.llamacpp/v1 + api_key: "sk-no-auth" + ``` +3. (No gen-apps.sh change needed — the `llamacpp` app already syncs the whole + directory recursively.) + +## TODO + +- `glm-4.7-flash`: still served by the external Ollama at `10.88.20.12:11434` + in `litellm/litellm.yaml`. Migrate once a GGUF source is confirmed (add a + `deployment-glm47-flash.yaml` + Service and flip the litellm entry). diff --git a/llamacpp/deployment-qwen36.yaml b/llamacpp/deployment-qwen36.yaml new file mode 100644 index 0000000..0c3da45 --- /dev/null +++ b/llamacpp/deployment-qwen36.yaml @@ -0,0 +1,148 @@ +# Qwen3.6 (27B, Q4_K_XL) served by llama.cpp's llama-server on the NUCBox APU. +# +# Hardware: AMD Ryzen AI Max 395 (Strix Halo) — integrated Radeon 8060S, +# 128 GiB unified memory (32 GiB RAM / 96 GiB VRAM via firmware). The Q4 model +# (~16 GiB) is fully offloaded to the GPU via the Vulkan backend, leaving plenty +# of VRAM for the KV cache. +# +# Image: ghcr.io/ggerganov/llama.cpp:server-vulkan bundles the Mesa/RADV Vulkan +# driver, which supports the Radeon 8060S (RDNA 3.5). Pin to a digest for +# production once the rolling tag is confirmed working. +# +# GPU access: the container mounts /dev/dri (the DRM render nodes) and runs +# privileged. This is the simplest reliable option on k3s without a Vulkan +# device plugin; tighten later with supplementalGroups if desired. +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llamacpp-qwen36 + namespace: llamacpp + labels: + app: llamacpp + model: qwen3.6 +spec: + replicas: 1 + strategy: + type: Recreate # never run two pods loading the same model into VRAM + selector: + matchLabels: + app: llamacpp + model: qwen3.6 + template: + metadata: + labels: + app: llamacpp + model: qwen3.6 + spec: + nodeSelector: + kubernetes.io/arch: amd64 + hardware: high-memory + initContainers: + # Idempotently download the GGUF into the shared models PVC on first boot. + # Exits immediately if the file is already present (pod restart / recreate). + - name: fetch-model + image: alpine:3.20 + command: ["/bin/sh", "-c"] + args: + - | + set -e + if [ -f "/models/$MODEL_FILE" ]; then + echo "Model $MODEL_FILE already present — skipping download." + exit 0 + fi + echo "Installing curl..." + apk add --no-cache curl + echo "Downloading $MODEL_FILE from $MODEL_URL ..." + curl -fL --retry 5 --retry-delay 5 -o "/models/$MODEL_FILE" "$MODEL_URL" + echo "Download complete: $(ls -lh /models/$MODEL_FILE)" + env: + - name: MODEL_URL + value: "https://huggingface.co/unsloth/Qwen3.6-27B-MTP-GGUF/resolve/main/Qwen3.6-27B-UD-Q4_K_XL.gguf" + - name: MODEL_FILE + value: "Qwen3.6-27B-UD-Q4_K_XL.gguf" + volumeMounts: + - name: models + mountPath: /models + containers: + - name: llama-server + image: ghcr.io/ggerganov/llama.cpp:server-vulkan + imagePullPolicy: IfNotPresent + args: + - -m=/models/Qwen3.6-27B-UD-Q4_K_XL.gguf + - --alias=qwen3.6 # /v1/models reports this name; matches the litellm alias + - --host=0.0.0.0 + - --port=8080 + - --jinja # use the GGUF's chat template (Qwen3 thinking format) + - -ngl=999 # offload ALL layers to the GPU (fits in 96 GiB VRAM) + - -c=32768 # total KV-cache context, split across parallel slots + - -np=4 # 4 parallel slots => 8192 tokens per concurrent request + - --cont-batching # continuous batching across slots + - --threads=8 # CPU threads for sampling/overhead (GPU does the heavy lifting) + ports: + - name: http + containerPort: 8080 + resources: + # The model weights + KV cache live in GPU VRAM (96 GiB pool) and are + # NOT counted against the cgroup memory limit. This limit only covers + # CPU-side overhead + the mmap'd GGUF file pages during load (~16 GiB, + # reclaimable). k8s sees ~32 GiB as the node's allocatable system RAM, + # so the request is kept low to stay schedulable alongside other pods. + # If the pod OOM-kills during load, the amdgpu driver may be counting + # some VRAM against the cgroup — raise the limit. + requests: + cpu: "1000m" + memory: "2Gi" + limits: + cpu: "4000m" + memory: "24Gi" + readinessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 180 # model load + Vulkan init can take a few minutes + periodSeconds: 30 + failureThreshold: 5 + securityContext: + # Vulkan on the AMD APU needs /dev/dri + the driver. Privileged is the + # simplest reliable path on k3s without a device plugin. + privileged: true + volumeMounts: + - name: models + mountPath: /models + readOnly: true + - name: dri + mountPath: /dev/dri + volumes: + - name: models + persistentVolumeClaim: + claimName: llamacpp-models + - name: dri + hostPath: + path: /dev/dri + type: Directory +--- +apiVersion: v1 +kind: Service +metadata: + name: llamacpp-qwen36 + namespace: llamacpp + labels: + app: llamacpp + model: qwen3.6 +spec: + type: ClusterIP + selector: + app: llamacpp + model: qwen3.6 + ports: + - name: http + port: 80 + targetPort: 8080 diff --git a/llamacpp/namespace.yaml b/llamacpp/namespace.yaml new file mode 100644 index 0000000..d31cdb7 --- /dev/null +++ b/llamacpp/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: llamacpp diff --git a/llamacpp/pv.yaml b/llamacpp/pv.yaml new file mode 100644 index 0000000..a0a8063 --- /dev/null +++ b/llamacpp/pv.yaml @@ -0,0 +1,47 @@ +# Shared model-weight storage for all llama.cpp pods. +# +# All llamacpp pods are pinned to the NUCBox (roger-nucbox-evo-x2) via +# nodeSelector, so a single hostPath PV on that node is correct and matches the +# existing postgres hostPath pattern. GGUF files are large (10s of GB); baking +# them into images would be wasteful, and an initContainer downloads them +# idempotently on first boot instead. +# +# nodeAffinity keeps the PV bound to the NUCBox even if labels change later. +apiVersion: v1 +kind: PersistentVolume +metadata: + name: llamacpp-models + labels: + type: local + app: llamacpp +spec: + storageClassName: manual + capacity: + storage: 100Gi + accessModes: + - ReadWriteMany + hostPath: + path: /data/llamacpp/models + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - roger-nucbox-evo-x2 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: llamacpp-models + namespace: llamacpp + labels: + app: llamacpp +spec: + storageClassName: manual + accessModes: + - ReadWriteMany + resources: + requests: + storage: 100Gi