set platform engineer url to platform-engineer.rogi.casa
This commit is contained in:
@@ -50,11 +50,11 @@ data:
|
|||||||
api_key: "sk-no-auth"
|
api_key: "sk-no-auth"
|
||||||
# Faster "flash" variant: Qwen3.6-35B-A3B is a MoE (3B active params), so
|
# Faster "flash" variant: Qwen3.6-35B-A3B is a MoE (3B active params), so
|
||||||
# inference is much faster than the dense 27B despite more total weights.
|
# inference is much faster than the dense 27B despite more total weights.
|
||||||
# See llamacpp/deployment-qwen36-36b-a3b.yaml.
|
# See llamacpp/deployment-qwen36-35b-a3b.yaml.
|
||||||
- model_name: qwen3.6-36b-a3b
|
- model_name: qwen3.6-35b-a3b
|
||||||
litellm_params:
|
litellm_params:
|
||||||
model: openai/qwen3.6-36b-a3b
|
model: openai/qwen3.6-35b-a3b
|
||||||
api_base: http://llamacpp-qwen36-36b-a3b.llamacpp/v1
|
api_base: http://llamacpp-qwen36-35b-a3b.llamacpp/v1
|
||||||
api_key: "sk-no-auth"
|
api_key: "sk-no-auth"
|
||||||
litellm_settings:
|
litellm_settings:
|
||||||
#set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production
|
#set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ to the NUCBox (`nodeSelector: {kubernetes.io/arch: amd64, hardware: high-memory}
|
|||||||
| Alias | Model | GGUF | Service |
|
| Alias | Model | GGUF | Service |
|
||||||
|-------------------|----------------------------|-------------------------------------------------------|-------------------------------------------|
|
|-------------------|----------------------------|-------------------------------------------------------|-------------------------------------------|
|
||||||
| `qwen3.6-27b` | Qwen3.6-27B (dense) | unsloth/Qwen3.6-27B-MTP-GGUF (Q4_K_XL, ~16 GiB) | `llamacpp-qwen36-27b.llamacpp:80` |
|
| `qwen3.6-27b` | Qwen3.6-27B (dense) | unsloth/Qwen3.6-27B-MTP-GGUF (Q4_K_XL, ~16 GiB) | `llamacpp-qwen36-27b.llamacpp:80` |
|
||||||
| `qwen3.6-36b-a3b` | Qwen3.6-35B-A3B (MoE/flash)| unsloth/Qwen3.6-35B-A3B-MTP-GGUF (Q4_K_XL, ~20 GiB) | `llamacpp-qwen36-36b-a3b.llamacpp:80` |
|
| `qwen3.6-35b-a3b` | Qwen3.6-35B-A3B (MoE/flash)| unsloth/Qwen3.6-35B-A3B-MTP-GGUF (Q4_K_XL, ~20 GiB) | `llamacpp-qwen36-35b-a3b.llamacpp:80` |
|
||||||
|
|
||||||
The 35B-A3B is a Mixture-of-Experts model (3B active params per token), so
|
The 35B-A3B is a Mixture-of-Experts model (3B active params per token), so
|
||||||
inference is significantly faster than the dense 27B despite more total weights —
|
inference is significantly faster than the dense 27B despite more total weights —
|
||||||
@@ -42,7 +42,7 @@ group GID) if desired.
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
kubectl logs -n llamacpp deploy/llamacpp-qwen36-27b | grep -iE 'vulkan|gpu|offload|device'
|
kubectl logs -n llamacpp deploy/llamacpp-qwen36-27b | grep -iE 'vulkan|gpu|offload|device'
|
||||||
kubectl logs -n llamacpp deploy/llamacpp-qwen36-36b-a3b | grep -iE 'vulkan|gpu|offload|device'
|
kubectl logs -n llamacpp deploy/llamacpp-qwen36-35b-a3b | grep -iE 'vulkan|gpu|offload|device'
|
||||||
```
|
```
|
||||||
|
|
||||||
If only a CPU device shows up, the container can't see the GPU — check that
|
If only a CPU device shows up, the container can't see the GPU — check that
|
||||||
@@ -55,7 +55,7 @@ Both models run simultaneously on the same 96 GiB VRAM pool. Approximate usage:
|
|||||||
| Model | Weights | KV cache (32k×4) | Subtotal |
|
| Model | Weights | KV cache (32k×4) | Subtotal |
|
||||||
|-------------------|----------|-------------------|----------|
|
|-------------------|----------|-------------------|----------|
|
||||||
| qwen3.6-27b | ~16 GiB | ~34 GiB | ~50 GiB |
|
| qwen3.6-27b | ~16 GiB | ~34 GiB | ~50 GiB |
|
||||||
| qwen3.6-36b-a3b | ~20 GiB | ~10 GiB | ~30 GiB |
|
| qwen3.6-35b-a3b | ~20 GiB | ~10 GiB | ~30 GiB |
|
||||||
| **Total** | | | **~80 GiB** |
|
| **Total** | | | **~80 GiB** |
|
||||||
|
|
||||||
~16 GiB headroom — comfortable but not infinite. If VRAM is exhausted ( Vulkan
|
~16 GiB headroom — comfortable but not infinite. If VRAM is exhausted ( Vulkan
|
||||||
|
|||||||
@@ -18,11 +18,11 @@
|
|||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
metadata:
|
metadata:
|
||||||
name: llamacpp-qwen36-36b-a3b
|
name: llamacpp-qwen36-35b-a3b
|
||||||
namespace: llamacpp
|
namespace: llamacpp
|
||||||
labels:
|
labels:
|
||||||
app: llamacpp
|
app: llamacpp
|
||||||
model: qwen3.6-36b-a3b
|
model: qwen3.6-35b-a3b
|
||||||
spec:
|
spec:
|
||||||
replicas: 1
|
replicas: 1
|
||||||
strategy:
|
strategy:
|
||||||
@@ -30,12 +30,12 @@ spec:
|
|||||||
selector:
|
selector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
app: llamacpp
|
app: llamacpp
|
||||||
model: qwen3.6-36b-a3b
|
model: qwen3.6-35b-a3b
|
||||||
template:
|
template:
|
||||||
metadata:
|
metadata:
|
||||||
labels:
|
labels:
|
||||||
app: llamacpp
|
app: llamacpp
|
||||||
model: qwen3.6-36b-a3b
|
model: qwen3.6-35b-a3b
|
||||||
spec:
|
spec:
|
||||||
nodeSelector:
|
nodeSelector:
|
||||||
kubernetes.io/arch: amd64
|
kubernetes.io/arch: amd64
|
||||||
@@ -76,7 +76,7 @@ spec:
|
|||||||
- -m # model file
|
- -m # model file
|
||||||
- /models/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf
|
- /models/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf
|
||||||
- --alias # /v1/models reports this name; matches the litellm alias
|
- --alias # /v1/models reports this name; matches the litellm alias
|
||||||
- qwen3.6-36b-a3b
|
- qwen3.6-35b-a3b
|
||||||
- --host
|
- --host
|
||||||
- 0.0.0.0
|
- 0.0.0.0
|
||||||
- --port
|
- --port
|
||||||
@@ -142,16 +142,16 @@ spec:
|
|||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: Service
|
kind: Service
|
||||||
metadata:
|
metadata:
|
||||||
name: llamacpp-qwen36-36b-a3b
|
name: llamacpp-qwen36-35b-a3b
|
||||||
namespace: llamacpp
|
namespace: llamacpp
|
||||||
labels:
|
labels:
|
||||||
app: llamacpp
|
app: llamacpp
|
||||||
model: qwen3.6-36b-a3b
|
model: qwen3.6-35b-a3b
|
||||||
spec:
|
spec:
|
||||||
type: ClusterIP
|
type: ClusterIP
|
||||||
selector:
|
selector:
|
||||||
app: llamacpp
|
app: llamacpp
|
||||||
model: qwen3.6-36b-a3b
|
model: qwen3.6-35b-a3b
|
||||||
ports:
|
ports:
|
||||||
- name: http
|
- name: http
|
||||||
port: 80
|
port: 80
|
||||||
@@ -44,7 +44,7 @@ platform-engineer/
|
|||||||
├── pvc.yaml # persistent /opt/data (HERMES_HOME)
|
├── pvc.yaml # persistent /opt/data (HERMES_HOME)
|
||||||
├── dockerfile # derived image: hermes-agent + kubectl + helm
|
├── dockerfile # derived image: hermes-agent + kubectl + helm
|
||||||
├── deployment.yaml # Deployment, schedules on amd64, mounts kube SA token
|
├── deployment.yaml # Deployment, schedules on amd64, mounts kube SA token
|
||||||
├── ingress.yaml # hermes.rogi.casa → dashboard (optional)
|
├── ingress.yaml # platform-engineer.rogi.casa → dashboard (optional)
|
||||||
└── README.md # this file
|
└── README.md # this file
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -196,7 +196,7 @@ HERMES_DASHBOARD_BASIC_AUTH_PASSWORD=<strong password>
|
|||||||
2–4 GiB; 1 GiB is fine without browser tools, which we keep off).
|
2–4 GiB; 1 GiB is fine without browser tools, which we keep off).
|
||||||
- Volume: PVC mounted at `/opt/data` (HERMES_HOME), RWX not needed (single pod).
|
- Volume: PVC mounted at `/opt/data` (HERMES_HOME), RWX not needed (single pod).
|
||||||
- Ports: 8642 (gateway API, internal only) and 9119 (dashboard) → exposed via
|
- Ports: 8642 (gateway API, internal only) and 9119 (dashboard) → exposed via
|
||||||
Ingress `hermes.rogi.casa` with TLS + basic-auth (already enforced by the
|
Ingress `platform-engineer.rogi.casa` with TLS + basic-auth (already enforced by the
|
||||||
`HERMES_DASHBOARD_BASIC_AUTH_*` env vars).
|
`HERMES_DASHBOARD_BASIC_AUTH_*` env vars).
|
||||||
- `imagePullSecrets: gitea-registry`.
|
- `imagePullSecrets: gitea-registry`.
|
||||||
- env from Secret; `HERMES_DASHBOARD=1`.
|
- env from Secret; `HERMES_DASHBOARD=1`.
|
||||||
@@ -283,7 +283,7 @@ Design rules baked into SOUL.md:
|
|||||||
|
|
||||||
1. **Notifications:** dedicated `platform-engineer` Discord bot → its own token
|
1. **Notifications:** dedicated `platform-engineer` Discord bot → its own token
|
||||||
in `secret.yaml` (`DISCORD_BOT_TOKEN`, `DISCORD_HOME_CHANNEL`).
|
in `secret.yaml` (`DISCORD_BOT_TOKEN`, `DISCORD_HOME_CHANNEL`).
|
||||||
2. **Dashboard:** public at `hermes.rogi.casa` (Traefik TLS + cert-manager + the
|
2. **Dashboard:** public at `platform-engineer.rogi.casa` (Traefik TLS + cert-manager + the
|
||||||
bundled Hermes basic-auth provider). Reach the dashboard on port 9119; the
|
bundled Hermes basic-auth provider). Reach the dashboard on port 9119; the
|
||||||
gateway API on 8642 is ClusterIP-only.
|
gateway API on 8642 is ClusterIP-only.
|
||||||
3. **Image:** derived image pushed to `git.rogi.casa/roger/hermes-agent`, pulled
|
3. **Image:** derived image pushed to `git.rogi.casa/roger/hermes-agent`, pulled
|
||||||
@@ -322,7 +322,7 @@ Design rules baked into SOUL.md:
|
|||||||
set the LiteLLM master key as `OPENAI_API_KEY` and a strong dashboard
|
set the LiteLLM master key as `OPENAI_API_KEY` and a strong dashboard
|
||||||
password + a 32-byte session secret.
|
password + a 32-byte session secret.
|
||||||
5. **Commit & push** the whole change. ArgoCD will create the namespace
|
5. **Commit & push** the whole change. ArgoCD will create the namespace
|
||||||
resources, deploy the pod, and bring up the ingress at `hermes.rogi.casa`.
|
resources, deploy the pod, and bring up the ingress at `platform-engineer.rogi.casa`.
|
||||||
6. **Seed the cron jobs:**
|
6. **Seed the cron jobs:**
|
||||||
`kubectl apply -f platform-engineer/cron-seed.yaml` (one-shot Job) — it waits
|
`kubectl apply -f platform-engineer/cron-seed.yaml` (one-shot Job) — it waits
|
||||||
for the hermes pod, then runs `hermes cron create ...` for each watchdog.
|
for the hermes pod, then runs `hermes cron create ...` for each watchdog.
|
||||||
@@ -357,7 +357,7 @@ Design rules baked into SOUL.md:
|
|||||||
| `dockerfile` | derived image: hermes-agent + kubectl + helm (linux/amd64) |
|
| `dockerfile` | derived image: hermes-agent + kubectl + helm (linux/amd64) |
|
||||||
| `build-and-push.sh` | builds & pushes the image to the Gitea registry |
|
| `build-and-push.sh` | builds & pushes the image to the Gitea registry |
|
||||||
| `deployment.yaml` | Deployment (1 replica, Recreate, pinned to amd64 NUC) + Service |
|
| `deployment.yaml` | Deployment (1 replica, Recreate, pinned to amd64 NUC) + Service |
|
||||||
| `ingress.yaml` | `hermes.rogi.casa` → dashboard (TLS + basic auth) |
|
| `ingress.yaml` | `platform-engineer.rogi.casa` → dashboard (TLS + basic auth) |
|
||||||
| `cron-seed.yaml` | one-shot Job that creates the Hermes cron schedule |
|
| `cron-seed.yaml` | one-shot Job that creates the Hermes cron schedule |
|
||||||
|
|
||||||
Also changed outside this directory:
|
Also changed outside this directory:
|
||||||
|
|||||||
@@ -16,11 +16,11 @@ data:
|
|||||||
auxiliary:
|
auxiliary:
|
||||||
compression:
|
compression:
|
||||||
provider: openai-api
|
provider: openai-api
|
||||||
model: qwen3.6-27b
|
model: qwen3.6-35b-a3b # flash model for speed-sensitive auxiliary tasks
|
||||||
base_url: "http://litellm-service.litellm:80/v1"
|
base_url: "http://litellm-service.litellm:80/v1"
|
||||||
title_generation:
|
title_generation:
|
||||||
provider: openai-api
|
provider: openai-api
|
||||||
model: qwen3.6-27b
|
model: qwen3.6-35b-a3b # flash model for speed-sensitive auxiliary tasks
|
||||||
base_url: "http://litellm-service.litellm:80/v1"
|
base_url: "http://litellm-service.litellm:80/v1"
|
||||||
|
|
||||||
terminal:
|
terminal:
|
||||||
|
|||||||
@@ -54,6 +54,12 @@ spec:
|
|||||||
|
|
||||||
exists() { kubectl -n platform-engineer exec "$POD" -- hermes cron list 2>/dev/null | grep -qi " $1 "; }
|
exists() { kubectl -n platform-engineer exec "$POD" -- hermes cron list 2>/dev/null | grep -qi " $1 "; }
|
||||||
|
|
||||||
|
# NOTE: `hermes cron create` has no --model flag. New jobs inherit the
|
||||||
|
# global default model from config.yaml at creation time. To pin a
|
||||||
|
# specific model per-job (e.g. the flash model qwen3.6-35b-a3b for all
|
||||||
|
# cron tasks), edit /opt/data/cron/jobs.json directly after seeding:
|
||||||
|
# kubectl exec deploy/hermes -- python3 -c "...set model field..."
|
||||||
|
# See llamacpp/README.md and the deployment notes for details.
|
||||||
create() {
|
create() {
|
||||||
name="$1"; schedule="$2"; deliver="$3"; prompt="$4"
|
name="$1"; schedule="$2"; deliver="$3"; prompt="$4"
|
||||||
if exists "$name"; then
|
if exists "$name"; then
|
||||||
|
|||||||
@@ -9,10 +9,10 @@ spec:
|
|||||||
ingressClassName: traefik
|
ingressClassName: traefik
|
||||||
tls:
|
tls:
|
||||||
- hosts:
|
- hosts:
|
||||||
- hermes.rogi.casa
|
- platform-engineer.rogi.casa
|
||||||
secretName: hermes-tls
|
secretName: platform-engineer-tls
|
||||||
rules:
|
rules:
|
||||||
- host: hermes.rogi.casa
|
- host: platform-engineer.rogi.casa
|
||||||
http:
|
http:
|
||||||
paths:
|
paths:
|
||||||
- path: /
|
- path: /
|
||||||
|
|||||||
Reference in New Issue
Block a user