platform-engineer-agent/k3s-cluster
1
0
Fork 0
forked from roger/k3s-cluster
Code Pull Requests Activity

loki + promtail new monitoring services

Browse Source
This commit is contained in:
Roger Oriol
2026-07-05 17:32:15 +02:00
parent 85c8cbfc31
commit 9fd7d02c7c
12 changed files with 568 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
argocd/argocd-cm.yaml Normal file
View File

@@ -0,0 +1,15 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: argocd-cm
namespace: argocd
labels:
app.kubernetes.io/name: argocd-cm
app.kubernetes.io/part-of: argocd
data:
# add an additional local user with apiKey and login capabilities
# apiKey - allows generating API keys
# login - allows to login using UI
accounts.roger: apiKey, login
# disables user. User is enabled by default
accounts.roger.enabled: "false"

14
litellm/litellm.yaml
View File

@@ -28,11 +28,19 @@ data:
model: ollama/glm-4.7-flash
api_base: http://10.88.20.12:11434
# Used by the platform-engineer Hermes agent (deployed in ns platform-engineer).
# model_name is the alias Hermes requests; the underlying Ollama model is qwen3.6:27b.
- model_name: qwen3.6:27b
# model_name is the alias Hermes requests; the underlying Ollama model is
# qwen3.6:latest (the fast non-27b tag). 27b is a slow reasoning model.
# `ollama_chat/` (not `ollama/`) uses Ollama's NATIVE /api/chat endpoint.
# `think: false` + `chat_template_kwargs.enable_thinking: false` disable
# Qwen3 thinking so the model emits content directly (otherwise the
# OpenAI-compat translation returns empty content with reasoning split off).
- model_name: qwen3.6
litellm_params:
model: ollama/qwen3.6:27b
model: ollama_chat/qwen3.6:latest
api_base: http://10.88.20.12:11434
think: false
chat_template_kwargs:
enable_thinking: false
litellm_settings:
#set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production
callbacks: ["arize_phoenix"]

5
monitoring/grafana-datasource.yaml
View File

@@ -13,3 +13,8 @@ data:
url: http://prometheus:9090
isDefault: true
editable: true
- name: Loki
type: loki
access: proxy
url: http://loki:3100
editable: true

95
monitoring/k8s-event-logger.yaml Normal file
View File

@@ -0,0 +1,95 @@
# k8s-event-exporter — watches Kubernetes Events and logs them to stdout as
# structured JSON. Promtail tails the logs and ships them to Loki with the
# label app=kubernetes-event-exporter. The Hermes agent queries them with LogQL:
# {app="kubernetes-event-exporter"} |= "BackOff"
#
# This avoids giving the agent any k8s API token for events — the exporter
# has its own narrow read-only SA, and the agent only talks to Loki.
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: k8s-event-exporter
namespace: monitoring
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: k8s-event-exporter
rules:
- apiGroups: [""]
resources:
- events
- pods
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: k8s-event-exporter
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: k8s-event-exporter
subjects:
- kind: ServiceAccount
name: k8s-event-exporter
namespace: monitoring
---
apiVersion: v1
kind: ConfigMap
metadata:
name: k8s-event-exporter-config
namespace: monitoring
data:
config.yaml: |
logLevel: info
logFormat: json
route:
routes:
- match:
- receiver: dump
receivers:
- name: dump
dump: {}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: k8s-event-exporter
namespace: monitoring
labels:
app: kubernetes-event-exporter
spec:
replicas: 1
selector:
matchLabels:
app: kubernetes-event-exporter
template:
metadata:
labels:
app: kubernetes-event-exporter
spec:
serviceAccountName: k8s-event-exporter
nodeSelector:
kubernetes.io/arch: amd64
containers:
- name: exporter
image: opsgenie/kubernetes-event-exporter:0.9
args:
- -conf=/config/config.yaml
volumeMounts:
- name: config
mountPath: /config
readOnly: true
resources:
requests:
memory: "32Mi"
cpu: "25m"
limits:
memory: "128Mi"
cpu: "100m"
volumes:
- name: config
configMap:
name: k8s-event-exporter-config

153
monitoring/loki.yaml Normal file
View File

@@ -0,0 +1,153 @@
# Loki — log aggregation (single-binary mode, local filesystem storage).
#
# Stores compressed, indexed pod logs shipped by Promtail. Queried by the
# platform-engineer Hermes agent via the HTTP API (LogQL) and by Grafana.
#
# Storage: 20 GiB local PVC, 1-week retention enforced by the compactor.
# Service: loki.monitoring:3100 (ClusterIP, no auth — homelab).
---
apiVersion: v1
kind: ConfigMap
metadata:
name: loki-config
namespace: monitoring
data:
loki.yaml: |
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
common:
path_prefix: /loki
replication_factor: 1
ring:
instance_addr: 127.0.0.1
kvstore:
store: inmemory
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
storage_config:
filesystem:
directory: /loki/chunks
tsdb_shipper:
active_index_directory: /loki/tsdb-index
cache_location: /loki/tsdb-cache
limits_config:
retention_period: 168h # 1 week
max_query_series: 10000
reject_old_samples: true
reject_old_samples_max_age: 168h
allow_structured_metadata: false # tsdb v13 compat
compactor:
working_directory: /loki/compactor
compaction_interval: 10m
retention_enabled: true
retention_delete_delay: 2h
retention_delete_worker_count: 50
delete_request_store: filesystem
analytics:
reporting_enabled: false
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: loki-data
namespace: monitoring
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 20Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: loki
namespace: monitoring
labels:
app: loki
spec:
replicas: 1
strategy:
type: Recreate # single-writer storage
selector:
matchLabels:
app: loki
template:
metadata:
labels:
app: loki
spec:
nodeSelector:
kubernetes.io/arch: amd64 # Loki image; runs on the NUC
containers:
- name: loki
image: grafana/loki:3.4.4
args:
- -config.file=/etc/loki/loki.yaml
ports:
- name: http
containerPort: 3100
volumeMounts:
- name: config
mountPath: /etc/loki
readOnly: true
- name: data
mountPath: /loki
resources:
requests:
memory: "512Mi"
cpu: "250m"
limits:
memory: "1Gi"
cpu: "1000m"
readinessProbe:
httpGet:
path: /ready
port: 3100
initialDelaySeconds: 30
periodSeconds: 10
failureThreshold: 5
livenessProbe:
httpGet:
path: /ready
port: 3100
initialDelaySeconds: 60
periodSeconds: 30
failureThreshold: 5
volumes:
- name: config
configMap:
name: loki-config
- name: data
persistentVolumeClaim:
claimName: loki-data
---
apiVersion: v1
kind: Service
metadata:
name: loki
namespace: monitoring
spec:
type: ClusterIP
selector:
app: loki
ports:
- name: http
port: 3100
targetPort: 3100

138
monitoring/promtail.yaml Normal file
View File

@@ -0,0 +1,138 @@
# Promtail — DaemonSet that tails pod logs on every node and ships them to Loki.
#
# Runs on ALL nodes (amd64 + arm). Multi-arch image. Reads /var/log/pods/*,
# attaches k8s labels (namespace, pod, container), ships to loki.monitoring:3100.
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: promtail
namespace: monitoring
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: promtail
rules:
- apiGroups: [""]
resources:
- nodes
- nodes/proxy
- services
- endpoints
- pods
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: promtail
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: promtail
subjects:
- kind: ServiceAccount
name: promtail
namespace: monitoring
---
apiVersion: v1
kind: ConfigMap
metadata:
name: promtail-config
namespace: monitoring
data:
promtail.yaml: |
server:
http_listen_port: 9080
grpc_listen_port: 0
positions:
filename: /tmp/positions.yaml
clients:
- url: http://loki.monitoring:3100/loki/api/v1/push
scrape_configs:
# Tail all container logs via /var/log/containers/*.log (symlinks to
# /var/log/pods/<ns>_<pod>_<uid>/<container>/<N>.log). Extract namespace,
# pod, container labels from the filename via pipeline_stages regex.
- job_name: kubernetes-containers
static_configs:
- targets:
- localhost
labels:
job: kube-containers
__path__: /var/log/containers/*.log
pipeline_stages:
- cri: {}
# k3s filename: <pod>_<namespace>_<container>-<hash>.log
- regex:
expression: '/var/log/containers/(?P<pod>[^_]+)_(?P<namespace>[^_]+)_(?P<container>[^-]+)-.*\.log'
source: filename
- labels:
pod:
namespace:
container:
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: promtail
namespace: monitoring
labels:
app: promtail
spec:
selector:
matchLabels:
app: promtail
template:
metadata:
labels:
app: promtail
spec:
serviceAccountName: promtail
tolerations:
- operator: Exists # run on every node including tainted Pis
containers:
- name: promtail
image: grafana/promtail:3.4.4
args:
- -config.file=/etc/promtail/promtail.yaml
- -config.expand-env=true
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: config
mountPath: /etc/promtail
readOnly: true
- name: positions
mountPath: /tmp
- name: pods-logs
mountPath: /var/log/pods
readOnly: true
- name: containers-logs
mountPath: /var/log/containers
readOnly: true
resources:
requests:
memory: "64Mi"
cpu: "50m"
limits:
memory: "256Mi"
cpu: "250m"
volumes:
- name: config
configMap:
name: promtail-config
- name: positions
emptyDir: {}
- name: pods-logs
hostPath:
path: /var/log/pods
- name: containers-logs
hostPath:
path: /var/log/containers

43
platform-engineer/build-and-push.sh
View File

@@ -1,43 +0,0 @@
#!/usr/bin/env bash
# Build & push the derived Hermes image (kubectl + helm).
#
# Two modes:
# ./build-and-push.sh push # build + push to the Gitea registry
# ./build-and-push.sh local # build + import directly into the NUC's k3s containerd
# # (no registry needed; pod is pinned to this node)
#
# Default (no arg): push.
set -euo pipefail
# Docker registry pushes can't go through the Cloudflare proxy (100 MB cap),
# so push to the DNS-only registry hostname instead of git.rogi.casa.
# Override with: REGISTRY=git.rogi.casa ./build-and-push.sh push (if grey-clouded)
REGISTRY="${REGISTRY:-registry.rogi.casa}"
REPO="roger/hermes-agent"
TAG="${TAG:-v1.35-1}"
IMAGE="${REGISTRY}/${REPO}:${TAG}"
MODE="${1:-push}"
cd "$(dirname "$0")"
echo "==> Building ${IMAGE}"
docker build --platform linux/amd64 -t "${IMAGE}" -f dockerfile .
case "$MODE" in
push)
echo "==> Pushing ${IMAGE}"
docker push "${IMAGE}"
echo "==> Done. If the pod can't pull, create the gitea-registry secret in the namespace."
;;
local)
# Requires k3s + being run on the node the pod schedules to (roger-nucbox-evo-x2).
echo "==> Importing into k3s containerd (requires sudo)"
docker save "${IMAGE}" | sudo k3s ctr images import -
echo "==> Done. Verify: sudo k3s ctr images ls | grep hermes-agent"
echo " deployment.yaml is set to imagePullPolicy: IfNotPresent"
;;
*)
echo "Usage: $0 {push|local}" >&2
exit 1
;;
esac

139
platform-engineer/configmap.yaml
View File

@@ -1,5 +1,4 @@
# Hermes configuration, SOUL.md, and the cron-seed script.
# Seeded into the PVC (/opt/data) by the initContainer on first boot only.
# Hermes configuration + SOUL.md + profile.d (seeded into the PVC on first boot).
---
apiVersion: v1
kind: ConfigMap
@@ -10,28 +9,26 @@ data:
config.yaml: |
model:
provider: openai-api
default: qwen3.6:27b
base_url: "https://litellm.rogi.casa/v1"
default: qwen3.6
base_url: "http://litellm-service.litellm:80/v1"
api_mode: chat_completions
# Cheap/fast model for auxiliary tasks (titling, compression).
auxiliary:
compression:
provider: openai-api
model: qwen3.6:27b
base_url: "https://litellm.rogi.casa/v1"
model: qwen3.6
base_url: "http://litellm-service.litellm:80/v1"
title_generation:
provider: openai-api
model: qwen3.6:27b
base_url: "https://litellm.rogi.casa/v1"
model: qwen3.6
base_url: "http://litellm-service.litellm:80/v1"
terminal:
backend: local
cwd: /workspace
cwd: /workspace/k3s-cluster
timeout: 180
home_mode: profile
# Unattended gateway → circuit-break on stuck tool-call loops.
tool_loop_guardrails:
hard_stop_enabled: true
hard_stop_after:
@@ -63,53 +60,87 @@ data:
## The cluster you look after
- **Nodes:**
- `raspberrypi` — control-plane, arm64 (4 GiB)
- `rpi2` — worker, arm, very low memory (~512 MiB)
- `roger-nucbox-evo-x2` — worker, amd64, 24 GiB (you run here)
- **GitOps:** ArgoCD owns every app from `https://git.rogi.casa/roger/k3s-cluster.git`.
Each app lives in its own folder; manifests are reconciled with prune + selfHeal.
- **Ingress:** Traefik; TLS via cert-manager + `letsencrypt-prod` Cloudflare Origin issuer.
- **LLM gateway:** LiteLLM at `https://litellm.rogi.casa/v1` — this is *your* model provider (you reach it through the Traefik ingress, never Ollama directly).
- **Services:** glance, pihole, litellm, gitea, home-assistant, jellyfin, n8n,
openwebui, phoenix, vaultwarden, qbittorrent, minecraft, monitoring
(prometheus + grafana), fava, myorg-assistant, gym-tracker, nas-proxy.
- **Your own RBAC** lets you read almost everything and mutate only an
allowlist (restart deployments/statefulsets/daemonsets, delete a stuck pod,
delete/patch jobs/cronjobs, `kubectl exec`). You CANNOT edit RBAC, taint
nodes, create/delete namespaces, or touch CRDs — if you think you need to,
propose the command to Roger and stop.
- **Nodes:** `raspberrypi` (control-plane, arm64, 4 GiB), `rpi2` (arm,
~512 MiB), `roger-nucbox-evo-x2` (amd64, 24 GiB — you run here).
- **GitOps:** ArgoCD owns every app from the git repo at `$GITEA_REPO_URL`.
The repo is cloned at `/workspace/k3s-cluster`. Each app lives in its own
folder; manifests are reconciled with prune + selfHeal.
- **Ingress:** Traefik; TLS via cert-manager + `letsencrypt-prod`.
- **Your model provider:** LiteLLM at `http://litellm-service.litellm:80/v1`
(reached in-cluster; never Ollama directly).
- **Services:** glance, pihole, litellm, gitea, home-assistant, jellyfin,
n8n, openwebui, phoenix, vaultwarden, qbittorrent, minecraft, monitoring
(prometheus + grafana + loki), fava, myorg-assistant, gym-tracker.
## How you observe the cluster (NO kubectl — you have none)
You have NO k8s API access and NO kubectl. Use these HTTP APIs instead:
1. **Prometheus** (metrics) at `http://prometheus.monitoring:9090/api/v1/query`
— PromQL via `curl -G -s "http://prometheus.monitoring:9090/api/v1/query" --data-urlencode "query=<PROMQL>"`
Examples:
- Node Ready: `kube_node_status_condition{condition="Ready",status="true"}`
- Node CPU/mem: `node_memory_MemAvailable_bytes`, `node_cpu_seconds_total`
- Pod restarts: `kube_pod_container_status_restarts_total`
- PVC usage: `kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes`
- Cert expiry: `certmanager_certificate_expiration_timestamp_seconds`
2. **Loki** (pod logs + events) at `http://loki.monitoring:3100/loki/api/v1/query_range`
— LogQL via `curl -G -s "http://loki.monitoring:3100/loki/api/v1/query_range" --data-urlencode "query=<LOGQL>" --data-urlencode "start=<unix_ns>" --data-urlencode "end=<unix_ns>" --data-urlencode "limit=50"`
Examples:
- Errors in a namespace: `{namespace="myorg-assistant"} |= "error"`
- CrashLoop across cluster: `{namespace=~".+"} |= "BackOff"`
- k8s events: `{app="k8s-event-logger"} |= "Warning"`
3. **ArgoCD API** at `https://argocd-server.argocd:443` — bearer token in
`$ARGOCD_API_TOKEN`. (Verify the cert with `--insecure` if needed since
it's the internal service.)
Examples:
- List apps: `curl -sk -H "Authorization: Bearer $ARGOCD_API_TOKEN" https://argocd-server.argocd:443/api/v1/applications`
- Sync an app: `curl -sk -X POST -H "Authorization: Bearer $ARGOCD_API_TOKEN" https://argocd-server.argocd:443/api/v1/applications/<app>/sync`
## How you remediate (git commit → ArgoCD sync)
You have NO k8s write access. Every fix is a git commit to the repo at
`/workspace/k3s-cluster` (which you `git push` to Gitea using `$GITEA_TOKEN`).
ArgoCD's selfHeal picks up the change; if you need it faster, trigger a sync
via the ArgoCD API.
Workflow:
cd /workspace/k3s-cluster
git pull
# ... edit the manifest(s) ...
git add -A && git commit -m "fix(<app>): <what changed>"
git push # uses the token in GITEA_REPO_URL / GITEA_TOKEN
# optionally trigger ArgoCD sync:
curl -sk -X POST -H "Authorization: Bearer $ARGOCD_API_TOKEN" \
https://argocd-server.argocd:443/api/v1/applications/<app>/sync
## Operating rules
1. **Read first, act second.** Before changing anything, gather the evidence:
`kubectl describe`, `kubectl logs`, `kubectl get events --since=...`,
`kubectl top`. Cite the exact resource (ns/name) and the exact command in
every report.
2. **Only safe, idempotent remediations.** Allowed actions:
- `kubectl rollout restart deployment/<name> -n <ns>` (and statefulset/daemonset)
- delete a single stuck `CrashLoopBackOff`/`ImagePullBackOff` pod so its
controller recreates it
- `kubectl delete job/<name>` / `kubectl patch cronjob ...`
Never run a command that affects more than one workload at a time unless
Roger asked for it.
3. **When in doubt, notify, don't act.** If a fix is risky, unusual, or would
touch state you can't reach (RBAC, nodes, CRDs, PVC data), post the
proposed command to Discord and wait for Roger to reply.
4. **Be quiet when healthy.** Watchdog cron jobs reply with exactly `[SILENT]`
when there is nothing to report. Failed jobs always deliver regardless.
5. **No runaway loops.** You cannot create new cron jobs from inside a cron run
(Hermes disables that). Do not try.
6. **Talk like an engineer.** Short, concrete, with resource names and
commands. No filler. When you fixed something, say what you did in one line.
7. **Respect GitOps.** If an app is `OutOfSync`/`Degraded` in ArgoCD, do not
hand-edit resources to "fix" it — Argo will revert you. Report it so Roger
can fix the source repo.
1. **Read first, act second.** Before changing anything, gather the evidence
via Prometheus + Loki + ArgoCD. Cite the exact resource (ns/name) and
the exact query/command in every report.
2. **GitOps is the ONLY write path.** Never try to use kubectl (you don't
have it). Every remediation is a git commit + push + optional ArgoCD sync
trigger. ArgoCD will reconcile; if it reverts you, your fix was wrong.
3. **Only safe, idempotent remediations.** Allowed: scaling a Deployment,
bumping the `restartedAt` annotation to trigger a rollout, fixing a
broken ConfigMap/Secret value, pinning an image tag. Never touch RBAC,
ArgoCD's own Application manifests, nodes, or CRDs.
4. **When in doubt, notify, don't act.** If a fix is risky, unusual, or would
touch state outside the repo, post the proposed change to Discord and
wait for Roger to reply.
5. **Be quiet when healthy.** Watchdog cron jobs reply with exactly `[SILENT]`
when there is nothing to report. Failed jobs always deliver.
6. **No runaway loops.** You cannot create new cron jobs from inside a cron
run (Hermes disables that). Do not try.
7. **Talk like an engineer.** Short, concrete, with resource names and
queries. No filler. When you fixed something, say what you did in one line.
8. **Respect GitOps.** If an app is `OutOfSync`/`Degraded`, check whether a
commit is stuck. Don't hand-edit resources — fix the source repo.
## How you reach Roger
Notifications go to Discord (your home channel). Cron jobs deliver there by
default (`deliver="discord"`). Keep messages under ~1800 chars; attach
longer logs as `kubectl logs ... > /opt/data/cron/output/<file>` and link
the path.
```
default (`deliver="discord"`). Keep messages under ~1800 chars.

40
platform-engineer/cron-seed.yaml
View File

@@ -1,9 +1,8 @@
# One-shot Job that seeds Hermes' built-in cron schedule on first install.
# Idempotent: skips job names that already exist.
#
# The agent's own cron jobs live in /opt/data/cron/jobs.json on the PVC and are
# NOT reconciled by ArgoCD (runtime state). Re-run this Job manually after a
# wipe to re-seed: kubectl job restart hermes-cron-seed -n platform-engineer
# Uses a `cron-seeder` SA scoped to pods/exec on the hermes pod ONLY (no k8s
# access for the agent itself).
---
apiVersion: batch/v1
kind: Job
@@ -13,8 +12,6 @@ metadata:
labels:
app: hermes
annotations:
# Job.spec.template is immutable — tell ArgoCD to Replace (delete+create)
# instead of patching, so edits to this Job sync cleanly under selfHeal.
argocd.argoproj.io/sync-options: Replace=true
argocd.argoproj.io/hook: Sync
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
@@ -26,20 +23,15 @@ spec:
labels:
app: hermes
spec:
serviceAccountName: platform-engineer
serviceAccountName: cron-seeder
restartPolicy: OnFailure
containers:
- name: seed
# alpine is tiny and always available; we install curl + download the
# right-arch kubectl binary at runtime (bitnami/kubectl tags are
# inconsistent across versions, so we avoid depending on them).
image: alpine:3.20
command: ["sh", "-c"]
args:
- |
set -e
# Install curl, then download kubectl for this node's architecture.
apk add --no-cache curl
ARCH=$(uname -m)
case "$ARCH" in
@@ -48,11 +40,9 @@ spec:
armv7l) KARCH=arm ;;
*) echo "unsupported arch: $ARCH" >&2; exit 1 ;;
esac
echo "Downloading kubectl for linux/$KARCH ..."
curl -fsSL -o /usr/local/bin/kubectl \
"https://dl.k8s.io/release/v1.35.0/bin/linux/${KARCH}/kubectl"
chmod +x /usr/local/bin/kubectl
kubectl version --client
echo "Waiting for hermes pod to be Ready..."
kubectl -n platform-engineer wait --for=condition=Ready pod -l app=hermes --timeout=300s || true
@@ -68,38 +58,34 @@ spec:
echo "cron job '$name' already exists — skipping"
else
echo "creating cron job '$name' ..."
# NOTE: the `hermes cron` CLI has no --provider/--model flags.
# Unpinned jobs snapshot the current global default (qwen3.6:27b)
# at creation, so they run fine. They only fail-closed if the
# global default is changed LATER (intended safety, #44585).
# To pin a job, use the `cronjob` agent tool inside a chat turn.
kubectl -n platform-engineer exec "$POD" -- hermes cron create "$schedule" "$prompt" \
--name "$name" --deliver "$deliver"
kubectl -n platform-engineer exec "$POD" -- hermes cron create "$schedule" "$prompt" --name "$name" --deliver "$deliver"
fi
}
NOW_NS='$(date +%s)000000000'
# ---- Watchdog checks (silent unless something is wrong) ----
create "cluster-health-check" "every 15m" "discord" \
"Run: kubectl get nodes; kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded; kubectl get events -A --field-selector type=Warning --since=20m. If everything is healthy and there are no Warning events, reply with exactly [SILENT]. Otherwise give a concise per-resource summary of what is wrong (node name, pod ns/name, phase, last event)."
"Check cluster health via HTTP APIs (you have NO kubectl). (1) Prometheus: curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=kube_node_status_condition{condition=\"Ready\",status!=\"true\"}' — if any node is NotReady, report it. (2) Prometheus: curl for kube_pod_status_phase{phase!=\"Running\"} to find pods not Running. (3) Loki: curl -G -s 'http://loki.monitoring:3100/loki/api/v1/query_range' --data-urlencode 'query={namespace=~\".+\"} |~ \"(?i)error|panic|crashloop|backoff\"' --data-urlencode 'start=$(date -d \"20 minutes ago\" +%s)000000000' --data-urlencode 'end=$(date +%s)000000000' --data-urlencode 'limit=20' — report any error lines with namespace/pod. (4) ArgoCD: curl -sk -H \"Authorization: Bearer \$ARGOCD_API_TOKEN\" 'https://argocd-server.argocd:443/api/v1/applications' — report any app not Synced+Healthy. If everything is healthy, reply with exactly [SILENT]. Otherwise give a concise per-resource summary."
create "pod-restart-loop" "every 10m" "discord" \
"Find pods in CrashLoopBackOff or ImagePullBackOff across all namespaces (kubectl get pods -A). For each, fetch kubectl logs (previous) and describe. If the cause is clearly transient (OOM kill, a one-off config parse error that will retry cleanly, a missing Secret the controller will recreate), attempt ONE safe remediation: kubectl rollout restart of the owning Deployment/StatefulSet/DaemonSet, OR delete the single stuck pod. Report what you did in one line per resource. If the cause is not clearly transient (bad image, missing config, auth failure), do NOT act — post the log excerpt and the proposed command and wait for Roger. If no such pods exist, reply [SILENT]."
"Find pods with high restart rates via Prometheus (NO kubectl): curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=topk(5, max_over_time(kube_pod_container_status_restarts_total[15m]))' — if any pod has >3 restarts in 15m, fetch its logs from Loki: curl -G -s 'http://loki.monitoring:3100/loki/api/v1/query_range' --data-urlencode 'query={namespace=\"<ns>\",pod=\"<pod>\"}' --data-urlencode 'start=<15m ago unix ns>' --data-urlencode 'end=<now unix ns>' --data-urlencode 'limit=30'. Diagnose the cause. If fixable via a manifest change (e.g., bump memory limit, fix a config value, bump restartedAt annotation), edit the file in /workspace/k3s-cluster, git add -A, git commit -m 'fix(<app>): <reason>', git push, then trigger ArgoCD sync: curl -sk -X POST -H 'Authorization: Bearer \$ARGOCD_API_TOKEN' 'https://argocd-server.argocd:443/api/v1/applications/<app>/sync'. Report what you did in one line. If not clearly fixable, post the log excerpt and proposed fix, and wait for Roger. If no high-restart pods, reply [SILENT]."
create "pvc-pressure" "every 30m" "discord" \
"Check cluster storage health: kubectl get pv,pvc -A; kubectl top nodes. Alert if any PVC is Pending/Lost or any node filesystem usage is over 85%. If all healthy, reply [SILENT]."
"Check storage health via Prometheus (NO kubectl): curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes' — alert on any PVC with <15% free. Also check node disk: curl for '1 - (node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})'. If any PVC or node disk is over 85% used, report it with the namespace/PVC name and percentage. If all healthy, reply [SILENT]."
create "argocd-sync-health" "every 1h" "discord" \
"Run: kubectl get applications -n argocd -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status. If every app is Synced and Healthy, reply [SILENT]. Otherwise list the OutOfSync/Degraded apps with their status. Do NOT hand-edit resources to fix them (Argo will revert) — just report."
"Check ArgoCD app health via API (NO kubectl): curl -sk -H 'Authorization: Bearer \$ARGOCD_API_TOKEN' 'https://argocd-server.argocd:443/api/v1/applications'. For each app, check syncStatus and healthStatus. If every app is Synced and Healthy, reply [SILENT]. Otherwise list the OutOfSync/Degraded apps with their status. If an app is OutOfSync and you believe a recent git push caused it, you may trigger a sync: curl -sk -X POST -H 'Authorization: Bearer \$ARGOCD_API_TOKEN' 'https://argocd-server.argocd:443/api/v1/applications/<app>/sync'. Do NOT hand-edit resources to fix them — fix the source repo."
create "cert-expiry" "0 9 * * *" "discord" \
"List all cert-manager Certificate resources (kubectl get certificates -A). For each, check notAfter. Alert on any certificate expiring in under 21 days. If none, reply [SILENT]."
"Check certificate expiry via Prometheus (NO kubectl): curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400' — this gives days until expiry. Alert on any certificate expiring in under 21 days, with its name and namespace. If none, reply [SILENT]."
create "node-resource-drift" "every 30m" "discord" \
"Run kubectl top nodes. If any node CPU or memory usage is over 90%, or any node is NotReady, report it with the numbers. Otherwise reply [SILENT]."
"Check node resources via Prometheus (NO kubectl): (1) Node CPU: curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (node)' (2) Node memory: curl for '1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)' by node (3) Node Ready: curl for 'kube_node_status_condition{condition=\"Ready\",status!=\"true\"}'. If any node is NotReady, or any node CPU>90% or memory>90%, report it with the numbers. Otherwise reply [SILENT]."
# ---- Daily report (always delivered) ----
create "daily-cluster-report" "0 8 * * *" "discord" \
"Produce a daily cluster report for Roger: (1) node count + Ready/NotReady; (2) top 5 pods by CPU and by memory across all namespaces (kubectl top pods -A --sort-by); (3) count of pods not Running; (4) ArgoCD apps OutOfSync or Degraded; (5) any certificates expiring within 30 days; (6) any recent Warning events (last 24h). Keep it under 1800 chars. Always deliver (no [SILENT])."
"Produce a daily cluster report for Roger using HTTP APIs (NO kubectl): (1) Node status: curl Prometheus for kube_node_status_condition{condition=\"Ready\"} — report Ready/NotReady per node. (2) Top pods by CPU/mem: curl Prometheus for topk(5, rate(container_cpu_usage_seconds_total[5m])) and topk(5, container_memory_working_set_bytes). (3) Pods not Running: curl for kube_pod_status_phase{phase!=\"Running\"} count by namespace. (4) ArgoCD apps: curl -sk -H 'Authorization: Bearer \$ARGOCD_API_TOKEN' 'https://argocd-server.argocd:443/api/v1/applications' — list any OutOfSync or Degraded. (5) Certificates expiring <30d: curl Prometheus for certmanager_certificate_expiration_timestamp_seconds. (6) Recent warnings: curl Loki for {app=\"k8s-event-logger\"} |= \"Warning\" in last 24h. Keep it under 1800 chars. Always deliver (no [SILENT])."
echo "Done. Listing all cron jobs:"
kubectl -n platform-engineer exec "$POD" -- hermes cron list

78
platform-engineer/deployment.yaml
View File

@@ -17,8 +17,8 @@ spec:
labels:
app: hermes
spec:
serviceAccountName: platform-engineer
# No imagePullSecrets — using the public stock Hermes image from Docker Hub.
# No serviceAccountName — the agent has NO k8s API access. It manages the
# cluster via git commits (→ ArgoCD sync) and reads via Loki/Prometheus/ArgoCD.
# Pin to the powerful amd64 node (image is linux/amd64; the NUC has 24 GiB).
nodeSelector:
@@ -42,29 +42,33 @@ spec:
topologyKey: kubernetes.io/hostname
initContainers:
# Download kubectl + helm into a shared emptyDir so the stock Hermes image
# (which doesn't ship kubectl) can still drive the cluster. Avoids building
# and pushing a custom image through a slow / size-capped registry.
- name: install-tools
image: curlimages/curl:8.12.1
# Clone the k3s-cluster repo into a persistent workspace so the agent can
# commit + push remediations. The token is injected via envFrom.
- name: git-clone
image: alpine/git:2.43.0
command: ["sh", "-c"]
args:
- |
set -e
echo "Downloading kubectl v1.35.0..."
curl -fsSL -o /tools/kubectl \
https://dl.k8s.io/release/v1.35.0/bin/linux/amd64/kubectl
chmod +x /tools/kubectl
echo "Downloading helm v3.16.3..."
curl -fsSL https://get.helm.sh/helm-v3.16.3-linux-amd64.tar.gz \
| tar -xz -C /tools --strip-components=1 linux-amd64/helm
chmod +x /tools/helm
echo "Tools installed:"; ls -la /tools
cd /workspace
if [ -d k3s-cluster/.git ]; then
echo "Repo exists, pulling latest..."
cd k3s-cluster && git pull --rebase || true
else
echo "Cloning repo..."
git clone "${GITEA_REPO_URL}" k3s-cluster
cd k3s-cluster
git config user.name "Platform Engineer"
git config user.email "platform-engineer@rogi.casa"
fi
envFrom:
- secretRef:
name: hermes-env
volumeMounts:
- name: tools
mountPath: /tools
- name: workspace
mountPath: /workspace
# Seed /opt/data with config.yaml + SOUL.md on first boot only.
# Seed /opt/data with config.yaml + SOUL.md + .env on first boot only.
# ArgoCD owns the manifests; the PVC is runtime state and is NOT reconciled.
- name: seed-data
image: busybox:1.36
@@ -73,14 +77,28 @@ spec:
- |
set -e
if [ ! -f /opt/data/config.yaml ]; then
echo "First boot: seeding /opt/data from ConfigMap..."
echo "First boot: seeding /opt/data from ConfigMap + env..."
cp /seed/config.yaml /opt/data/config.yaml
cp /seed/SOUL.md /opt/data/SOUL.md
chmod 600 /opt/data/config.yaml
# Write .env from the injected Secret env vars so the s6 gateway
# finds API keys (the hermes container reads keys from /opt/data/.env).
: > /opt/data/.env
chmod 600 /opt/data/.env
for k in OPENAI_API_KEY OPENAI_BASE_URL DISCORD_BOT_TOKEN DISCORD_HOME_CHANNEL \
GITEA_TOKEN GITEA_REPO_URL ARGOCD_API_TOKEN ARGOCD_SERVER \
HERMES_DASHBOARD HERMES_DASHBOARD_BASIC_AUTH_USERNAME \
HERMES_DASHBOARD_BASIC_AUTH_PASSWORD HERMES_DASHBOARD_BASIC_AUTH_SECRET; do
eval "v=\${$k:-}"
[ -n "$v" ] && echo "$k=$v" >> /opt/data/.env
done
else
echo "/opt/data already initialized — leaving runtime state intact."
fi
mkdir -p /opt/data/home/.kube /opt/data/cron/output /opt/data/scripts /workspace
mkdir -p /opt/data/home/.kube /opt/data/cron/output /opt/data/scripts
envFrom:
- secretRef:
name: hermes-env
volumeMounts:
- name: data
mountPath: /opt/data
@@ -93,8 +111,7 @@ spec:
imagePullPolicy: Always
# IMPORTANT: do NOT set `command:` — it would override the image's
# ENTRYPOINT (/init, s6-overlay), which sets up the hermes user, seeds
# config on first boot, and supervises the gateway. The image's CMD
# (main-wrapper.sh) already routes `gateway run` through s6.
# config on first boot, and supervises the gateway.
args: ["gateway", "run"]
ports:
- name: gateway
@@ -105,21 +122,13 @@ spec:
- secretRef:
name: hermes-env
env:
# k3s injects KUBERNETES_SERVICE_HOST/PORT + the SA token automatically;
# kubectl inside the pod authenticates as the platform-engineer SA.
- name: HERMES_HOME
value: /opt/data
# Put the initContainer-installed kubectl/helm on PATH for the hermes user.
- name: PATH
value: /opt/hermes/bin:/opt/hermes/.venv/bin:/tools:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
volumeMounts:
- name: data
mountPath: /opt/data
- name: workspace
mountPath: /workspace
- name: tools
mountPath: /tools
readOnly: true
resources:
requests:
memory: "512Mi"
@@ -129,10 +138,7 @@ spec:
cpu: "1000m"
livenessProbe:
# Probe the dashboard port (9119, always enabled via HERMES_DASHBOARD=1
# and binds 0.0.0.0). The gateway API on 8642 is off by default
# (API_SERVER_ENABLED not set), so 9119 is the reliable liveness signal.
# s6 auto-restarts the gateway itself; this probe only catches a wedged
# container.
# and binds 0.0.0.0). The gateway API on 8642 is off by default.
tcpSocket:
port: 9119
initialDelaySeconds: 90
@@ -148,8 +154,6 @@ spec:
claimName: hermes-data
- name: workspace
emptyDir: {}
- name: tools
emptyDir: {}
- name: seed
configMap:
name: hermes-seed

31
platform-engineer/dockerfile
View File

@@ -1,31 +0,0 @@
# Derived Hermes Agent image with kubectl + helm so the agent can drive the
# k3s cluster from inside the container (terminal backend = local).
#
# Build & push to the Gitea registry:
# docker build -t git.rogi.casa/roger/hermes-agent:v1.35-1 -f dockerfile .
# docker push git.rogi.casa/roger/hermes-agent:v1.35-1
#
# This image targets linux/amd64 (the agent pod is pinned to the amd64 NUC).
FROM nousresearch/hermes-agent:latest
USER root
# kubectl (v1.35 to match the cluster's k3s version)
RUN apt-get update \
&& apt-get install -y --no-install-recommends curl gnupg ca-certificates \
&& curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.35/deb/Release.key \
| gpg --dearmor -o /usr/share/keyrings/kubernetes-apt-keyring.gpg \
&& echo 'deb [signed-by=/usr/share/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.35/deb/ /' \
> /etc/apt/sources.list.d/kubernetes.list \
&& apt-get update \
&& apt-get install -y --no-install-recommends kubectl \
# helm
&& curl -fsSL https://get.helm.sh/helm-v3.16.3-linux-amd64.tar.gz \
| tar -xz -C /usr/local/bin --strip-components=1 linux-amd64/helm \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Hermes' own CLI/kubeconfig helper dir for tool subprocesses
RUN mkdir -p /opt/data/home/.kube
USER hermes

108
platform-engineer/rbac.yaml
View File

@@ -1,111 +1,27 @@
# Least-privilege RBAC for the Platform Engineer Hermes agent.
# Minimal RBAC for the cron-seed Job ONLY.
#
# The agent can READ almost everything cluster-wide, but can only MUTATE a
# narrow allowlist of safe, idempotent resources (restart deployments, delete a
# stuck pod so its controller recreates it, etc.). It CANNOT touch RBAC, nodes,
# namespaces, CRDs, or other namespaces' Secrets beyond read.
# The Hermes agent itself has NO k8s RBAC — it manages the cluster via git
# commits (→ ArgoCD sync) and reads state via Loki / Prometheus / ArgoCD APIs.
#
# The cron-seed Job needs to `kubectl exec` into the hermes pod to run
# `hermes cron create ...` (the only way to seed Hermes' internal cron).
# Scoped to this namespace, pods/exec on the hermes pod only.
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: platform-engineer
name: cron-seeder
namespace: platform-engineer
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
kind: Role
metadata:
name: platform-engineer
name: cron-seeder
namespace: platform-engineer
rules:
# ---- Broad read access (cluster-wide) ----
- apiGroups: [""]
resources:
- nodes
- nodes/proxy
- services
- endpoints
- pods
- pods/log
- configmaps
- secrets
- persistentvolumeclaims
- persistentvolumes
- namespaces
- events
- replicationcontrollers
verbs: ["get", "list", "watch"]
- apiGroups: ["apps"]
resources:
- deployments
- statefulsets
- daemonsets
- replicasets
verbs: ["get", "list", "watch"]
- apiGroups: ["batch"]
resources:
- jobs
- cronjobs
verbs: ["get", "list", "watch"]
- apiGroups: ["networking.k8s.io"]
resources:
- ingresses
verbs: ["get", "list", "watch"]
- apiGroups: ["autoscaling"]
resources:
- horizontalpodautoscalers
verbs: ["get", "list", "watch"]
- apiGroups: ["argoproj.io"]
resources:
- applications
- appprojects
verbs: ["get", "list", "watch"]
- apiGroups: ["cert-manager.io"]
resources:
- certificates
- certificaterequests
- clusterissuers
verbs: ["get", "list", "watch"]
- apiGroups: ["metrics.k8s.io"]
resources:
- pods
- nodes
verbs: ["get", "list"]
# ---- Metrics / health endpoints ----
- nonResourceURLs: ["/metrics", "/metrics/*"]
verbs: ["get"]
# ---- Narrow mutate allowlist (idempotent, safe remediation) ----
# Restart a stuck pod by deleting it (its controller recreates it).
- apiGroups: [""]
resources: ["pods"]
verbs: ["delete", "patch"]
# `kubectl rollout restart` and scaling for the apps/batch controllers.
- apiGroups: ["apps"]
resources:
- deployments
- statefulsets
- daemonsets
- replicasets
verbs: ["patch", "update"]
- apiGroups: ["batch"]
resources:
- jobs
- cronjobs
verbs: ["patch", "update", "delete"]
# Exec into pods for log-style / debug inspection (granted per request #5).
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: platform-engineer
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: platform-engineer
subjects:
- kind: ServiceAccount
name: platform-engineer
namespace: platform-engineer