From 9fd7d02c7c3320a31548c6f5c26e69615c785c27 Mon Sep 17 00:00:00 2001 From: Roger Oriol Date: Sun, 5 Jul 2026 17:32:15 +0200 Subject: [PATCH] loki + promtail new monitoring services --- argocd/argocd-cm.yaml | 15 +++ litellm/litellm.yaml | 14 ++- monitoring/grafana-datasource.yaml | 5 + monitoring/k8s-event-logger.yaml | 95 +++++++++++++++++ monitoring/loki.yaml | 153 ++++++++++++++++++++++++++++ monitoring/promtail.yaml | 138 +++++++++++++++++++++++++ platform-engineer/build-and-push.sh | 43 -------- platform-engineer/configmap.yaml | 139 +++++++++++++++---------- platform-engineer/cron-seed.yaml | 40 +++----- platform-engineer/deployment.yaml | 78 +++++++------- platform-engineer/dockerfile | 31 ------ platform-engineer/rbac.yaml | 108 +++----------------- 12 files changed, 568 insertions(+), 291 deletions(-) create mode 100644 argocd/argocd-cm.yaml create mode 100644 monitoring/k8s-event-logger.yaml create mode 100644 monitoring/loki.yaml create mode 100644 monitoring/promtail.yaml delete mode 100755 platform-engineer/build-and-push.sh delete mode 100644 platform-engineer/dockerfile diff --git a/argocd/argocd-cm.yaml b/argocd/argocd-cm.yaml new file mode 100644 index 0000000..decfc5f --- /dev/null +++ b/argocd/argocd-cm.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: argocd-cm + namespace: argocd + labels: + app.kubernetes.io/name: argocd-cm + app.kubernetes.io/part-of: argocd +data: + # add an additional local user with apiKey and login capabilities + # apiKey - allows generating API keys + # login - allows to login using UI + accounts.roger: apiKey, login + # disables user. User is enabled by default + accounts.roger.enabled: "false" diff --git a/litellm/litellm.yaml b/litellm/litellm.yaml index 7db10de..1b9e0ec 100644 --- a/litellm/litellm.yaml +++ b/litellm/litellm.yaml @@ -28,11 +28,19 @@ data: model: ollama/glm-4.7-flash api_base: http://10.88.20.12:11434 # Used by the platform-engineer Hermes agent (deployed in ns platform-engineer). - # model_name is the alias Hermes requests; the underlying Ollama model is qwen3.6:27b. - - model_name: qwen3.6:27b + # model_name is the alias Hermes requests; the underlying Ollama model is + # qwen3.6:latest (the fast non-27b tag). 27b is a slow reasoning model. + # `ollama_chat/` (not `ollama/`) uses Ollama's NATIVE /api/chat endpoint. + # `think: false` + `chat_template_kwargs.enable_thinking: false` disable + # Qwen3 thinking so the model emits content directly (otherwise the + # OpenAI-compat translation returns empty content with reasoning split off). + - model_name: qwen3.6 litellm_params: - model: ollama/qwen3.6:27b + model: ollama_chat/qwen3.6:latest api_base: http://10.88.20.12:11434 + think: false + chat_template_kwargs: + enable_thinking: false litellm_settings: #set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production callbacks: ["arize_phoenix"] diff --git a/monitoring/grafana-datasource.yaml b/monitoring/grafana-datasource.yaml index 85803a8..9b7021b 100644 --- a/monitoring/grafana-datasource.yaml +++ b/monitoring/grafana-datasource.yaml @@ -13,3 +13,8 @@ data: url: http://prometheus:9090 isDefault: true editable: true + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + editable: true diff --git a/monitoring/k8s-event-logger.yaml b/monitoring/k8s-event-logger.yaml new file mode 100644 index 0000000..31fa451 --- /dev/null +++ b/monitoring/k8s-event-logger.yaml @@ -0,0 +1,95 @@ +# k8s-event-exporter — watches Kubernetes Events and logs them to stdout as +# structured JSON. Promtail tails the logs and ships them to Loki with the +# label app=kubernetes-event-exporter. The Hermes agent queries them with LogQL: +# {app="kubernetes-event-exporter"} |= "BackOff" +# +# This avoids giving the agent any k8s API token for events — the exporter +# has its own narrow read-only SA, and the agent only talks to Loki. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: k8s-event-exporter + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: k8s-event-exporter +rules: + - apiGroups: [""] + resources: + - events + - pods + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: k8s-event-exporter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: k8s-event-exporter +subjects: + - kind: ServiceAccount + name: k8s-event-exporter + namespace: monitoring +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: k8s-event-exporter-config + namespace: monitoring +data: + config.yaml: | + logLevel: info + logFormat: json + route: + routes: + - match: + - receiver: dump + receivers: + - name: dump + dump: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: k8s-event-exporter + namespace: monitoring + labels: + app: kubernetes-event-exporter +spec: + replicas: 1 + selector: + matchLabels: + app: kubernetes-event-exporter + template: + metadata: + labels: + app: kubernetes-event-exporter + spec: + serviceAccountName: k8s-event-exporter + nodeSelector: + kubernetes.io/arch: amd64 + containers: + - name: exporter + image: opsgenie/kubernetes-event-exporter:0.9 + args: + - -conf=/config/config.yaml + volumeMounts: + - name: config + mountPath: /config + readOnly: true + resources: + requests: + memory: "32Mi" + cpu: "25m" + limits: + memory: "128Mi" + cpu: "100m" + volumes: + - name: config + configMap: + name: k8s-event-exporter-config diff --git a/monitoring/loki.yaml b/monitoring/loki.yaml new file mode 100644 index 0000000..9e22914 --- /dev/null +++ b/monitoring/loki.yaml @@ -0,0 +1,153 @@ +# Loki — log aggregation (single-binary mode, local filesystem storage). +# +# Stores compressed, indexed pod logs shipped by Promtail. Queried by the +# platform-engineer Hermes agent via the HTTP API (LogQL) and by Grafana. +# +# Storage: 20 GiB local PVC, 1-week retention enforced by the compactor. +# Service: loki.monitoring:3100 (ClusterIP, no auth — homelab). +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: loki-config + namespace: monitoring +data: + loki.yaml: | + auth_enabled: false + + server: + http_listen_port: 3100 + grpc_listen_port: 9096 + + common: + path_prefix: /loki + replication_factor: 1 + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + + schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + + storage_config: + filesystem: + directory: /loki/chunks + tsdb_shipper: + active_index_directory: /loki/tsdb-index + cache_location: /loki/tsdb-cache + + limits_config: + retention_period: 168h # 1 week + max_query_series: 10000 + reject_old_samples: true + reject_old_samples_max_age: 168h + allow_structured_metadata: false # tsdb v13 compat + + compactor: + working_directory: /loki/compactor + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + retention_delete_worker_count: 50 + delete_request_store: filesystem + + analytics: + reporting_enabled: false +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: loki-data + namespace: monitoring +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: loki + namespace: monitoring + labels: + app: loki +spec: + replicas: 1 + strategy: + type: Recreate # single-writer storage + selector: + matchLabels: + app: loki + template: + metadata: + labels: + app: loki + spec: + nodeSelector: + kubernetes.io/arch: amd64 # Loki image; runs on the NUC + containers: + - name: loki + image: grafana/loki:3.4.4 + args: + - -config.file=/etc/loki/loki.yaml + ports: + - name: http + containerPort: 3100 + volumeMounts: + - name: config + mountPath: /etc/loki + readOnly: true + - name: data + mountPath: /loki + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "1000m" + readinessProbe: + httpGet: + path: /ready + port: 3100 + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 5 + livenessProbe: + httpGet: + path: /ready + port: 3100 + initialDelaySeconds: 60 + periodSeconds: 30 + failureThreshold: 5 + volumes: + - name: config + configMap: + name: loki-config + - name: data + persistentVolumeClaim: + claimName: loki-data +--- +apiVersion: v1 +kind: Service +metadata: + name: loki + namespace: monitoring +spec: + type: ClusterIP + selector: + app: loki + ports: + - name: http + port: 3100 + targetPort: 3100 diff --git a/monitoring/promtail.yaml b/monitoring/promtail.yaml new file mode 100644 index 0000000..d291925 --- /dev/null +++ b/monitoring/promtail.yaml @@ -0,0 +1,138 @@ +# Promtail — DaemonSet that tails pod logs on every node and ships them to Loki. +# +# Runs on ALL nodes (amd64 + arm). Multi-arch image. Reads /var/log/pods/*, +# attaches k8s labels (namespace, pod, container), ships to loki.monitoring:3100. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: promtail + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: promtail +rules: + - apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: promtail +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: promtail +subjects: + - kind: ServiceAccount + name: promtail + namespace: monitoring +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: promtail-config + namespace: monitoring +data: + promtail.yaml: | + server: + http_listen_port: 9080 + grpc_listen_port: 0 + + positions: + filename: /tmp/positions.yaml + + clients: + - url: http://loki.monitoring:3100/loki/api/v1/push + + scrape_configs: + # Tail all container logs via /var/log/containers/*.log (symlinks to + # /var/log/pods/__//.log). Extract namespace, + # pod, container labels from the filename via pipeline_stages regex. + - job_name: kubernetes-containers + static_configs: + - targets: + - localhost + labels: + job: kube-containers + __path__: /var/log/containers/*.log + pipeline_stages: + - cri: {} + # k3s filename: __-.log + - regex: + expression: '/var/log/containers/(?P[^_]+)_(?P[^_]+)_(?P[^-]+)-.*\.log' + source: filename + - labels: + pod: + namespace: + container: +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: promtail + namespace: monitoring + labels: + app: promtail +spec: + selector: + matchLabels: + app: promtail + template: + metadata: + labels: + app: promtail + spec: + serviceAccountName: promtail + tolerations: + - operator: Exists # run on every node including tainted Pis + containers: + - name: promtail + image: grafana/promtail:3.4.4 + args: + - -config.file=/etc/promtail/promtail.yaml + - -config.expand-env=true + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: config + mountPath: /etc/promtail + readOnly: true + - name: positions + mountPath: /tmp + - name: pods-logs + mountPath: /var/log/pods + readOnly: true + - name: containers-logs + mountPath: /var/log/containers + readOnly: true + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "256Mi" + cpu: "250m" + volumes: + - name: config + configMap: + name: promtail-config + - name: positions + emptyDir: {} + - name: pods-logs + hostPath: + path: /var/log/pods + - name: containers-logs + hostPath: + path: /var/log/containers diff --git a/platform-engineer/build-and-push.sh b/platform-engineer/build-and-push.sh deleted file mode 100755 index 98faec1..0000000 --- a/platform-engineer/build-and-push.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Build & push the derived Hermes image (kubectl + helm). -# -# Two modes: -# ./build-and-push.sh push # build + push to the Gitea registry -# ./build-and-push.sh local # build + import directly into the NUC's k3s containerd -# # (no registry needed; pod is pinned to this node) -# -# Default (no arg): push. -set -euo pipefail - -# Docker registry pushes can't go through the Cloudflare proxy (100 MB cap), -# so push to the DNS-only registry hostname instead of git.rogi.casa. -# Override with: REGISTRY=git.rogi.casa ./build-and-push.sh push (if grey-clouded) -REGISTRY="${REGISTRY:-registry.rogi.casa}" -REPO="roger/hermes-agent" -TAG="${TAG:-v1.35-1}" -IMAGE="${REGISTRY}/${REPO}:${TAG}" -MODE="${1:-push}" - -cd "$(dirname "$0")" - -echo "==> Building ${IMAGE}" -docker build --platform linux/amd64 -t "${IMAGE}" -f dockerfile . - -case "$MODE" in - push) - echo "==> Pushing ${IMAGE}" - docker push "${IMAGE}" - echo "==> Done. If the pod can't pull, create the gitea-registry secret in the namespace." - ;; - local) - # Requires k3s + being run on the node the pod schedules to (roger-nucbox-evo-x2). - echo "==> Importing into k3s containerd (requires sudo)" - docker save "${IMAGE}" | sudo k3s ctr images import - - echo "==> Done. Verify: sudo k3s ctr images ls | grep hermes-agent" - echo " deployment.yaml is set to imagePullPolicy: IfNotPresent" - ;; - *) - echo "Usage: $0 {push|local}" >&2 - exit 1 - ;; -esac diff --git a/platform-engineer/configmap.yaml b/platform-engineer/configmap.yaml index d7e37de..c6cfcca 100644 --- a/platform-engineer/configmap.yaml +++ b/platform-engineer/configmap.yaml @@ -1,5 +1,4 @@ -# Hermes configuration, SOUL.md, and the cron-seed script. -# Seeded into the PVC (/opt/data) by the initContainer on first boot only. +# Hermes configuration + SOUL.md + profile.d (seeded into the PVC on first boot). --- apiVersion: v1 kind: ConfigMap @@ -10,28 +9,26 @@ data: config.yaml: | model: provider: openai-api - default: qwen3.6:27b - base_url: "https://litellm.rogi.casa/v1" + default: qwen3.6 + base_url: "http://litellm-service.litellm:80/v1" api_mode: chat_completions - # Cheap/fast model for auxiliary tasks (titling, compression). auxiliary: compression: provider: openai-api - model: qwen3.6:27b - base_url: "https://litellm.rogi.casa/v1" + model: qwen3.6 + base_url: "http://litellm-service.litellm:80/v1" title_generation: provider: openai-api - model: qwen3.6:27b - base_url: "https://litellm.rogi.casa/v1" + model: qwen3.6 + base_url: "http://litellm-service.litellm:80/v1" terminal: backend: local - cwd: /workspace + cwd: /workspace/k3s-cluster timeout: 180 home_mode: profile - # Unattended gateway → circuit-break on stuck tool-call loops. tool_loop_guardrails: hard_stop_enabled: true hard_stop_after: @@ -63,53 +60,87 @@ data: ## The cluster you look after - - **Nodes:** - - `raspberrypi` — control-plane, arm64 (4 GiB) - - `rpi2` — worker, arm, very low memory (~512 MiB) - - `roger-nucbox-evo-x2` — worker, amd64, 24 GiB (you run here) - - **GitOps:** ArgoCD owns every app from `https://git.rogi.casa/roger/k3s-cluster.git`. - Each app lives in its own folder; manifests are reconciled with prune + selfHeal. - - **Ingress:** Traefik; TLS via cert-manager + `letsencrypt-prod` Cloudflare Origin issuer. - - **LLM gateway:** LiteLLM at `https://litellm.rogi.casa/v1` — this is *your* model provider (you reach it through the Traefik ingress, never Ollama directly). - - **Services:** glance, pihole, litellm, gitea, home-assistant, jellyfin, n8n, - openwebui, phoenix, vaultwarden, qbittorrent, minecraft, monitoring - (prometheus + grafana), fava, myorg-assistant, gym-tracker, nas-proxy. - - **Your own RBAC** lets you read almost everything and mutate only an - allowlist (restart deployments/statefulsets/daemonsets, delete a stuck pod, - delete/patch jobs/cronjobs, `kubectl exec`). You CANNOT edit RBAC, taint - nodes, create/delete namespaces, or touch CRDs — if you think you need to, - propose the command to Roger and stop. + - **Nodes:** `raspberrypi` (control-plane, arm64, 4 GiB), `rpi2` (arm, + ~512 MiB), `roger-nucbox-evo-x2` (amd64, 24 GiB — you run here). + - **GitOps:** ArgoCD owns every app from the git repo at `$GITEA_REPO_URL`. + The repo is cloned at `/workspace/k3s-cluster`. Each app lives in its own + folder; manifests are reconciled with prune + selfHeal. + - **Ingress:** Traefik; TLS via cert-manager + `letsencrypt-prod`. + - **Your model provider:** LiteLLM at `http://litellm-service.litellm:80/v1` + (reached in-cluster; never Ollama directly). + - **Services:** glance, pihole, litellm, gitea, home-assistant, jellyfin, + n8n, openwebui, phoenix, vaultwarden, qbittorrent, minecraft, monitoring + (prometheus + grafana + loki), fava, myorg-assistant, gym-tracker. + + ## How you observe the cluster (NO kubectl — you have none) + + You have NO k8s API access and NO kubectl. Use these HTTP APIs instead: + + 1. **Prometheus** (metrics) at `http://prometheus.monitoring:9090/api/v1/query` + — PromQL via `curl -G -s "http://prometheus.monitoring:9090/api/v1/query" --data-urlencode "query="` + Examples: + - Node Ready: `kube_node_status_condition{condition="Ready",status="true"}` + - Node CPU/mem: `node_memory_MemAvailable_bytes`, `node_cpu_seconds_total` + - Pod restarts: `kube_pod_container_status_restarts_total` + - PVC usage: `kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes` + - Cert expiry: `certmanager_certificate_expiration_timestamp_seconds` + + 2. **Loki** (pod logs + events) at `http://loki.monitoring:3100/loki/api/v1/query_range` + — LogQL via `curl -G -s "http://loki.monitoring:3100/loki/api/v1/query_range" --data-urlencode "query=" --data-urlencode "start=" --data-urlencode "end=" --data-urlencode "limit=50"` + Examples: + - Errors in a namespace: `{namespace="myorg-assistant"} |= "error"` + - CrashLoop across cluster: `{namespace=~".+"} |= "BackOff"` + - k8s events: `{app="k8s-event-logger"} |= "Warning"` + + 3. **ArgoCD API** at `https://argocd-server.argocd:443` — bearer token in + `$ARGOCD_API_TOKEN`. (Verify the cert with `--insecure` if needed since + it's the internal service.) + Examples: + - List apps: `curl -sk -H "Authorization: Bearer $ARGOCD_API_TOKEN" https://argocd-server.argocd:443/api/v1/applications` + - Sync an app: `curl -sk -X POST -H "Authorization: Bearer $ARGOCD_API_TOKEN" https://argocd-server.argocd:443/api/v1/applications//sync` + + ## How you remediate (git commit → ArgoCD sync) + + You have NO k8s write access. Every fix is a git commit to the repo at + `/workspace/k3s-cluster` (which you `git push` to Gitea using `$GITEA_TOKEN`). + ArgoCD's selfHeal picks up the change; if you need it faster, trigger a sync + via the ArgoCD API. + + Workflow: + cd /workspace/k3s-cluster + git pull + # ... edit the manifest(s) ... + git add -A && git commit -m "fix(): " + git push # uses the token in GITEA_REPO_URL / GITEA_TOKEN + # optionally trigger ArgoCD sync: + curl -sk -X POST -H "Authorization: Bearer $ARGOCD_API_TOKEN" \ + https://argocd-server.argocd:443/api/v1/applications//sync ## Operating rules - 1. **Read first, act second.** Before changing anything, gather the evidence: - `kubectl describe`, `kubectl logs`, `kubectl get events --since=...`, - `kubectl top`. Cite the exact resource (ns/name) and the exact command in - every report. - 2. **Only safe, idempotent remediations.** Allowed actions: - - `kubectl rollout restart deployment/ -n ` (and statefulset/daemonset) - - delete a single stuck `CrashLoopBackOff`/`ImagePullBackOff` pod so its - controller recreates it - - `kubectl delete job/` / `kubectl patch cronjob ...` - Never run a command that affects more than one workload at a time unless - Roger asked for it. - 3. **When in doubt, notify, don't act.** If a fix is risky, unusual, or would - touch state you can't reach (RBAC, nodes, CRDs, PVC data), post the - proposed command to Discord and wait for Roger to reply. - 4. **Be quiet when healthy.** Watchdog cron jobs reply with exactly `[SILENT]` - when there is nothing to report. Failed jobs always deliver regardless. - 5. **No runaway loops.** You cannot create new cron jobs from inside a cron run - (Hermes disables that). Do not try. - 6. **Talk like an engineer.** Short, concrete, with resource names and - commands. No filler. When you fixed something, say what you did in one line. - 7. **Respect GitOps.** If an app is `OutOfSync`/`Degraded` in ArgoCD, do not - hand-edit resources to "fix" it — Argo will revert you. Report it so Roger - can fix the source repo. + 1. **Read first, act second.** Before changing anything, gather the evidence + via Prometheus + Loki + ArgoCD. Cite the exact resource (ns/name) and + the exact query/command in every report. + 2. **GitOps is the ONLY write path.** Never try to use kubectl (you don't + have it). Every remediation is a git commit + push + optional ArgoCD sync + trigger. ArgoCD will reconcile; if it reverts you, your fix was wrong. + 3. **Only safe, idempotent remediations.** Allowed: scaling a Deployment, + bumping the `restartedAt` annotation to trigger a rollout, fixing a + broken ConfigMap/Secret value, pinning an image tag. Never touch RBAC, + ArgoCD's own Application manifests, nodes, or CRDs. + 4. **When in doubt, notify, don't act.** If a fix is risky, unusual, or would + touch state outside the repo, post the proposed change to Discord and + wait for Roger to reply. + 5. **Be quiet when healthy.** Watchdog cron jobs reply with exactly `[SILENT]` + when there is nothing to report. Failed jobs always deliver. + 6. **No runaway loops.** You cannot create new cron jobs from inside a cron + run (Hermes disables that). Do not try. + 7. **Talk like an engineer.** Short, concrete, with resource names and + queries. No filler. When you fixed something, say what you did in one line. + 8. **Respect GitOps.** If an app is `OutOfSync`/`Degraded`, check whether a + commit is stuck. Don't hand-edit resources — fix the source repo. ## How you reach Roger Notifications go to Discord (your home channel). Cron jobs deliver there by - default (`deliver="discord"`). Keep messages under ~1800 chars; attach - longer logs as `kubectl logs ... > /opt/data/cron/output/` and link - the path. - ``` + default (`deliver="discord"`). Keep messages under ~1800 chars. diff --git a/platform-engineer/cron-seed.yaml b/platform-engineer/cron-seed.yaml index ecf98ce..9ea96d6 100644 --- a/platform-engineer/cron-seed.yaml +++ b/platform-engineer/cron-seed.yaml @@ -1,9 +1,8 @@ # One-shot Job that seeds Hermes' built-in cron schedule on first install. # Idempotent: skips job names that already exist. # -# The agent's own cron jobs live in /opt/data/cron/jobs.json on the PVC and are -# NOT reconciled by ArgoCD (runtime state). Re-run this Job manually after a -# wipe to re-seed: kubectl job restart hermes-cron-seed -n platform-engineer +# Uses a `cron-seeder` SA scoped to pods/exec on the hermes pod ONLY (no k8s +# access for the agent itself). --- apiVersion: batch/v1 kind: Job @@ -13,8 +12,6 @@ metadata: labels: app: hermes annotations: - # Job.spec.template is immutable — tell ArgoCD to Replace (delete+create) - # instead of patching, so edits to this Job sync cleanly under selfHeal. argocd.argoproj.io/sync-options: Replace=true argocd.argoproj.io/hook: Sync argocd.argoproj.io/hook-delete-policy: BeforeHookCreation @@ -26,20 +23,15 @@ spec: labels: app: hermes spec: - serviceAccountName: platform-engineer + serviceAccountName: cron-seeder restartPolicy: OnFailure containers: - name: seed - # alpine is tiny and always available; we install curl + download the - # right-arch kubectl binary at runtime (bitnami/kubectl tags are - # inconsistent across versions, so we avoid depending on them). image: alpine:3.20 command: ["sh", "-c"] args: - | set -e - - # Install curl, then download kubectl for this node's architecture. apk add --no-cache curl ARCH=$(uname -m) case "$ARCH" in @@ -48,11 +40,9 @@ spec: armv7l) KARCH=arm ;; *) echo "unsupported arch: $ARCH" >&2; exit 1 ;; esac - echo "Downloading kubectl for linux/$KARCH ..." curl -fsSL -o /usr/local/bin/kubectl \ "https://dl.k8s.io/release/v1.35.0/bin/linux/${KARCH}/kubectl" chmod +x /usr/local/bin/kubectl - kubectl version --client echo "Waiting for hermes pod to be Ready..." kubectl -n platform-engineer wait --for=condition=Ready pod -l app=hermes --timeout=300s || true @@ -68,38 +58,34 @@ spec: echo "cron job '$name' already exists — skipping" else echo "creating cron job '$name' ..." - # NOTE: the `hermes cron` CLI has no --provider/--model flags. - # Unpinned jobs snapshot the current global default (qwen3.6:27b) - # at creation, so they run fine. They only fail-closed if the - # global default is changed LATER (intended safety, #44585). - # To pin a job, use the `cronjob` agent tool inside a chat turn. - kubectl -n platform-engineer exec "$POD" -- hermes cron create "$schedule" "$prompt" \ - --name "$name" --deliver "$deliver" + kubectl -n platform-engineer exec "$POD" -- hermes cron create "$schedule" "$prompt" --name "$name" --deliver "$deliver" fi } + NOW_NS='$(date +%s)000000000' + # ---- Watchdog checks (silent unless something is wrong) ---- create "cluster-health-check" "every 15m" "discord" \ - "Run: kubectl get nodes; kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded; kubectl get events -A --field-selector type=Warning --since=20m. If everything is healthy and there are no Warning events, reply with exactly [SILENT]. Otherwise give a concise per-resource summary of what is wrong (node name, pod ns/name, phase, last event)." + "Check cluster health via HTTP APIs (you have NO kubectl). (1) Prometheus: curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=kube_node_status_condition{condition=\"Ready\",status!=\"true\"}' — if any node is NotReady, report it. (2) Prometheus: curl for kube_pod_status_phase{phase!=\"Running\"} to find pods not Running. (3) Loki: curl -G -s 'http://loki.monitoring:3100/loki/api/v1/query_range' --data-urlencode 'query={namespace=~\".+\"} |~ \"(?i)error|panic|crashloop|backoff\"' --data-urlencode 'start=$(date -d \"20 minutes ago\" +%s)000000000' --data-urlencode 'end=$(date +%s)000000000' --data-urlencode 'limit=20' — report any error lines with namespace/pod. (4) ArgoCD: curl -sk -H \"Authorization: Bearer \$ARGOCD_API_TOKEN\" 'https://argocd-server.argocd:443/api/v1/applications' — report any app not Synced+Healthy. If everything is healthy, reply with exactly [SILENT]. Otherwise give a concise per-resource summary." create "pod-restart-loop" "every 10m" "discord" \ - "Find pods in CrashLoopBackOff or ImagePullBackOff across all namespaces (kubectl get pods -A). For each, fetch kubectl logs (previous) and describe. If the cause is clearly transient (OOM kill, a one-off config parse error that will retry cleanly, a missing Secret the controller will recreate), attempt ONE safe remediation: kubectl rollout restart of the owning Deployment/StatefulSet/DaemonSet, OR delete the single stuck pod. Report what you did in one line per resource. If the cause is not clearly transient (bad image, missing config, auth failure), do NOT act — post the log excerpt and the proposed command and wait for Roger. If no such pods exist, reply [SILENT]." + "Find pods with high restart rates via Prometheus (NO kubectl): curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=topk(5, max_over_time(kube_pod_container_status_restarts_total[15m]))' — if any pod has >3 restarts in 15m, fetch its logs from Loki: curl -G -s 'http://loki.monitoring:3100/loki/api/v1/query_range' --data-urlencode 'query={namespace=\"\",pod=\"\"}' --data-urlencode 'start=<15m ago unix ns>' --data-urlencode 'end=' --data-urlencode 'limit=30'. Diagnose the cause. If fixable via a manifest change (e.g., bump memory limit, fix a config value, bump restartedAt annotation), edit the file in /workspace/k3s-cluster, git add -A, git commit -m 'fix(): ', git push, then trigger ArgoCD sync: curl -sk -X POST -H 'Authorization: Bearer \$ARGOCD_API_TOKEN' 'https://argocd-server.argocd:443/api/v1/applications//sync'. Report what you did in one line. If not clearly fixable, post the log excerpt and proposed fix, and wait for Roger. If no high-restart pods, reply [SILENT]." create "pvc-pressure" "every 30m" "discord" \ - "Check cluster storage health: kubectl get pv,pvc -A; kubectl top nodes. Alert if any PVC is Pending/Lost or any node filesystem usage is over 85%. If all healthy, reply [SILENT]." + "Check storage health via Prometheus (NO kubectl): curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes' — alert on any PVC with <15% free. Also check node disk: curl for '1 - (node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})'. If any PVC or node disk is over 85% used, report it with the namespace/PVC name and percentage. If all healthy, reply [SILENT]." create "argocd-sync-health" "every 1h" "discord" \ - "Run: kubectl get applications -n argocd -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status. If every app is Synced and Healthy, reply [SILENT]. Otherwise list the OutOfSync/Degraded apps with their status. Do NOT hand-edit resources to fix them (Argo will revert) — just report." + "Check ArgoCD app health via API (NO kubectl): curl -sk -H 'Authorization: Bearer \$ARGOCD_API_TOKEN' 'https://argocd-server.argocd:443/api/v1/applications'. For each app, check syncStatus and healthStatus. If every app is Synced and Healthy, reply [SILENT]. Otherwise list the OutOfSync/Degraded apps with their status. If an app is OutOfSync and you believe a recent git push caused it, you may trigger a sync: curl -sk -X POST -H 'Authorization: Bearer \$ARGOCD_API_TOKEN' 'https://argocd-server.argocd:443/api/v1/applications//sync'. Do NOT hand-edit resources to fix them — fix the source repo." create "cert-expiry" "0 9 * * *" "discord" \ - "List all cert-manager Certificate resources (kubectl get certificates -A). For each, check notAfter. Alert on any certificate expiring in under 21 days. If none, reply [SILENT]." + "Check certificate expiry via Prometheus (NO kubectl): curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400' — this gives days until expiry. Alert on any certificate expiring in under 21 days, with its name and namespace. If none, reply [SILENT]." create "node-resource-drift" "every 30m" "discord" \ - "Run kubectl top nodes. If any node CPU or memory usage is over 90%, or any node is NotReady, report it with the numbers. Otherwise reply [SILENT]." + "Check node resources via Prometheus (NO kubectl): (1) Node CPU: curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (node)' (2) Node memory: curl for '1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)' by node (3) Node Ready: curl for 'kube_node_status_condition{condition=\"Ready\",status!=\"true\"}'. If any node is NotReady, or any node CPU>90% or memory>90%, report it with the numbers. Otherwise reply [SILENT]." # ---- Daily report (always delivered) ---- create "daily-cluster-report" "0 8 * * *" "discord" \ - "Produce a daily cluster report for Roger: (1) node count + Ready/NotReady; (2) top 5 pods by CPU and by memory across all namespaces (kubectl top pods -A --sort-by); (3) count of pods not Running; (4) ArgoCD apps OutOfSync or Degraded; (5) any certificates expiring within 30 days; (6) any recent Warning events (last 24h). Keep it under 1800 chars. Always deliver (no [SILENT])." + "Produce a daily cluster report for Roger using HTTP APIs (NO kubectl): (1) Node status: curl Prometheus for kube_node_status_condition{condition=\"Ready\"} — report Ready/NotReady per node. (2) Top pods by CPU/mem: curl Prometheus for topk(5, rate(container_cpu_usage_seconds_total[5m])) and topk(5, container_memory_working_set_bytes). (3) Pods not Running: curl for kube_pod_status_phase{phase!=\"Running\"} count by namespace. (4) ArgoCD apps: curl -sk -H 'Authorization: Bearer \$ARGOCD_API_TOKEN' 'https://argocd-server.argocd:443/api/v1/applications' — list any OutOfSync or Degraded. (5) Certificates expiring <30d: curl Prometheus for certmanager_certificate_expiration_timestamp_seconds. (6) Recent warnings: curl Loki for {app=\"k8s-event-logger\"} |= \"Warning\" in last 24h. Keep it under 1800 chars. Always deliver (no [SILENT])." echo "Done. Listing all cron jobs:" kubectl -n platform-engineer exec "$POD" -- hermes cron list diff --git a/platform-engineer/deployment.yaml b/platform-engineer/deployment.yaml index 6770e9f..cac637b 100644 --- a/platform-engineer/deployment.yaml +++ b/platform-engineer/deployment.yaml @@ -17,8 +17,8 @@ spec: labels: app: hermes spec: - serviceAccountName: platform-engineer - # No imagePullSecrets — using the public stock Hermes image from Docker Hub. + # No serviceAccountName — the agent has NO k8s API access. It manages the + # cluster via git commits (→ ArgoCD sync) and reads via Loki/Prometheus/ArgoCD. # Pin to the powerful amd64 node (image is linux/amd64; the NUC has 24 GiB). nodeSelector: @@ -42,29 +42,33 @@ spec: topologyKey: kubernetes.io/hostname initContainers: - # Download kubectl + helm into a shared emptyDir so the stock Hermes image - # (which doesn't ship kubectl) can still drive the cluster. Avoids building - # and pushing a custom image through a slow / size-capped registry. - - name: install-tools - image: curlimages/curl:8.12.1 + # Clone the k3s-cluster repo into a persistent workspace so the agent can + # commit + push remediations. The token is injected via envFrom. + - name: git-clone + image: alpine/git:2.43.0 command: ["sh", "-c"] args: - | set -e - echo "Downloading kubectl v1.35.0..." - curl -fsSL -o /tools/kubectl \ - https://dl.k8s.io/release/v1.35.0/bin/linux/amd64/kubectl - chmod +x /tools/kubectl - echo "Downloading helm v3.16.3..." - curl -fsSL https://get.helm.sh/helm-v3.16.3-linux-amd64.tar.gz \ - | tar -xz -C /tools --strip-components=1 linux-amd64/helm - chmod +x /tools/helm - echo "Tools installed:"; ls -la /tools + cd /workspace + if [ -d k3s-cluster/.git ]; then + echo "Repo exists, pulling latest..." + cd k3s-cluster && git pull --rebase || true + else + echo "Cloning repo..." + git clone "${GITEA_REPO_URL}" k3s-cluster + cd k3s-cluster + git config user.name "Platform Engineer" + git config user.email "platform-engineer@rogi.casa" + fi + envFrom: + - secretRef: + name: hermes-env volumeMounts: - - name: tools - mountPath: /tools + - name: workspace + mountPath: /workspace - # Seed /opt/data with config.yaml + SOUL.md on first boot only. + # Seed /opt/data with config.yaml + SOUL.md + .env on first boot only. # ArgoCD owns the manifests; the PVC is runtime state and is NOT reconciled. - name: seed-data image: busybox:1.36 @@ -73,14 +77,28 @@ spec: - | set -e if [ ! -f /opt/data/config.yaml ]; then - echo "First boot: seeding /opt/data from ConfigMap..." + echo "First boot: seeding /opt/data from ConfigMap + env..." cp /seed/config.yaml /opt/data/config.yaml cp /seed/SOUL.md /opt/data/SOUL.md chmod 600 /opt/data/config.yaml + # Write .env from the injected Secret env vars so the s6 gateway + # finds API keys (the hermes container reads keys from /opt/data/.env). + : > /opt/data/.env + chmod 600 /opt/data/.env + for k in OPENAI_API_KEY OPENAI_BASE_URL DISCORD_BOT_TOKEN DISCORD_HOME_CHANNEL \ + GITEA_TOKEN GITEA_REPO_URL ARGOCD_API_TOKEN ARGOCD_SERVER \ + HERMES_DASHBOARD HERMES_DASHBOARD_BASIC_AUTH_USERNAME \ + HERMES_DASHBOARD_BASIC_AUTH_PASSWORD HERMES_DASHBOARD_BASIC_AUTH_SECRET; do + eval "v=\${$k:-}" + [ -n "$v" ] && echo "$k=$v" >> /opt/data/.env + done else echo "/opt/data already initialized — leaving runtime state intact." fi - mkdir -p /opt/data/home/.kube /opt/data/cron/output /opt/data/scripts /workspace + mkdir -p /opt/data/home/.kube /opt/data/cron/output /opt/data/scripts + envFrom: + - secretRef: + name: hermes-env volumeMounts: - name: data mountPath: /opt/data @@ -93,8 +111,7 @@ spec: imagePullPolicy: Always # IMPORTANT: do NOT set `command:` — it would override the image's # ENTRYPOINT (/init, s6-overlay), which sets up the hermes user, seeds - # config on first boot, and supervises the gateway. The image's CMD - # (main-wrapper.sh) already routes `gateway run` through s6. + # config on first boot, and supervises the gateway. args: ["gateway", "run"] ports: - name: gateway @@ -105,21 +122,13 @@ spec: - secretRef: name: hermes-env env: - # k3s injects KUBERNETES_SERVICE_HOST/PORT + the SA token automatically; - # kubectl inside the pod authenticates as the platform-engineer SA. - name: HERMES_HOME value: /opt/data - # Put the initContainer-installed kubectl/helm on PATH for the hermes user. - - name: PATH - value: /opt/hermes/bin:/opt/hermes/.venv/bin:/tools:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin volumeMounts: - name: data mountPath: /opt/data - name: workspace mountPath: /workspace - - name: tools - mountPath: /tools - readOnly: true resources: requests: memory: "512Mi" @@ -129,10 +138,7 @@ spec: cpu: "1000m" livenessProbe: # Probe the dashboard port (9119, always enabled via HERMES_DASHBOARD=1 - # and binds 0.0.0.0). The gateway API on 8642 is off by default - # (API_SERVER_ENABLED not set), so 9119 is the reliable liveness signal. - # s6 auto-restarts the gateway itself; this probe only catches a wedged - # container. + # and binds 0.0.0.0). The gateway API on 8642 is off by default. tcpSocket: port: 9119 initialDelaySeconds: 90 @@ -148,8 +154,6 @@ spec: claimName: hermes-data - name: workspace emptyDir: {} - - name: tools - emptyDir: {} - name: seed configMap: name: hermes-seed diff --git a/platform-engineer/dockerfile b/platform-engineer/dockerfile deleted file mode 100644 index 06c4409..0000000 --- a/platform-engineer/dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -# Derived Hermes Agent image with kubectl + helm so the agent can drive the -# k3s cluster from inside the container (terminal backend = local). -# -# Build & push to the Gitea registry: -# docker build -t git.rogi.casa/roger/hermes-agent:v1.35-1 -f dockerfile . -# docker push git.rogi.casa/roger/hermes-agent:v1.35-1 -# -# This image targets linux/amd64 (the agent pod is pinned to the amd64 NUC). -FROM nousresearch/hermes-agent:latest - -USER root - -# kubectl (v1.35 to match the cluster's k3s version) -RUN apt-get update \ - && apt-get install -y --no-install-recommends curl gnupg ca-certificates \ - && curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.35/deb/Release.key \ - | gpg --dearmor -o /usr/share/keyrings/kubernetes-apt-keyring.gpg \ - && echo 'deb [signed-by=/usr/share/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.35/deb/ /' \ - > /etc/apt/sources.list.d/kubernetes.list \ - && apt-get update \ - && apt-get install -y --no-install-recommends kubectl \ - # helm - && curl -fsSL https://get.helm.sh/helm-v3.16.3-linux-amd64.tar.gz \ - | tar -xz -C /usr/local/bin --strip-components=1 linux-amd64/helm \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Hermes' own CLI/kubeconfig helper dir for tool subprocesses -RUN mkdir -p /opt/data/home/.kube - -USER hermes diff --git a/platform-engineer/rbac.yaml b/platform-engineer/rbac.yaml index e03b954..0accfab 100644 --- a/platform-engineer/rbac.yaml +++ b/platform-engineer/rbac.yaml @@ -1,111 +1,27 @@ -# Least-privilege RBAC for the Platform Engineer Hermes agent. +# Minimal RBAC for the cron-seed Job ONLY. # -# The agent can READ almost everything cluster-wide, but can only MUTATE a -# narrow allowlist of safe, idempotent resources (restart deployments, delete a -# stuck pod so its controller recreates it, etc.). It CANNOT touch RBAC, nodes, -# namespaces, CRDs, or other namespaces' Secrets beyond read. +# The Hermes agent itself has NO k8s RBAC — it manages the cluster via git +# commits (→ ArgoCD sync) and reads state via Loki / Prometheus / ArgoCD APIs. +# +# The cron-seed Job needs to `kubectl exec` into the hermes pod to run +# `hermes cron create ...` (the only way to seed Hermes' internal cron). +# Scoped to this namespace, pods/exec on the hermes pod only. --- apiVersion: v1 kind: ServiceAccount metadata: - name: platform-engineer + name: cron-seeder namespace: platform-engineer --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole +kind: Role metadata: - name: platform-engineer + name: cron-seeder + namespace: platform-engineer rules: - # ---- Broad read access (cluster-wide) ---- - - apiGroups: [""] - resources: - - nodes - - nodes/proxy - - services - - endpoints - - pods - - pods/log - - configmaps - - secrets - - persistentvolumeclaims - - persistentvolumes - - namespaces - - events - - replicationcontrollers - verbs: ["get", "list", "watch"] - - apiGroups: ["apps"] - resources: - - deployments - - statefulsets - - daemonsets - - replicasets - verbs: ["get", "list", "watch"] - - apiGroups: ["batch"] - resources: - - jobs - - cronjobs - verbs: ["get", "list", "watch"] - - apiGroups: ["networking.k8s.io"] - resources: - - ingresses - verbs: ["get", "list", "watch"] - - apiGroups: ["autoscaling"] - resources: - - horizontalpodautoscalers - verbs: ["get", "list", "watch"] - - apiGroups: ["argoproj.io"] - resources: - - applications - - appprojects - verbs: ["get", "list", "watch"] - - apiGroups: ["cert-manager.io"] - resources: - - certificates - - certificaterequests - - clusterissuers - verbs: ["get", "list", "watch"] - - apiGroups: ["metrics.k8s.io"] - resources: - - pods - - nodes - verbs: ["get", "list"] - - # ---- Metrics / health endpoints ---- - - nonResourceURLs: ["/metrics", "/metrics/*"] - verbs: ["get"] - - # ---- Narrow mutate allowlist (idempotent, safe remediation) ---- - # Restart a stuck pod by deleting it (its controller recreates it). - apiGroups: [""] resources: ["pods"] - verbs: ["delete", "patch"] - # `kubectl rollout restart` and scaling for the apps/batch controllers. - - apiGroups: ["apps"] - resources: - - deployments - - statefulsets - - daemonsets - - replicasets - verbs: ["patch", "update"] - - apiGroups: ["batch"] - resources: - - jobs - - cronjobs - verbs: ["patch", "update", "delete"] - # Exec into pods for log-style / debug inspection (granted per request #5). + verbs: ["get", "list"] - apiGroups: [""] resources: ["pods/exec"] verbs: ["create"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: platform-engineer -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: platform-engineer -subjects: - - kind: ServiceAccount - name: platform-engineer - namespace: platform-engineer