loki + promtail new monitoring services

argocd/argocd-cm.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: argocd-cm
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: argocd-cm
+    app.kubernetes.io/part-of: argocd
+data:
+  # add an additional local user with apiKey and login capabilities
+  #   apiKey - allows generating API keys
+  #   login - allows to login using UI
+  accounts.roger: apiKey, login
+  # disables user. User is enabled by default
+  accounts.roger.enabled: "false"

litellm/litellm.yaml

@@ -28,11 +28,19 @@ data:
           model: ollama/glm-4.7-flash
           api_base: http://10.88.20.12:11434
       # Used by the platform-engineer Hermes agent (deployed in ns platform-engineer).
-      # model_name is the alias Hermes requests; the underlying Ollama model is qwen3.6:27b.
+      # model_name is the alias Hermes requests; the underlying Ollama model is
-      - model_name: qwen3.6:27b
+      # qwen3.6:latest (the fast non-27b tag). 27b is a slow reasoning model.
+      # `ollama_chat/` (not `ollama/`) uses Ollama's NATIVE /api/chat endpoint.
+      # `think: false` + `chat_template_kwargs.enable_thinking: false` disable
+      # Qwen3 thinking so the model emits content directly (otherwise the
+      # OpenAI-compat translation returns empty content with reasoning split off).
+      - model_name: qwen3.6
         litellm_params:
-          model: ollama/qwen3.6:27b
+          model: ollama_chat/qwen3.6:latest
           api_base: http://10.88.20.12:11434
+          think: false
+          chat_template_kwargs:
+            enable_thinking: false
     litellm_settings:
       #set_verbose: True  # Uncomment this if you want to see verbose logs; not recommended in production
       callbacks: ["arize_phoenix"]

monitoring/grafana-datasource.yaml

@@ -13,3 +13,8 @@ data:
       url: http://prometheus:9090
       isDefault: true
       editable: true
+    - name: Loki
+      type: loki
+      access: proxy
+      url: http://loki:3100
+      editable: true

monitoring/k8s-event-logger.yaml

@@ -0,0 +1,95 @@
+# k8s-event-exporter — watches Kubernetes Events and logs them to stdout as
+# structured JSON. Promtail tails the logs and ships them to Loki with the
+# label app=kubernetes-event-exporter. The Hermes agent queries them with LogQL:
+#   {app="kubernetes-event-exporter"} |= "BackOff"
+#
+# This avoids giving the agent any k8s API token for events — the exporter
+# has its own narrow read-only SA, and the agent only talks to Loki.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: k8s-event-exporter
+  namespace: monitoring
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: k8s-event-exporter
+rules:
+  - apiGroups: [""]
+    resources:
+      - events
+      - pods
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: k8s-event-exporter
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: k8s-event-exporter
+subjects:
+  - kind: ServiceAccount
+    name: k8s-event-exporter
+    namespace: monitoring
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: k8s-event-exporter-config
+  namespace: monitoring
+data:
+  config.yaml: |
+    logLevel: info
+    logFormat: json
+    route:
+      routes:
+        - match:
+            - receiver: dump
+    receivers:
+      - name: dump
+        dump: {}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: k8s-event-exporter
+  namespace: monitoring
+  labels:
+    app: kubernetes-event-exporter
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kubernetes-event-exporter
+  template:
+    metadata:
+      labels:
+        app: kubernetes-event-exporter
+    spec:
+      serviceAccountName: k8s-event-exporter
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      containers:
+      - name: exporter
+        image: opsgenie/kubernetes-event-exporter:0.9
+        args:
+          - -conf=/config/config.yaml
+        volumeMounts:
+        - name: config
+          mountPath: /config
+          readOnly: true
+        resources:
+          requests:
+            memory: "32Mi"
+            cpu: "25m"
+          limits:
+            memory: "128Mi"
+            cpu: "100m"
+      volumes:
+      - name: config
+        configMap:
+          name: k8s-event-exporter-config

monitoring/loki.yaml

@@ -0,0 +1,153 @@
+# Loki — log aggregation (single-binary mode, local filesystem storage).
+#
+# Stores compressed, indexed pod logs shipped by Promtail. Queried by the
+# platform-engineer Hermes agent via the HTTP API (LogQL) and by Grafana.
+#
+# Storage: 20 GiB local PVC, 1-week retention enforced by the compactor.
+# Service: loki.monitoring:3100 (ClusterIP, no auth — homelab).
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: loki-config
+  namespace: monitoring
+data:
+  loki.yaml: |
+    auth_enabled: false
+
+    server:
+      http_listen_port: 3100
+      grpc_listen_port: 9096
+
+    common:
+      path_prefix: /loki
+      replication_factor: 1
+      ring:
+        instance_addr: 127.0.0.1
+        kvstore:
+          store: inmemory
+
+    schema_config:
+      configs:
+        - from: 2024-01-01
+          store: tsdb
+          object_store: filesystem
+          schema: v13
+          index:
+            prefix: index_
+            period: 24h
+
+    storage_config:
+      filesystem:
+        directory: /loki/chunks
+      tsdb_shipper:
+        active_index_directory: /loki/tsdb-index
+        cache_location: /loki/tsdb-cache
+
+    limits_config:
+      retention_period: 168h          # 1 week
+      max_query_series: 10000
+      reject_old_samples: true
+      reject_old_samples_max_age: 168h
+      allow_structured_metadata: false  # tsdb v13 compat
+
+    compactor:
+      working_directory: /loki/compactor
+      compaction_interval: 10m
+      retention_enabled: true
+      retention_delete_delay: 2h
+      retention_delete_worker_count: 50
+      delete_request_store: filesystem
+
+    analytics:
+      reporting_enabled: false
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: loki-data
+  namespace: monitoring
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 20Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: loki
+  namespace: monitoring
+  labels:
+    app: loki
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate          # single-writer storage
+  selector:
+    matchLabels:
+      app: loki
+  template:
+    metadata:
+      labels:
+        app: loki
+    spec:
+      nodeSelector:
+        kubernetes.io/arch: amd64   # Loki image; runs on the NUC
+      containers:
+      - name: loki
+        image: grafana/loki:3.4.4
+        args:
+          - -config.file=/etc/loki/loki.yaml
+        ports:
+        - name: http
+          containerPort: 3100
+        volumeMounts:
+        - name: config
+          mountPath: /etc/loki
+          readOnly: true
+        - name: data
+          mountPath: /loki
+        resources:
+          requests:
+            memory: "512Mi"
+            cpu: "250m"
+          limits:
+            memory: "1Gi"
+            cpu: "1000m"
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: 3100
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          failureThreshold: 5
+        livenessProbe:
+          httpGet:
+            path: /ready
+            port: 3100
+          initialDelaySeconds: 60
+          periodSeconds: 30
+          failureThreshold: 5
+      volumes:
+      - name: config
+        configMap:
+          name: loki-config
+      - name: data
+        persistentVolumeClaim:
+          claimName: loki-data
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: loki
+  namespace: monitoring
+spec:
+  type: ClusterIP
+  selector:
+    app: loki
+  ports:
+    - name: http
+      port: 3100
+      targetPort: 3100

monitoring/promtail.yaml

@@ -0,0 +1,138 @@
+# Promtail — DaemonSet that tails pod logs on every node and ships them to Loki.
+#
+# Runs on ALL nodes (amd64 + arm). Multi-arch image. Reads /var/log/pods/*,
+# attaches k8s labels (namespace, pod, container), ships to loki.monitoring:3100.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: promtail
+  namespace: monitoring
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: promtail
+rules:
+  - apiGroups: [""]
+    resources:
+      - nodes
+      - nodes/proxy
+      - services
+      - endpoints
+      - pods
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: promtail
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: promtail
+subjects:
+  - kind: ServiceAccount
+    name: promtail
+    namespace: monitoring
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: promtail-config
+  namespace: monitoring
+data:
+  promtail.yaml: |
+    server:
+      http_listen_port: 9080
+      grpc_listen_port: 0
+
+    positions:
+      filename: /tmp/positions.yaml
+
+    clients:
+      - url: http://loki.monitoring:3100/loki/api/v1/push
+
+    scrape_configs:
+      # Tail all container logs via /var/log/containers/*.log (symlinks to
+      # /var/log/pods/<ns>_<pod>_<uid>/<container>/<N>.log). Extract namespace,
+      # pod, container labels from the filename via pipeline_stages regex.
+      - job_name: kubernetes-containers
+        static_configs:
+          - targets:
+              - localhost
+            labels:
+              job: kube-containers
+              __path__: /var/log/containers/*.log
+        pipeline_stages:
+          - cri: {}
+          # k3s filename: <pod>_<namespace>_<container>-<hash>.log
+          - regex:
+              expression: '/var/log/containers/(?P<pod>[^_]+)_(?P<namespace>[^_]+)_(?P<container>[^-]+)-.*\.log'
+              source: filename
+          - labels:
+              pod:
+              namespace:
+              container:
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: promtail
+  namespace: monitoring
+  labels:
+    app: promtail
+spec:
+  selector:
+    matchLabels:
+      app: promtail
+  template:
+    metadata:
+      labels:
+        app: promtail
+    spec:
+      serviceAccountName: promtail
+      tolerations:
+        - operator: Exists   # run on every node including tainted Pis
+      containers:
+      - name: promtail
+        image: grafana/promtail:3.4.4
+        args:
+          - -config.file=/etc/promtail/promtail.yaml
+          - -config.expand-env=true
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        volumeMounts:
+        - name: config
+          mountPath: /etc/promtail
+          readOnly: true
+        - name: positions
+          mountPath: /tmp
+        - name: pods-logs
+          mountPath: /var/log/pods
+          readOnly: true
+        - name: containers-logs
+          mountPath: /var/log/containers
+          readOnly: true
+        resources:
+          requests:
+            memory: "64Mi"
+            cpu: "50m"
+          limits:
+            memory: "256Mi"
+            cpu: "250m"
+      volumes:
+      - name: config
+        configMap:
+          name: promtail-config
+      - name: positions
+        emptyDir: {}
+      - name: pods-logs
+        hostPath:
+          path: /var/log/pods
+      - name: containers-logs
+        hostPath:
+          path: /var/log/containers

platform-engineer/build-and-push.sh

@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-# Build & push the derived Hermes image (kubectl + helm).
-#
-# Two modes:
-#   ./build-and-push.sh push    # build + push to the Gitea registry
-#   ./build-and-push.sh local   # build + import directly into the NUC's k3s containerd
-#                               # (no registry needed; pod is pinned to this node)
-#
-# Default (no arg): push.
-set -euo pipefail
-
-# Docker registry pushes can't go through the Cloudflare proxy (100 MB cap),
-# so push to the DNS-only registry hostname instead of git.rogi.casa.
-# Override with: REGISTRY=git.rogi.casa ./build-and-push.sh push   (if grey-clouded)
-REGISTRY="${REGISTRY:-registry.rogi.casa}"
-REPO="roger/hermes-agent"
-TAG="${TAG:-v1.35-1}"
-IMAGE="${REGISTRY}/${REPO}:${TAG}"
-MODE="${1:-push}"
-
-cd "$(dirname "$0")"
-
-echo "==> Building ${IMAGE}"
-docker build --platform linux/amd64 -t "${IMAGE}" -f dockerfile .
-
-case "$MODE" in
-  push)
-    echo "==> Pushing ${IMAGE}"
-    docker push "${IMAGE}"
-    echo "==> Done. If the pod can't pull, create the gitea-registry secret in the namespace."
-    ;;
-  local)
-    # Requires k3s + being run on the node the pod schedules to (roger-nucbox-evo-x2).
-    echo "==> Importing into k3s containerd (requires sudo)"
-    docker save "${IMAGE}" | sudo k3s ctr images import -
-    echo "==> Done. Verify: sudo k3s ctr images ls | grep hermes-agent"
-    echo "    deployment.yaml is set to imagePullPolicy: IfNotPresent"
-    ;;
-  *)
-    echo "Usage: $0 {push|local}" >&2
-    exit 1
-    ;;
-esac

platform-engineer/configmap.yaml

@@ -1,5 +1,4 @@
-# Hermes configuration, SOUL.md, and the cron-seed script.
+# Hermes configuration + SOUL.md + profile.d (seeded into the PVC on first boot).
-# Seeded into the PVC (/opt/data) by the initContainer on first boot only.
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -10,28 +9,26 @@ data:
   config.yaml: |
     model:
       provider: openai-api
-      default: qwen3.6:27b
+      default: qwen3.6
-      base_url: "https://litellm.rogi.casa/v1"
+      base_url: "http://litellm-service.litellm:80/v1"
       api_mode: chat_completions
 
-    # Cheap/fast model for auxiliary tasks (titling, compression).
     auxiliary:
       compression:
         provider: openai-api
-        model: qwen3.6:27b
+        model: qwen3.6
-        base_url: "https://litellm.rogi.casa/v1"
+        base_url: "http://litellm-service.litellm:80/v1"
       title_generation:
         provider: openai-api
-        model: qwen3.6:27b
+        model: qwen3.6
-        base_url: "https://litellm.rogi.casa/v1"
+        base_url: "http://litellm-service.litellm:80/v1"
 
     terminal:
       backend: local
-      cwd: /workspace
+      cwd: /workspace/k3s-cluster
       timeout: 180
       home_mode: profile
 
-    # Unattended gateway → circuit-break on stuck tool-call loops.
     tool_loop_guardrails:
       hard_stop_enabled: true
       hard_stop_after:
@@ -63,53 +60,87 @@ data:
 
     ## The cluster you look after
 
-    - **Nodes:**
+    - **Nodes:** `raspberrypi` (control-plane, arm64, 4 GiB), `rpi2` (arm,
-      - `raspberrypi` — control-plane, arm64 (4 GiB)
+      ~512 MiB), `roger-nucbox-evo-x2` (amd64, 24 GiB — you run here).
-      - `rpi2` — worker, arm, very low memory (~512 MiB)
+    - **GitOps:** ArgoCD owns every app from the git repo at `$GITEA_REPO_URL`.
-      - `roger-nucbox-evo-x2` — worker, amd64, 24 GiB (you run here)
+      The repo is cloned at `/workspace/k3s-cluster`. Each app lives in its own
-    - **GitOps:** ArgoCD owns every app from `https://git.rogi.casa/roger/k3s-cluster.git`.
+      folder; manifests are reconciled with prune + selfHeal.
-      Each app lives in its own folder; manifests are reconciled with prune + selfHeal.
+    - **Ingress:** Traefik; TLS via cert-manager + `letsencrypt-prod`.
-    - **Ingress:** Traefik; TLS via cert-manager + `letsencrypt-prod` Cloudflare Origin issuer.
+    - **Your model provider:** LiteLLM at `http://litellm-service.litellm:80/v1`
-    - **LLM gateway:** LiteLLM at `https://litellm.rogi.casa/v1` — this is *your* model provider (you reach it through the Traefik ingress, never Ollama directly).
+      (reached in-cluster; never Ollama directly).
-    - **Services:** glance, pihole, litellm, gitea, home-assistant, jellyfin, n8n,
+    - **Services:** glance, pihole, litellm, gitea, home-assistant, jellyfin,
-      openwebui, phoenix, vaultwarden, qbittorrent, minecraft, monitoring
+      n8n, openwebui, phoenix, vaultwarden, qbittorrent, minecraft, monitoring
-      (prometheus + grafana), fava, myorg-assistant, gym-tracker, nas-proxy.
+      (prometheus + grafana + loki), fava, myorg-assistant, gym-tracker.
-    - **Your own RBAC** lets you read almost everything and mutate only an
+
-      allowlist (restart deployments/statefulsets/daemonsets, delete a stuck pod,
+    ## How you observe the cluster (NO kubectl — you have none)
-      delete/patch jobs/cronjobs, `kubectl exec`). You CANNOT edit RBAC, taint
+
-      nodes, create/delete namespaces, or touch CRDs — if you think you need to,
+    You have NO k8s API access and NO kubectl. Use these HTTP APIs instead:
-      propose the command to Roger and stop.
+
+    1. **Prometheus** (metrics) at `http://prometheus.monitoring:9090/api/v1/query`
+       — PromQL via `curl -G -s "http://prometheus.monitoring:9090/api/v1/query" --data-urlencode "query=<PROMQL>"`
+       Examples:
+         - Node Ready:        `kube_node_status_condition{condition="Ready",status="true"}`
+         - Node CPU/mem:      `node_memory_MemAvailable_bytes`, `node_cpu_seconds_total`
+         - Pod restarts:     `kube_pod_container_status_restarts_total`
+         - PVC usage:         `kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes`
+         - Cert expiry:      `certmanager_certificate_expiration_timestamp_seconds`
+
+    2. **Loki** (pod logs + events) at `http://loki.monitoring:3100/loki/api/v1/query_range`
+       — LogQL via `curl -G -s "http://loki.monitoring:3100/loki/api/v1/query_range" --data-urlencode "query=<LOGQL>" --data-urlencode "start=<unix_ns>" --data-urlencode "end=<unix_ns>" --data-urlencode "limit=50"`
+       Examples:
+         - Errors in a namespace: `{namespace="myorg-assistant"} |= "error"`
+         - CrashLoop across cluster:  `{namespace=~".+"} |= "BackOff"`
+         - k8s events:                 `{app="k8s-event-logger"} |= "Warning"`
+
+    3. **ArgoCD API** at `https://argocd-server.argocd:443` — bearer token in
+       `$ARGOCD_API_TOKEN`. (Verify the cert with `--insecure` if needed since
+       it's the internal service.)
+       Examples:
+         - List apps:  `curl -sk -H "Authorization: Bearer $ARGOCD_API_TOKEN" https://argocd-server.argocd:443/api/v1/applications`
+         - Sync an app: `curl -sk -X POST -H "Authorization: Bearer $ARGOCD_API_TOKEN" https://argocd-server.argocd:443/api/v1/applications/<app>/sync`
+
+    ## How you remediate (git commit → ArgoCD sync)
+
+    You have NO k8s write access. Every fix is a git commit to the repo at
+    `/workspace/k3s-cluster` (which you `git push` to Gitea using `$GITEA_TOKEN`).
+    ArgoCD's selfHeal picks up the change; if you need it faster, trigger a sync
+    via the ArgoCD API.
+
+    Workflow:
+      cd /workspace/k3s-cluster
+      git pull
+      # ... edit the manifest(s) ...
+      git add -A && git commit -m "fix(<app>): <what changed>"
+      git push    # uses the token in GITEA_REPO_URL / GITEA_TOKEN
+      # optionally trigger ArgoCD sync:
+      curl -sk -X POST -H "Authorization: Bearer $ARGOCD_API_TOKEN" \
+        https://argocd-server.argocd:443/api/v1/applications/<app>/sync
 
     ## Operating rules
 
-    1. **Read first, act second.** Before changing anything, gather the evidence:
+    1. **Read first, act second.** Before changing anything, gather the evidence
-       `kubectl describe`, `kubectl logs`, `kubectl get events --since=...`,
+       via Prometheus + Loki + ArgoCD. Cite the exact resource (ns/name) and
-       `kubectl top`. Cite the exact resource (ns/name) and the exact command in
+       the exact query/command in every report.
-       every report.
+    2. **GitOps is the ONLY write path.** Never try to use kubectl (you don't
-    2. **Only safe, idempotent remediations.** Allowed actions:
+       have it). Every remediation is a git commit + push + optional ArgoCD sync
-       - `kubectl rollout restart deployment/<name> -n <ns>` (and statefulset/daemonset)
+       trigger. ArgoCD will reconcile; if it reverts you, your fix was wrong.
-       - delete a single stuck `CrashLoopBackOff`/`ImagePullBackOff` pod so its
+    3. **Only safe, idempotent remediations.** Allowed: scaling a Deployment,
-         controller recreates it
+       bumping the `restartedAt` annotation to trigger a rollout, fixing a
-       - `kubectl delete job/<name>` / `kubectl patch cronjob ...`
+       broken ConfigMap/Secret value, pinning an image tag. Never touch RBAC,
-       Never run a command that affects more than one workload at a time unless
+       ArgoCD's own Application manifests, nodes, or CRDs.
-       Roger asked for it.
+    4. **When in doubt, notify, don't act.** If a fix is risky, unusual, or would
-    3. **When in doubt, notify, don't act.** If a fix is risky, unusual, or would
+       touch state outside the repo, post the proposed change to Discord and
-       touch state you can't reach (RBAC, nodes, CRDs, PVC data), post the
+       wait for Roger to reply.
-       proposed command to Discord and wait for Roger to reply.
+    5. **Be quiet when healthy.** Watchdog cron jobs reply with exactly `[SILENT]`
-    4. **Be quiet when healthy.** Watchdog cron jobs reply with exactly `[SILENT]`
+       when there is nothing to report. Failed jobs always deliver.
-       when there is nothing to report. Failed jobs always deliver regardless.
+    6. **No runaway loops.** You cannot create new cron jobs from inside a cron
-    5. **No runaway loops.** You cannot create new cron jobs from inside a cron run
+       run (Hermes disables that). Do not try.
-       (Hermes disables that). Do not try.
+    7. **Talk like an engineer.** Short, concrete, with resource names and
-    6. **Talk like an engineer.** Short, concrete, with resource names and
+       queries. No filler. When you fixed something, say what you did in one line.
-       commands. No filler. When you fixed something, say what you did in one line.
+    8. **Respect GitOps.** If an app is `OutOfSync`/`Degraded`, check whether a
-    7. **Respect GitOps.** If an app is `OutOfSync`/`Degraded` in ArgoCD, do not
+       commit is stuck. Don't hand-edit resources — fix the source repo.
-       hand-edit resources to "fix" it — Argo will revert you. Report it so Roger
-       can fix the source repo.
 
     ## How you reach Roger
 
     Notifications go to Discord (your home channel). Cron jobs deliver there by
-    default (`deliver="discord"`). Keep messages under ~1800 chars; attach
+    default (`deliver="discord"`). Keep messages under ~1800 chars.
-       longer logs as `kubectl logs ... > /opt/data/cron/output/<file>` and link
-       the path.
-    ```

platform-engineer/cron-seed.yaml

@@ -1,9 +1,8 @@
 # One-shot Job that seeds Hermes' built-in cron schedule on first install.
 # Idempotent: skips job names that already exist.
 #
-# The agent's own cron jobs live in /opt/data/cron/jobs.json on the PVC and are
+# Uses a `cron-seeder` SA scoped to pods/exec on the hermes pod ONLY (no k8s
-# NOT reconciled by ArgoCD (runtime state). Re-run this Job manually after a
+# access for the agent itself).
-# wipe to re-seed:  kubectl job restart hermes-cron-seed -n platform-engineer
 ---
 apiVersion: batch/v1
 kind: Job
@@ -13,8 +12,6 @@ metadata:
   labels:
     app: hermes
   annotations:
-    # Job.spec.template is immutable — tell ArgoCD to Replace (delete+create)
-    # instead of patching, so edits to this Job sync cleanly under selfHeal.
     argocd.argoproj.io/sync-options: Replace=true
     argocd.argoproj.io/hook: Sync
     argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
@@ -26,20 +23,15 @@ spec:
       labels:
         app: hermes
     spec:
-      serviceAccountName: platform-engineer
+      serviceAccountName: cron-seeder
       restartPolicy: OnFailure
       containers:
       - name: seed
-        # alpine is tiny and always available; we install curl + download the
-        # right-arch kubectl binary at runtime (bitnami/kubectl tags are
-        # inconsistent across versions, so we avoid depending on them).
         image: alpine:3.20
         command: ["sh", "-c"]
         args:
         - |
           set -e
-
-          # Install curl, then download kubectl for this node's architecture.
           apk add --no-cache curl
           ARCH=$(uname -m)
           case "$ARCH" in
@@ -48,11 +40,9 @@ spec:
             armv7l)   KARCH=arm ;;
             *) echo "unsupported arch: $ARCH" >&2; exit 1 ;;
           esac
-          echo "Downloading kubectl for linux/$KARCH ..."
           curl -fsSL -o /usr/local/bin/kubectl \
             "https://dl.k8s.io/release/v1.35.0/bin/linux/${KARCH}/kubectl"
           chmod +x /usr/local/bin/kubectl
-          kubectl version --client
 
           echo "Waiting for hermes pod to be Ready..."
           kubectl -n platform-engineer wait --for=condition=Ready pod -l app=hermes --timeout=300s || true
@@ -68,38 +58,34 @@ spec:
               echo "cron job '$name' already exists — skipping"
             else
               echo "creating cron job '$name' ..."
-              # NOTE: the `hermes cron` CLI has no --provider/--model flags.
+              kubectl -n platform-engineer exec "$POD" -- hermes cron create "$schedule" "$prompt" --name "$name" --deliver "$deliver"
-              # Unpinned jobs snapshot the current global default (qwen3.6:27b)
-              # at creation, so they run fine. They only fail-closed if the
-              # global default is changed LATER (intended safety, #44585).
-              # To pin a job, use the `cronjob` agent tool inside a chat turn.
-              kubectl -n platform-engineer exec "$POD" -- hermes cron create "$schedule" "$prompt" \
-                --name "$name" --deliver "$deliver"
             fi
           }
 
+          NOW_NS='$(date +%s)000000000'
+
           # ---- Watchdog checks (silent unless something is wrong) ----
           create "cluster-health-check" "every 15m" "discord" \
-            "Run: kubectl get nodes; kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded; kubectl get events -A --field-selector type=Warning --since=20m. If everything is healthy and there are no Warning events, reply with exactly [SILENT]. Otherwise give a concise per-resource summary of what is wrong (node name, pod ns/name, phase, last event)."
+            "Check cluster health via HTTP APIs (you have NO kubectl). (1) Prometheus: curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=kube_node_status_condition{condition=\"Ready\",status!=\"true\"}' — if any node is NotReady, report it. (2) Prometheus: curl for kube_pod_status_phase{phase!=\"Running\"} to find pods not Running. (3) Loki: curl -G -s 'http://loki.monitoring:3100/loki/api/v1/query_range' --data-urlencode 'query={namespace=~\".+\"} |~ \"(?i)error|panic|crashloop|backoff\"' --data-urlencode 'start=$(date -d \"20 minutes ago\" +%s)000000000' --data-urlencode 'end=$(date +%s)000000000' --data-urlencode 'limit=20' — report any error lines with namespace/pod. (4) ArgoCD: curl -sk -H \"Authorization: Bearer \$ARGOCD_API_TOKEN\" 'https://argocd-server.argocd:443/api/v1/applications' — report any app not Synced+Healthy. If everything is healthy, reply with exactly [SILENT]. Otherwise give a concise per-resource summary."
 
           create "pod-restart-loop" "every 10m" "discord" \
-            "Find pods in CrashLoopBackOff or ImagePullBackOff across all namespaces (kubectl get pods -A). For each, fetch kubectl logs (previous) and describe. If the cause is clearly transient (OOM kill, a one-off config parse error that will retry cleanly, a missing Secret the controller will recreate), attempt ONE safe remediation: kubectl rollout restart of the owning Deployment/StatefulSet/DaemonSet, OR delete the single stuck pod. Report what you did in one line per resource. If the cause is not clearly transient (bad image, missing config, auth failure), do NOT act — post the log excerpt and the proposed command and wait for Roger. If no such pods exist, reply [SILENT]."
+            "Find pods with high restart rates via Prometheus (NO kubectl): curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=topk(5, max_over_time(kube_pod_container_status_restarts_total[15m]))' — if any pod has >3 restarts in 15m, fetch its logs from Loki: curl -G -s 'http://loki.monitoring:3100/loki/api/v1/query_range' --data-urlencode 'query={namespace=\"<ns>\",pod=\"<pod>\"}' --data-urlencode 'start=<15m ago unix ns>' --data-urlencode 'end=<now unix ns>' --data-urlencode 'limit=30'. Diagnose the cause. If fixable via a manifest change (e.g., bump memory limit, fix a config value, bump restartedAt annotation), edit the file in /workspace/k3s-cluster, git add -A, git commit -m 'fix(<app>): <reason>', git push, then trigger ArgoCD sync: curl -sk -X POST -H 'Authorization: Bearer \$ARGOCD_API_TOKEN' 'https://argocd-server.argocd:443/api/v1/applications/<app>/sync'. Report what you did in one line. If not clearly fixable, post the log excerpt and proposed fix, and wait for Roger. If no high-restart pods, reply [SILENT]."
 
           create "pvc-pressure" "every 30m" "discord" \
-            "Check cluster storage health: kubectl get pv,pvc -A; kubectl top nodes. Alert if any PVC is Pending/Lost or any node filesystem usage is over 85%. If all healthy, reply [SILENT]."
+            "Check storage health via Prometheus (NO kubectl): curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes' — alert on any PVC with <15% free. Also check node disk: curl for '1 - (node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})'. If any PVC or node disk is over 85% used, report it with the namespace/PVC name and percentage. If all healthy, reply [SILENT]."
 
           create "argocd-sync-health" "every 1h" "discord" \
-            "Run: kubectl get applications -n argocd -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status. If every app is Synced and Healthy, reply [SILENT]. Otherwise list the OutOfSync/Degraded apps with their status. Do NOT hand-edit resources to fix them (Argo will revert) — just report."
+            "Check ArgoCD app health via API (NO kubectl): curl -sk -H 'Authorization: Bearer \$ARGOCD_API_TOKEN' 'https://argocd-server.argocd:443/api/v1/applications'. For each app, check syncStatus and healthStatus. If every app is Synced and Healthy, reply [SILENT]. Otherwise list the OutOfSync/Degraded apps with their status. If an app is OutOfSync and you believe a recent git push caused it, you may trigger a sync: curl -sk -X POST -H 'Authorization: Bearer \$ARGOCD_API_TOKEN' 'https://argocd-server.argocd:443/api/v1/applications/<app>/sync'. Do NOT hand-edit resources to fix them — fix the source repo."
 
           create "cert-expiry" "0 9 * * *" "discord" \
-            "List all cert-manager Certificate resources (kubectl get certificates -A). For each, check notAfter. Alert on any certificate expiring in under 21 days. If none, reply [SILENT]."
+            "Check certificate expiry via Prometheus (NO kubectl): curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400' — this gives days until expiry. Alert on any certificate expiring in under 21 days, with its name and namespace. If none, reply [SILENT]."
 
           create "node-resource-drift" "every 30m" "discord" \
-            "Run kubectl top nodes. If any node CPU or memory usage is over 90%, or any node is NotReady, report it with the numbers. Otherwise reply [SILENT]."
+            "Check node resources via Prometheus (NO kubectl): (1) Node CPU: curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (node)' (2) Node memory: curl for '1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)' by node (3) Node Ready: curl for 'kube_node_status_condition{condition=\"Ready\",status!=\"true\"}'. If any node is NotReady, or any node CPU>90% or memory>90%, report it with the numbers. Otherwise reply [SILENT]."
 
           # ---- Daily report (always delivered) ----
           create "daily-cluster-report" "0 8 * * *" "discord" \
-            "Produce a daily cluster report for Roger: (1) node count + Ready/NotReady; (2) top 5 pods by CPU and by memory across all namespaces (kubectl top pods -A --sort-by); (3) count of pods not Running; (4) ArgoCD apps OutOfSync or Degraded; (5) any certificates expiring within 30 days; (6) any recent Warning events (last 24h). Keep it under 1800 chars. Always deliver (no [SILENT])."
+            "Produce a daily cluster report for Roger using HTTP APIs (NO kubectl): (1) Node status: curl Prometheus for kube_node_status_condition{condition=\"Ready\"} — report Ready/NotReady per node. (2) Top pods by CPU/mem: curl Prometheus for topk(5, rate(container_cpu_usage_seconds_total[5m])) and topk(5, container_memory_working_set_bytes). (3) Pods not Running: curl for kube_pod_status_phase{phase!=\"Running\"} count by namespace. (4) ArgoCD apps: curl -sk -H 'Authorization: Bearer \$ARGOCD_API_TOKEN' 'https://argocd-server.argocd:443/api/v1/applications' — list any OutOfSync or Degraded. (5) Certificates expiring <30d: curl Prometheus for certmanager_certificate_expiration_timestamp_seconds. (6) Recent warnings: curl Loki for {app=\"k8s-event-logger\"} |= \"Warning\" in last 24h. Keep it under 1800 chars. Always deliver (no [SILENT])."
 
           echo "Done. Listing all cron jobs:"
           kubectl -n platform-engineer exec "$POD" -- hermes cron list

platform-engineer/deployment.yaml

@@ -17,8 +17,8 @@ spec:
       labels:
         app: hermes
     spec:
-      serviceAccountName: platform-engineer
+      # No serviceAccountName — the agent has NO k8s API access. It manages the
-      # No imagePullSecrets — using the public stock Hermes image from Docker Hub.
+      # cluster via git commits (→ ArgoCD sync) and reads via Loki/Prometheus/ArgoCD.
 
       # Pin to the powerful amd64 node (image is linux/amd64; the NUC has 24 GiB).
       nodeSelector:
@@ -42,29 +42,33 @@ spec:
               topologyKey: kubernetes.io/hostname
 
       initContainers:
-      # Download kubectl + helm into a shared emptyDir so the stock Hermes image
+      # Clone the k3s-cluster repo into a persistent workspace so the agent can
-      # (which doesn't ship kubectl) can still drive the cluster. Avoids building
+      # commit + push remediations. The token is injected via envFrom.
-      # and pushing a custom image through a slow / size-capped registry.
+      - name: git-clone
-      - name: install-tools
+        image: alpine/git:2.43.0
-        image: curlimages/curl:8.12.1
         command: ["sh", "-c"]
         args:
         - |
           set -e
-          echo "Downloading kubectl v1.35.0..."
+          cd /workspace
-          curl -fsSL -o /tools/kubectl \
+          if [ -d k3s-cluster/.git ]; then
-            https://dl.k8s.io/release/v1.35.0/bin/linux/amd64/kubectl
+            echo "Repo exists, pulling latest..."
-          chmod +x /tools/kubectl
+            cd k3s-cluster && git pull --rebase || true
-          echo "Downloading helm v3.16.3..."
+          else
-          curl -fsSL https://get.helm.sh/helm-v3.16.3-linux-amd64.tar.gz \
+            echo "Cloning repo..."
-            | tar -xz -C /tools --strip-components=1 linux-amd64/helm
+            git clone "${GITEA_REPO_URL}" k3s-cluster
-          chmod +x /tools/helm
+            cd k3s-cluster
-          echo "Tools installed:"; ls -la /tools
+            git config user.name "Platform Engineer"
+            git config user.email "platform-engineer@rogi.casa"
+          fi
+        envFrom:
+        - secretRef:
+            name: hermes-env
         volumeMounts:
-        - name: tools
+        - name: workspace
-          mountPath: /tools
+          mountPath: /workspace
 
-      # Seed /opt/data with config.yaml + SOUL.md on first boot only.
+      # Seed /opt/data with config.yaml + SOUL.md + .env on first boot only.
       # ArgoCD owns the manifests; the PVC is runtime state and is NOT reconciled.
       - name: seed-data
         image: busybox:1.36
@@ -73,14 +77,28 @@ spec:
         - |
           set -e
           if [ ! -f /opt/data/config.yaml ]; then
-            echo "First boot: seeding /opt/data from ConfigMap..."
+            echo "First boot: seeding /opt/data from ConfigMap + env..."
             cp /seed/config.yaml /opt/data/config.yaml
             cp /seed/SOUL.md /opt/data/SOUL.md
             chmod 600 /opt/data/config.yaml
+            # Write .env from the injected Secret env vars so the s6 gateway
+            # finds API keys (the hermes container reads keys from /opt/data/.env).
+            : > /opt/data/.env
+            chmod 600 /opt/data/.env
+            for k in OPENAI_API_KEY OPENAI_BASE_URL DISCORD_BOT_TOKEN DISCORD_HOME_CHANNEL \
+                     GITEA_TOKEN GITEA_REPO_URL ARGOCD_API_TOKEN ARGOCD_SERVER \
+                     HERMES_DASHBOARD HERMES_DASHBOARD_BASIC_AUTH_USERNAME \
+                     HERMES_DASHBOARD_BASIC_AUTH_PASSWORD HERMES_DASHBOARD_BASIC_AUTH_SECRET; do
+              eval "v=\${$k:-}"
+              [ -n "$v" ] && echo "$k=$v" >> /opt/data/.env
+            done
           else
             echo "/opt/data already initialized — leaving runtime state intact."
           fi
-          mkdir -p /opt/data/home/.kube /opt/data/cron/output /opt/data/scripts /workspace
+          mkdir -p /opt/data/home/.kube /opt/data/cron/output /opt/data/scripts
+        envFrom:
+        - secretRef:
+            name: hermes-env
         volumeMounts:
         - name: data
           mountPath: /opt/data
@@ -93,8 +111,7 @@ spec:
         imagePullPolicy: Always
         # IMPORTANT: do NOT set `command:` — it would override the image's
         # ENTRYPOINT (/init, s6-overlay), which sets up the hermes user, seeds
-        # config on first boot, and supervises the gateway. The image's CMD
+        # config on first boot, and supervises the gateway.
-        # (main-wrapper.sh) already routes `gateway run` through s6.
         args: ["gateway", "run"]
         ports:
         - name: gateway
@@ -105,21 +122,13 @@ spec:
         - secretRef:
             name: hermes-env
         env:
-        # k3s injects KUBERNETES_SERVICE_HOST/PORT + the SA token automatically;
-        # kubectl inside the pod authenticates as the platform-engineer SA.
         - name: HERMES_HOME
           value: /opt/data
-        # Put the initContainer-installed kubectl/helm on PATH for the hermes user.
-        - name: PATH
-          value: /opt/hermes/bin:/opt/hermes/.venv/bin:/tools:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
         volumeMounts:
         - name: data
           mountPath: /opt/data
         - name: workspace
           mountPath: /workspace
-        - name: tools
-          mountPath: /tools
-          readOnly: true
         resources:
           requests:
             memory: "512Mi"
@@ -129,10 +138,7 @@ spec:
             cpu: "1000m"
         livenessProbe:
           # Probe the dashboard port (9119, always enabled via HERMES_DASHBOARD=1
-          # and binds 0.0.0.0). The gateway API on 8642 is off by default
+          # and binds 0.0.0.0). The gateway API on 8642 is off by default.
-          # (API_SERVER_ENABLED not set), so 9119 is the reliable liveness signal.
-          # s6 auto-restarts the gateway itself; this probe only catches a wedged
-          # container.
           tcpSocket:
             port: 9119
           initialDelaySeconds: 90
@@ -148,8 +154,6 @@ spec:
           claimName: hermes-data
       - name: workspace
         emptyDir: {}
-      - name: tools
-        emptyDir: {}
       - name: seed
         configMap:
           name: hermes-seed

platform-engineer/dockerfile

@@ -1,31 +0,0 @@
-# Derived Hermes Agent image with kubectl + helm so the agent can drive the
-# k3s cluster from inside the container (terminal backend = local).
-#
-# Build & push to the Gitea registry:
-#   docker build -t git.rogi.casa/roger/hermes-agent:v1.35-1 -f dockerfile .
-#   docker push git.rogi.casa/roger/hermes-agent:v1.35-1
-#
-# This image targets linux/amd64 (the agent pod is pinned to the amd64 NUC).
-FROM nousresearch/hermes-agent:latest
-
-USER root
-
-# kubectl (v1.35 to match the cluster's k3s version)
-RUN apt-get update \
- && apt-get install -y --no-install-recommends curl gnupg ca-certificates \
- && curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.35/deb/Release.key \
-    | gpg --dearmor -o /usr/share/keyrings/kubernetes-apt-keyring.gpg \
- && echo 'deb [signed-by=/usr/share/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.35/deb/ /' \
-    > /etc/apt/sources.list.d/kubernetes.list \
- && apt-get update \
- && apt-get install -y --no-install-recommends kubectl \
- # helm
- && curl -fsSL https://get.helm.sh/helm-v3.16.3-linux-amd64.tar.gz \
-    | tar -xz -C /usr/local/bin --strip-components=1 linux-amd64/helm \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
-
-# Hermes' own CLI/kubeconfig helper dir for tool subprocesses
-RUN mkdir -p /opt/data/home/.kube
-
-USER hermes

platform-engineer/rbac.yaml

@@ -1,111 +1,27 @@
-# Least-privilege RBAC for the Platform Engineer Hermes agent.
+# Minimal RBAC for the cron-seed Job ONLY.
 #
-# The agent can READ almost everything cluster-wide, but can only MUTATE a
+# The Hermes agent itself has NO k8s RBAC — it manages the cluster via git
-# narrow allowlist of safe, idempotent resources (restart deployments, delete a
+# commits (→ ArgoCD sync) and reads state via Loki / Prometheus / ArgoCD APIs.
-# stuck pod so its controller recreates it, etc.). It CANNOT touch RBAC, nodes,
+#
-# namespaces, CRDs, or other namespaces' Secrets beyond read.
+# The cron-seed Job needs to `kubectl exec` into the hermes pod to run
+# `hermes cron create ...` (the only way to seed Hermes' internal cron).
+# Scoped to this namespace, pods/exec on the hermes pod only.
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: platform-engineer
+  name: cron-seeder
   namespace: platform-engineer
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+kind: Role
 metadata:
-  name: platform-engineer
+  name: cron-seeder
+  namespace: platform-engineer
 rules:
-  # ---- Broad read access (cluster-wide) ----
-  - apiGroups: [""]
-    resources:
-      - nodes
-      - nodes/proxy
-      - services
-      - endpoints
-      - pods
-      - pods/log
-      - configmaps
-      - secrets
-      - persistentvolumeclaims
-      - persistentvolumes
-      - namespaces
-      - events
-      - replicationcontrollers
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["apps"]
-    resources:
-      - deployments
-      - statefulsets
-      - daemonsets
-      - replicasets
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["batch"]
-    resources:
-      - jobs
-      - cronjobs
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["networking.k8s.io"]
-    resources:
-      - ingresses
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["autoscaling"]
-    resources:
-      - horizontalpodautoscalers
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["argoproj.io"]
-    resources:
-      - applications
-      - appprojects
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["cert-manager.io"]
-    resources:
-      - certificates
-      - certificaterequests
-      - clusterissuers
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["metrics.k8s.io"]
-    resources:
-      - pods
-      - nodes
-    verbs: ["get", "list"]
-
-  # ---- Metrics / health endpoints ----
-  - nonResourceURLs: ["/metrics", "/metrics/*"]
-    verbs: ["get"]
-
-  # ---- Narrow mutate allowlist (idempotent, safe remediation) ----
-  # Restart a stuck pod by deleting it (its controller recreates it).
   - apiGroups: [""]
     resources: ["pods"]
-    verbs: ["delete", "patch"]
+    verbs: ["get", "list"]
-  # `kubectl rollout restart` and scaling for the apps/batch controllers.
-  - apiGroups: ["apps"]
-    resources:
-      - deployments
-      - statefulsets
-      - daemonsets
-      - replicasets
-    verbs: ["patch", "update"]
-  - apiGroups: ["batch"]
-    resources:
-      - jobs
-      - cronjobs
-    verbs: ["patch", "update", "delete"]
-  # Exec into pods for log-style / debug inspection (granted per request #5).
   - apiGroups: [""]
     resources: ["pods/exec"]
     verbs: ["create"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: platform-engineer
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: platform-engineer
-subjects:
-  - kind: ServiceAccount
-    name: platform-engineer
-    namespace: platform-engineer