# One-shot Job that seeds Hermes' built-in cron schedule on first install. # Idempotent: skips job names that already exist. # # Cron prompts are deliberately written as plain-English instructions (no inline # curl commands) to avoid tripping Hermes' threat-pattern scanner, which blocks # cron prompts containing curl+auth-header patterns. The exact API endpoints and # query examples are documented in the agent's SOUL.md instead. --- apiVersion: batch/v1 kind: Job metadata: name: hermes-cron-seed namespace: platform-engineer labels: app: hermes annotations: argocd.argoproj.io/sync-options: Replace=true argocd.argoproj.io/hook: Sync argocd.argoproj.io/hook-delete-policy: BeforeHookCreation spec: backoffLimit: 4 ttlSecondsAfterFinished: 86400 template: metadata: labels: app: hermes spec: serviceAccountName: cron-seeder restartPolicy: OnFailure containers: - name: seed image: alpine:3.20 command: ["sh", "-c"] args: - | set -e apk add --no-cache curl ARCH=$(uname -m) case "$ARCH" in x86_64) KARCH=amd64 ;; aarch64) KARCH=arm64 ;; armv7l) KARCH=arm ;; *) echo "unsupported arch: $ARCH" >&2; exit 1 ;; esac curl -fsSL -o /usr/local/bin/kubectl \ "https://dl.k8s.io/release/v1.35.0/bin/linux/${KARCH}/kubectl" chmod +x /usr/local/bin/kubectl echo "Waiting for hermes pod to be Ready..." kubectl -n platform-engineer wait --for=condition=Ready pod -l app=hermes --timeout=300s || true POD=$(kubectl -n platform-engineer get pod -l app=hermes -o jsonpath='{.items[0].metadata.name}') echo "Using pod: $POD" exists() { kubectl -n platform-engineer exec "$POD" -- hermes cron list 2>/dev/null | grep -qi " $1 "; } # NOTE: `hermes cron create` has no --model flag. New jobs inherit the # global default model from config.yaml at creation time. To pin a # specific model per-job (e.g. deepseek-v4-flash-0731 for all # cron tasks), edit /opt/data/cron/jobs.json directly after seeding: # kubectl exec deploy/hermes -- python3 -c "...set model field..." # See llamacpp/README.md and the deployment notes for details. create() { name="$1"; schedule="$2"; deliver="$3"; prompt="$4" if exists "$name"; then echo "cron job '$name' already exists — skipping" else echo "creating cron job '$name' ..." kubectl -n platform-engineer exec "$POD" -- hermes cron create "$schedule" "$prompt" --name "$name" --deliver "$deliver" fi } # ---- Watchdog checks (silent unless something is wrong) ---- # NOTE: the old single "cluster-health-check" combined 4 heavy HTTP # gathers (Prometheus + a cluster-wide raw Loki log dump) into one # cron session and exceeded the flash model's per-request context. # The 35b-a3b llama-server runs -c 131072 -np 4, so each request only # gets ~32k tokens of KV cache (NOT 132k). It has been split into # focused sub-checks so each run stays small, and the log check now # uses LogQL aggregation instead of dumping raw lines cluster-wide. create "node-health-check" "every 6h" "discord" \ "Check node health using the Prometheus API documented in your SOUL.md. Query kube_node_status_condition{condition=\"Ready\",status!=\"true\"}. If all nodes are Ready, reply with exactly [SILENT]. Otherwise list each NotReady node by name. Keep it short; do not paste raw JSON — summarize with jq." create "pod-phase-check" "every 6h" "discord" \ "Check pod health using the Prometheus API documented in your SOUL.md. Query kube_pod_status_phase{phase!=\"Running\"} and exclude Completed/Succeeded pods (jobs). If all Running, reply with exactly [SILENT]. Otherwise list each non-Running pod as namespace/pod:phase, grouped by namespace. Summarize with jq; do not dump raw JSON." create "pod-error-log-check" "every 6h" "discord" \ "Check for recent crash-loop/error logs using the Loki API documented in your SOUL.md. IMPORTANT context discipline: do NOT fetch raw log lines across all namespaces — that overflows your context window. First run a LogQL aggregation that returns only counts: sum by (namespace, pod) (count_over_time({namespace=~\".+\"} |~ \"(?i)backoff|crashloop|panic\" [20m])). Parse the counts with jq. If every count is zero, reply with exactly [SILENT]. If any namespace/pod has a non-zero count, fetch at most 10 sample lines for THAT pod only (limit=10) to identify the cause. Summarize concisely; never paste more than a handful of lines." # (ArgoCD health is already covered by the argocd-sync-health cron below.) create "pod-restart-loop" "every 1h" "discord" \ "Find pods with high restart rates using the Prometheus API documented in your SOUL.md. Run increase(kube_pod_container_status_restarts_total[15m]) and list only pods whose value is greater than 3. If none, reply [SILENT]. For any pod that qualifies, IMPORTANT context discipline: do NOT fetch raw logs across the cluster or over a wide time window — that overflows your context window (a raw cluster-wide Loki dump can exceed 60k tokens). Fetch at most 10 sample lines for THAT specific pod only, using Loki query_range with limit=10 over the last 15 minutes, filtered to error/crash patterns (e.g. {namespace=\"\",pod=\"\"} |~ \"(?i)error|panic|crash|oom|backoff\"). Summarize the root cause in one line; never paste more than a handful of log lines. If the cause is clearly fixable via a manifest change such as bumping a memory limit, fixing a config value, or bumping the restartedAt annotation, make the edit in /workspace/k3s-cluster, commit and push, then trigger an ArgoCD sync via the API. Report what you did in one line. If not clearly fixable, post the log excerpt and proposed fix, and wait for Roger." create "pvc-pressure" "every 1d" "discord" \ "Check storage health using the Prometheus API documented in your SOUL.md. Alert if any PVC has less than 15 percent free space, or if any node filesystem is over 85 percent full. If all healthy, reply [SILENT]." create "argocd-sync-health" "every 6h" "discord" \ "Check ArgoCD app health using the API documented in your SOUL.md. If every app is Synced and Healthy, reply [SILENT]. Otherwise list the OutOfSync or Degraded apps with their status. If an app is OutOfSync and you believe a recent git push caused it, you may trigger a sync via the API. Do NOT hand-edit resources to fix them — fix the source repo." create "cert-expiry" "0 7 * * *" "discord" \ "Check certificate expiry using the Prometheus API documented in your SOUL.md. Alert on any certificate expiring in under 21 days, with its name and namespace. If none, reply [SILENT]." create "node-resource-drift" "every 1d" "discord" \ "Check node resources using the Prometheus API documented in your SOUL.md. Alert if any node is NotReady, or if any node has CPU over 90 percent or memory over 90 percent. Otherwise reply [SILENT]." # ---- Daily report (always delivered) ---- # NOTE: Hermes' cron runs in UTC (no TZ set on the container). The # cluster is at UTC+2, so fixed daily times are expressed in UTC and # shifted back 2h from the intended local wall-clock time: # 08:00 local -> 0 6 (daily cluster report) # 09:00 local -> 0 7 (cert expiry check) # Relative schedules (every 6h / every 1h / every 1d) are # timezone-independent. create "daily-cluster-report" "0 6 * * *" "discord" \ "Produce a daily cluster report for Roger using the HTTP APIs documented in your SOUL.md. Include: (1) node count and Ready/NotReady status per node, (2) top 5 pods by CPU and by memory, (3) count of pods not Running grouped by namespace, (4) any ArgoCD apps that are OutOfSync or Degraded, (5) any certificates expiring within 30 days, (6) recent error/crash activity: IMPORTANT context discipline — do NOT fetch raw log lines across the cluster over 24h — that overflows your context window. First run a LogQL aggregation that returns only counts: sum by (namespace, pod) (count_over_time({namespace=~\".+\"} |~ \"(?i)error|panic|crash|oom|backoff\" [24h])). Parse the counts with jq and summarize the top few noisiest pods by namespace/pod:count. If one stands out, fetch at most 5 sample lines for THAT pod only (limit=5) to identify the cause. Never paste more than a handful of lines total. Keep the whole report under 1800 chars. Always deliver (no [SILENT])." echo "Done. Listing all cron jobs:" kubectl -n platform-engineer exec "$POD" -- hermes cron list