# One-shot Job that seeds Hermes' built-in cron schedule on first install.
# Idempotent: skips job names that already exist.
#
# Uses a `cron-seeder` SA scoped to pods/exec on the hermes pod ONLY (no k8s
# access for the agent itself).
---
apiVersion: batch/v1
kind: Job
metadata:
  name: hermes-cron-seed
  namespace: platform-engineer
  labels:
    app: hermes
  annotations:
    argocd.argoproj.io/sync-options: Replace=true
    argocd.argoproj.io/hook: Sync
    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
spec:
  backoffLimit: 4
  ttlSecondsAfterFinished: 86400
  template:
    metadata:
      labels:
        app: hermes
    spec:
      serviceAccountName: cron-seeder
      restartPolicy: OnFailure
      containers:
      - name: seed
        image: alpine:3.20
        command: ["sh", "-c"]
        args:
        - |
          set -e
          apk add --no-cache curl
          ARCH=$(uname -m)
          case "$ARCH" in
            x86_64)   KARCH=amd64 ;;
            aarch64)  KARCH=arm64 ;;
            armv7l)   KARCH=arm ;;
            *) echo "unsupported arch: $ARCH" >&2; exit 1 ;;
          esac
          curl -fsSL -o /usr/local/bin/kubectl \
            "https://dl.k8s.io/release/v1.35.0/bin/linux/${KARCH}/kubectl"
          chmod +x /usr/local/bin/kubectl

          echo "Waiting for hermes pod to be Ready..."
          kubectl -n platform-engineer wait --for=condition=Ready pod -l app=hermes --timeout=300s || true

          POD=$(kubectl -n platform-engineer get pod -l app=hermes -o jsonpath='{.items[0].metadata.name}')
          echo "Using pod: $POD"

          exists() { kubectl -n platform-engineer exec "$POD" -- hermes cron list 2>/dev/null | grep -qi " $1 "; }

          create() {
            name="$1"; schedule="$2"; deliver="$3"; prompt="$4"
            if exists "$name"; then
              echo "cron job '$name' already exists — skipping"
            else
              echo "creating cron job '$name' ..."
              kubectl -n platform-engineer exec "$POD" -- hermes cron create "$schedule" "$prompt" --name "$name" --deliver "$deliver"
            fi
          }

          NOW_NS='$(date +%s)000000000'

          # ---- Watchdog checks (silent unless something is wrong) ----
          create "cluster-health-check" "every 15m" "discord" \
            "Check cluster health via HTTP APIs (you have NO kubectl). (1) Prometheus: curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=kube_node_status_condition{condition=\"Ready\",status!=\"true\"}' — if any node is NotReady, report it. (2) Prometheus: curl for kube_pod_status_phase{phase!=\"Running\"} to find pods not Running. (3) Loki: curl -G -s 'http://loki.monitoring:3100/loki/api/v1/query_range' --data-urlencode 'query={namespace=~\".+\"} |~ \"(?i)error|panic|crashloop|backoff\"' --data-urlencode 'start=$(date -d \"20 minutes ago\" +%s)000000000' --data-urlencode 'end=$(date +%s)000000000' --data-urlencode 'limit=20' — report any error lines with namespace/pod. (4) ArgoCD: curl -sk -H \"Authorization: Bearer \$ARGOCD_API_TOKEN\" 'https://argocd-server.argocd:443/api/v1/applications' — report any app not Synced+Healthy. If everything is healthy, reply with exactly [SILENT]. Otherwise give a concise per-resource summary."

          create "pod-restart-loop" "every 10m" "discord" \
            "Find pods with high restart rates via Prometheus (NO kubectl): curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=topk(5, max_over_time(kube_pod_container_status_restarts_total[15m]))' — if any pod has >3 restarts in 15m, fetch its logs from Loki: curl -G -s 'http://loki.monitoring:3100/loki/api/v1/query_range' --data-urlencode 'query={namespace=\"<ns>\",pod=\"<pod>\"}' --data-urlencode 'start=<15m ago unix ns>' --data-urlencode 'end=<now unix ns>' --data-urlencode 'limit=30'. Diagnose the cause. If fixable via a manifest change (e.g., bump memory limit, fix a config value, bump restartedAt annotation), edit the file in /workspace/k3s-cluster, git add -A, git commit -m 'fix(<app>): <reason>', git push, then trigger ArgoCD sync: curl -sk -X POST -H 'Authorization: Bearer \$ARGOCD_API_TOKEN' 'https://argocd-server.argocd:443/api/v1/applications/<app>/sync'. Report what you did in one line. If not clearly fixable, post the log excerpt and proposed fix, and wait for Roger. If no high-restart pods, reply [SILENT]."

          create "pvc-pressure" "every 30m" "discord" \
            "Check storage health via Prometheus (NO kubectl): curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes' — alert on any PVC with <15% free. Also check node disk: curl for '1 - (node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})'. If any PVC or node disk is over 85% used, report it with the namespace/PVC name and percentage. If all healthy, reply [SILENT]."

          create "argocd-sync-health" "every 1h" "discord" \
            "Check ArgoCD app health via API (NO kubectl): curl -sk -H 'Authorization: Bearer \$ARGOCD_API_TOKEN' 'https://argocd-server.argocd:443/api/v1/applications'. For each app, check syncStatus and healthStatus. If every app is Synced and Healthy, reply [SILENT]. Otherwise list the OutOfSync/Degraded apps with their status. If an app is OutOfSync and you believe a recent git push caused it, you may trigger a sync: curl -sk -X POST -H 'Authorization: Bearer \$ARGOCD_API_TOKEN' 'https://argocd-server.argocd:443/api/v1/applications/<app>/sync'. Do NOT hand-edit resources to fix them — fix the source repo."

          create "cert-expiry" "0 9 * * *" "discord" \
            "Check certificate expiry via Prometheus (NO kubectl): curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400' — this gives days until expiry. Alert on any certificate expiring in under 21 days, with its name and namespace. If none, reply [SILENT]."

          create "node-resource-drift" "every 30m" "discord" \
            "Check node resources via Prometheus (NO kubectl): (1) Node CPU: curl -G -s 'http://prometheus.monitoring:9090/api/v1/query' --data-urlencode 'query=1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (node)' (2) Node memory: curl for '1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)' by node (3) Node Ready: curl for 'kube_node_status_condition{condition=\"Ready\",status!=\"true\"}'. If any node is NotReady, or any node CPU>90% or memory>90%, report it with the numbers. Otherwise reply [SILENT]."

          # ---- Daily report (always delivered) ----
          create "daily-cluster-report" "0 8 * * *" "discord" \
            "Produce a daily cluster report for Roger using HTTP APIs (NO kubectl): (1) Node status: curl Prometheus for kube_node_status_condition{condition=\"Ready\"} — report Ready/NotReady per node. (2) Top pods by CPU/mem: curl Prometheus for topk(5, rate(container_cpu_usage_seconds_total[5m])) and topk(5, container_memory_working_set_bytes). (3) Pods not Running: curl for kube_pod_status_phase{phase!=\"Running\"} count by namespace. (4) ArgoCD apps: curl -sk -H 'Authorization: Bearer \$ARGOCD_API_TOKEN' 'https://argocd-server.argocd:443/api/v1/applications' — list any OutOfSync or Degraded. (5) Certificates expiring <30d: curl Prometheus for certmanager_certificate_expiration_timestamp_seconds. (6) Recent warnings: curl Loki for {app=\"k8s-event-logger\"} |= \"Warning\" in last 24h. Keep it under 1800 chars. Always deliver (no [SILENT])."

          echo "Done. Listing all cron jobs:"
          kubectl -n platform-engineer exec "$POD" -- hermes cron list