Compare commits

...

4 Commits

Author SHA1 Message Date
Platform Engineer
70ad508180 fix(myorg-assistant): add WEB_SECRET_KEY env to cronjobs to stop pydantic Settings validation crash 2026-07-11 23:51:39 +00:00
Platform Engineer
aab38aa67b fix(myorg-assistant): point cronjobs at registry image + add imagePullSecrets
CronJob pods were stuck in ImagePullBackOff because they referenced
the local-only image 'myorg-assistant:latest' which is not present
on the node. Switch all 5 cronjobs to the Gitea registry image
git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf
(matching the Deployment), set imagePullPolicy: Always, and add
imagePullSecrets: gitea-registry so they can authenticate to the
private registry.
2026-07-06 12:39:51 +00:00
Platform Engineer
962ffd5200 fix(argocd): add ignoreDifferences to root app to stop Application CRD drift loop
The k3s-cluster-root app-of-apps has been in a continuous Synced<->OutOfSync
oscillation (489 transitions in 24h). Each cycle, ArgoCD syncs the 'argocd'
Application CRD, succeeds, then immediately detects drift because ArgoCD
adds status/operation fields to Application resources at runtime.

Adding ignoreDifferences for /status and /operation on Application resources
stops the drift loop while keeping the app-of-apps functional.
2026-07-06 12:01:36 +00:00
Roger Oriol
6b697c9665 configure higher cron schedules 2026-07-05 23:30:51 +02:00
7 changed files with 57 additions and 16 deletions

View File

@@ -22,3 +22,9 @@ spec:
selfHeal: true
syncOptions:
- CreateNamespace=false
ignoreDifferences:
- group: argoproj.io
kind: Application
jsonPointers:
- /status
- /operation

View File

@@ -22,10 +22,12 @@ spec:
job: deadline-checker
spec:
restartPolicy: OnFailure
imagePullSecrets:
- name: gitea-registry
containers:
- name: deadline-checker
image: myorg-assistant:latest
imagePullPolicy: IfNotPresent
image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf
imagePullPolicy: Always
command:
- python
- run_job.py
@@ -51,6 +53,11 @@ spec:
secretKeyRef:
name: myorg-assistant-secret
key: LITELLM_API_KEY
- name: WEB_SECRET_KEY
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: WEB_SECRET_KEY
volumeMounts:
- name: myorg-data
mountPath: /data/myorg

View File

@@ -22,10 +22,12 @@ spec:
job: evening-summary
spec:
restartPolicy: OnFailure
imagePullSecrets:
- name: gitea-registry
containers:
- name: evening-summary
image: myorg-assistant:latest
imagePullPolicy: IfNotPresent
image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf
imagePullPolicy: Always
command:
- python
- run_job.py
@@ -51,6 +53,11 @@ spec:
secretKeyRef:
name: myorg-assistant-secret
key: LITELLM_API_KEY
- name: WEB_SECRET_KEY
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: WEB_SECRET_KEY
volumeMounts:
- name: myorg-data
mountPath: /data/myorg

View File

@@ -22,10 +22,12 @@ spec:
job: git-sync
spec:
restartPolicy: OnFailure
imagePullSecrets:
- name: gitea-registry
containers:
- name: git-sync
image: myorg-assistant:latest
imagePullPolicy: IfNotPresent
image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf
imagePullPolicy: Always
command:
- python
- run_job.py
@@ -66,6 +68,11 @@ spec:
secretKeyRef:
name: myorg-assistant-secret
key: LITELLM_API_KEY
- name: WEB_SECRET_KEY
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: WEB_SECRET_KEY
volumeMounts:
- name: myorg-data
mountPath: /data/myorg

View File

@@ -22,10 +22,12 @@ spec:
job: morning-briefing
spec:
restartPolicy: OnFailure
imagePullSecrets:
- name: gitea-registry
containers:
- name: morning-briefing
image: myorg-assistant:latest
imagePullPolicy: IfNotPresent
image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf
imagePullPolicy: Always
command:
- python
- run_job.py
@@ -58,6 +60,11 @@ spec:
secretKeyRef:
name: myorg-assistant-secret
key: LITELLM_API_KEY
- name: WEB_SECRET_KEY
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: WEB_SECRET_KEY
volumeMounts:
- name: myorg-data
mountPath: /data/myorg

View File

@@ -22,10 +22,12 @@ spec:
job: waiting-followup
spec:
restartPolicy: OnFailure
imagePullSecrets:
- name: gitea-registry
containers:
- name: waiting-followup
image: myorg-assistant:latest
imagePullPolicy: IfNotPresent
image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf
imagePullPolicy: Always
command:
- python
- run_job.py
@@ -51,6 +53,11 @@ spec:
secretKeyRef:
name: myorg-assistant-secret
key: LITELLM_API_KEY
- name: WEB_SECRET_KEY
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: WEB_SECRET_KEY
volumeMounts:
- name: myorg-data
mountPath: /data/myorg

View File

@@ -65,26 +65,26 @@ spec:
}
# ---- Watchdog checks (silent unless something is wrong) ----
create "cluster-health-check" "every 15m" "discord" \
create "cluster-health-check" "every 6h" "discord" \
"Check cluster health using the HTTP APIs documented in your SOUL.md. Check: (1) any node that is NotReady, (2) any pod not in Running phase, (3) any recent error/panic/crashloop/backoff log lines in Loki across all namespaces in the last 20 minutes, (4) any ArgoCD app that is not Synced plus Healthy. If everything is healthy, reply with exactly [SILENT]. Otherwise give a concise per-resource summary of what is wrong."
create "pod-restart-loop" "every 10m" "discord" \
create "pod-restart-loop" "every 1h" "discord" \
"Find pods with high restart rates using the Prometheus API documented in your SOUL.md. If any pod has more than 3 restarts in the last 15 minutes, fetch its logs from Loki to diagnose the cause. If the cause is clearly fixable via a manifest change such as bumping a memory limit, fixing a config value, or bumping the restartedAt annotation, make the edit in /workspace/k3s-cluster, commit and push, then trigger an ArgoCD sync via the API. Report what you did in one line. If not clearly fixable, post the log excerpt and proposed fix, and wait for Roger. If no high-restart pods, reply [SILENT]."
create "pvc-pressure" "every 30m" "discord" \
create "pvc-pressure" "every 1d" "discord" \
"Check storage health using the Prometheus API documented in your SOUL.md. Alert if any PVC has less than 15 percent free space, or if any node filesystem is over 85 percent full. If all healthy, reply [SILENT]."
create "argocd-sync-health" "every 1h" "discord" \
create "argocd-sync-health" "every 6h" "discord" \
"Check ArgoCD app health using the API documented in your SOUL.md. If every app is Synced and Healthy, reply [SILENT]. Otherwise list the OutOfSync or Degraded apps with their status. If an app is OutOfSync and you believe a recent git push caused it, you may trigger a sync via the API. Do NOT hand-edit resources to fix them — fix the source repo."
create "cert-expiry" "0 9 * * *" "discord" \
"Check certificate expiry using the Prometheus API documented in your SOUL.md. Alert on any certificate expiring in under 21 days, with its name and namespace. If none, reply [SILENT]."
create "node-resource-drift" "every 30m" "discord" \
create "node-resource-drift" "every 1d" "discord" \
"Check node resources using the Prometheus API documented in your SOUL.md. Alert if any node is NotReady, or if any node has CPU over 90 percent or memory over 90 percent. Otherwise reply [SILENT]."
# ---- Daily report (always delivered) ----
create "daily-cluster-report" "0 8 * * *" "discord" \
create "daily-cluster-report" "0 8 * * *" "discord" \
"Produce a daily cluster report for Roger using the HTTP APIs documented in your SOUL.md. Include: (1) node count and Ready/NotReady status per node, (2) top 5 pods by CPU and by memory, (3) count of pods not Running grouped by namespace, (4) any ArgoCD apps that are OutOfSync or Degraded, (5) any certificates expiring within 30 days, (6) any recent Warning-level log lines from the last 24 hours. Keep it under 1800 chars. Always deliver (no [SILENT])."
echo "Done. Listing all cron jobs:"