Compare commits

..

3 Commits

Author SHA1 Message Date
Platform Engineer
aab38aa67b fix(myorg-assistant): point cronjobs at registry image + add imagePullSecrets
CronJob pods were stuck in ImagePullBackOff because they referenced
the local-only image 'myorg-assistant:latest' which is not present
on the node. Switch all 5 cronjobs to the Gitea registry image
git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf
(matching the Deployment), set imagePullPolicy: Always, and add
imagePullSecrets: gitea-registry so they can authenticate to the
private registry.
2026-07-06 12:39:51 +00:00
Platform Engineer
962ffd5200 fix(argocd): add ignoreDifferences to root app to stop Application CRD drift loop
The k3s-cluster-root app-of-apps has been in a continuous Synced<->OutOfSync
oscillation (489 transitions in 24h). Each cycle, ArgoCD syncs the 'argocd'
Application CRD, succeeds, then immediately detects drift because ArgoCD
adds status/operation fields to Application resources at runtime.

Adding ignoreDifferences for /status and /operation on Application resources
stops the drift loop while keeping the app-of-apps functional.
2026-07-06 12:01:36 +00:00
Roger Oriol
6b697c9665 configure higher cron schedules 2026-07-05 23:30:51 +02:00
7 changed files with 32 additions and 16 deletions

View File

@@ -22,3 +22,9 @@ spec:
selfHeal: true selfHeal: true
syncOptions: syncOptions:
- CreateNamespace=false - CreateNamespace=false
ignoreDifferences:
- group: argoproj.io
kind: Application
jsonPointers:
- /status
- /operation

View File

@@ -22,10 +22,12 @@ spec:
job: deadline-checker job: deadline-checker
spec: spec:
restartPolicy: OnFailure restartPolicy: OnFailure
imagePullSecrets:
- name: gitea-registry
containers: containers:
- name: deadline-checker - name: deadline-checker
image: myorg-assistant:latest image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf
imagePullPolicy: IfNotPresent imagePullPolicy: Always
command: command:
- python - python
- run_job.py - run_job.py

View File

@@ -22,10 +22,12 @@ spec:
job: evening-summary job: evening-summary
spec: spec:
restartPolicy: OnFailure restartPolicy: OnFailure
imagePullSecrets:
- name: gitea-registry
containers: containers:
- name: evening-summary - name: evening-summary
image: myorg-assistant:latest image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf
imagePullPolicy: IfNotPresent imagePullPolicy: Always
command: command:
- python - python
- run_job.py - run_job.py

View File

@@ -22,10 +22,12 @@ spec:
job: git-sync job: git-sync
spec: spec:
restartPolicy: OnFailure restartPolicy: OnFailure
imagePullSecrets:
- name: gitea-registry
containers: containers:
- name: git-sync - name: git-sync
image: myorg-assistant:latest image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf
imagePullPolicy: IfNotPresent imagePullPolicy: Always
command: command:
- python - python
- run_job.py - run_job.py

View File

@@ -22,10 +22,12 @@ spec:
job: morning-briefing job: morning-briefing
spec: spec:
restartPolicy: OnFailure restartPolicy: OnFailure
imagePullSecrets:
- name: gitea-registry
containers: containers:
- name: morning-briefing - name: morning-briefing
image: myorg-assistant:latest image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf
imagePullPolicy: IfNotPresent imagePullPolicy: Always
command: command:
- python - python
- run_job.py - run_job.py

View File

@@ -22,10 +22,12 @@ spec:
job: waiting-followup job: waiting-followup
spec: spec:
restartPolicy: OnFailure restartPolicy: OnFailure
imagePullSecrets:
- name: gitea-registry
containers: containers:
- name: waiting-followup - name: waiting-followup
image: myorg-assistant:latest image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf
imagePullPolicy: IfNotPresent imagePullPolicy: Always
command: command:
- python - python
- run_job.py - run_job.py

View File

@@ -65,22 +65,22 @@ spec:
} }
# ---- Watchdog checks (silent unless something is wrong) ---- # ---- Watchdog checks (silent unless something is wrong) ----
create "cluster-health-check" "every 15m" "discord" \ create "cluster-health-check" "every 6h" "discord" \
"Check cluster health using the HTTP APIs documented in your SOUL.md. Check: (1) any node that is NotReady, (2) any pod not in Running phase, (3) any recent error/panic/crashloop/backoff log lines in Loki across all namespaces in the last 20 minutes, (4) any ArgoCD app that is not Synced plus Healthy. If everything is healthy, reply with exactly [SILENT]. Otherwise give a concise per-resource summary of what is wrong." "Check cluster health using the HTTP APIs documented in your SOUL.md. Check: (1) any node that is NotReady, (2) any pod not in Running phase, (3) any recent error/panic/crashloop/backoff log lines in Loki across all namespaces in the last 20 minutes, (4) any ArgoCD app that is not Synced plus Healthy. If everything is healthy, reply with exactly [SILENT]. Otherwise give a concise per-resource summary of what is wrong."
create "pod-restart-loop" "every 10m" "discord" \ create "pod-restart-loop" "every 1h" "discord" \
"Find pods with high restart rates using the Prometheus API documented in your SOUL.md. If any pod has more than 3 restarts in the last 15 minutes, fetch its logs from Loki to diagnose the cause. If the cause is clearly fixable via a manifest change such as bumping a memory limit, fixing a config value, or bumping the restartedAt annotation, make the edit in /workspace/k3s-cluster, commit and push, then trigger an ArgoCD sync via the API. Report what you did in one line. If not clearly fixable, post the log excerpt and proposed fix, and wait for Roger. If no high-restart pods, reply [SILENT]." "Find pods with high restart rates using the Prometheus API documented in your SOUL.md. If any pod has more than 3 restarts in the last 15 minutes, fetch its logs from Loki to diagnose the cause. If the cause is clearly fixable via a manifest change such as bumping a memory limit, fixing a config value, or bumping the restartedAt annotation, make the edit in /workspace/k3s-cluster, commit and push, then trigger an ArgoCD sync via the API. Report what you did in one line. If not clearly fixable, post the log excerpt and proposed fix, and wait for Roger. If no high-restart pods, reply [SILENT]."
create "pvc-pressure" "every 30m" "discord" \ create "pvc-pressure" "every 1d" "discord" \
"Check storage health using the Prometheus API documented in your SOUL.md. Alert if any PVC has less than 15 percent free space, or if any node filesystem is over 85 percent full. If all healthy, reply [SILENT]." "Check storage health using the Prometheus API documented in your SOUL.md. Alert if any PVC has less than 15 percent free space, or if any node filesystem is over 85 percent full. If all healthy, reply [SILENT]."
create "argocd-sync-health" "every 1h" "discord" \ create "argocd-sync-health" "every 6h" "discord" \
"Check ArgoCD app health using the API documented in your SOUL.md. If every app is Synced and Healthy, reply [SILENT]. Otherwise list the OutOfSync or Degraded apps with their status. If an app is OutOfSync and you believe a recent git push caused it, you may trigger a sync via the API. Do NOT hand-edit resources to fix them — fix the source repo." "Check ArgoCD app health using the API documented in your SOUL.md. If every app is Synced and Healthy, reply [SILENT]. Otherwise list the OutOfSync or Degraded apps with their status. If an app is OutOfSync and you believe a recent git push caused it, you may trigger a sync via the API. Do NOT hand-edit resources to fix them — fix the source repo."
create "cert-expiry" "0 9 * * *" "discord" \ create "cert-expiry" "0 9 * * *" "discord" \
"Check certificate expiry using the Prometheus API documented in your SOUL.md. Alert on any certificate expiring in under 21 days, with its name and namespace. If none, reply [SILENT]." "Check certificate expiry using the Prometheus API documented in your SOUL.md. Alert on any certificate expiring in under 21 days, with its name and namespace. If none, reply [SILENT]."
create "node-resource-drift" "every 30m" "discord" \ create "node-resource-drift" "every 1d" "discord" \
"Check node resources using the Prometheus API documented in your SOUL.md. Alert if any node is NotReady, or if any node has CPU over 90 percent or memory over 90 percent. Otherwise reply [SILENT]." "Check node resources using the Prometheus API documented in your SOUL.md. Alert if any node is NotReady, or if any node has CPU over 90 percent or memory over 90 percent. Otherwise reply [SILENT]."
# ---- Daily report (always delivered) ---- # ---- Daily report (always delivered) ----