From 962ffd5200ad7516076fc3ad6f25a490cf0b051f Mon Sep 17 00:00:00 2001 From: Platform Engineer Date: Mon, 6 Jul 2026 12:01:36 +0000 Subject: [PATCH 1/3] fix(argocd): add ignoreDifferences to root app to stop Application CRD drift loop The k3s-cluster-root app-of-apps has been in a continuous Synced<->OutOfSync oscillation (489 transitions in 24h). Each cycle, ArgoCD syncs the 'argocd' Application CRD, succeeds, then immediately detects drift because ArgoCD adds status/operation fields to Application resources at runtime. Adding ignoreDifferences for /status and /operation on Application resources stops the drift loop while keeping the app-of-apps functional. --- argocd-bootstrap.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/argocd-bootstrap.yaml b/argocd-bootstrap.yaml index a005d07..7f33391 100644 --- a/argocd-bootstrap.yaml +++ b/argocd-bootstrap.yaml @@ -22,3 +22,9 @@ spec: selfHeal: true syncOptions: - CreateNamespace=false + ignoreDifferences: + - group: argoproj.io + kind: Application + jsonPointers: + - /status + - /operation -- 2.49.1 From aab38aa67b9fcb94160e08e1f7a34a05d979a94b Mon Sep 17 00:00:00 2001 From: Platform Engineer Date: Mon, 6 Jul 2026 12:39:51 +0000 Subject: [PATCH 2/3] fix(myorg-assistant): point cronjobs at registry image + add imagePullSecrets CronJob pods were stuck in ImagePullBackOff because they referenced the local-only image 'myorg-assistant:latest' which is not present on the node. Switch all 5 cronjobs to the Gitea registry image git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf (matching the Deployment), set imagePullPolicy: Always, and add imagePullSecrets: gitea-registry so they can authenticate to the private registry. --- myorg-assistant/cronjobs/deadline-checker.yaml | 6 ++++-- myorg-assistant/cronjobs/evening-summary.yaml | 6 ++++-- myorg-assistant/cronjobs/git-sync.yaml | 6 ++++-- myorg-assistant/cronjobs/morning-briefing.yaml | 6 ++++-- myorg-assistant/cronjobs/waiting-followup.yaml | 6 ++++-- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/myorg-assistant/cronjobs/deadline-checker.yaml b/myorg-assistant/cronjobs/deadline-checker.yaml index d6e0c2a..bde44b5 100644 --- a/myorg-assistant/cronjobs/deadline-checker.yaml +++ b/myorg-assistant/cronjobs/deadline-checker.yaml @@ -22,10 +22,12 @@ spec: job: deadline-checker spec: restartPolicy: OnFailure + imagePullSecrets: + - name: gitea-registry containers: - name: deadline-checker - image: myorg-assistant:latest - imagePullPolicy: IfNotPresent + image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf + imagePullPolicy: Always command: - python - run_job.py diff --git a/myorg-assistant/cronjobs/evening-summary.yaml b/myorg-assistant/cronjobs/evening-summary.yaml index f4e35d5..2a0e432 100644 --- a/myorg-assistant/cronjobs/evening-summary.yaml +++ b/myorg-assistant/cronjobs/evening-summary.yaml @@ -22,10 +22,12 @@ spec: job: evening-summary spec: restartPolicy: OnFailure + imagePullSecrets: + - name: gitea-registry containers: - name: evening-summary - image: myorg-assistant:latest - imagePullPolicy: IfNotPresent + image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf + imagePullPolicy: Always command: - python - run_job.py diff --git a/myorg-assistant/cronjobs/git-sync.yaml b/myorg-assistant/cronjobs/git-sync.yaml index 45d42ea..8659d4b 100644 --- a/myorg-assistant/cronjobs/git-sync.yaml +++ b/myorg-assistant/cronjobs/git-sync.yaml @@ -22,10 +22,12 @@ spec: job: git-sync spec: restartPolicy: OnFailure + imagePullSecrets: + - name: gitea-registry containers: - name: git-sync - image: myorg-assistant:latest - imagePullPolicy: IfNotPresent + image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf + imagePullPolicy: Always command: - python - run_job.py diff --git a/myorg-assistant/cronjobs/morning-briefing.yaml b/myorg-assistant/cronjobs/morning-briefing.yaml index b73bb9a..59bc97b 100644 --- a/myorg-assistant/cronjobs/morning-briefing.yaml +++ b/myorg-assistant/cronjobs/morning-briefing.yaml @@ -22,10 +22,12 @@ spec: job: morning-briefing spec: restartPolicy: OnFailure + imagePullSecrets: + - name: gitea-registry containers: - name: morning-briefing - image: myorg-assistant:latest - imagePullPolicy: IfNotPresent + image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf + imagePullPolicy: Always command: - python - run_job.py diff --git a/myorg-assistant/cronjobs/waiting-followup.yaml b/myorg-assistant/cronjobs/waiting-followup.yaml index 33d70c0..1366a31 100644 --- a/myorg-assistant/cronjobs/waiting-followup.yaml +++ b/myorg-assistant/cronjobs/waiting-followup.yaml @@ -22,10 +22,12 @@ spec: job: waiting-followup spec: restartPolicy: OnFailure + imagePullSecrets: + - name: gitea-registry containers: - name: waiting-followup - image: myorg-assistant:latest - imagePullPolicy: IfNotPresent + image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf + imagePullPolicy: Always command: - python - run_job.py -- 2.49.1 From 70ad5081802fb2ae2307477536b4039b35d715ee Mon Sep 17 00:00:00 2001 From: Platform Engineer Date: Sat, 11 Jul 2026 23:51:39 +0000 Subject: [PATCH 3/3] fix(myorg-assistant): add WEB_SECRET_KEY env to cronjobs to stop pydantic Settings validation crash --- myorg-assistant/cronjobs/deadline-checker.yaml | 5 +++++ myorg-assistant/cronjobs/evening-summary.yaml | 5 +++++ myorg-assistant/cronjobs/git-sync.yaml | 5 +++++ myorg-assistant/cronjobs/morning-briefing.yaml | 5 +++++ myorg-assistant/cronjobs/waiting-followup.yaml | 5 +++++ 5 files changed, 25 insertions(+) diff --git a/myorg-assistant/cronjobs/deadline-checker.yaml b/myorg-assistant/cronjobs/deadline-checker.yaml index bde44b5..3f30659 100644 --- a/myorg-assistant/cronjobs/deadline-checker.yaml +++ b/myorg-assistant/cronjobs/deadline-checker.yaml @@ -53,6 +53,11 @@ spec: secretKeyRef: name: myorg-assistant-secret key: LITELLM_API_KEY + - name: WEB_SECRET_KEY + valueFrom: + secretKeyRef: + name: myorg-assistant-secret + key: WEB_SECRET_KEY volumeMounts: - name: myorg-data mountPath: /data/myorg diff --git a/myorg-assistant/cronjobs/evening-summary.yaml b/myorg-assistant/cronjobs/evening-summary.yaml index 2a0e432..418e2ee 100644 --- a/myorg-assistant/cronjobs/evening-summary.yaml +++ b/myorg-assistant/cronjobs/evening-summary.yaml @@ -53,6 +53,11 @@ spec: secretKeyRef: name: myorg-assistant-secret key: LITELLM_API_KEY + - name: WEB_SECRET_KEY + valueFrom: + secretKeyRef: + name: myorg-assistant-secret + key: WEB_SECRET_KEY volumeMounts: - name: myorg-data mountPath: /data/myorg diff --git a/myorg-assistant/cronjobs/git-sync.yaml b/myorg-assistant/cronjobs/git-sync.yaml index 8659d4b..cfd5377 100644 --- a/myorg-assistant/cronjobs/git-sync.yaml +++ b/myorg-assistant/cronjobs/git-sync.yaml @@ -68,6 +68,11 @@ spec: secretKeyRef: name: myorg-assistant-secret key: LITELLM_API_KEY + - name: WEB_SECRET_KEY + valueFrom: + secretKeyRef: + name: myorg-assistant-secret + key: WEB_SECRET_KEY volumeMounts: - name: myorg-data mountPath: /data/myorg diff --git a/myorg-assistant/cronjobs/morning-briefing.yaml b/myorg-assistant/cronjobs/morning-briefing.yaml index 59bc97b..0bef6a8 100644 --- a/myorg-assistant/cronjobs/morning-briefing.yaml +++ b/myorg-assistant/cronjobs/morning-briefing.yaml @@ -60,6 +60,11 @@ spec: secretKeyRef: name: myorg-assistant-secret key: LITELLM_API_KEY + - name: WEB_SECRET_KEY + valueFrom: + secretKeyRef: + name: myorg-assistant-secret + key: WEB_SECRET_KEY volumeMounts: - name: myorg-data mountPath: /data/myorg diff --git a/myorg-assistant/cronjobs/waiting-followup.yaml b/myorg-assistant/cronjobs/waiting-followup.yaml index 1366a31..dae1158 100644 --- a/myorg-assistant/cronjobs/waiting-followup.yaml +++ b/myorg-assistant/cronjobs/waiting-followup.yaml @@ -53,6 +53,11 @@ spec: secretKeyRef: name: myorg-assistant-secret key: LITELLM_API_KEY + - name: WEB_SECRET_KEY + valueFrom: + secretKeyRef: + name: myorg-assistant-secret + key: WEB_SECRET_KEY volumeMounts: - name: myorg-data mountPath: /data/myorg -- 2.49.1