From fc1b4383c11daa27ce05530739e3e4b3ea31bef6 Mon Sep 17 00:00:00 2001 From: Platform Engineer Date: Mon, 6 Jul 2026 12:01:36 +0000 Subject: [PATCH 1/7] fix(argocd): add ignoreDifferences to root app to stop Application CRD drift loop The k3s-cluster-root app-of-apps has been in a continuous Synced<->OutOfSync oscillation (489 transitions in 24h). Each cycle, ArgoCD syncs the 'argocd' Application CRD, succeeds, then immediately detects drift because ArgoCD adds status/operation fields to Application resources at runtime. Adding ignoreDifferences for /status and /operation on Application resources stops the drift loop while keeping the app-of-apps functional. --- argocd-bootstrap.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/argocd-bootstrap.yaml b/argocd-bootstrap.yaml index a005d07..7f33391 100644 --- a/argocd-bootstrap.yaml +++ b/argocd-bootstrap.yaml @@ -22,3 +22,9 @@ spec: selfHeal: true syncOptions: - CreateNamespace=false + ignoreDifferences: + - group: argoproj.io + kind: Application + jsonPointers: + - /status + - /operation -- 2.49.1 From 0794153e56760138750c7219c79724c3b3e07630 Mon Sep 17 00:00:00 2001 From: Platform Engineer Date: Mon, 6 Jul 2026 12:39:51 +0000 Subject: [PATCH 2/7] fix(myorg-assistant): point cronjobs at registry image + add imagePullSecrets CronJob pods were stuck in ImagePullBackOff because they referenced the local-only image 'myorg-assistant:latest' which is not present on the node. Switch all 5 cronjobs to the Gitea registry image git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf (matching the Deployment), set imagePullPolicy: Always, and add imagePullSecrets: gitea-registry so they can authenticate to the private registry. --- myorg-assistant/cronjobs/deadline-checker.yaml | 6 ++++-- myorg-assistant/cronjobs/evening-summary.yaml | 6 ++++-- myorg-assistant/cronjobs/git-sync.yaml | 6 ++++-- myorg-assistant/cronjobs/morning-briefing.yaml | 6 ++++-- myorg-assistant/cronjobs/waiting-followup.yaml | 6 ++++-- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/myorg-assistant/cronjobs/deadline-checker.yaml b/myorg-assistant/cronjobs/deadline-checker.yaml index d6e0c2a..bde44b5 100644 --- a/myorg-assistant/cronjobs/deadline-checker.yaml +++ b/myorg-assistant/cronjobs/deadline-checker.yaml @@ -22,10 +22,12 @@ spec: job: deadline-checker spec: restartPolicy: OnFailure + imagePullSecrets: + - name: gitea-registry containers: - name: deadline-checker - image: myorg-assistant:latest - imagePullPolicy: IfNotPresent + image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf + imagePullPolicy: Always command: - python - run_job.py diff --git a/myorg-assistant/cronjobs/evening-summary.yaml b/myorg-assistant/cronjobs/evening-summary.yaml index f4e35d5..2a0e432 100644 --- a/myorg-assistant/cronjobs/evening-summary.yaml +++ b/myorg-assistant/cronjobs/evening-summary.yaml @@ -22,10 +22,12 @@ spec: job: evening-summary spec: restartPolicy: OnFailure + imagePullSecrets: + - name: gitea-registry containers: - name: evening-summary - image: myorg-assistant:latest - imagePullPolicy: IfNotPresent + image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf + imagePullPolicy: Always command: - python - run_job.py diff --git a/myorg-assistant/cronjobs/git-sync.yaml b/myorg-assistant/cronjobs/git-sync.yaml index 45d42ea..8659d4b 100644 --- a/myorg-assistant/cronjobs/git-sync.yaml +++ b/myorg-assistant/cronjobs/git-sync.yaml @@ -22,10 +22,12 @@ spec: job: git-sync spec: restartPolicy: OnFailure + imagePullSecrets: + - name: gitea-registry containers: - name: git-sync - image: myorg-assistant:latest - imagePullPolicy: IfNotPresent + image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf + imagePullPolicy: Always command: - python - run_job.py diff --git a/myorg-assistant/cronjobs/morning-briefing.yaml b/myorg-assistant/cronjobs/morning-briefing.yaml index b73bb9a..59bc97b 100644 --- a/myorg-assistant/cronjobs/morning-briefing.yaml +++ b/myorg-assistant/cronjobs/morning-briefing.yaml @@ -22,10 +22,12 @@ spec: job: morning-briefing spec: restartPolicy: OnFailure + imagePullSecrets: + - name: gitea-registry containers: - name: morning-briefing - image: myorg-assistant:latest - imagePullPolicy: IfNotPresent + image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf + imagePullPolicy: Always command: - python - run_job.py diff --git a/myorg-assistant/cronjobs/waiting-followup.yaml b/myorg-assistant/cronjobs/waiting-followup.yaml index 33d70c0..1366a31 100644 --- a/myorg-assistant/cronjobs/waiting-followup.yaml +++ b/myorg-assistant/cronjobs/waiting-followup.yaml @@ -22,10 +22,12 @@ spec: job: waiting-followup spec: restartPolicy: OnFailure + imagePullSecrets: + - name: gitea-registry containers: - name: waiting-followup - image: myorg-assistant:latest - imagePullPolicy: IfNotPresent + image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf + imagePullPolicy: Always command: - python - run_job.py -- 2.49.1 From ce08365e06d1ce36af0265078b8f48eda69f5da2 Mon Sep 17 00:00:00 2001 From: Roger Oriol Date: Tue, 7 Jul 2026 23:52:37 +0200 Subject: [PATCH 3/7] revert to qwen3.6 for platform engineer --- platform-engineer/configmap.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platform-engineer/configmap.yaml b/platform-engineer/configmap.yaml index c9d3444..08f8480 100644 --- a/platform-engineer/configmap.yaml +++ b/platform-engineer/configmap.yaml @@ -9,18 +9,18 @@ data: config.yaml: | model: provider: openai-api - default: z-ai/glm-5.2 + default: qwen3.6 base_url: "http://litellm-service.litellm:80/v1" api_mode: chat_completions auxiliary: compression: provider: openai-api - model: z-ai/glm-5.2 + model: qwen3.6 base_url: "http://litellm-service.litellm:80/v1" title_generation: provider: openai-api - model: z-ai/glm-5.2 + model: qwen3.6 base_url: "http://litellm-service.litellm:80/v1" terminal: -- 2.49.1 From 81dfe6fd6092c31c487e610b7ec1a8d02f6724c1 Mon Sep 17 00:00:00 2001 From: Roger Oriol Date: Thu, 9 Jul 2026 00:45:10 +0200 Subject: [PATCH 4/7] fix argocd rbac and create agents.md --- AGENTS.md | 246 +++++++++++++++++++++++++++++++++++++ argocd/argocd-rbac-cm.yaml | 29 +++++ 2 files changed, 275 insertions(+) create mode 100644 AGENTS.md create mode 100644 argocd/argocd-rbac-cm.yaml diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..d304e00 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,246 @@ +# AGENTS.md - Guide for Coding Agents + +This file provides essential information for AI coding agents working with this Kubernetes cluster project. + +## Project Overview + +This repository contains Kubernetes manifests for a K3s cluster running self-hosted services on the `rogi.casa` domain. The cluster is managed via **GitOps using ArgoCD** - all changes to the cluster are deployed automatically from this Git repository. + +**⚠️ CRITICAL: Permission Model** + +You **DO NOT** have permission to push changes to this repository. Before applying any changes to the cluster: +1. Make the necessary code changes to the manifests +2. Clearly present the changes to the user +3. Ask the user to review and push the changes +4. Wait for confirmation that changes have been pushed +5. Only then will ArgoCD automatically deploy the changes to the cluster + +## Architecture & GitOps Workflow + +### ArgoCD App-of-Apps Pattern + +This project uses ArgoCD's "app-of-apps" pattern: + +``` +argocd-bootstrap.yaml (root Application) + ↓ +argocd/apps/ (directory containing all Application manifests) + ↓ +Individual Applications (one per service directory) + ↓ +Kubernetes manifests in each service directory (e.g., pihole/, homeassistant/) +``` + +### Deployment Flow + +1. You make changes to Kubernetes manifests in the repository +2. User reviews and pushes changes to the `main` branch +3. ArgoCD detects changes (automatically or on sync) +4. ArgoCD applies changes to the cluster with `prune: true` and `selfHeal: true` +5. Cluster state converges to match the Git state + +### Key Files + +- **`argocd-bootstrap.yaml`**: The root Application that bootstraps ArgoCD. Points to `argocd/apps/` directory. This is the only file that needs manual `kubectl apply` during initial setup. +- **`argocd/apps/project.yaml`**: ArgoCD AppProject defining permissions for all applications +- **`argocd/apps/*.yaml`**: Individual ArgoCD Application manifests (one per service) +- **`argocd/gen-apps.sh`**: Script to regenerate all ArgoCD manifests from the `APPS` array + +## Repository Structure + +``` +k3s-cluster/ +├── argocd-bootstrap.yaml # Root ArgoCD Application (app-of-apps) +├── argocd/ +│ ├── apps/ # Individual ArgoCD Application manifests +│ │ ├── project.yaml # AppProject definition +│ │ ├── pihole.yaml # Application for pihole/ +│ │ ├── homeassistant.yaml # Application for homeassistant/ +│ │ └── ... # One per service +│ ├── gen-apps.sh # Generates argocd/apps/* manifests +│ └── ingress.yaml # ArgoCD's own ingress +├── / # Each service has its own directory +│ ├── namespace.yaml # (Optional) Namespace definition +│ ├── deployment.yaml # Main deployment/statefulset +│ ├── service.yaml # Service definition +│ ├── ingress.yaml # Ingress configuration +│ ├── configmap.yaml # (Optional) ConfigMaps +│ ├── pvc.yaml # (Optional) PersistentVolumeClaims +│ └── secret.yaml # (Optional) Secrets (rarely committed) +├── cert-manager/ # cert-manager installation manifests +├── nas/ # External NAS service configuration +├── monitoring/ # Prometheus + Grafana stack +└── README.md # Comprehensive project documentation +``` + +## Current Services + +The cluster runs these services (each in its own directory): + +- **argocd** - GitOps continuous delivery platform +- **cert-manager** - SSL certificate management (Let's Encrypt) +- **fava** - Beancount accounting web interface +- **gitea** - Self-hosted Git server +- **glance** - Personal dashboard +- **gym-tracker** - Workout tracking application +- **homeassistant** - Home automation +- **jellyfin** - Media server +- **litellm** - LLM proxy +- **minecraft-server** - Minecraft server +- **monitoring** - Prometheus + Grafana +- **myorg-assistant** - Organization assistant +- **n8n** - Workflow automation +- **nas** - External NAS proxy +- **openwebui** - Web UI for LLMs +- **phoenix** - AI observability platform +- **pihole** - Network-wide ad blocking +- **platform-engineer** - Platform engineering tools +- **qbittorrent** - Torrent client +- **searxng** - Meta search engine +- **vaultwarden** - Password manager (Bitwarden compatible) + +## How to Make Changes + +### Adding a New Service + +1. Create a new directory: `mkdir new-service` +2. Create Kubernetes manifests in `new-service/`: + - `namespace.yaml` (if dedicated namespace needed) + - `deployment.yaml` or `statefulset.yaml` + - `service.yaml` + - `ingress.yaml` + - Any ConfigMaps, Secrets, PVCs needed +3. Add the service to `argocd/gen-apps.sh`: + - Add a line to the `APPS` array: `"new-service|namespace|new-service|true|true"` + - Format: `name|namespace|path|recurse|validate` +4. Run `./argocd/gen-apps.sh` to regenerate ArgoCD manifests +5. **Present changes to user for review and push** + +### Modifying an Existing Service + +1. Edit the relevant manifest(s) in the service directory +2. If changing ArgoCD configuration, also update `argocd/gen-apps.sh` and regenerate +3. **Present changes to user for review and push** + +### Removing a Service + +1. Remove the service directory: `rm -rf service-name/` +2. Remove from `APPS` array in `argocd/gen-apps.sh` +3. Run `./argocd/gen-apps.sh` to regenerate +4. **Present changes to user for review and push** +5. ArgoCD will automatically prune the resources from the cluster + +## Common Patterns + +### Ingress Configuration + +Each service has its own `ingress.yaml` with: +- `ingressClassName: traefik` (K3s default) +- TLS configured with `cert-manager.io/cluster-issuer: letsencrypt-prod` +- Host-based routing (e.g., `pihole.rogi.casa`) + +Example: +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: pihole + namespace: pihole + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod +spec: + ingressClassName: traefik + tls: + - hosts: + - pihole.rogi.casa + secretName: pihole-tls + rules: + - host: pihole.rogi.casa + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: pihole-web + port: + number: 80 +``` + +### Resource Management + +- Each service typically has its own namespace +- Use ResourceRequests and Limits for all containers +- PVCs for persistent data +- ConfigMaps for configuration files + +## Important Notes + +### What You CAN Do + +- Read and understand all manifests +- Create new manifest files +- Modify existing manifest files +- Run `./argocd/gen-apps.sh` to regenerate ArgoCD manifests +- Explain how the cluster works +- Troubleshoot issues by reading manifests + +### What You CANNOT Do + +- Push changes to the Git repository (no push permissions) +- Directly apply manifests with `kubectl apply` (unless explicitly asked) +- Access the Kubernetes cluster directly (unless explicitly configured) +- Create secrets that should remain private (those are managed manually) + +### Secrets Management + +Secrets are generally **not committed to the repository**. They must be created manually in the cluster: +```bash +kubectl create secret docker-registry gitea-registry \ + --docker-server=gitea.rogi.casa \ + --docker-username= \ + --docker-password= \ + -n +``` + +## Workflow Summary + +When asked to make changes: + +1. **Understand** the current state by reading relevant files +2. **Modify** the manifests (create/edit files) +3. **Regenerate** ArgoCD manifests if needed (`./argocd/gen-apps.sh`) +4. **Present** the changes clearly to the user: + ``` + I've made the following changes: + - Modified pihole/deployment.yaml to update image version + - Regenerated argocd/apps/pihole.yaml + + Please review and push these changes to deploy them. + ``` +5. **Wait** for user confirmation that changes are pushed +6. **Verify** (if possible) that ArgoCD has synced the changes + +## Useful Commands (for reference) + +```bash +# Regenerate ArgoCD manifests after modifying gen-apps.sh +./argocd/gen-apps.sh + +# Check ArgoCD applications status (requires kubectl access) +kubectl get applications -n argocd + +# View logs of a pod (requires kubectl access) +kubectl logs -n + +# Check ingress status (requires kubectl access) +kubectl get ingress -n +``` + +## Questions? + +If you're unsure about anything: +1. Read the comprehensive `README.md` in the repository root +2. Check existing service directories for examples +3. Ask the user for clarification before making changes +4. Remember: **never push without explicit user review and approval** diff --git a/argocd/argocd-rbac-cm.yaml b/argocd/argocd-rbac-cm.yaml new file mode 100644 index 0000000..cbdf65e --- /dev/null +++ b/argocd/argocd-rbac-cm.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: argocd-rbac-cm + namespace: argocd + labels: + app.kubernetes.io/name: argocd-rbac-cm + app.kubernetes.io/part-of: argocd +data: + policy.csv: | + # Grant platform-engineer read-only access to applications + g, platform-engineer, role:readonly + + # Custom policy for platform-engineer with application read permissions + p, role:platform-engineer, applications, get, *, allow + p, role:platform-engineer, applications, list, *, allow + p, role:platform-engineer, clusters, get, *, allow + p, role:platform-engineer, clusters, list, *, allow + p, role:platform-engineer, repositories, get, *, allow + p, role:platform-engineer, repositories, list, *, allow + p, role:platform-engineer, projects, get, *, allow + p, role:platform-engineer, projects, list, *, allow + g, platform-engineer, role:platform-engineer + + # Default policy - deny by default (ArgoCD default) + policy.default: role:readonly + + # Enable RBAC + rbac.enabled: "true" -- 2.49.1 From 04b736287b543640a6cbdf57b8492fa9f79d69c4 Mon Sep 17 00:00:00 2001 From: Roger Oriol Date: Thu, 9 Jul 2026 23:48:47 +0200 Subject: [PATCH 5/7] fix argocd redirect --- argocd/argocd-cm.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/argocd/argocd-cm.yaml b/argocd/argocd-cm.yaml index be8a47d..2a2965a 100644 --- a/argocd/argocd-cm.yaml +++ b/argocd/argocd-cm.yaml @@ -7,6 +7,11 @@ metadata: app.kubernetes.io/name: argocd-cm app.kubernetes.io/part-of: argocd data: + # Serve HTTP (no redirect to HTTPS) so the TLS-terminating Traefik ingress works. + # Without this, argocd-server redirects HTTP->HTTPS, causing an infinite + # redirect loop behind the ingress (argocd.rogi.casa unreachable). + server.insecure: "true" + # add an additional local user with apiKey and login capabilities # apiKey - allows generating API keys # login - allows to login using UI -- 2.49.1 From a5b90994a403ba8174380ed9fc11c2bdfed2f0ad Mon Sep 17 00:00:00 2001 From: Roger Oriol Date: Sat, 11 Jul 2026 19:22:57 +0200 Subject: [PATCH 6/7] fix myorg assistant --- myorg-assistant/cronjobs/deadline-checker.yaml | 3 ++- myorg-assistant/cronjobs/evening-summary.yaml | 3 ++- myorg-assistant/cronjobs/git-sync.yaml | 3 ++- myorg-assistant/cronjobs/morning-briefing.yaml | 3 ++- myorg-assistant/cronjobs/waiting-followup.yaml | 3 ++- myorg-assistant/deployment.yaml | 2 +- platform-engineer/deployment.yaml | 1 + 7 files changed, 12 insertions(+), 6 deletions(-) diff --git a/myorg-assistant/cronjobs/deadline-checker.yaml b/myorg-assistant/cronjobs/deadline-checker.yaml index bde44b5..2de9e0a 100644 --- a/myorg-assistant/cronjobs/deadline-checker.yaml +++ b/myorg-assistant/cronjobs/deadline-checker.yaml @@ -30,7 +30,8 @@ spec: imagePullPolicy: Always command: - python - - run_job.py + - -c + - "from src.scheduler.jobs import run_job; import sys; run_job(sys.argv[1])" - deadline-checker env: - name: MYORG_REPO_PATH diff --git a/myorg-assistant/cronjobs/evening-summary.yaml b/myorg-assistant/cronjobs/evening-summary.yaml index 2a0e432..1a43eae 100644 --- a/myorg-assistant/cronjobs/evening-summary.yaml +++ b/myorg-assistant/cronjobs/evening-summary.yaml @@ -30,7 +30,8 @@ spec: imagePullPolicy: Always command: - python - - run_job.py + - -c + - "from src.scheduler.jobs import run_job; import sys; run_job(sys.argv[1])" - evening-summary env: - name: MYORG_REPO_PATH diff --git a/myorg-assistant/cronjobs/git-sync.yaml b/myorg-assistant/cronjobs/git-sync.yaml index 8659d4b..ae609c7 100644 --- a/myorg-assistant/cronjobs/git-sync.yaml +++ b/myorg-assistant/cronjobs/git-sync.yaml @@ -30,7 +30,8 @@ spec: imagePullPolicy: Always command: - python - - run_job.py + - -c + - "from src.scheduler.jobs import run_job; import sys; run_job(sys.argv[1])" - git-sync env: - name: MYORG_REPO_PATH diff --git a/myorg-assistant/cronjobs/morning-briefing.yaml b/myorg-assistant/cronjobs/morning-briefing.yaml index 59bc97b..517a1db 100644 --- a/myorg-assistant/cronjobs/morning-briefing.yaml +++ b/myorg-assistant/cronjobs/morning-briefing.yaml @@ -30,7 +30,8 @@ spec: imagePullPolicy: Always command: - python - - run_job.py + - -c + - "from src.scheduler.jobs import run_job; import sys; run_job(sys.argv[1])" - morning-briefing env: # From ConfigMap diff --git a/myorg-assistant/cronjobs/waiting-followup.yaml b/myorg-assistant/cronjobs/waiting-followup.yaml index 1366a31..c3fde1e 100644 --- a/myorg-assistant/cronjobs/waiting-followup.yaml +++ b/myorg-assistant/cronjobs/waiting-followup.yaml @@ -30,7 +30,8 @@ spec: imagePullPolicy: Always command: - python - - run_job.py + - -c + - "from src.scheduler.jobs import run_job; import sys; run_job(sys.argv[1])" - waiting-followup env: - name: MYORG_REPO_PATH diff --git a/myorg-assistant/deployment.yaml b/myorg-assistant/deployment.yaml index 1981a00..e3e6948 100644 --- a/myorg-assistant/deployment.yaml +++ b/myorg-assistant/deployment.yaml @@ -34,7 +34,7 @@ spec: git config user.name "${GIT_USERNAME}" git config user.email "${GIT_USERNAME}@rogi.casa" git config credential.helper store - echo "https://${GIT_USERNAME}:${GIT_TOKEN}@gitea.rogi.casa" > ~/.git-credentials + echo "https://${GIT_USERNAME}:${GIT_TOKEN}@git.rogi.casa" > ~/.git-credentials else echo "Repository already exists, pulling latest changes..." cd /data/myorg diff --git a/platform-engineer/deployment.yaml b/platform-engineer/deployment.yaml index cac637b..eff1fae 100644 --- a/platform-engineer/deployment.yaml +++ b/platform-engineer/deployment.yaml @@ -86,6 +86,7 @@ spec: : > /opt/data/.env chmod 600 /opt/data/.env for k in OPENAI_API_KEY OPENAI_BASE_URL DISCORD_BOT_TOKEN DISCORD_HOME_CHANNEL \ + DISCORD_ALLOW_ALL_USERS DISCORD_FREE_RESPONSE_CHANNELS \ GITEA_TOKEN GITEA_REPO_URL ARGOCD_API_TOKEN ARGOCD_SERVER \ HERMES_DASHBOARD HERMES_DASHBOARD_BASIC_AUTH_USERNAME \ HERMES_DASHBOARD_BASIC_AUTH_PASSWORD HERMES_DASHBOARD_BASIC_AUTH_SECRET; do -- 2.49.1 From e420aa380802309f3aefb89d145ab941eeefe36d Mon Sep 17 00:00:00 2001 From: platform-engineer Date: Sun, 12 Jul 2026 23:38:48 +0000 Subject: [PATCH 7/7] fix(argocd): drop redundant directory.recurse=false so k3s-cluster-root self-heal converges --- argocd/apps/argocd.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/argocd/apps/argocd.yaml b/argocd/apps/argocd.yaml index 4372df5..59200d5 100644 --- a/argocd/apps/argocd.yaml +++ b/argocd/apps/argocd.yaml @@ -11,8 +11,6 @@ spec: repoURL: https://git.rogi.casa/roger/k3s-cluster.git targetRevision: main path: argocd - directory: - recurse: false destination: server: https://kubernetes.default.svc namespace: argocd -- 2.49.1