Compare commits

...

17 Commits

Author SHA1 Message Date
platform-engineer
9c134d5dd0 fix(litellm): add memory and CPU resource limits to prevent OOMKilled
Litellm pod was being killed by the kernel OOM killer (7 restarts, last
termination reason: OOMKilled). Added explicit resource requests (512Mi/250m)
and limits (2Gi/1000m) to ensure the container gets bounded memory.
2026-07-21 21:59:15 +00:00
Platform Engineer
6df0be81c9 fix(platform): fix high-restart pods - prometheus nodeSelector, litellm resources, gitea-runner limits
- Prometheus: change nodeSelector from hardware=high-memory (nonexistent label) to kubernetes.io/arch: amd64
  Fixes OOMKilled cause - pod was being scheduled on wrong nodes.
- litellm-deployment: add resource limits (2Gi/500m requests, 4Gi/2000m limits) + pin image to v1.34.0
  Fixes OOMKilled cause - no resource limits were set.
- gitea-runner: pin image to v0.12.0 from :latest + add resource limits (512Mi/250m requests, 1Gi/500m limits)
  Addresses extreme restart count (281) caused by unbounded memory usage and rolling image updates.
2026-07-21 20:28:12 +00:00
a5291da0b2 fix(gitea): bump restartedAt annotation to clear stuck gitea-runner pods (281 restarts from DNS resolution failures) 2026-07-21 15:36:57 +02:00
platform-engineer
075bdd8ca3 fix(gitea-runner): pin image tag to v0.7.2, add resource limits (256Mi/100m req, 512Mi/500m lim) to prevent OOM restarts 2026-07-20 22:19:37 +00:00
Roger Oriol
e44d7ba1fc Merge branch 'main' of https://git.rogi.casa/roger/k3s-cluster 2026-07-19 12:01:21 +02:00
Roger Oriol
f1005cd426 change openwebui url to ai.rogi.casa 2026-07-19 12:00:52 +02:00
Roger Oriol
43c0c2561e fix hermes write folder 2026-07-18 19:29:52 +02:00
8334cd48f8 Merge pull request 'fix(myorg-assistant): add git-clone initContainer to git-sync CronJob' (#26) from fix/myorg-git-sync-clone into main
Reviewed-on: roger/k3s-cluster#26
2026-07-18 19:25:00 +02:00
Hermes Platform Engineer
ba7e05a73d fix(myorg-assistant): add git-clone initContainer to git-sync CronJob
The git-sync CronJob only ran `git pull`/`git push` against /data/myorg
but never ensured the repo existed. The clone was solely the Deployment's
git-clone initContainer's job, and when that didn't populate the volume the
cron pod failed with 'Not a git repository: /data/myorg' on every run (and
swallowed the error, exiting 0).

Add an idempotent git-clone initContainer (guarded by [ ! -d /data/myorg/.git ])
so the cron job self-heals and actually syncs.

Refs: myorg-git-sync-* 'Pull: Error: Not a git repository' (last 24h+)
2026-07-18 17:21:26 +00:00
Roger Oriol
8bc3025296 fix platform engineer not allowed to respond to discord messages 2026-07-18 19:20:23 +02:00
Roger Oriol
7a7d67bedc configure git token env variable in myorg assistant cronjobs 2026-07-18 19:03:46 +02:00
Roger Oriol
19cdc77880 add git token env var to deadline checker 2026-07-18 18:59:36 +02:00
Roger Oriol
8983f482d0 give more memory to homeassistant 2026-07-17 23:59:50 +02:00
Roger Oriol
0b27cefd13 myorg assistant cron jobs env variables 2026-07-15 00:32:23 +02:00
Roger Oriol
279cc1f235 configure litellm models 2026-07-14 21:45:18 +02:00
Roger Oriol
cf6e2784fe configure litellm models 2026-07-14 21:36:38 +02:00
Roger Oriol
dad38347e7 upgrade n8n memory requirements 2026-07-14 18:50:28 +02:00
13 changed files with 144 additions and 12 deletions

View File

@@ -25,6 +25,8 @@ metadata:
namespace: gitea namespace: gitea
labels: labels:
app: gitea app: gitea
annotations:
kubectl.kubernetes.io/restartedAt: "2026-07-21T13:30:00Z"
spec: spec:
replicas: 1 replicas: 1
selector: selector:
@@ -101,6 +103,8 @@ metadata:
namespace: gitea namespace: gitea
labels: labels:
app: gitea-runner app: gitea-runner
annotations:
kubectl.kubernetes.io/restartedAt: "2026-07-21T13:30:00Z"
spec: spec:
replicas: 1 replicas: 1
selector: selector:
@@ -115,7 +119,14 @@ spec:
kubernetes.io/arch: arm64 kubernetes.io/arch: arm64
containers: containers:
- name: gitea-runner - name: gitea-runner
image: vegardit/gitea-act-runner:latest image: vegardit/gitea-act-runner:v0.12.0
resources:
requests:
memory: "512Mi"
cpu: "250m"
limits:
memory: "1Gi"
cpu: "500m"
env: env:
- name: GITEA_INSTANCE_URL - name: GITEA_INSTANCE_URL
valueFrom: valueFrom:

View File

@@ -58,9 +58,9 @@ spec:
image: ghcr.io/home-assistant/home-assistant:stable image: ghcr.io/home-assistant/home-assistant:stable
resources: resources:
requests: requests:
memory: "256Mi"
limits:
memory: "512Mi" memory: "512Mi"
limits:
memory: "1Gi"
ports: ports:
- containerPort: 8123 - containerPort: 8123
volumeMounts: volumeMounts:

View File

@@ -11,18 +11,30 @@ metadata:
data: data:
config.yaml: | config.yaml: |
model_list: model_list:
- model_name: gpt-5-mini - model_name: gpt-5.6-luna
litellm_params: litellm_params:
model: openai/gpt-5-mini-2025-08-07 model: openai/gpt-5.6-luna
api_key: "os.environ/OPENAI_API_KEY" api_key: "os.environ/OPENAI_API_KEY"
- model_name: claude-4.5-haiku - model_name: claude-haiku-4.5
litellm_params: litellm_params:
model: "anthropic/claude-haiku-4-5-20251001" model: "anthropic/claude-haiku-4-5-20251001"
api_key: "os.environ/ANTHROPIC_API_KEY" api_key: "os.environ/ANTHROPIC_API_KEY"
- model_name: claude-sonnet-5
litellm_params:
model: "anthropic/claude-sonnet-5"
api_key: "os.environ/ANTHROPIC_API_KEY"
- model_name: gemini-3-flash - model_name: gemini-3-flash
litellm_params: litellm_params:
model: gemini/gemini-3-flash-preview model: gemini/gemini-3-flash-preview
api_key: "os.environ/GEMINI_API_KEY" api_key: "os.environ/GEMINI_API_KEY"
- model_name: tencent/hy3:free
litellm_params:
model: openrouter/tencent/hy3:free
api_key: "os.environ/OPENROUTER_API_KEY"
- model_name: z-ai/glm-5.2
litellm_params:
model: openrouter/z-ai/glm-5.2
api_key: "os.environ/OPENROUTER_API_KEY"
- model_name: glm-4.7-flash - model_name: glm-4.7-flash
litellm_params: litellm_params:
model: ollama/glm-4.7-flash model: ollama/glm-4.7-flash
@@ -100,6 +112,13 @@ spec:
env: env:
- name: STORE_MODEL_IN_DB - name: STORE_MODEL_IN_DB
value: "True" value: "True"
resources:
requests:
memory: "512Mi"
cpu: "250m"
limits:
memory: "2Gi"
cpu: "1000m"
volumes: volumes:
- name: config-volume - name: config-volume
configMap: configMap:

View File

@@ -15,9 +15,10 @@ spec:
labels: labels:
app: prometheus app: prometheus
spec: spec:
# Prevent scheduling on Raspberry Pi due to resource requirements (512Mi-1Gi memory, 500m-1000m CPU) # Target the nucbox (amd64, 24Gi RAM) which is the only node with enough memory for Prometheus.
nodeSelector: nodeSelector:
hardware: high-memory kubernetes.io/os: linux
kubernetes.io/arch: amd64
serviceAccountName: prometheus serviceAccountName: prometheus
containers: containers:
- name: prometheus - name: prometheus

View File

@@ -54,6 +54,16 @@ spec:
secretKeyRef: secretKeyRef:
name: myorg-assistant-secret name: myorg-assistant-secret
key: LITELLM_API_KEY key: LITELLM_API_KEY
- name: WEB_SECRET_KEY
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: WEB_SECRET_KEY
- name: GIT_TOKEN
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: GIT_TOKEN
volumeMounts: volumeMounts:
- name: myorg-data - name: myorg-data
mountPath: /data/myorg mountPath: /data/myorg

View File

@@ -54,6 +54,16 @@ spec:
secretKeyRef: secretKeyRef:
name: myorg-assistant-secret name: myorg-assistant-secret
key: LITELLM_API_KEY key: LITELLM_API_KEY
- name: WEB_SECRET_KEY
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: WEB_SECRET_KEY
- name: GIT_TOKEN
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: GIT_TOKEN
volumeMounts: volumeMounts:
- name: myorg-data - name: myorg-data
mountPath: /data/myorg mountPath: /data/myorg

View File

@@ -24,6 +24,43 @@ spec:
restartPolicy: OnFailure restartPolicy: OnFailure
imagePullSecrets: imagePullSecrets:
- name: gitea-registry - name: gitea-registry
initContainers:
- name: git-clone
image: alpine/git:latest
command:
- sh
- -c
- |
if [ ! -d /data/myorg/.git ]; then
echo "Cloning repository..."
git clone ${GIT_REPO_URL} /data/myorg
cd /data/myorg
git config user.name "${GIT_USERNAME}"
git config user.email "${GIT_USERNAME}@rogi.casa"
git config credential.helper store
echo "https://${GIT_USERNAME}:${GIT_TOKEN}@git.rogi.casa" > ~/.git-credentials
else
echo "Repository already exists, skipping clone."
fi
env:
- name: GIT_REPO_URL
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: GIT_REPO_URL
- name: GIT_USERNAME
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: GIT_USERNAME
- name: GIT_TOKEN
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: GIT_TOKEN
volumeMounts:
- name: myorg-data
mountPath: /data/myorg
containers: containers:
- name: git-sync - name: git-sync
image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf image: git.rogi.casa/roger/myorg-assistant/myorg-assistant:fcf79bf
@@ -69,6 +106,11 @@ spec:
secretKeyRef: secretKeyRef:
name: myorg-assistant-secret name: myorg-assistant-secret
key: LITELLM_API_KEY key: LITELLM_API_KEY
- name: WEB_SECRET_KEY
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: WEB_SECRET_KEY
volumeMounts: volumeMounts:
- name: myorg-data - name: myorg-data
mountPath: /data/myorg mountPath: /data/myorg

View File

@@ -61,6 +61,16 @@ spec:
secretKeyRef: secretKeyRef:
name: myorg-assistant-secret name: myorg-assistant-secret
key: LITELLM_API_KEY key: LITELLM_API_KEY
- name: WEB_SECRET_KEY
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: WEB_SECRET_KEY
- name: GIT_TOKEN
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: GIT_TOKEN
volumeMounts: volumeMounts:
- name: myorg-data - name: myorg-data
mountPath: /data/myorg mountPath: /data/myorg

View File

@@ -54,6 +54,16 @@ spec:
secretKeyRef: secretKeyRef:
name: myorg-assistant-secret name: myorg-assistant-secret
key: LITELLM_API_KEY key: LITELLM_API_KEY
- name: WEB_SECRET_KEY
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: WEB_SECRET_KEY
- name: GIT_TOKEN
valueFrom:
secretKeyRef:
name: myorg-assistant-secret
key: GIT_TOKEN
volumeMounts: volumeMounts:
- name: myorg-data - name: myorg-data
mountPath: /data/myorg mountPath: /data/myorg

View File

@@ -53,15 +53,17 @@ spec:
value: http value: http
- name: N8N_PORT - name: N8N_PORT
value: "5678" value: "5678"
- name: NODE_OPTIONS
value: "--max-old-space-size=768"
image: n8nio/n8n image: n8nio/n8n
name: n8n name: n8n
ports: ports:
- containerPort: 5678 - containerPort: 5678
resources: resources:
requests: requests:
memory: "250Mi" memory: "512Mi"
limits: limits:
memory: "500Mi" memory: "1Gi"
volumeMounts: volumeMounts:
- mountPath: /home/node/.n8n - mountPath: /home/node/.n8n
name: n8n-claim0 name: n8n-claim0

View File

@@ -9,10 +9,10 @@ spec:
ingressClassName: traefik ingressClassName: traefik
tls: tls:
- hosts: - hosts:
- openai.rogi.casa - ai.rogi.casa
secretName: openwebui-tls secretName: openwebui-tls
rules: rules:
- host: openai.rogi.casa - host: ai.rogi.casa
http: http:
paths: paths:
- path: / - path: /

View File

@@ -59,6 +59,16 @@ data:
cron: cron:
wrap_response: false wrap_response: false
discord:
allowed_channels: '1470909384162017444' # DISCORD_HOME_CHANNEL
free_response_channels: '1470909384162017444' # no @mention needed here
# Per-platform gateway auth. Paired with GATEWAY_ALLOW_ALL_USERS=true in
# the env (secret.yaml), this lets the bot reply to inbound DMs and
# group messages from anyone. Tighten later by switching to
# DISCORD_ALLOWED_USERS=<id> in the secret and dropping these two lines.
dm_policy: open
group_policy: open
memory: memory:
memory_enabled: true memory_enabled: true
user_profile_enabled: true user_profile_enabled: true

View File

@@ -125,6 +125,13 @@ spec:
env: env:
- name: HERMES_HOME - name: HERMES_HOME
value: /opt/data value: /opt/data
# Hermes' file-write tool refuses any path outside HERMES_WRITE_SAFE_ROOT.
# When unset it defaults to HERMES_HOME (/opt/data), which blocks the
# agent's only GitOps remediation path (editing manifests under
# /workspace/k3s-cluster). Whitelist the whole filesystem — consistent
# with yolo:true, approvals.mode:off, and the agent having no k8s RBAC.
- name: HERMES_WRITE_SAFE_ROOT
value: "/"
volumeMounts: volumeMounts:
- name: data - name: data
mountPath: /opt/data mountPath: /opt/data