loki + promtail new monitoring services

monitoring/grafana-datasource.yaml

@@ -13,3 +13,8 @@ data:
       url: http://prometheus:9090
       isDefault: true
       editable: true
+    - name: Loki
+      type: loki
+      access: proxy
+      url: http://loki:3100
+      editable: true

monitoring/k8s-event-logger.yaml

@@ -0,0 +1,95 @@
+# k8s-event-exporter — watches Kubernetes Events and logs them to stdout as
+# structured JSON. Promtail tails the logs and ships them to Loki with the
+# label app=kubernetes-event-exporter. The Hermes agent queries them with LogQL:
+#   {app="kubernetes-event-exporter"} |= "BackOff"
+#
+# This avoids giving the agent any k8s API token for events — the exporter
+# has its own narrow read-only SA, and the agent only talks to Loki.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: k8s-event-exporter
+  namespace: monitoring
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: k8s-event-exporter
+rules:
+  - apiGroups: [""]
+    resources:
+      - events
+      - pods
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: k8s-event-exporter
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: k8s-event-exporter
+subjects:
+  - kind: ServiceAccount
+    name: k8s-event-exporter
+    namespace: monitoring
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: k8s-event-exporter-config
+  namespace: monitoring
+data:
+  config.yaml: |
+    logLevel: info
+    logFormat: json
+    route:
+      routes:
+        - match:
+            - receiver: dump
+    receivers:
+      - name: dump
+        dump: {}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: k8s-event-exporter
+  namespace: monitoring
+  labels:
+    app: kubernetes-event-exporter
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kubernetes-event-exporter
+  template:
+    metadata:
+      labels:
+        app: kubernetes-event-exporter
+    spec:
+      serviceAccountName: k8s-event-exporter
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      containers:
+      - name: exporter
+        image: opsgenie/kubernetes-event-exporter:0.9
+        args:
+          - -conf=/config/config.yaml
+        volumeMounts:
+        - name: config
+          mountPath: /config
+          readOnly: true
+        resources:
+          requests:
+            memory: "32Mi"
+            cpu: "25m"
+          limits:
+            memory: "128Mi"
+            cpu: "100m"
+      volumes:
+      - name: config
+        configMap:
+          name: k8s-event-exporter-config

monitoring/loki.yaml

@@ -0,0 +1,153 @@
+# Loki — log aggregation (single-binary mode, local filesystem storage).
+#
+# Stores compressed, indexed pod logs shipped by Promtail. Queried by the
+# platform-engineer Hermes agent via the HTTP API (LogQL) and by Grafana.
+#
+# Storage: 20 GiB local PVC, 1-week retention enforced by the compactor.
+# Service: loki.monitoring:3100 (ClusterIP, no auth — homelab).
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: loki-config
+  namespace: monitoring
+data:
+  loki.yaml: |
+    auth_enabled: false
+
+    server:
+      http_listen_port: 3100
+      grpc_listen_port: 9096
+
+    common:
+      path_prefix: /loki
+      replication_factor: 1
+      ring:
+        instance_addr: 127.0.0.1
+        kvstore:
+          store: inmemory
+
+    schema_config:
+      configs:
+        - from: 2024-01-01
+          store: tsdb
+          object_store: filesystem
+          schema: v13
+          index:
+            prefix: index_
+            period: 24h
+
+    storage_config:
+      filesystem:
+        directory: /loki/chunks
+      tsdb_shipper:
+        active_index_directory: /loki/tsdb-index
+        cache_location: /loki/tsdb-cache
+
+    limits_config:
+      retention_period: 168h          # 1 week
+      max_query_series: 10000
+      reject_old_samples: true
+      reject_old_samples_max_age: 168h
+      allow_structured_metadata: false  # tsdb v13 compat
+
+    compactor:
+      working_directory: /loki/compactor
+      compaction_interval: 10m
+      retention_enabled: true
+      retention_delete_delay: 2h
+      retention_delete_worker_count: 50
+      delete_request_store: filesystem
+
+    analytics:
+      reporting_enabled: false
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: loki-data
+  namespace: monitoring
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 20Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: loki
+  namespace: monitoring
+  labels:
+    app: loki
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate          # single-writer storage
+  selector:
+    matchLabels:
+      app: loki
+  template:
+    metadata:
+      labels:
+        app: loki
+    spec:
+      nodeSelector:
+        kubernetes.io/arch: amd64   # Loki image; runs on the NUC
+      containers:
+      - name: loki
+        image: grafana/loki:3.4.4
+        args:
+          - -config.file=/etc/loki/loki.yaml
+        ports:
+        - name: http
+          containerPort: 3100
+        volumeMounts:
+        - name: config
+          mountPath: /etc/loki
+          readOnly: true
+        - name: data
+          mountPath: /loki
+        resources:
+          requests:
+            memory: "512Mi"
+            cpu: "250m"
+          limits:
+            memory: "1Gi"
+            cpu: "1000m"
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: 3100
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          failureThreshold: 5
+        livenessProbe:
+          httpGet:
+            path: /ready
+            port: 3100
+          initialDelaySeconds: 60
+          periodSeconds: 30
+          failureThreshold: 5
+      volumes:
+      - name: config
+        configMap:
+          name: loki-config
+      - name: data
+        persistentVolumeClaim:
+          claimName: loki-data
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: loki
+  namespace: monitoring
+spec:
+  type: ClusterIP
+  selector:
+    app: loki
+  ports:
+    - name: http
+      port: 3100
+      targetPort: 3100

monitoring/promtail.yaml

@@ -0,0 +1,138 @@
+# Promtail — DaemonSet that tails pod logs on every node and ships them to Loki.
+#
+# Runs on ALL nodes (amd64 + arm). Multi-arch image. Reads /var/log/pods/*,
+# attaches k8s labels (namespace, pod, container), ships to loki.monitoring:3100.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: promtail
+  namespace: monitoring
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: promtail
+rules:
+  - apiGroups: [""]
+    resources:
+      - nodes
+      - nodes/proxy
+      - services
+      - endpoints
+      - pods
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: promtail
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: promtail
+subjects:
+  - kind: ServiceAccount
+    name: promtail
+    namespace: monitoring
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: promtail-config
+  namespace: monitoring
+data:
+  promtail.yaml: |
+    server:
+      http_listen_port: 9080
+      grpc_listen_port: 0
+
+    positions:
+      filename: /tmp/positions.yaml
+
+    clients:
+      - url: http://loki.monitoring:3100/loki/api/v1/push
+
+    scrape_configs:
+      # Tail all container logs via /var/log/containers/*.log (symlinks to
+      # /var/log/pods/<ns>_<pod>_<uid>/<container>/<N>.log). Extract namespace,
+      # pod, container labels from the filename via pipeline_stages regex.
+      - job_name: kubernetes-containers
+        static_configs:
+          - targets:
+              - localhost
+            labels:
+              job: kube-containers
+              __path__: /var/log/containers/*.log
+        pipeline_stages:
+          - cri: {}
+          # k3s filename: <pod>_<namespace>_<container>-<hash>.log
+          - regex:
+              expression: '/var/log/containers/(?P<pod>[^_]+)_(?P<namespace>[^_]+)_(?P<container>[^-]+)-.*\.log'
+              source: filename
+          - labels:
+              pod:
+              namespace:
+              container:
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: promtail
+  namespace: monitoring
+  labels:
+    app: promtail
+spec:
+  selector:
+    matchLabels:
+      app: promtail
+  template:
+    metadata:
+      labels:
+        app: promtail
+    spec:
+      serviceAccountName: promtail
+      tolerations:
+        - operator: Exists   # run on every node including tainted Pis
+      containers:
+      - name: promtail
+        image: grafana/promtail:3.4.4
+        args:
+          - -config.file=/etc/promtail/promtail.yaml
+          - -config.expand-env=true
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        volumeMounts:
+        - name: config
+          mountPath: /etc/promtail
+          readOnly: true
+        - name: positions
+          mountPath: /tmp
+        - name: pods-logs
+          mountPath: /var/log/pods
+          readOnly: true
+        - name: containers-logs
+          mountPath: /var/log/containers
+          readOnly: true
+        resources:
+          requests:
+            memory: "64Mi"
+            cpu: "50m"
+          limits:
+            memory: "256Mi"
+            cpu: "250m"
+      volumes:
+      - name: config
+        configMap:
+          name: promtail-config
+      - name: positions
+        emptyDir: {}
+      - name: pods-logs
+        hostPath:
+          path: /var/log/pods
+      - name: containers-logs
+        hostPath:
+          path: /var/log/containers