feat(observability): add loki, tempo and pod log collection

deploy/observability/grafana-datasources.yaml

@@ -0,0 +1,35 @@
+# kube-prometheus-stack's Grafana sidecar loads labeled data sources.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nereus-grafana-datasources
+  labels:
+    grafana_datasource: "1"
+data:
+  nereus.yaml: |
+    apiVersion: 1
+    datasources:
+      - name: Loki
+        uid: loki
+        type: loki
+        access: proxy
+        url: http://loki.observability.svc.cluster.local:3100
+        editable: false
+        jsonData:
+          derivedFields:
+            - name: TraceID
+              matcherRegex: '"trace_id":"([a-f0-9]{32})"'
+              datasourceUid: tempo
+              url: '$${__value.raw}'
+      - name: Tempo
+        uid: tempo
+        type: tempo
+        access: proxy
+        url: http://tempo.observability.svc.cluster.local:3200
+        editable: false
+        jsonData:
+          tracesToLogsV2:
+            datasourceUid: loki
+            spanStartTimeShift: -1m
+            spanEndTimeShift: 1m
+            filterByTraceID: true

deploy/observability/kustomization.yaml

@@ -6,4 +6,8 @@ kind: Kustomization
 namespace: observability
 
 resources:
+  - grafana-datasources.yaml
+  - loki.yaml
   - otel-collector.yaml
+  - otel-log-collector.yaml
+  - tempo.yaml

deploy/observability/loki.yaml

@@ -0,0 +1,113 @@
+# Single-process Loki for the two-node project cluster.
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: loki
+  labels:
+    app.kubernetes.io/name: loki
+data:
+  loki.yaml: |
+    auth_enabled: false
+
+    server:
+      http_listen_port: 3100
+
+    common:
+      path_prefix: /var/loki
+      replication_factor: 1
+      ring:
+        kvstore:
+          store: inmemory
+
+    schema_config:
+      configs:
+        - from: 2024-01-01
+          store: tsdb
+          object_store: filesystem
+          schema: v13
+          index:
+            prefix: index_
+            period: 24h
+
+    storage_config:
+      filesystem:
+        directory: /var/loki/chunks
+
+    analytics:
+      reporting_enabled: false
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: loki
+  labels:
+    app.kubernetes.io/name: loki
+spec:
+  selector:
+    app.kubernetes.io/name: loki
+  ports:
+    - name: http
+      port: 3100
+      targetPort: http
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: loki
+  labels:
+    app.kubernetes.io/name: loki
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: loki
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: loki
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsUser: 10001
+        runAsGroup: 10001
+        runAsNonRoot: true
+        fsGroup: 10001
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: loki
+          image: grafana/loki:3.7.6
+          args: ["-config.file=/etc/loki/loki.yaml"]
+          ports:
+            - name: http
+              containerPort: 3100
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          volumeMounts:
+            - name: config
+              mountPath: /etc/loki
+              readOnly: true
+            - name: data
+              mountPath: /var/loki
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          resources:
+            requests:
+              cpu: 25m
+              memory: 128Mi
+            limits:
+              memory: 384Mi
+      volumes:
+        - name: config
+          configMap:
+            name: loki
+        - name: data
+          emptyDir: {}

deploy/observability/otel-collector.yaml

@@ -1,9 +1,8 @@
 # OTLP collector for the cluster. The API's OTEL_EXPORTER_OTLP_ENDPOINT
 # points here.
 #
-# Traces only. The log pipeline in observability/otel-collector/config.yaml
-# needs a DaemonSet with hostPath access to /var/log/pods plus RBAC, and Loki
-# isn't in the cluster yet.
+# Pod logs use a separate node collector because this Deployment does not have
+# access to the hosts' /var/log/pods directories.
 ---
 apiVersion: v1
 kind: ConfigMap

deploy/observability/otel-log-collector.yaml

@@ -0,0 +1,93 @@
+# One collector per node tails only Nereus container logs.
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nereus-otel-log-collector
+  labels:
+    app.kubernetes.io/name: nereus-otel-log-collector
+data:
+  config.yaml: |
+    receivers:
+      file_log/nereus:
+        include:
+          - /var/log/pods/nereus_*/*/*.log
+        include_file_path: true
+        start_at: beginning
+        operators:
+          - type: container
+            id: parse-container-log
+
+    processors:
+      memory_limiter:
+        check_interval: 1s
+        limit_mib: 96
+      batch:
+        timeout: 5s
+
+    exporters:
+      otlp_http/loki:
+        endpoint: http://loki.observability.svc.cluster.local:3100/otlp
+
+    service:
+      pipelines:
+        logs:
+          receivers: [file_log/nereus]
+          processors: [memory_limiter, batch]
+          exporters: [otlp_http/loki]
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nereus-otel-log-collector
+  labels:
+    app.kubernetes.io/name: nereus-otel-log-collector
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: nereus-otel-log-collector
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: nereus-otel-log-collector
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsUser: 10001
+        runAsGroup: 10001
+        runAsNonRoot: true
+        # k3s: /var/log/pods is 0750 root:root, container logs 0640 root:root.
+        # Group root has r-x / r--, so gid 0 is enough to tail them as non-root.
+        supplementalGroups: [0]
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: otel-collector
+          image: otel/opentelemetry-collector-contrib:0.159.0
+          args: ["--config=/conf/config.yaml"]
+          volumeMounts:
+            - name: config
+              mountPath: /conf
+              readOnly: true
+            - name: pod-logs
+              mountPath: /var/log/pods
+              readOnly: true
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          resources:
+            requests:
+              cpu: 20m
+              memory: 64Mi
+            limits:
+              memory: 128Mi
+      volumes:
+        - name: config
+          configMap:
+            name: nereus-otel-log-collector
+        - name: pod-logs
+          hostPath:
+            path: /var/log/pods
+            type: Directory

deploy/observability/tempo.yaml

@@ -0,0 +1,107 @@
+# Single-process Tempo with local storage for short-lived lab traces.
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tempo
+  labels:
+    app.kubernetes.io/name: tempo
+data:
+  tempo.yaml: |
+    server:
+      http_listen_port: 3200
+
+    distributor:
+      receivers:
+        otlp:
+          protocols:
+            grpc:
+              endpoint: 0.0.0.0:4317
+
+    storage:
+      trace:
+        backend: local
+        wal:
+          path: /var/tempo/wal
+        local:
+          path: /var/tempo/traces
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: tempo
+  labels:
+    app.kubernetes.io/name: tempo
+spec:
+  selector:
+    app.kubernetes.io/name: tempo
+  ports:
+    - name: http
+      port: 3200
+      targetPort: http
+    - name: otlp-grpc
+      port: 4317
+      targetPort: otlp-grpc
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tempo
+  labels:
+    app.kubernetes.io/name: tempo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: tempo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: tempo
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsUser: 10001
+        runAsGroup: 10001
+        runAsNonRoot: true
+        fsGroup: 10001
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: tempo
+          image: grafana/tempo:3.0.3
+          args: ["-config.file=/etc/tempo/tempo.yaml"]
+          ports:
+            - name: http
+              containerPort: 3200
+            - name: otlp-grpc
+              containerPort: 4317
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          volumeMounts:
+            - name: config
+              mountPath: /etc/tempo
+              readOnly: true
+            - name: data
+              mountPath: /var/tempo
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          resources:
+            requests:
+              cpu: 25m
+              memory: 128Mi
+            limits:
+              memory: 384Mi
+      volumes:
+        - name: config
+          configMap:
+            name: tempo
+        - name: data
+          emptyDir: {}