From 724fc55f04f318f087ae1647bd67aef59ef21f90 Mon Sep 17 00:00:00 2001 From: Fi3w0 Date: Thu, 27 Aug 2026 21:22:19 +0200 Subject: [PATCH] feat(observability): add loki, tempo and pod log collection --- deploy/observability/grafana-datasources.yaml | 35 ++++++ deploy/observability/kustomization.yaml | 4 + deploy/observability/loki.yaml | 113 ++++++++++++++++++ deploy/observability/otel-collector.yaml | 5 +- deploy/observability/otel-log-collector.yaml | 93 ++++++++++++++ deploy/observability/tempo.yaml | 107 +++++++++++++++++ 6 files changed, 354 insertions(+), 3 deletions(-) create mode 100644 deploy/observability/grafana-datasources.yaml create mode 100644 deploy/observability/loki.yaml create mode 100644 deploy/observability/otel-log-collector.yaml create mode 100644 deploy/observability/tempo.yaml diff --git a/deploy/observability/grafana-datasources.yaml b/deploy/observability/grafana-datasources.yaml new file mode 100644 index 0000000..c243568 --- /dev/null +++ b/deploy/observability/grafana-datasources.yaml @@ -0,0 +1,35 @@ +# kube-prometheus-stack's Grafana sidecar loads labeled data sources. +apiVersion: v1 +kind: ConfigMap +metadata: + name: nereus-grafana-datasources + labels: + grafana_datasource: "1" +data: + nereus.yaml: | + apiVersion: 1 + datasources: + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki.observability.svc.cluster.local:3100 + editable: false + jsonData: + derivedFields: + - name: TraceID + matcherRegex: '"trace_id":"([a-f0-9]{32})"' + datasourceUid: tempo + url: '$${__value.raw}' + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo.observability.svc.cluster.local:3200 + editable: false + jsonData: + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: -1m + spanEndTimeShift: 1m + filterByTraceID: true diff --git a/deploy/observability/kustomization.yaml b/deploy/observability/kustomization.yaml index 4c3bb12..887bb45 100644 --- a/deploy/observability/kustomization.yaml +++ b/deploy/observability/kustomization.yaml @@ -6,4 +6,8 @@ kind: Kustomization namespace: observability resources: + - grafana-datasources.yaml + - loki.yaml - otel-collector.yaml + - otel-log-collector.yaml + - tempo.yaml diff --git a/deploy/observability/loki.yaml b/deploy/observability/loki.yaml new file mode 100644 index 0000000..5553255 --- /dev/null +++ b/deploy/observability/loki.yaml @@ -0,0 +1,113 @@ +# Single-process Loki for the two-node project cluster. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: loki + labels: + app.kubernetes.io/name: loki +data: + loki.yaml: | + auth_enabled: false + + server: + http_listen_port: 3100 + + common: + path_prefix: /var/loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + + schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + + storage_config: + filesystem: + directory: /var/loki/chunks + + analytics: + reporting_enabled: false +--- +apiVersion: v1 +kind: Service +metadata: + name: loki + labels: + app.kubernetes.io/name: loki +spec: + selector: + app.kubernetes.io/name: loki + ports: + - name: http + port: 3100 + targetPort: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: loki + labels: + app.kubernetes.io/name: loki +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: loki + template: + metadata: + labels: + app.kubernetes.io/name: loki + spec: + automountServiceAccountToken: false + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + runAsNonRoot: true + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: loki + image: grafana/loki:3.7.6 + args: ["-config.file=/etc/loki/loki.yaml"] + ports: + - name: http + containerPort: 3100 + readinessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + volumeMounts: + - name: config + mountPath: /etc/loki + readOnly: true + - name: data + mountPath: /var/loki + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + resources: + requests: + cpu: 25m + memory: 128Mi + limits: + memory: 384Mi + volumes: + - name: config + configMap: + name: loki + - name: data + emptyDir: {} diff --git a/deploy/observability/otel-collector.yaml b/deploy/observability/otel-collector.yaml index 7504bba..b894460 100644 --- a/deploy/observability/otel-collector.yaml +++ b/deploy/observability/otel-collector.yaml @@ -1,9 +1,8 @@ # OTLP collector for the cluster. The API's OTEL_EXPORTER_OTLP_ENDPOINT # points here. # -# Traces only. The log pipeline in observability/otel-collector/config.yaml -# needs a DaemonSet with hostPath access to /var/log/pods plus RBAC, and Loki -# isn't in the cluster yet. +# Pod logs use a separate node collector because this Deployment does not have +# access to the hosts' /var/log/pods directories. --- apiVersion: v1 kind: ConfigMap diff --git a/deploy/observability/otel-log-collector.yaml b/deploy/observability/otel-log-collector.yaml new file mode 100644 index 0000000..e34f2b7 --- /dev/null +++ b/deploy/observability/otel-log-collector.yaml @@ -0,0 +1,93 @@ +# One collector per node tails only Nereus container logs. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nereus-otel-log-collector + labels: + app.kubernetes.io/name: nereus-otel-log-collector +data: + config.yaml: | + receivers: + file_log/nereus: + include: + - /var/log/pods/nereus_*/*/*.log + include_file_path: true + start_at: beginning + operators: + - type: container + id: parse-container-log + + processors: + memory_limiter: + check_interval: 1s + limit_mib: 96 + batch: + timeout: 5s + + exporters: + otlp_http/loki: + endpoint: http://loki.observability.svc.cluster.local:3100/otlp + + service: + pipelines: + logs: + receivers: [file_log/nereus] + processors: [memory_limiter, batch] + exporters: [otlp_http/loki] +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nereus-otel-log-collector + labels: + app.kubernetes.io/name: nereus-otel-log-collector +spec: + selector: + matchLabels: + app.kubernetes.io/name: nereus-otel-log-collector + template: + metadata: + labels: + app.kubernetes.io/name: nereus-otel-log-collector + spec: + automountServiceAccountToken: false + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + runAsNonRoot: true + # k3s: /var/log/pods is 0750 root:root, container logs 0640 root:root. + # Group root has r-x / r--, so gid 0 is enough to tail them as non-root. + supplementalGroups: [0] + seccompProfile: + type: RuntimeDefault + containers: + - name: otel-collector + image: otel/opentelemetry-collector-contrib:0.159.0 + args: ["--config=/conf/config.yaml"] + volumeMounts: + - name: config + mountPath: /conf + readOnly: true + - name: pod-logs + mountPath: /var/log/pods + readOnly: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + resources: + requests: + cpu: 20m + memory: 64Mi + limits: + memory: 128Mi + volumes: + - name: config + configMap: + name: nereus-otel-log-collector + - name: pod-logs + hostPath: + path: /var/log/pods + type: Directory diff --git a/deploy/observability/tempo.yaml b/deploy/observability/tempo.yaml new file mode 100644 index 0000000..db86c5f --- /dev/null +++ b/deploy/observability/tempo.yaml @@ -0,0 +1,107 @@ +# Single-process Tempo with local storage for short-lived lab traces. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: tempo + labels: + app.kubernetes.io/name: tempo +data: + tempo.yaml: | + server: + http_listen_port: 3200 + + distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + + storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/traces +--- +apiVersion: v1 +kind: Service +metadata: + name: tempo + labels: + app.kubernetes.io/name: tempo +spec: + selector: + app.kubernetes.io/name: tempo + ports: + - name: http + port: 3200 + targetPort: http + - name: otlp-grpc + port: 4317 + targetPort: otlp-grpc +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tempo + labels: + app.kubernetes.io/name: tempo +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: tempo + template: + metadata: + labels: + app.kubernetes.io/name: tempo + spec: + automountServiceAccountToken: false + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + runAsNonRoot: true + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: tempo + image: grafana/tempo:3.0.3 + args: ["-config.file=/etc/tempo/tempo.yaml"] + ports: + - name: http + containerPort: 3200 + - name: otlp-grpc + containerPort: 4317 + readinessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + volumeMounts: + - name: config + mountPath: /etc/tempo + readOnly: true + - name: data + mountPath: /var/tempo + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + resources: + requests: + cpu: 25m + memory: 128Mi + limits: + memory: 384Mi + volumes: + - name: config + configMap: + name: tempo + - name: data + emptyDir: {}