From a622739053fc98d6f5bcdcfaaf164840902ab1f6 Mon Sep 17 00:00:00 2001
From: Fi3w0 <alex.lazarevych@icloud.com>
Date: Mon, 24 Aug 2026 23:17:53 +0200
Subject: [PATCH] feat(obs): add telemetry stack

---
 observability/README.md                       | 23 +++++
 observability/alerts/nereus.yaml              | 46 ++++++++++
 observability/compose.yaml                    | 57 ++++++++++++
 .../grafana/dashboards/nereus-api.json        | 92 +++++++++++++++++++
 .../provisioning/dashboards/dashboards.yaml   | 10 ++
 .../provisioning/datasources/datasources.yaml | 41 +++++++++
 observability/loki/loki.yaml                  | 28 ++++++
 observability/otel-collector/config.yaml      | 44 +++++++++
 observability/otel-collector/local.yaml       | 25 +++++
 observability/prometheus/prometheus.yaml      | 12 +++
 observability/tempo/tempo.yaml                | 17 ++++
 11 files changed, 395 insertions(+)
 create mode 100644 observability/README.md
 create mode 100644 observability/alerts/nereus.yaml
 create mode 100644 observability/compose.yaml
 create mode 100644 observability/grafana/dashboards/nereus-api.json
 create mode 100644 observability/grafana/provisioning/dashboards/dashboards.yaml
 create mode 100644 observability/grafana/provisioning/datasources/datasources.yaml
 create mode 100644 observability/loki/loki.yaml
 create mode 100644 observability/otel-collector/config.yaml
 create mode 100644 observability/otel-collector/local.yaml
 create mode 100644 observability/prometheus/prometheus.yaml
 create mode 100644 observability/tempo/tempo.yaml

diff --git a/observability/README.md b/observability/README.md
new file mode 100644
index 0000000..b5f6618
--- /dev/null
+++ b/observability/README.md
@@ -0,0 +1,23 @@
+# Nereus observability
+
+Start the application and its local observability overlay with one command:
+
+```bash
+docker compose -f compose.yaml -f observability/compose.yaml up -d
+```
+
+The local endpoints are:
+
+- Grafana: `http://127.0.0.1:13000`
+- Prometheus: `http://127.0.0.1:19090`
+- API: `http://127.0.0.1:18080`
+
+Grafana uses anonymous administrator access only in this local overlay. No
+password or credential is stored in the repository. Stop the complete stack
+with the same two Compose files and `down`.
+
+`otel-collector/local.yaml` handles local API traces. `otel-collector/config.yaml`
+is the cluster configuration: it receives traces and tails only pod logs under
+the `nereus` namespace path, then exports traces to Tempo and logs to Loki. Log
+collection and trace-to-log links therefore require the real Kubernetes
+environment and are not simulated in local Compose.
diff --git a/observability/alerts/nereus.yaml b/observability/alerts/nereus.yaml
new file mode 100644
index 0000000..d42f372
--- /dev/null
+++ b/observability/alerts/nereus.yaml
@@ -0,0 +1,46 @@
+groups:
+  - name: nereus-api
+    rules:
+      - alert: NereusAPIHighErrorRate
+        expr: |
+          sum(rate(nereus_http_requests_total{path=~"/api/v1/.*",status=~"5.."}[2m]))
+          /
+          clamp_min(sum(rate(nereus_http_requests_total{path=~"/api/v1/.*"}[2m])), 0.001)
+          > 0.05
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: Nereus API error rate exceeds 5 percent
+          description: The API has returned more than 5 percent HTTP 5xx responses for one minute.
+
+      - alert: NereusAPIHighLatency
+        expr: |
+          histogram_quantile(
+            0.95,
+            sum by (le) (rate(nereus_http_request_duration_seconds_bucket{path=~"/api/v1/.*"}[5m]))
+          ) > 1
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: Nereus API p95 latency exceeds one second
+          description: The API p95 request duration has exceeded one second for two minutes.
+
+      - alert: NereusAPIReadinessFailing
+        expr: sum(rate(nereus_http_requests_total{path="/readyz",status="503"}[2m])) > 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: Nereus API cannot reach its database
+          description: Readiness requests have returned HTTP 503 for one minute.
+
+      - alert: NereusAPIDown
+        expr: up{job="nereus-api"} == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: Prometheus cannot scrape the Nereus API
+          description: The Nereus API metrics endpoint has been unreachable for one minute.
diff --git a/observability/compose.yaml b/observability/compose.yaml
new file mode 100644
index 0000000..7ced71e
--- /dev/null
+++ b/observability/compose.yaml
@@ -0,0 +1,57 @@
+services:
+  prometheus:
+    image: prom/prometheus:v3.14.0
+    command:
+      - --config.file=/etc/prometheus/prometheus.yaml
+      - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time=6h
+    ports:
+      - "${PROMETHEUS_PORT:-19090}:9090"
+    volumes:
+      - ./observability/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
+      - ./observability/alerts:/etc/prometheus/rules:ro
+      - prometheus-data:/prometheus
+
+  tempo:
+    image: grafana/tempo:3.0.3
+    command: ["-config.file=/etc/tempo.yaml"]
+    volumes:
+      - ./observability/tempo/tempo.yaml:/etc/tempo.yaml:ro
+    tmpfs:
+      - /var/tempo:uid=10001,gid=10001,mode=0755
+
+  loki:
+    image: grafana/loki:3.7.6
+    command: ["-config.file=/etc/loki/local-config.yaml"]
+    volumes:
+      - ./observability/loki/loki.yaml:/etc/loki/local-config.yaml:ro
+    tmpfs:
+      - /var/loki:uid=10001,gid=10001,mode=0755
+
+  otel-collector:
+    volumes:
+      - ./observability/otel-collector/local.yaml:/etc/otelcol-contrib/config.yaml:ro
+    depends_on:
+      - tempo
+      - loki
+
+  grafana:
+    image: grafana/grafana:13.2.0
+    environment:
+      GF_AUTH_ANONYMOUS_ENABLED: "true"
+      GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
+      GF_AUTH_DISABLE_LOGIN_FORM: "true"
+    ports:
+      - "${GRAFANA_PORT:-13000}:3000"
+    volumes:
+      - ./observability/grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
+      - grafana-data:/var/lib/grafana
+    depends_on:
+      - prometheus
+      - tempo
+      - loki
+
+volumes:
+  prometheus-data:
+  grafana-data:
diff --git a/observability/grafana/dashboards/nereus-api.json b/observability/grafana/dashboards/nereus-api.json
new file mode 100644
index 0000000..2158494
--- /dev/null
+++ b/observability/grafana/dashboards/nereus-api.json
@@ -0,0 +1,92 @@
+{
+  "annotations": {"list": []},
+  "editable": false,
+  "graphTooltip": 1,
+  "panels": [
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "reqps"}},
+      "gridPos": {"h": 8, "w": 8, "x": 0, "y": 0},
+      "id": 1,
+      "targets": [{"expr": "sum by (version) (rate(nereus_http_requests_total{path=~\"/api/v1/.*\",version=~\"$version\"}[1m]))", "legendFormat": "{{version}}", "refId": "A"}],
+      "title": "API request rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}},
+      "gridPos": {"h": 8, "w": 8, "x": 8, "y": 0},
+      "id": 2,
+      "targets": [{"expr": "sum by (version) (rate(nereus_http_requests_total{path=~\"/api/v1/.*\",status=~\"5..\",version=~\"$version\"}[1m])) / clamp_min(sum by (version) (rate(nereus_http_requests_total{path=~\"/api/v1/.*\",version=~\"$version\"}[1m])), 0.001)", "legendFormat": "{{version}}", "refId": "A"}],
+      "title": "API error rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "s"}},
+      "gridPos": {"h": 8, "w": 8, "x": 16, "y": 0},
+      "id": 3,
+      "targets": [{"expr": "histogram_quantile(0.95, sum by (le, version) (rate(nereus_http_request_duration_seconds_bucket{path=~\"/api/v1/.*\",version=~\"$version\"}[5m])))", "legendFormat": "p95 {{version}}", "refId": "A"}],
+      "title": "API p95 latency",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "s"}},
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
+      "id": 4,
+      "targets": [{"expr": "histogram_quantile(0.95, sum by (le, operation) (rate(nereus_db_query_duration_seconds_bucket[5m])))", "legendFormat": "{{operation}}", "refId": "A"}],
+      "title": "Database p95 latency",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "ops"}},
+      "gridPos": {"h": 8, "w": 6, "x": 12, "y": 8},
+      "id": 5,
+      "targets": [{"expr": "sum(rate(nereus_readings_ingested_total[1m]))", "refId": "A"}],
+      "title": "Readings ingested",
+      "type": "stat"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "gridPos": {"h": 8, "w": 6, "x": 18, "y": 8},
+      "id": 6,
+      "targets": [{"expr": "nereus_buoys_active", "refId": "A"}],
+      "title": "Active buoys",
+      "type": "stat"
+    },
+    {
+      "datasource": {"type": "loki", "uid": "loki"},
+      "gridPos": {"h": 9, "w": 24, "x": 0, "y": 16},
+      "id": 7,
+      "targets": [{"expr": "{k8s_namespace_name=\"nereus\"} | json", "refId": "A"}],
+      "title": "Correlated API logs",
+      "type": "logs"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 42,
+  "tags": ["nereus"],
+  "templating": {
+    "list": [
+      {
+        "allValue": ".*",
+        "current": {"text": "All", "value": "$__all"},
+        "datasource": {"type": "prometheus", "uid": "prometheus"},
+        "definition": "label_values(nereus_http_requests_total, version)",
+        "includeAll": true,
+        "label": "Version",
+        "name": "version",
+        "query": {"query": "label_values(nereus_http_requests_total, version)", "refId": "StandardVariableQuery"},
+        "refresh": 1,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {"from": "now-30m", "to": "now"},
+  "timezone": "browser",
+  "title": "Nereus API",
+  "uid": "nereus-api",
+  "version": 1
+}
diff --git a/observability/grafana/provisioning/dashboards/dashboards.yaml b/observability/grafana/provisioning/dashboards/dashboards.yaml
new file mode 100644
index 0000000..938f7c8
--- /dev/null
+++ b/observability/grafana/provisioning/dashboards/dashboards.yaml
@@ -0,0 +1,10 @@
+apiVersion: 1
+
+providers:
+  - name: Nereus
+    folder: Nereus
+    type: file
+    disableDeletion: true
+    editable: false
+    options:
+      path: /var/lib/grafana/dashboards
diff --git a/observability/grafana/provisioning/datasources/datasources.yaml b/observability/grafana/provisioning/datasources/datasources.yaml
new file mode 100644
index 0000000..c4a2403
--- /dev/null
+++ b/observability/grafana/provisioning/datasources/datasources.yaml
@@ -0,0 +1,41 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    uid: prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
+    jsonData:
+      timeInterval: 5s
+
+  - name: Loki
+    uid: loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    editable: false
+    jsonData:
+      derivedFields:
+        - name: TraceID
+          matcherRegex: '"trace_id":"([a-f0-9]{32})"'
+          datasourceUid: tempo
+          url: '${__value.raw}'
+
+  - name: Tempo
+    uid: tempo
+    type: tempo
+    access: proxy
+    url: http://tempo:3200
+    editable: false
+    jsonData:
+      tracesToLogsV2:
+        datasourceUid: loki
+        spanStartTimeShift: -1m
+        spanEndTimeShift: 1m
+        tags:
+          - key: service.name
+            value: service_name
+        filterByTraceID: true
diff --git a/observability/loki/loki.yaml b/observability/loki/loki.yaml
new file mode 100644
index 0000000..3bdab9e
--- /dev/null
+++ b/observability/loki/loki.yaml
@@ -0,0 +1,28 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  path_prefix: /var/loki
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  filesystem:
+    directory: /var/loki/chunks
+
+analytics:
+  reporting_enabled: false
diff --git a/observability/otel-collector/config.yaml b/observability/otel-collector/config.yaml
new file mode 100644
index 0000000..971118a
--- /dev/null
+++ b/observability/otel-collector/config.yaml
@@ -0,0 +1,44 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+  file_log/nereus:
+    include:
+      - /var/log/pods/nereus_*/*/*.log
+    include_file_path: true
+    operators:
+      - type: container
+        id: parse-container-log
+
+processors:
+  memory_limiter:
+    check_interval: 1s
+    limit_mib: 256
+  k8s_attributes:
+    extract:
+      metadata:
+        - k8s.namespace.name
+        - k8s.pod.name
+        - k8s.container.name
+  batch:
+    timeout: 5s
+
+exporters:
+  otlp_grpc/tempo:
+    endpoint: tempo:4317
+    tls:
+      insecure: true
+  otlp_http/loki:
+    endpoint: http://loki:3100/otlp
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [memory_limiter, k8s_attributes, batch]
+      exporters: [otlp_grpc/tempo]
+    logs:
+      receivers: [file_log/nereus]
+      processors: [memory_limiter, k8s_attributes, batch]
+      exporters: [otlp_http/loki]
diff --git a/observability/otel-collector/local.yaml b/observability/otel-collector/local.yaml
new file mode 100644
index 0000000..3518d79
--- /dev/null
+++ b/observability/otel-collector/local.yaml
@@ -0,0 +1,25 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+
+processors:
+  memory_limiter:
+    check_interval: 1s
+    limit_mib: 256
+  batch:
+    timeout: 5s
+
+exporters:
+  otlp_grpc/tempo:
+    endpoint: tempo:4317
+    tls:
+      insecure: true
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [memory_limiter, batch]
+      exporters: [otlp_grpc/tempo]
diff --git a/observability/prometheus/prometheus.yaml b/observability/prometheus/prometheus.yaml
new file mode 100644
index 0000000..aac288c
--- /dev/null
+++ b/observability/prometheus/prometheus.yaml
@@ -0,0 +1,12 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+
+rule_files:
+  - /etc/prometheus/rules/*.yaml
+
+scrape_configs:
+  - job_name: nereus-api
+    metrics_path: /metrics
+    static_configs:
+      - targets: [api:8080]
diff --git a/observability/tempo/tempo.yaml b/observability/tempo/tempo.yaml
new file mode 100644
index 0000000..7737d0a
--- /dev/null
+++ b/observability/tempo/tempo.yaml
@@ -0,0 +1,17 @@
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:4317
+
+storage:
+  trace:
+    backend: local
+    wal:
+      path: /var/tempo/wal
+    local:
+      path: /var/tempo/traces