fix(ci): use a valid mirror toggle

.forgejo/workflows/pipeline.yml

@@ -18,7 +18,7 @@ jobs:
         run: |
           docker run --rm -v "$PWD:/repo" \
             ghcr.io/gitleaks/gitleaks:v8.30.1 \
-            detect --source /repo --redact -v
+            detect --source /repo --config /repo/.gitleaks.toml --redact -v
 
       - name: Test API
         run: |
@@ -51,6 +51,18 @@ jobs:
           docker run --rm -v "$PWD:/repo" -w /repo bash:5.3 \
             bash -ec 'for file in scripts/k3d/lab.sh scripts/provision/bootstrap.sh scripts/provision/lab.sh; do bash -n "$file"; done'
 
+      - name: Validate Terraform
+        run: |
+          for stack in terraform/infra terraform/platform
+          do
+            docker run --rm -v "$PWD:/work" -w "/work/$stack" \
+              hashicorp/terraform:1.15.9 fmt -check
+            docker run --rm -v "$PWD:/work" -w "/work/$stack" \
+              hashicorp/terraform:1.15.9 init -backend=false -lockfile=readonly
+            docker run --rm -v "$PWD:/work" -w "/work/$stack" \
+              hashicorp/terraform:1.15.9 validate
+          done
+
       - name: Validate Compose and Kubernetes configuration
         env:
           CLUSTER_API_UPSTREAM: 192.0.2.1:8080
@@ -124,9 +136,6 @@ jobs:
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     needs: publish
     runs-on: docker
-    env:
-      API_IMAGE: git.fiwlabs.dev/fiwdev/nereus-api:${{ github.sha }}
-      LOADGEN_IMAGE: git.fiwlabs.dev/fiwdev/nereus-loadgen:${{ github.sha }}
     steps:
       - name: Check out repository
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
@@ -160,15 +169,26 @@ jobs:
             --from-file=.dockerconfigjson=/work/.ci/docker/config.json \
             --dry-run=client -o yaml | kube apply -f -
 
+          kube apply -k deploy/secrets
+          kube wait sealedsecret.bitnami.com/nereus-db --namespace nereus \
+            --for=condition=Synced --timeout=2m
           kube get secret nereus-db --namespace nereus >/dev/null
           kube apply -k deploy/observability
-          kube apply -k deploy/overlays/prod
-          kube patch rollout.argoproj.io nereus-api --namespace nereus --type json \
-            --patch "[{\"op\":\"replace\",\"path\":\"/spec/template/spec/containers/0/image\",\"value\":\"$API_IMAGE\"}]"
-          kube set image deployment/nereus-loadgen --namespace nereus \
-            "loadgen=$LOADGEN_IMAGE"
-          kube set image deployment/nereus-loadgen-preview --namespace nereus \
-            "loadgen=$LOADGEN_IMAGE"
+
+          # Set the built tag before applying. Applying the overlay first would
+          # roll out its pinned 0.1.0 tag and then roll out a second time on the
+          # patch, running the pre-promotion analysis against the wrong image.
+          install -d .ci/release
+          cat >.ci/release/kustomization.yaml <<'EOF'
+          resources:
+            - ../../deploy/overlays/prod
+          images:
+            - name: git.fiwlabs.dev/fiwdev/nereus-api
+              newTag: ${{ github.sha }}
+            - name: git.fiwlabs.dev/fiwdev/nereus-loadgen
+              newTag: ${{ github.sha }}
+          EOF
+          kube apply -k .ci/release
 
           kube rollout status deployment/nereus-loadgen --namespace nereus --timeout=5m
           kube rollout status deployment/nereus-loadgen-preview --namespace nereus --timeout=5m
@@ -179,3 +199,31 @@ jobs:
       - name: Remove cluster credentials
         if: always()
         run: rm -rf .ci
+
+  mirror:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main' && vars.MIRROR_ENABLED == 'true'
+    needs: deploy
+    runs-on: docker
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          fetch-depth: 0
+
+      - name: Push main to GitHub
+        env:
+          GITHUB_MIRROR_SSH_KEY: ${{ secrets.GITHUB_MIRROR_SSH_KEY }}
+        run: |
+          test -n "$GITHUB_MIRROR_SSH_KEY"
+          install -d -m 0700 .ci/mirror
+          printf '%s\n' "$GITHUB_MIRROR_SSH_KEY" >.ci/mirror/id_ed25519
+          chmod 0600 .ci/mirror/id_ed25519
+          cat >.ci/mirror/known_hosts <<'EOF'
+          github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
+          EOF
+          GIT_SSH_COMMAND="ssh -i $PWD/.ci/mirror/id_ed25519 -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$PWD/.ci/mirror/known_hosts" \
+            git push git@github.com:Fi3w0/Nereus.git HEAD:main
+
+      - name: Remove GitHub credentials
+        if: always()
+        run: rm -rf .ci

.github/workflows/ci.yml

@@ -38,7 +38,7 @@ jobs:
         run: |
           docker run --rm -v "$PWD:/repo" \
             ghcr.io/gitleaks/gitleaks:v8.30.1 \
-            detect --source /repo --redact -v
+            detect --source /repo --config /repo/.gitleaks.toml --redact -v
 
   go:
     name: vet, test, lint
@@ -74,3 +74,21 @@ jobs:
         if: steps.probe.outputs.ready == 'true'
         with:
           working-directory: apps/api
+
+  terraform:
+    name: terraform
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: format and validate
+        run: |
+          for stack in terraform/infra terraform/platform
+          do
+            docker run --rm -v "$PWD:/work" -w "/work/$stack" \
+              hashicorp/terraform:1.15.9 fmt -check
+            docker run --rm -v "$PWD:/work" -w "/work/$stack" \
+              hashicorp/terraform:1.15.9 init -backend=false -lockfile=readonly
+            docker run --rm -v "$PWD:/work" -w "/work/$stack" \
+              hashicorp/terraform:1.15.9 validate
+          done

.gitignore

@@ -85,6 +85,7 @@ CLAUDE.local.md
 *.tfstate.*
 *.tfvars
 !*.tfvars.example
+terraform/**/inventory.yml
 crash.log
 crash.*.log
 override.tf

.gitleaks.toml

@@ -0,0 +1,12 @@
+title = "Nereus gitleaks configuration"
+
+[extend]
+useDefault = true
+
+[[allowlists]]
+description = "Encrypted SealedSecret values are ciphertext, not plaintext credentials."
+condition = "AND"
+targetRules = ["generic-api-key"]
+regexTarget = "line"
+paths = ['''^deploy/secrets/[^/]+-sealed\.yaml$''']
+regexes = ['''^\s+(POSTGRES_PASSWORD|DATABASE_URL):\s+Ag[A-Za-z0-9+/=]+\s*$''']

README.md

@@ -0,0 +1,388 @@
+# Nereus
+
+Nereus is an ocean buoy telemetry API that exists so there is something real to
+deploy. It stores buoys and their readings, exposes Prometheus metrics and
+OpenTelemetry traces, and can be told to start failing on demand. The
+application is deliberately small and boring. The project is everything wrapped
+around it: custom images, a two-node k3s cluster, CI/CD on a self-hosted runner,
+blue-green deploys that abort themselves when the new version is bad, and logs,
+metrics and traces that actually join up.
+
+This is my final DevOps project for Tokio School. The fictional company in the
+brief is TechWave Solutions, which fits the water theme, so I ran with it.
+
+**If you only run one thing, run the k3d lab.** It puts a real multi-node k3s
+inside Docker and proves the automated rollback end to end on any OS with Docker
+and 8 GB of RAM:
+
+```bash
+scripts/k3d/lab.sh up
+scripts/k3d/lab.sh prove
+```
+
+## What the API does
+
+Go 1.26, `chi` for routing, `pgx/v5` against PostgreSQL 17, no ORM. Migrations
+are plain SQL files applied on startup and written to be idempotent, so a pod
+restart or a second replica does not break anything.
+
+```
+GET  /healthz                              liveness, 200 whenever the process is up
+GET  /readyz                               readiness, 200 only once the database is reachable
+GET  /metrics                              Prometheus exposition
+
+GET    /api/v1/buoys                       list
+POST   /api/v1/buoys                       create
+GET    /api/v1/buoys/{id}                  fetch one
+DELETE /api/v1/buoys/{id}                  delete
+
+GET  /api/v1/readings?buoy_id=&from=&to=   list, paginated
+POST /api/v1/readings                      create
+GET  /api/v1/readings/aggregate?window=1h  deliberately expensive GROUP BY
+```
+
+The split between `/healthz` and `/readyz` is the part that matters for
+Kubernetes. The service starts and serves `/healthz` even with the database
+down, so a PostgreSQL outage makes pods unready instead of killing them in a
+restart loop. `/api/v1/readings/aggregate` runs a real grouped query over a time
+window and is not optimised on purpose, because the dashboards need something
+that produces visible latency.
+
+`CHAOS_ERROR_RATE` is a float between 0 and 1. Above zero, that fraction of
+`/api/v1/*` requests return HTTP 500 with a JSON body. Health endpoints are never
+affected. This is the entire mechanism for simulating a bad release, and it is
+controlled by an environment variable rather than an admin endpoint so that
+turning it on is a deployment, not a runtime toggle.
+
+`apps/loadgen/` is a small Go binary that hits the API continuously with a
+weighted mix of reads, writes and aggregates. It is not decoration. The rollback
+analysis computes an error rate from Prometheus, and with no traffic there are
+no samples and the analysis has nothing to decide on.
+
+## Running it
+
+There are three levels, and they exist because nobody grading this is going to
+install KVM and download a Fedora image.
+
+### Level 1: Docker Compose, about three minutes
+
+```bash
+docker compose up --build
+```
+
+API, PostgreSQL and an OTel Collector, with the API on `127.0.0.1:18080` and the
+load generator already producing traffic. Add the observability overlay for
+Grafana on `127.0.0.1:13000` and Prometheus on `127.0.0.1:19090`:
+
+```bash
+docker compose -f compose.yaml -f observability/compose.yaml up -d
+```
+
+### Level 2: k3d, real Kubernetes, any OS with Docker
+
+```bash
+scripts/k3d/lab.sh up      # one server, two agents, Argo Rollouts, kube-prometheus-stack
+scripts/k3d/lab.sh prove   # promote a healthy version, then abort a bad one
+scripts/k3d/lab.sh destroy
+```
+
+`prove` is the interesting one. It drives a blue-green Rollout through a real
+AnalysisRun that queries a real Prometheus over the network, first with a query
+that returns a healthy value and then with one that returns a failing value, and
+it fails loudly if the bad revision ever reaches the active Service.
+
+Be clear about what is faked here: the harness in
+`scripts/k3d/analysis-harness/` uses a throwaway nginx Rollout and a hardcoded
+metric value. Argo Rollouts, the AnalysisRun, the Prometheus query and the abort
+are all real. Only the number is synthetic. The real gate lives in
+`deploy/rollouts/analysistemplate.yaml` and ran against the real API on the
+Fedora cluster.
+
+### Level 3: two Fedora 44 machines
+
+This is my actual cluster. `scripts/provision/` holds an Ansible playbook that
+turns two already-installed Fedora 44 hosts into a k3s server and agent:
+
+```bash
+cp scripts/provision/inventory.example.yml scripts/provision/inventory.yml
+# edit addresses, CIDRs and SSH user
+K3S_TOKEN="$(openssl rand -hex 32)" scripts/provision/bootstrap.sh scripts/provision/inventory.yml
+```
+
+The playbook is idempotent and reruns clean. The token is passed through the
+process environment and never written to the inventory or the repository.
+
+`scripts/provision/lab.sh` builds the same two hosts as local QEMU VMs from the
+Fedora 44 cloud image, with 2 vCPUs, 3 GiB of RAM and a 30 GiB thin disk each.
+Those numbers are the lowest I actually tested, not a recommendation. VM state
+and its SSH key live under `~/.local/state/nereus-lab`, outside the repo.
+
+## How a bad deploy gets killed
+
+This is the part I would defend first, so it is worth spelling out.
+
+The API is an Argo Rollouts `Rollout`, not a Deployment. New pods come up
+alongside the old ones and take no production traffic. `nereus-api-active` points
+at the stable version and `nereus-api-preview` at the candidate. A second load
+generator drives the preview Service, because an analysis that measures nothing
+would treat "no samples" as success.
+
+Before promotion, a `prePromotionAnalysis` runs this query against Prometheus:
+
+```promql
+(
+  sum(rate(nereus_http_requests_total{service="nereus-api-preview",status=~"5.."}[1m]))
+  or vector(0)
+)
+/
+sum(rate(nereus_http_requests_total{service="nereus-api-preview"}[1m]))
+```
+
+Five samples, 20 seconds apart, after a 30 second delay to let the new pods take
+traffic. The success condition is `len(result) == 0 || result[0] < 0.05`, and
+`failureLimit: 0` means a single bad sample aborts. The `or vector(0)` keeps the
+division defined when there are no errors at all, which is otherwise an empty
+result rather than a zero.
+
+When it aborts, the active Service selector never moves. `scaleDownDelaySeconds:
+30` keeps the old ReplicaSet warm so the fallback is instant rather than a cold
+start. From `docs/evidence.md`, the two AnalysisRuns from the mechanism proof:
+
+```text
+probe-746bbb94df-2-pre   Successful   vector(0.0)
+probe-556d5b659b-3-pre   Failed       vector(1.0)
+```
+
+To trigger it for real, deploy a version with `CHAOS_ERROR_RATE` above 0.05 and
+watch it refuse to promote.
+
+## Architecture
+
+```
+Workstation (CachyOS)
+└── libvirt/KVM
+    ├── nereus-node1   Fedora 44, k3s server
+    └── nereus-node2   Fedora 44, k3s agent
+
+Mini PC (always on)
+├── Forgejo at git.fiwlabs.dev, behind Traefik with automatic TLS
+├── Forgejo Runner, docker mode, same LAN as the cluster
+├── Container registry, part of Forgejo
+└── Static dashboard at nereus.fiwlabs.dev, nginx behind Traefik
+
+Mirror to GitHub, which is the link I hand in.
+```
+
+Inside the cluster:
+
+```
+Traefik (ships with k3s)
+   └── Ingress → nereus-api-active
+                    ├── stable ReplicaSet   (real traffic)
+                    └── preview ReplicaSet  (candidate, no traffic)
+
+PostgreSQL      StatefulSet on a local-path PVC
+loadgen         one Deployment against active, one against preview
+
+observability/
+   kube-prometheus-stack    Prometheus, Grafana, node-exporter, kube-state-metrics
+   OTel Collector           Deployment, receives OTLP traces, exports to Tempo
+   OTel Collector           DaemonSet, tails pod logs, exports to Loki
+   Tempo                    traces
+   Loki                     logs
+```
+
+The web dashboard runs on the Mini PC, not in the cluster, and that is
+deliberate. The cluster is off most of the time. If the page lived inside it,
+the domain would be dead 95% of the time and there would be nothing to show.
+Traefik on the Mini PC serves the static files at `/` and proxies `/api/v1/*`,
+`/healthz` and `/readyz` through to the cluster, so everything is same-origin
+with no CORS and no hardcoded hostnames. With the cluster off, those paths return
+502 and the page renders the last known values greyed out and keeps polling. An
+unreachable API is a normal state for that page, not an error.
+
+## Why I built it this way
+
+**distroless, not alpine.** The final image is
+`gcr.io/distroless/static:nonroot` with a statically linked binary copied in and
+nothing else. No shell, no package manager, no busybox. If someone gets code
+execution there is nothing to pivot with. It also happens to be tiny: the API
+image is under 8 MB, with the exact figure recorded in `docs/evidence.md`.
+
+**k3s, not full Kubernetes.** Same API, one binary, Traefik and a local-path
+provisioner included, and it runs on two 3 GiB VMs. Nothing in this project uses
+an API that k3s does not have.
+
+**On-premise, not EKS or AKS.** The brief asks for a managed cloud cluster. I
+did not do that, and the reason is cost and control rather than difficulty: I
+have hardware sitting here and no interest in paying for a load balancer to prove
+a point about Terraform. The manifests are plain Kubernetes and would apply to a
+managed cluster unchanged. This is the deviation from the brief I expect to be
+asked about, so it is stated here rather than buried.
+
+**Forgejo Actions, not GitHub Actions.** The brief allows the tool of your
+choice. The runner lives on the same LAN as the cluster, which means the deploy
+job can reach the Kubernetes API without exposing it to the internet or
+punching a hole through a firewall for a hosted runner. The registry is part of
+the same Forgejo instance, so images never leave the network either. The GitHub
+mirror still runs its own workflow so the repo a reviewer opens shows green
+checks, and that workflow consumes no secrets by design.
+
+**The cluster is ephemeral, on purpose.** I bring it up, record, and shut it
+down. That is not a gap in the project, it is the evidence that the
+infrastructure as code works. If the whole thing rebuilds from the repository
+with a command, then the repository really is the source of truth.
+
+**Everything runs non-root with a read-only root filesystem**, including the
+pieces where that is inconvenient. The log collector was the awkward one. k3s
+writes `/var/log/pods` as `0750 root:root` and each container log as `0640
+root:root`, so a collector running as UID 10001 cannot even traverse the
+directory. Adding the `DAC_READ_SEARCH` capability looks like the fix and does
+nothing, because Kubernetes sets no ambient capabilities and the capability is
+cleared on exec for a non-root user. The container ended up with
+`CapEff: 0000000000000000`, matched no files, and reported no error at all. The
+fix is `supplementalGroups: [0]`, since group root already has read on those
+files. Non-root, no capabilities, read-only mount.
+
+## Secrets and security
+
+No secret is committed, and none is read by any tooling that logs. Gitleaks
+scans both the working tree and the committed history on every push and a hit
+fails the build. Trivy scans the release configuration and the built images for
+HIGH and CRITICAL findings.
+
+The `nereus-db` Secret holds the PostgreSQL password and DSN. Production gets
+it from the encrypted `deploy/secrets/nereus-db-sealed.yaml`; the Sealed Secrets
+controller is the only component that can decrypt it. The current Fedora
+cluster was reset onto a fresh PVC and verified with that committed resource.
+Development can still create an ephemeral Secret directly. See
+`deploy/secrets/README.md` for both workflows.
+
+The CI pipeline never uses a cluster-admin kubeconfig. It authenticates as a
+`deployer` ServiceAccount with RBAC scoped to the `nereus` namespace, supplied
+as an encrypted `KUBECONFIG_B64` repository secret, written to the workspace with
+mode `0600` and deleted in an `always()` step. The registry pull secret is
+created through a pipe so the generated manifest is never logged or written to
+disk.
+
+Containers drop all capabilities, disallow privilege escalation, run with a
+read-only root filesystem and a `RuntimeDefault` seccomp profile, and mount
+`tmpfs` where a writable path is genuinely needed. SELinux stays enforcing on
+both Fedora nodes.
+
+## Observability
+
+Metrics, logs and traces come from the same request and can be walked between.
+
+The API exports five metrics with fixed names, since the dashboards and the
+rollback query depend on them literally:
+
+```
+nereus_http_requests_total{method, path, status, version}    counter
+nereus_http_request_duration_seconds{method, path, version}  histogram
+nereus_db_query_duration_seconds{operation}                  histogram
+nereus_readings_ingested_total                               counter
+nereus_buoys_active                                          gauge
+```
+
+`path` is always the chi route template, `/api/v1/buoys/{id}` rather than the
+resolved path, because resolved paths give unbounded cardinality and will
+eventually take Prometheus down.
+
+Traces go out over OTLP/gRPC to the collector Deployment and on to Tempo. Every
+handler is a span and every database query is a child span. If
+`OTEL_EXPORTER_OTLP_ENDPOINT` is unset, tracing is a no-op, so a missing
+collector can never stop the service from starting.
+
+Logs are structured JSON on stdout via `log/slog`. A DaemonSet collector tails
+`/var/log/pods` on each node and ships to Loki. Every request logs one line
+carrying its `trace_id`, and Grafana's Loki data source has a derived field that
+turns that value into a link into Tempo, with the reverse link configured on the
+Tempo side. A verified example, log line and the trace it resolves to, is in
+`docs/evidence.md`.
+
+Alert rules for error rate, latency, readiness failures and scrape failure are in
+`observability/alerts/nereus.yaml`. cAdvisor and node-exporter metrics arrive
+with kube-prometheus-stack rather than being installed separately.
+
+## CI/CD
+
+`.forgejo/workflows/pipeline.yml` runs on pushes and pull requests to `main`.
+
+Verification runs the Go tests with the race detector, `go vet`, a pinned
+`golangci-lint`, gitleaks over history and tree, a kustomize build of every
+overlay, and a Trivy config scan. Everything runs as a plain container image
+rather than a marketplace action, because Forgejo resolves actions from
+`code.forgejo.org` and most third-party ones simply are not mirrored there.
+
+A push to `main` then builds the three images, tags them with the immutable
+commit SHA, scans them, pushes them to the Forgejo registry, and deploys the
+production overlay. The deploy job waits on the Rollout, which means a failed
+pre-promotion analysis fails the pipeline.
+
+## Where each requirement from the brief lives
+
+| Requirement | Where |
+|---|---|
+| Custom, optimised Docker images | `build/*.Dockerfile`, multi-stage into distroless, API under 8 MB |
+| Docker Compose | `compose.yaml`, plus `observability/compose.yaml` and `build/compose.mini.yaml` |
+| Kubernetes orchestration | k3s, two Fedora 44 nodes; k3d for a portable equivalent |
+| Deployments, Services, Ingress, ConfigMaps, Secrets | `deploy/base/`, composed with kustomize overlays |
+| Infrastructure as code | `terraform/infra/` for libvirt machines, `terraform/platform/` for cluster controllers, and `scripts/provision/` for host configuration |
+| CI/CD pipeline | `.forgejo/workflows/pipeline.yml`, self-hosted runner |
+| Container registry | Forgejo registry at `git.fiwlabs.dev`, TLS from Traefik |
+| Secure credential handling | Gitleaks, Trivy, namespace-scoped `deployer` kubeconfig, and a committed encrypted `nereus-db` SealedSecret |
+| Blue-green deployment | Argo Rollouts, `deploy/rollouts/rollout.yaml` |
+| Automated rollback | `deploy/rollouts/analysistemplate.yaml`, Prometheus-driven abort |
+| OpenTelemetry Collector | `deploy/observability/otel-collector.yaml` for traces, `otel-log-collector.yaml` for logs |
+| Prometheus | kube-prometheus-stack, scraped through a ServiceMonitor |
+| Grafana dashboards | `observability/dashboards/`, data sources in `deploy/observability/grafana-datasources.yaml` |
+| Loki | `deploy/observability/loki.yaml` |
+| cAdvisor and node-exporter | included with kube-prometheus-stack |
+| Alerting | `observability/alerts/nereus.yaml` |
+
+## What is not done
+
+Stated plainly so it does not have to be discovered.
+
+Discord routing is configured but disabled until an owner-provided webhook is
+sealed. The Sealed Secrets controller key also needs an off-repository backup;
+without it, a rebuilt cluster cannot decrypt the committed database resource.
+
+The remaining acceptance gap is environmental: the reproducible fresh-VM path
+has not been rerun on a second workstation with no dependencies installed.
+After final sign-off, the local QEMU lab and packages installed only for it can
+be removed.
+
+## Fedora traps worth knowing
+
+Three things broke the cluster in ways that took longer to find than to fix, all
+of them handled by the playbook now.
+
+| Symptom | Cause | Fix |
+|---|---|---|
+| Pods on different nodes cannot reach each other, DNS fails strangely | firewalld blocks VXLAN | open `8472/udp` between nodes |
+| kubelet fails with permission errors | SELinux enforcing without the policy | `dnf install k3s-selinux` |
+| k3s complains at startup | zram swap enabled | `systemctl disable --now zram-generator` |
+
+## Repo layout
+
+```
+apps/api/          Go service
+apps/loadgen/      traffic generator
+apps/web/          static dashboard, runs on the Mini PC
+build/             Dockerfiles and the Mini PC compose file
+deploy/base/       kustomize base
+deploy/overlays/   dev and prod
+deploy/rollouts/   Rollout and AnalysisTemplate
+deploy/observability/  Loki, Tempo, collectors, Grafana data sources
+observability/     dashboards, alert rules, collector configs, local compose
+scripts/k3d/       portable rollback lab
+scripts/provision/ Ansible roles, bootstrap, QEMU lab
+terraform/         libvirt machines, Ansible inventory and cluster-wide Helm releases
+docs/              roadmap, evidence, CI/CD and Mini PC notes
+```
+
+`AGENTS.md` holds the rules the AI agents working in this repo had to follow.
+`PLAN.md` is my own working plan, in Spanish, and it is a worklog rather than
+documentation.

deploy/observability/grafana-datasources.yaml

@@ -0,0 +1,35 @@
+# kube-prometheus-stack's Grafana sidecar loads labeled data sources.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nereus-grafana-datasources
+  labels:
+    grafana_datasource: "1"
+data:
+  nereus.yaml: |
+    apiVersion: 1
+    datasources:
+      - name: Loki
+        uid: loki
+        type: loki
+        access: proxy
+        url: http://loki.observability.svc.cluster.local:3100
+        editable: false
+        jsonData:
+          derivedFields:
+            - name: TraceID
+              matcherRegex: '"trace_id":"([a-f0-9]{32})"'
+              datasourceUid: tempo
+              url: '$${__value.raw}'
+      - name: Tempo
+        uid: tempo
+        type: tempo
+        access: proxy
+        url: http://tempo.observability.svc.cluster.local:3200
+        editable: false
+        jsonData:
+          tracesToLogsV2:
+            datasourceUid: loki
+            spanStartTimeShift: -1m
+            spanEndTimeShift: 1m
+            filterByTraceID: true

deploy/observability/kustomization.yaml

@@ -6,4 +6,8 @@ kind: Kustomization
 namespace: observability
 
 resources:
+  - grafana-datasources.yaml
+  - loki.yaml
   - otel-collector.yaml
+  - otel-log-collector.yaml
+  - tempo.yaml

deploy/observability/loki.yaml

@@ -0,0 +1,113 @@
+# Single-process Loki for the two-node project cluster.
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: loki
+  labels:
+    app.kubernetes.io/name: loki
+data:
+  loki.yaml: |
+    auth_enabled: false
+
+    server:
+      http_listen_port: 3100
+
+    common:
+      path_prefix: /var/loki
+      replication_factor: 1
+      ring:
+        kvstore:
+          store: inmemory
+
+    schema_config:
+      configs:
+        - from: 2024-01-01
+          store: tsdb
+          object_store: filesystem
+          schema: v13
+          index:
+            prefix: index_
+            period: 24h
+
+    storage_config:
+      filesystem:
+        directory: /var/loki/chunks
+
+    analytics:
+      reporting_enabled: false
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: loki
+  labels:
+    app.kubernetes.io/name: loki
+spec:
+  selector:
+    app.kubernetes.io/name: loki
+  ports:
+    - name: http
+      port: 3100
+      targetPort: http
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: loki
+  labels:
+    app.kubernetes.io/name: loki
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: loki
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: loki
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsUser: 10001
+        runAsGroup: 10001
+        runAsNonRoot: true
+        fsGroup: 10001
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: loki
+          image: grafana/loki:3.7.6
+          args: ["-config.file=/etc/loki/loki.yaml"]
+          ports:
+            - name: http
+              containerPort: 3100
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          volumeMounts:
+            - name: config
+              mountPath: /etc/loki
+              readOnly: true
+            - name: data
+              mountPath: /var/loki
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          resources:
+            requests:
+              cpu: 25m
+              memory: 128Mi
+            limits:
+              memory: 384Mi
+      volumes:
+        - name: config
+          configMap:
+            name: loki
+        - name: data
+          emptyDir: {}

deploy/observability/otel-collector.yaml

@@ -1,9 +1,8 @@
 # OTLP collector for the cluster. The API's OTEL_EXPORTER_OTLP_ENDPOINT
 # points here.
 #
-# Traces only. The log pipeline in observability/otel-collector/config.yaml
-# needs a DaemonSet with hostPath access to /var/log/pods plus RBAC, and Loki
-# isn't in the cluster yet.
+# Pod logs use a separate node collector because this Deployment does not have
+# access to the hosts' /var/log/pods directories.
 ---
 apiVersion: v1
 kind: ConfigMap

deploy/observability/otel-log-collector.yaml

@@ -0,0 +1,98 @@
+# One collector per node tails only Nereus container logs.
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nereus-otel-log-collector
+  labels:
+    app.kubernetes.io/name: nereus-otel-log-collector
+data:
+  config.yaml: |
+    receivers:
+      filelog/nereus:
+        include:
+          - /var/log/pods/nereus_*/*/*.log
+        include_file_path: true
+        start_at: beginning
+        operators:
+          - type: container
+            id: parse-container-log
+
+    processors:
+      memory_limiter:
+        check_interval: 1s
+        limit_mib: 96
+      batch:
+        timeout: 5s
+        # Loki's internal gRPC push limit is 4 MiB. start_at: beginning replays
+        # whole files after a restart, so an uncapped batch exceeds it and the
+        # exporter retries the same oversized request forever.
+        send_batch_size: 1000
+        send_batch_max_size: 2000
+
+    exporters:
+      otlphttp/loki:
+        endpoint: http://loki.observability.svc.cluster.local:3100/otlp
+
+    service:
+      pipelines:
+        logs:
+          receivers: [filelog/nereus]
+          processors: [memory_limiter, batch]
+          exporters: [otlphttp/loki]
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nereus-otel-log-collector
+  labels:
+    app.kubernetes.io/name: nereus-otel-log-collector
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: nereus-otel-log-collector
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: nereus-otel-log-collector
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsUser: 10001
+        runAsGroup: 10001
+        runAsNonRoot: true
+        # k3s: /var/log/pods is 0750 root:root, container logs 0640 root:root.
+        # Group root has r-x / r--, so gid 0 is enough to tail them as non-root.
+        supplementalGroups: [0]
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: otel-collector
+          image: otel/opentelemetry-collector-contrib:0.159.0
+          args: ["--config=/conf/config.yaml"]
+          volumeMounts:
+            - name: config
+              mountPath: /conf
+              readOnly: true
+            - name: pod-logs
+              mountPath: /var/log/pods
+              readOnly: true
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          resources:
+            requests:
+              cpu: 20m
+              memory: 64Mi
+            limits:
+              memory: 128Mi
+      volumes:
+        - name: config
+          configMap:
+            name: nereus-otel-log-collector
+        - name: pod-logs
+          hostPath:
+            path: /var/log/pods
+            type: Directory

deploy/observability/tempo.yaml

@@ -0,0 +1,107 @@
+# Single-process Tempo with local storage for short-lived lab traces.
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tempo
+  labels:
+    app.kubernetes.io/name: tempo
+data:
+  tempo.yaml: |
+    server:
+      http_listen_port: 3200
+
+    distributor:
+      receivers:
+        otlp:
+          protocols:
+            grpc:
+              endpoint: 0.0.0.0:4317
+
+    storage:
+      trace:
+        backend: local
+        wal:
+          path: /var/tempo/wal
+        local:
+          path: /var/tempo/traces
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: tempo
+  labels:
+    app.kubernetes.io/name: tempo
+spec:
+  selector:
+    app.kubernetes.io/name: tempo
+  ports:
+    - name: http
+      port: 3200
+      targetPort: http
+    - name: otlp-grpc
+      port: 4317
+      targetPort: otlp-grpc
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tempo
+  labels:
+    app.kubernetes.io/name: tempo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: tempo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: tempo
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsUser: 10001
+        runAsGroup: 10001
+        runAsNonRoot: true
+        fsGroup: 10001
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: tempo
+          image: grafana/tempo:3.0.3
+          args: ["-config.file=/etc/tempo/tempo.yaml"]
+          ports:
+            - name: http
+              containerPort: 3200
+            - name: otlp-grpc
+              containerPort: 4317
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          volumeMounts:
+            - name: config
+              mountPath: /etc/tempo
+              readOnly: true
+            - name: data
+              mountPath: /var/tempo
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          resources:
+            requests:
+              cpu: 25m
+              memory: 128Mi
+            limits:
+              memory: 384Mi
+      volumes:
+        - name: config
+          configMap:
+            name: tempo
+        - name: data
+          emptyDir: {}

deploy/overlays/prod/kustomization.yaml

@@ -8,6 +8,7 @@ namespace: nereus
 resources:
   - ../../base
   - ../../rollouts
+  - ../../secrets
 
 images:
   - name: nereus-api
@@ -41,11 +42,3 @@ patches:
             labelSelector:
               matchLabels:
                 app.kubernetes.io/name: nereus-api
-
-  - target:
-      kind: ConfigMap
-      name: nereus-api-config
-    patch: |
-      - op: replace
-        path: /data/LOG_LEVEL
-        value: "warn"

deploy/secrets/README.md

@@ -24,12 +24,50 @@ Postgres only reads `POSTGRES_PASSWORD` when it initialises the data directory.
 
 ## prod (sealed)
 
-Write the plaintext to `nereus-db.plain.yaml` (gitignored), then:
+Install the controller through `terraform/platform` first. It is named
+`sealed-secrets-controller`, which is the default expected by `kubeseal`.
 
-```fish
-kubeseal --format yaml < nereus-db.plain.yaml > nereus-db-sealed.yaml
+Create the database Secret locally and pipe it straight into `kubeseal`. The
+plaintext exists only in the pipeline and the password variable:
+
+```bash
+db_password="$(openssl rand -hex 16)"
+kubectl create secret generic nereus-db --namespace nereus \
+  --from-literal=POSTGRES_PASSWORD="$db_password" \
+  --from-literal=DATABASE_URL="postgres://nereus:${db_password}@nereus-postgres:5432/nereus?sslmode=disable" \
+  --dry-run=client -o yaml | \
+kubeseal --format yaml >deploy/secrets/nereus-db-sealed.yaml
+unset db_password
 ```
 
-Commit only `nereus-db-sealed.yaml` and add it to the prod overlay. The
-controller isn't installed yet — `kubectl get crd | grep sealed` comes back
-empty.
+Commit only `nereus-db-sealed.yaml` and add it to the prod overlay. Back up the
+controller key outside the repository before relying on sealed secrets for
+recovery. Losing that key makes every committed `SealedSecret` undecryptable.
+
+On a cluster that already has a manually created `nereus-db` Secret, do not
+apply the sealed replacement in place. PostgreSQL keeps the password used when
+its data directory was initialized. Cut over during a deliberate database
+reset: remove the old Secret and disposable PVC, apply the `SealedSecret`, then
+start PostgreSQL against the empty volume. Fresh clusters can apply the prod
+overlay directly.
+
+## Discord alert routing
+
+Alertmanager reads its Discord webhook from the `nereus-discord` Secret as a
+mounted file. Create the webhook in Discord, keep it in the shell environment,
+and seal it without writing plaintext to disk:
+
+```bash
+test -n "${DISCORD_WEBHOOK_URL:-}"
+printf '%s' "$DISCORD_WEBHOOK_URL" | \
+kubectl create secret generic nereus-discord --namespace observability \
+  --from-file=webhook-url=/dev/stdin --dry-run=client -o yaml | \
+kubeseal --format yaml >deploy/secrets/nereus-discord-sealed.yaml
+unset DISCORD_WEBHOOK_URL
+```
+
+Apply the sealed secret, confirm it produced the `nereus-discord` Secret, then
+set `enable_discord = true` in the platform Terraform variables and apply
+again. Discord routing defaults off so a missing owner-provided webhook cannot
+stop Alertmanager from starting. The committed opt-in values route warning and
+critical Nereus alerts to Discord and discard alerts that do not match.

deploy/secrets/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - nereus-db-sealed.yaml

deploy/secrets/nereus-db-sealed.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  name: nereus-db
+  namespace: nereus
+spec:
+  encryptedData:
+    DATABASE_URL: AgBYadb8AxzvkAIu3JumuybqEGw1rxowhl6RjAOBWl0talN1Fx1+VuMWhb2Thvl0ood8oDbY5fvGx0jD3zEctq3Vxl+22BEzFcea+KLdiOPbJ4tKoOSz3m8MCRbd95DhZwISChhxjBEwoZBdkKQ4rorDFJytx2cmhTlnIxlUKE+m3FHhYYZ/firF74EbhjP9Udk05Dg7COCyrlhyKEXvYv9md/e/PJe7LmMSFHudhV4k8lvg5Uv4helkTDis35GubdICfF1n4BefxIUbyV+uy8XcjU0RBh3lN3o03ZBV/OmL9n2aC1zJIRQZpl1PogWDf5QsUK/AUtlZdB8L1v06sWCk/U4IeQXNEo24HcwKDToL7yVjU8ykF5X5aGLEO+5ATdPKCnB2LsnMF3g1xDiw6p9q8IexLag3c7aUSHwDFmj8Q3j8ky8IgB2cGIb3QzHBFQkGrPvgTdxdeWPcQ+k2vIoOuBOkG6W3W0plJcN72i1/vPnF4ge9xFAzEe2a4Jmiq7gk0dwSZArFPesLeLRZBfTM776WGgrCpNv8PRl+f76KhxaP0hi0F85UClIR6z97H9WdtD/ZvE7gTCjAryz0SnB1yOt79cUlYYmxsqYkYPCmtdlVR+s6vzljr6uUOum3fHeRMzIO2w58g4vqGQF+Uwojk3vmjkUROncf+BRP1Hcd3pBy/OadBkxXInt27ezJcDyoH89HdmBukx7keXonUl3QbVN3ReHwqqAoqSGfwxtYYz5+puSdgzaDqS98r8PhooeNTJaEPvbmYtNMUiTkMv1ZsWgYXltnTZGdjweCJue1xRVIg3SCtWp5vODK3rJg
+    POSTGRES_PASSWORD: AgBtyba0ZixyNnSL5rmn9QdlSWaVHL9Nx29KMKI5lpgu0JLdvRgXbQkZWCYnFi+gLul4FU3eQVsqqvHPxisYe0O0gKj6POToP8dFT5snJWD5ikU0SZX6PWPp94VXrAeLlsYW33V1eoWXxKLbWjX33pCExGVwDoSu0pAEuixtuaUjtv32FsHMAd70SY2uPlcJWeHhi6696agBH9Q3/sBR0m0fikRF4OFz3OC77HQ7GDDkda71KU34uqHs9Cbz30ePmu+xys0HM41Pi3ahguXYg4J1jdwTrjyAde5cAo61efrFNaqRteXFNkDenh4T7QaRfAZbkNYGXQ6Gtt85B3h1AM9592Mo3yBzjuEsAXr5HevITwFM6KZhoK8nVCzmqqSBlNU6DkHyRJxE3q+QWUu17vuG4bAWrd01ua6NfbzFwZb1q4sfszMLp62W1DI3y3S4vg5z9Zhnb9Sa78uPAAR5OTqGx/zMSseJrJX8FXBkA/raMtXFbUn1j1JnLfxmdsOW3LrjhTm7SoMH1Hr4IRf4iYAGgpmak4a/2gSFw9fN77LJI8GlJ/qRN9GcE4mmaLw2gojb60nav+PW0iBRbdHxTcxYiwdyvc7WxFGOHqrkZkEYcVZ2B7lJQMD3uNwC0Xd5rIVpE+lwX4gHMBRbCI6+4OZMGzKnUOWqSsoKS4RptQruKe1b8sXRB4hQYpZ4qhewujM01NSamdz6ZW5Ttp7IlBA2s0yVbOlXQfesDXg77tQ0iQ==
+  template:
+    metadata:
+      name: nereus-db
+      namespace: nereus

docs/ci-cd.md

@@ -21,9 +21,18 @@ Forgejo UI:
 - `REGISTRY_USERNAME`: account allowed to push the three Nereus packages.
 - `REGISTRY_PASSWORD`: package-scoped token for that account.
 - `KUBECONFIG_B64`: base64 encoding of a least-privilege deployment kubeconfig.
+- `GITHUB_MIRROR_SSH_KEY`: private half of a write-enabled deploy key for
+  `git@github.com:Fi3w0/Nereus.git`.
+
+After registering that deploy key, set the Forgejo Actions repository variable
+`MIRROR_ENABLED` to `true`. The mirror job stays skipped until both sides
+are configured, so an absent GitHub credential cannot break production deploys.
 
 The workflow never prints these values. It writes the kubeconfig to the
 ephemeral workspace with mode `0600`, removes it in an `always()` step, and
 creates the Kubernetes registry pull secret through a pipe so its generated
-manifest is not logged or committed. The pre-existing `nereus-db` Secret is
-required and is only checked for presence.
+manifest is not logged or committed. The deploy job applies the committed
+database SealedSecret and waits for the controller to synchronize it before
+starting the application rollout. After a successful production deployment,
+the mirror job pushes `main` to GitHub over SSH and removes its temporary key in
+an `always()` step.

docs/evidence.md

@@ -57,6 +57,81 @@ probe-556d5b659b-3-pre   Failed       vector(1.0)
 After the failed analysis, the active Service selector remained on the healthy
 revision `746bbb94df`.
 
+## Log and trace correlation
+
+Captured on 2026-08-27 from the two-node Fedora cluster. Loki and Tempo run in
+the `observability` namespace, and a DaemonSet OTel Collector tails
+`/var/log/pods` on both nodes.
+
+The collector reads those files as non-root. k3s writes `/var/log/pods` as
+`0750 root:root` and each container log as `0640 root:root`, so the pod runs
+with `supplementalGroups: [0]` and keeps `runAsNonRoot: true`, `drop: ["ALL"]`,
+a read-only root filesystem, and a read-only host mount. Granting
+`DAC_READ_SEARCH` instead does not work: Kubernetes sets no ambient
+capabilities, so the capability is cleared on exec and the container runs with
+`CapEff: 0000000000000000`, matching no files and reporting no error.
+
+Loki received logs from every Nereus container:
+
+```text
+k8s_container_name   ["api", "loadgen", "postgres"]
+k8s_namespace_name   ["nereus"]
+```
+
+A single API request log line carries the trace identifier:
+
+```text
+{"time":"2026-08-27T19:21:24.107201Z","level":"INFO","msg":"request",
+ "method":"GET","path":"/api/v1/readings","status":200,"duration_ms":2,
+ "trace_id":"fd7812d8463298f50b4570374b03dfdb"}
+```
+
+Querying Tempo for that same identifier returned the matching trace, which is
+the hop the provisioned Grafana data sources make in both directions:
+
+```text
+nereus-api   http.request     SPAN_KIND_SERVER
+nereus-api   list_readings    SPAN_KIND_INTERNAL
+```
+
+Both data sources are provisioned by the labelled `nereus-grafana-datasources`
+ConfigMap, which the kube-prometheus-stack Grafana sidecar loads as
+`nereus.yaml`. The correlation requires the API at `LOG_LEVEL=info`, because the
+per-request log line is the only one that carries `trace_id`.
+
+Two collector defects were found and fixed on the way to this result, and both
+belong in the report as corrective actions. The first is the capability problem
+above. The second only appears after a restart: `start_at: beginning` makes the
+receiver replay whole log files, the batch processor had no size cap, and Loki
+refused the resulting push.
+
+```text
+HTTP 503, ResourceExhausted: grpc: received message larger than max
+(4378796 vs. 4194304)
+```
+
+The exporter retries that request unchanged, so the backlog never drains.
+Capping the batch with `send_batch_size: 1000` and `send_batch_max_size: 2000`
+keeps every push under Loki's 4 MiB gRPC limit. What remains after a restart is
+a `Permanent` rejection of replayed lines older than Loki's acceptance window,
+which drops those lines rather than blocking, and leaves live traffic
+unaffected.
+
+## Sealed database cutover
+
+Verified on 2026-08-28 against the two-node Fedora cluster. PostgreSQL was
+stopped, its disposable PVC and manually managed Secret were removed, and the
+committed `nereus-db` SealedSecret created the replacement Secret. The
+controller reported `Synced=True`, the new 5 GiB PVC bound, PostgreSQL reached
+one ready replica, three restarted API pods became ready, and both load
+generators became healthy after reseeding the empty database. The first
+analysis attempts correctly rejected stale reset errors. After one full clean
+Prometheus window, AnalysisRun `nereus-api-75d884b4dd-19-pre` passed all five
+measurements and promoted revision `75d884b4dd` to the active Service.
+
+No plaintext credential was written to the repository, Terraform state, shell
+output, or evidence.
+
 ## Presentation commands
 
 ```bash

docs/roadmap.md

@@ -1,6 +1,6 @@
 # Nereus delivery roadmap
 
-Updated: 2026-08-26
+Updated: 2026-08-28
 
 `[x]` means locally verified. `[~]` means implemented but awaiting a real
 integration environment. `[ ]` means not implemented. `[H]` is human-owned and
@@ -115,15 +115,16 @@ task. Mark a task `[x]` only after its stated verification passes.
 
 - [x] Receive OTLP/gRPC traces in the Collector.
 - [x] Export traces from the Collector to Tempo.
-- [~] Ship structured Kubernetes pod logs to Loki with the cluster Collector.
+- [x] Ship structured Kubernetes pod logs to Loki with the cluster Collector.
 - [x] Start Loki locally and verify its Grafana data source readiness.
 - [x] Configure Prometheus to scrape `/metrics`.
 - [x] Add request-rate, error-rate, and latency panels.
 - [x] Add database-operation latency panels.
 - [x] Add readings-ingested and active-buoy panels.
 - [x] Add a deployed-version dashboard variable.
-- [~] Correlate logs and traces using `trace_id` after cluster log ingestion exists.
+- [x] Correlate logs and traces using `trace_id` after cluster log ingestion exists.
 - [x] Add API error-rate, latency, readiness, and scrape alerts.
+- [~] Route warning and critical alerts to Discord; configuration validates and awaits the owner-provided sealed webhook.
 - [x] Load the dashboard and evaluate all six PromQL queries.
 - [x] Trigger chaos and distinguish the failing version in Grafana.
 
@@ -177,7 +178,7 @@ task. Mark a task `[x]` only after its stated verification passes.
 - [x] Execute `lab.sh check` after a workstation reboot.
 - [x] Test recovery from an interrupted image download and validate the resumed QCOW2.
 - [x] Test recovery after node 2 is deleted manually and rejoin it with a fresh identity.
-- [ ] Test on a workstation with no dependencies installed.
+- [~] Test on a workstation with no dependencies installed; the reproducible fresh-VM path is implemented and awaits a second host.
 - [x] Decide whether an aarch64 image path is required; keep the lab x86_64-only.
 - [ ] After final sign-off, destroy the QEMU lab and remove packages installed only for it with Pacman's dependency-aware cleanup.
 
@@ -198,8 +199,8 @@ task. Mark a task `[x]` only after its stated verification passes.
 
 ## Human-owned deployment and infrastructure
 
-- [H] Define infrastructure modules under `terraform/`.
-- [H] Export provisioned addresses for Ansible inventory.
+- [x] Define libvirt infrastructure and Helm platform stacks under `terraform/`.
+- [x] Export stable node addresses to an Ansible inventory without provisioner hooks.
 - [H] Create Kubernetes base resources and overlays. Agent-authored 2026-08-21 at the owner's request; builds and server dry-runs pass, not applied.
 - [H] Create API and load-generator workloads. Agent-authored 2026-08-21; not applied.
 - [H] Create active and preview Services. Agent-authored 2026-08-21; not applied.
@@ -207,7 +208,7 @@ task. Mark a task `[x]` only after its stated verification passes.
 - [H] Create the real error-rate AnalysisTemplate. Agent-authored 2026-08-21; PromQL never evaluated against real series.
 - [H] Configure blue-green promotion and automated rollback. Agent-authored 2026-08-21; promotion and abort paths untested with the real API.
 - [H] Maintain Forgejo Actions and GitHub mirror workflows.
-- [H] Create sealed secrets from off-repository plaintext inputs.
+- [x] Install Sealed Secrets, commit the encrypted database resource, and verify the fresh-volume cutover.
 
 ## Forgejo CI/CD
 

scripts/provision/README.md

@@ -37,10 +37,11 @@ can later be managed by Ansible without rebuilding it.
 ## Terraform path
 
 Terraform is responsible only for creating machines, networks, and addresses.
-After `terraform apply`, put its resulting addresses into `inventory.yml` and
-run this playbook. Keeping configuration out of provisioner hooks makes the
-same Ansible workflow usable for physical hardware, VMs, and manually created
-hosts.
+`terraform/infra` creates two Fedora 44 libvirt machines and writes its resulting
+addresses to `terraform/infra/inventory.yml`. Pass that file to `bootstrap.sh`
+after `terraform apply`. Keeping configuration out of provisioner hooks makes
+the same Ansible workflow usable for physical hardware, VMs, and manually
+created hosts.
 
 ## One-command local QEMU lab
 

terraform/infra/.terraform.lock.hcl

@@ -0,0 +1,44 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/dmacvicar/libvirt" {
+  version     = "0.8.3"
+  constraints = "0.8.3"
+  hashes = [
+    "h1:Tttxr3E9O75MM+dDmq5sYHQEw29PwtIj+XDj/5drdfE=",
+    "zh:06ff0169beafd1891dc5a30616983abd32004a4f570d1d3dbb5851d84bd1c007",
+    "zh:2dbdd726d0987cda73b56ecdfbcb98a67485e86a7a44aec976c0081b7239d89d",
+    "zh:2e195a7bbdfcc13c45460571a5ba848a5c1e746b477c8381058767560f0ac93b",
+    "zh:3952da13080018c5aec498b73e343c4c22ad884afb8c983138fb7255617aa991",
+    "zh:478841bcf57df938726ddb90f55c7953fad09db4f6348747519afe7fc84b403b",
+    "zh:53bce78b03a82c4782acfe1f32c2b46a68fa5fb2fb90d4a5392c90b436b44244",
+    "zh:5c157f23e9768c67cddf9e847a571adca441607cb5adfb96dbfdd626ceadf92c",
+    "zh:6bc78d631959fb695664966851308e140c38f3f5cf648dd89756320c2d91765d",
+    "zh:8605d7d6915190836802654920a8eea3d751ae437273c4f4476dc0ebb9167a1d",
+    "zh:8b66a22b97331c2a56aed092fd39152d06ad957fd4810aa3f0c4ade0f9b15755",
+    "zh:92586a47a04082f70bb33f722672127a287caeed109beaaca2668e2e1d6a9caf",
+    "zh:99a9ee414f5c4268e287660ce8edec2efcba1f79351f83791b64c7e5ab04f569",
+    "zh:b7cff09fe74b0eb63b5b9aa94de5b33dadbd006d6d5b9578ac476039ea20b062",
+    "zh:d4188a343ff32c0e03ff28c7e84abce0f43cad2fdbcd9046eaafc247429039ff",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/local" {
+  version     = "2.5.3"
+  constraints = "2.5.3"
+  hashes = [
+    "h1:1Nkh16jQJMp0EuDmvP/96f5Unnir0z12WyDuoR6HjMo=",
+    "zh:284d4b5b572eacd456e605e94372f740f6de27b71b4e1fd49b63745d8ecd4927",
+    "zh:40d9dfc9c549e406b5aab73c023aa485633c1b6b730c933d7bcc2fa67fd1ae6e",
+    "zh:6243509bb208656eb9dc17d3c525c89acdd27f08def427a0dce22d5db90a4c8b",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:885d85869f927853b6fe330e235cd03c337ac3b933b0d9ae827ec32fa1fdcdbf",
+    "zh:bab66af51039bdfcccf85b25fe562cbba2f54f6b3812202f4873ade834ec201d",
+    "zh:c505ff1bf9442a889ac7dca3ac05a8ee6f852e0118dd9a61796a2f6ff4837f09",
+    "zh:d36c0b5770841ddb6eaf0499ba3de48e5d4fc99f4829b6ab66b0fab59b1aaf4f",
+    "zh:ddb6a407c7f3ec63efb4dad5f948b54f7f4434ee1a2607a49680d494b1776fe1",
+    "zh:e0dafdd4500bec23d3ff221e3a9b60621c5273e5df867bc59ef6b7e41f5c91f6",
+    "zh:ece8742fd2882a8fc9d6efd20e2590010d43db386b920b2a9c220cfecc18de47",
+    "zh:f4c6b3eb8f39105004cf720e202f04f57e3578441cfb76ca27611139bc116a82",
+  ]
+}

terraform/infra/README.md

@@ -0,0 +1,19 @@
+# Libvirt infrastructure
+
+This stack creates a dedicated NAT network and two Fedora 44 x86_64 machines.
+It stops at the machine boundary. Ansible remains responsible for Fedora and
+k3s configuration.
+
+Keep real values in an untracked `terraform.tfvars` file:
+
+```bash
+cp terraform.tfvars.example terraform.tfvars
+terraform init
+terraform apply
+K3S_TOKEN="$(openssl rand -hex 32)" \
+  ../../scripts/provision/bootstrap.sh inventory.yml
+```
+
+The generated `inventory.yml`, Terraform state, and variable file are ignored.
+The SSH public key is safe to place in Terraform input; the matching private key
+is referenced by absolute path and is never read into Terraform state.

terraform/infra/cloud-init.yaml.tftpl

@@ -0,0 +1,12 @@
+#cloud-config
+hostname: ${hostname}
+manage_etc_hosts: true
+users:
+  - name: fedora
+    groups: [wheel]
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    shell: /bin/bash
+    ssh_authorized_keys:
+      - ${ssh_public_key}
+ssh_pwauth: false
+disable_root: true

terraform/infra/inventory.yml.tftpl

@@ -0,0 +1,24 @@
+all:
+  vars:
+    ansible_user: fedora
+    ansible_ssh_private_key_file: ${private_key_file}
+    ansible_ssh_common_args: "-o StrictHostKeyChecking=accept-new"
+    k3s_version: v1.33.4+k3s1
+    k3s_cluster_cidr: 10.42.0.0/16
+    k3s_service_cidr: 10.43.0.0/16
+    k3s_node_cidr: ${node_cidr}
+    k3s_operator_cidrs:
+%{ for cidr in operator_cidrs ~}
+      - ${cidr}
+%{ endfor ~}
+  children:
+    k3s_server:
+      hosts:
+        node1:
+          ansible_host: ${server_address}
+          k3s_node_ip: ${server_address}
+    k3s_agent:
+      hosts:
+        node2:
+          ansible_host: ${agent_address}
+          k3s_node_ip: ${agent_address}

terraform/infra/main.tf

@@ -0,0 +1,97 @@
+locals {
+  nodes = {
+    nereus-node1 = {
+      address = var.node_addresses["nereus-node1"]
+      role    = "server"
+    }
+    nereus-node2 = {
+      address = var.node_addresses["nereus-node2"]
+      role    = "agent"
+    }
+  }
+}
+
+resource "libvirt_network" "nereus" {
+  name      = "nereus"
+  mode      = "nat"
+  domain    = "nereus.test"
+  addresses = [var.network_cidr]
+
+  dhcp {
+    enabled = true
+  }
+
+  dns {
+    enabled = true
+  }
+}
+
+resource "libvirt_volume" "fedora" {
+  name   = "nereus-fedora-44-base.qcow2"
+  pool   = "default"
+  source = var.fedora_image_url
+  format = "qcow2"
+}
+
+resource "libvirt_volume" "node" {
+  for_each       = local.nodes
+  name           = "${each.key}.qcow2"
+  pool           = "default"
+  base_volume_id = libvirt_volume.fedora.id
+  size           = 30 * 1024 * 1024 * 1024
+}
+
+resource "libvirt_cloudinit_disk" "node" {
+  for_each = local.nodes
+  name     = "${each.key}-cloud-init.iso"
+  pool     = "default"
+  user_data = templatefile("${path.module}/cloud-init.yaml.tftpl", {
+    hostname       = each.key
+    ssh_public_key = var.ssh_public_key
+  })
+}
+
+resource "libvirt_domain" "node" {
+  for_each = local.nodes
+
+  name      = each.key
+  memory    = 3072
+  vcpu      = 2
+  autostart = true
+  cloudinit = libvirt_cloudinit_disk.node[each.key].id
+
+  disk {
+    volume_id = libvirt_volume.node[each.key].id
+  }
+
+  network_interface {
+    network_id     = libvirt_network.nereus.id
+    hostname       = each.key
+    addresses      = [each.value.address]
+    wait_for_lease = true
+  }
+
+  console {
+    type        = "pty"
+    target_type = "serial"
+    target_port = "0"
+  }
+
+  graphics {
+    type        = "spice"
+    listen_type = "address"
+    autoport    = true
+  }
+}
+
+resource "local_sensitive_file" "inventory" {
+  filename        = "${path.module}/inventory.yml"
+  file_permission = "0600"
+  content = templatefile("${path.module}/inventory.yml.tftpl", {
+    private_key_file = var.ssh_private_key_file
+    operator_cidrs   = var.operator_cidrs
+    node_cidr        = var.network_cidr
+    server_address   = local.nodes["nereus-node1"].address
+    agent_address    = local.nodes["nereus-node2"].address
+  })
+}

terraform/infra/outputs.tf

@@ -0,0 +1,9 @@
+output "node_addresses" {
+  description = "Addresses passed to Ansible."
+  value       = { for name, node in local.nodes : name => node.address }
+}
+
+output "ansible_inventory" {
+  description = "Generated inventory consumed by scripts/provision/bootstrap.sh."
+  value       = local_sensitive_file.inventory.filename
+}

terraform/infra/terraform.tfvars.example

@@ -0,0 +1,4 @@
+ssh_public_key       = "replace with one public key"
+ssh_private_key_file = "/absolute/path/to/the/matching/private/key"
+
+operator_cidrs = ["192.168.123.1/32"]

terraform/infra/variables.tf

@@ -0,0 +1,42 @@
+variable "libvirt_uri" {
+  description = "Libvirt connection used to create the Fedora machines."
+  type        = string
+  default     = "qemu:///system"
+}
+
+variable "fedora_image_url" {
+  description = "Fedora 44 Cloud Base Generic x86_64 image URL."
+  type        = string
+  default     = "https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2"
+}
+
+variable "ssh_public_key" {
+  description = "Public SSH key installed for the fedora user."
+  type        = string
+}
+
+variable "ssh_private_key_file" {
+  description = "Absolute private-key path written into the generated Ansible inventory."
+  type        = string
+}
+
+variable "operator_cidrs" {
+  description = "Networks allowed to administer k3s."
+  type        = list(string)
+  default     = ["192.168.123.1/32"]
+}
+
+variable "network_cidr" {
+  description = "Dedicated libvirt network for the two nodes."
+  type        = string
+  default     = "192.168.123.0/24"
+}
+
+variable "node_addresses" {
+  description = "Stable addresses assigned to each node."
+  type        = map(string)
+  default = {
+    nereus-node1 = "192.168.123.10"
+    nereus-node2 = "192.168.123.11"
+  }
+}

terraform/infra/versions.tf

@@ -0,0 +1,18 @@
+terraform {
+  required_version = ">= 1.10.0"
+
+  required_providers {
+    libvirt = {
+      source  = "dmacvicar/libvirt"
+      version = "0.8.3"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = "2.5.3"
+    }
+  }
+}
+
+provider "libvirt" {
+  uri = var.libvirt_uri
+}

terraform/platform/.terraform.lock.hcl

@@ -0,0 +1,22 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/helm" {
+  version     = "3.1.1"
+  constraints = "3.1.1"
+  hashes = [
+    "h1:5b2ojWKT0noujHiweCds37ZreRFRQLNaErdJLusJN88=",
+    "zh:1a6d5ce931708aec29d1f3d9e360c2a0c35ba5a54d03eeaff0ce3ca597cd0275",
+    "zh:3411919ba2a5941801e677f0fea08bdd0ae22ba3c9ce3309f55554699e06524a",
+    "zh:81b36138b8f2320dc7f877b50f9e38f4bc614affe68de885d322629dd0d16a29",
+    "zh:95a2a0a497a6082ee06f95b38bd0f0d6924a65722892a856cfd914c0d117f104",
+    "zh:9d3e78c2d1bb46508b972210ad706dd8c8b106f8b206ecf096cd211c54f46990",
+    "zh:a79139abf687387a6efdbbb04289a0a8e7eaca2bd91cdc0ce68ea4f3286c2c34",
+    "zh:aaa8784be125fbd50c48d84d6e171d3fb6ef84a221dbc5165c067ce05faab4c8",
+    "zh:afecd301f469975c9d8f350cc482fe656e082b6ab0f677d1a816c3c615837cc1",
+    "zh:c54c22b18d48ff9053d899d178d9ffef7d9d19785d9bf310a07d648b7aac075b",
+    "zh:db2eefd55aea48e73384a555c72bac3f7d428e24147bedb64e1a039398e5b903",
+    "zh:ee61666a233533fd2be971091cecc01650561f1585783c381b6f6e8a390198a4",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+  ]
+}

terraform/platform/README.md

@@ -0,0 +1,20 @@
+# Cluster platform
+
+This stack installs Argo Rollouts, Sealed Secrets, and kube-prometheus-stack on
+an existing k3s cluster. Use it on a fresh cluster, or import matching Helm
+releases before applying it to a cluster where they already exist.
+
+```bash
+cp terraform.tfvars.example terraform.tfvars
+terraform init
+terraform apply
+```
+
+The kubeconfig stays outside the repository. Terraform records only its path.
+Application and observability manifests remain under `deploy/` and are applied
+after the cluster-wide controllers are ready.
+
+Discord routing is deliberately off by default so a missing webhook cannot
+stop Alertmanager. After applying `nereus-discord-sealed.yaml` and confirming
+that it produced the `nereus-discord` Secret, set `enable_discord = true` and
+apply this stack again.

terraform/platform/alertmanager-discord.values.yaml

@@ -0,0 +1,23 @@
+alertmanager:
+  alertmanagerSpec:
+    secrets:
+      - nereus-discord
+  config:
+    global:
+      resolve_timeout: 5m
+    route:
+      receiver: discard
+      group_by: [alertname, namespace]
+      group_wait: 30s
+      group_interval: 5m
+      repeat_interval: 4h
+      routes:
+        - receiver: discord
+          matchers:
+            - severity=~"warning|critical"
+    receivers:
+      - name: discard
+      - name: discord
+        discord_configs:
+          - webhook_url_file: /etc/alertmanager/secrets/nereus-discord/webhook-url
+            send_resolved: true

terraform/platform/kube-prometheus-stack.values.yaml

@@ -0,0 +1,27 @@
+kubeEtcd:
+  enabled: false
+kubeProxy:
+  enabled: false
+
+prometheus:
+  prometheusSpec:
+    retention: 6h
+    serviceMonitorSelectorNilUsesHelmValues: false
+    podMonitorSelectorNilUsesHelmValues: false
+    ruleSelectorNilUsesHelmValues: false
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          accessModes: [ReadWriteOnce]
+          resources:
+            requests:
+              storage: 5Gi
+
+grafana:
+  defaultDashboardsTimezone: browser
+
+alertmanager:
+  alertmanagerSpec:
+    resources:
+      requests: {cpu: 25m, memory: 64Mi}
+      limits: {memory: 256Mi}

terraform/platform/main.tf

@@ -0,0 +1,44 @@
+resource "helm_release" "argo_rollouts" {
+  name             = "argo-rollouts"
+  repository       = "https://argoproj.github.io/argo-helm"
+  chart            = "argo-rollouts"
+  version          = "2.41.1"
+  namespace        = "argo-rollouts"
+  create_namespace = true
+  wait             = true
+  timeout          = 300
+}
+
+resource "helm_release" "sealed_secrets" {
+  name             = "sealed-secrets"
+  repository       = "https://bitnami-labs.github.io/sealed-secrets"
+  chart            = "sealed-secrets"
+  version          = "2.19.3"
+  namespace        = "kube-system"
+  create_namespace = false
+  wait             = true
+  timeout          = 300
+
+  values = [yamlencode({
+    fullnameOverride = "sealed-secrets-controller"
+    resources = {
+      requests = { cpu = "25m", memory = "64Mi" }
+      limits   = { memory = "256Mi" }
+    }
+  })]
+}
+
+resource "helm_release" "kube_prometheus_stack" {
+  name             = "kube-prometheus-stack"
+  repository       = "https://prometheus-community.github.io/helm-charts"
+  chart            = "kube-prometheus-stack"
+  version          = "88.5.2"
+  namespace        = "observability"
+  create_namespace = true
+  wait             = true
+  timeout          = 600
+  values = concat(
+    [file("${path.module}/kube-prometheus-stack.values.yaml")],
+    var.enable_discord ? [file("${path.module}/alertmanager-discord.values.yaml")] : []
+  )
+}

terraform/platform/terraform.tfvars.example

@@ -0,0 +1,2 @@
+kubeconfig_path = "/absolute/path/to/kubeconfig"
+enable_discord   = false

terraform/platform/variables.tf

@@ -0,0 +1,10 @@
+variable "kubeconfig_path" {
+  description = "Absolute path to the operator kubeconfig. Its contents never enter Terraform state."
+  type        = string
+}
+
+variable "enable_discord" {
+  description = "Enable Discord routing after the nereus-discord SealedSecret has created its Secret."
+  type        = bool
+  default     = false
+}

terraform/platform/versions.tf

@@ -0,0 +1,16 @@
+terraform {
+  required_version = ">= 1.10.0"
+
+  required_providers {
+    helm = {
+      source  = "hashicorp/helm"
+      version = "3.1.1"
+    }
+  }
+}
+
+provider "helm" {
+  kubernetes = {
+    config_path = var.kubeconfig_path
+  }
+}