feat(deploy): add blue-green release manifests

deploy/base/api-services.yaml

@@ -0,0 +1,41 @@
+# Active and preview Services for the blue-green Rollout.
+#
+# Argo Rollouts rewrites both selectors at runtime to pin them to the right
+# ReplicaSet. What's below is just the starting state.
+#
+# Same app.kubernetes.io/name on both so one ServiceMonitor covers them, which
+# is what gives the analysis query a `service` label to filter on.
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nereus-api-active
+  labels:
+    app.kubernetes.io/name: nereus-api
+    app.kubernetes.io/part-of: nereus
+    app.kubernetes.io/component: active
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: nereus-api
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nereus-api-preview
+  labels:
+    app.kubernetes.io/name: nereus-api
+    app.kubernetes.io/part-of: nereus
+    app.kubernetes.io/component: preview
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: nereus-api
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http

deploy/base/configmap.yaml

@@ -0,0 +1,18 @@
+# Non-secret API config. The DSN lives in the nereus-db Secret.
+#
+# Values must be quoted -- data is map[string]string, and an unquoted 8080
+# gets rejected as an integer.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nereus-api-config
+  labels:
+    app.kubernetes.io/name: nereus-api
+    app.kubernetes.io/part-of: nereus
+data:
+  PORT: "8080"
+  LOG_LEVEL: "info"
+  # main.go only enables tracing when this is set.
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://nereus-otel-collector.observability.svc.cluster.local:4317"
+  # Patched to a nonzero value on the preview to trigger the rollback demo.
+  CHAOS_ERROR_RATE: "0.0"

deploy/base/ingress.yaml

@@ -0,0 +1,40 @@
+# Traefik ingress. Reachable at http://localhost:8080 in the k3d lab.
+#
+# Active Service only -- the preview stays internal so real traffic can't hit
+# an unpromoted version. /metrics is not exposed either.
+#
+# No host, so any Host header matches. A real hostname belongs in the prod
+# overlay.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: nereus-api
+  labels:
+    app.kubernetes.io/name: nereus-api
+    app.kubernetes.io/part-of: nereus
+spec:
+  ingressClassName: traefik
+  rules:
+    - http:
+        paths:
+          - path: /api/v1
+            pathType: Prefix
+            backend:
+              service:
+                name: nereus-api-active
+                port:
+                  name: http
+          - path: /healthz
+            pathType: Exact
+            backend:
+              service:
+                name: nereus-api-active
+                port:
+                  name: http
+          - path: /readyz
+            pathType: Exact
+            backend:
+              service:
+                name: nereus-api-active
+                port:
+                  name: http

deploy/base/kustomization.yaml

@@ -0,0 +1,18 @@
+# Plain Kubernetes only -- no Rollouts CRDs, so this still applies on a
+# cluster without Argo installed.
+#
+# The nereus-db Secret is not here; see deploy/secrets/README.md.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# Sets the namespace on everything below, so the files don't repeat it.
+namespace: nereus
+
+resources:
+  - namespace.yaml
+  - configmap.yaml
+  - postgres.yaml
+  - api-services.yaml
+  - servicemonitor.yaml
+  - loadgen.yaml
+  - ingress.yaml

deploy/base/loadgen.yaml

@@ -0,0 +1,100 @@
+# Load generator. A plain Deployment -- nothing to promote, and it has to keep
+# running straight through an API rollout.
+#
+# Points at the active Service so its traffic never lands on the preview.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nereus-loadgen
+  labels:
+    app.kubernetes.io/name: nereus-loadgen
+    app.kubernetes.io/part-of: nereus
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: nereus-loadgen
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: nereus-loadgen
+        app.kubernetes.io/part-of: nereus
+    spec:
+      imagePullSecrets:
+        - name: nereus-registry
+      securityContext:
+        # distroless nonroot.
+        runAsUser: 65532
+        runAsGroup: 65532
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: loadgen
+          image: nereus-loadgen:0.1.0
+          # Side-loaded with `k3d image import`; there's no registry.
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: TARGET_URL
+              value: http://nereus-api-active.nereus.svc.cluster.local:8080
+            - name: RPS
+              value: "5"
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          resources:
+            requests:
+              cpu: 25m
+              memory: 32Mi
+            limits:
+              memory: 128Mi
+---
+# Keeps request samples flowing through the preview while analysis runs.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nereus-loadgen-preview
+  labels:
+    app.kubernetes.io/name: nereus-loadgen-preview
+    app.kubernetes.io/part-of: nereus
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: nereus-loadgen-preview
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: nereus-loadgen-preview
+        app.kubernetes.io/part-of: nereus
+    spec:
+      imagePullSecrets:
+        - name: nereus-registry
+      securityContext:
+        runAsUser: 65532
+        runAsGroup: 65532
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: loadgen
+          image: nereus-loadgen:0.1.0
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: TARGET_URL
+              value: http://nereus-api-preview.nereus.svc.cluster.local:8080
+            - name: RPS
+              value: "5"
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          resources:
+            requests:
+              cpu: 25m
+              memory: 32Mi
+            limits:
+              memory: 128Mi

deploy/base/namespace.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: nereus
+  labels:
+    app.kubernetes.io/name: nereus
+    app.kubernetes.io/part-of: nereus

deploy/base/postgres.yaml

@@ -0,0 +1,113 @@
+# Postgres 17, StatefulSet + headless Service.
+#
+# StatefulSet rather than Deployment for the stable pod name and a PVC that
+# survives the pod.
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nereus-postgres
+  labels:
+    app.kubernetes.io/name: nereus-postgres
+    app.kubernetes.io/part-of: nereus
+spec:
+  clusterIP: None
+  selector:
+    app.kubernetes.io/name: nereus-postgres
+  ports:
+    - name: postgres
+      port: 5432
+      targetPort: postgres
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: nereus-postgres
+  labels:
+    app.kubernetes.io/name: nereus-postgres
+    app.kubernetes.io/part-of: nereus
+spec:
+  serviceName: nereus-postgres
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: nereus-postgres
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: nereus-postgres
+        app.kubernetes.io/part-of: nereus
+    spec:
+      securityContext:
+        # postgres:17-alpine runs as uid 70. The Debian tags use 999.
+        runAsUser: 70
+        runAsGroup: 70
+        fsGroup: 70
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: postgres
+          image: postgres:17-alpine
+          ports:
+            - name: postgres
+              containerPort: 5432
+          env:
+            - name: POSTGRES_DB
+              value: nereus
+            - name: POSTGRES_USER
+              value: nereus
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: nereus-db
+                  key: POSTGRES_PASSWORD
+            # Subdirectory, or initdb trips over lost+found on the volume.
+            - name: PGDATA
+              value: /var/lib/postgresql/data/pgdata
+          volumeMounts:
+            - name: data
+              mountPath: /var/lib/postgresql/data
+            - name: runtime
+              mountPath: /var/run/postgresql
+            - name: tmp
+              mountPath: /tmp
+          readinessProbe:
+            exec:
+              command: ["pg_isready", "-U", "nereus", "-d", "nereus"]
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 6
+          livenessProbe:
+            exec:
+              command: ["pg_isready", "-U", "nereus", "-d", "nereus"]
+            initialDelaySeconds: 30
+            periodSeconds: 15
+            timeoutSeconds: 5
+            failureThreshold: 3
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              memory: 1Gi
+      volumes:
+        - name: runtime
+          emptyDir: {}
+        - name: tmp
+          emptyDir:
+            sizeLimit: 64Mi
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 5Gi

deploy/base/servicemonitor.yaml

@@ -0,0 +1,24 @@
+# Prometheus Operator CRD, not core Kubernetes. Works because the k3d values
+# set serviceMonitorSelectorNilUsesHelmValues: false.
+#
+# Matches both Services, so each pod is scraped once per Service. That's the
+# point -- the `service` label is how the analysis isolates the preview.
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: nereus-api
+  labels:
+    app.kubernetes.io/name: nereus-api
+    app.kubernetes.io/part-of: nereus
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: nereus-api
+  namespaceSelector:
+    matchNames:
+      - nereus
+  endpoints:
+    - port: http
+      path: /metrics
+      interval: 15s
+      scrapeTimeout: 10s

deploy/observability/kustomization.yaml

@@ -0,0 +1,9 @@
+# Lands in `observability`, alongside kube-prometheus-stack.
+#   kubectl apply -k deploy/observability
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: observability
+
+resources:
+  - otel-collector.yaml

deploy/observability/otel-collector.yaml

@@ -0,0 +1,109 @@
+# OTLP collector for the cluster. The API's OTEL_EXPORTER_OTLP_ENDPOINT
+# points here.
+#
+# Traces only. The log pipeline in observability/otel-collector/config.yaml
+# needs a DaemonSet with hostPath access to /var/log/pods plus RBAC, and Loki
+# isn't in the cluster yet.
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nereus-otel-collector
+  labels:
+    app.kubernetes.io/name: nereus-otel-collector
+data:
+  config.yaml: |
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:4317
+
+    processors:
+      memory_limiter:
+        check_interval: 1s
+        limit_mib: 256
+      batch:
+        timeout: 5s
+
+    exporters:
+      otlp/tempo:
+        endpoint: tempo.observability.svc.cluster.local:4317
+        tls:
+          insecure: true
+      # Keeps traces visible in `kubectl logs` until Tempo is deployed.
+      debug:
+        verbosity: basic
+
+    service:
+      pipelines:
+        traces:
+          receivers: [otlp]
+          processors: [memory_limiter, batch]
+          exporters: [otlp/tempo, debug]
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nereus-otel-collector
+  labels:
+    app.kubernetes.io/name: nereus-otel-collector
+spec:
+  selector:
+    app.kubernetes.io/name: nereus-otel-collector
+  ports:
+    - name: otlp-grpc
+      port: 4317
+      targetPort: otlp-grpc
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nereus-otel-collector
+  labels:
+    app.kubernetes.io/name: nereus-otel-collector
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: nereus-otel-collector
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: nereus-otel-collector
+      annotations:
+        # Restarts the pod when the config changes; the collector doesn't reload.
+        checksum/config: placeholder
+    spec:
+      securityContext:
+        runAsUser: 10001
+        runAsGroup: 10001
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: otel-collector
+          image: otel/opentelemetry-collector-contrib:0.159.0
+          args: ["--config=/conf/config.yaml"]
+          ports:
+            - name: otlp-grpc
+              containerPort: 4317
+          volumeMounts:
+            - name: config
+              mountPath: /conf
+              readOnly: true
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+            limits:
+              memory: 384Mi
+      volumes:
+        - name: config
+          configMap:
+            name: nereus-otel-collector

deploy/overlays/dev/kustomization.yaml

@@ -0,0 +1,30 @@
+# dev -- the local k3d lab.
+#   kubectl apply -k deploy/overlays/dev
+#
+# Needs the images imported and the nereus-db Secret created first.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: nereus
+
+resources:
+  - ../../base
+  - ../../rollouts
+
+patches:
+  # One replica is easier to watch swap over, and enough on a laptop.
+  - target:
+      kind: Rollout
+      name: nereus-api
+    patch: |
+      - op: replace
+        path: /spec/replicas
+        value: 1
+
+  - target:
+      kind: StatefulSet
+      name: nereus-postgres
+    patch: |
+      - op: replace
+        path: /spec/volumeClaimTemplates/0/spec/resources/requests/storage
+        value: 2Gi

deploy/overlays/prod/kustomization.yaml

@@ -0,0 +1,51 @@
+# prod overlay -- the two Fedora nodes.
+#   kubectl apply -k deploy/overlays/prod
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: nereus
+
+resources:
+  - ../../base
+  - ../../rollouts
+
+images:
+  - name: nereus-api
+    newName: git.fiwlabs.dev/fiwdev/nereus-api
+    newTag: 0.1.0
+  - name: nereus-loadgen
+    newName: git.fiwlabs.dev/fiwdev/nereus-loadgen
+    newTag: 0.1.0
+
+patches:
+  # 3 replicas so a node can go down mid-rollout.
+  - target:
+      kind: Rollout
+      name: nereus-api
+    patch: |
+      - op: replace
+        path: /spec/replicas
+        value: 3
+
+  # Spread the API across both nodes.
+  - target:
+      kind: Rollout
+      name: nereus-api
+    patch: |
+      - op: add
+        path: /spec/template/spec/topologySpreadConstraints
+        value:
+          - maxSkew: 1
+            topologyKey: kubernetes.io/hostname
+            whenUnsatisfiable: ScheduleAnyway
+            labelSelector:
+              matchLabels:
+                app.kubernetes.io/name: nereus-api
+
+  - target:
+      kind: ConfigMap
+      name: nereus-api-config
+    patch: |
+      - op: replace
+        path: /data/LOG_LEVEL
+        value: "warn"

deploy/rollouts/analysistemplate.yaml

@@ -0,0 +1,36 @@
+# The real error-rate gate, replacing the vector(0.0) harness in
+# scripts/k3d/analysis-harness/.
+#
+# Runs before promotion. Every sample has to pass; one failure aborts and the
+# active Service never moves.
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: nereus-api-error-rate
+  labels:
+    app.kubernetes.io/name: nereus-api
+    app.kubernetes.io/part-of: nereus
+spec:
+  args:
+    - name: service
+  metrics:
+    - name: error-rate
+      # Let the new pods take traffic first.
+      initialDelay: 30s
+      interval: 20s
+      count: 5
+      failureLimit: 0
+      # Empty result means no traffic yet -- not a failure.
+      successCondition: "len(result) == 0 || result[0] < 0.05"
+      provider:
+        prometheus:
+          address: http://kube-prometheus-stack-prometheus.observability.svc.cluster.local:9090
+          # 5xx as a fraction of all responses on the preview.
+          # `or vector(0)` keeps the division defined when there are no errors.
+          query: |
+            (
+              sum(rate(nereus_http_requests_total{service="{{args.service}}",status=~"5.."}[1m]))
+              or vector(0)
+            )
+            /
+            sum(rate(nereus_http_requests_total{service="{{args.service}}"}[1m]))

deploy/rollouts/kustomization.yaml

@@ -0,0 +1,8 @@
+# Argo Rollouts resources, kept separate from base/ so the plain-Kubernetes
+# manifests stay applicable on a cluster without the Rollouts CRDs installed.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - rollout.yaml
+  - analysistemplate.yaml

deploy/rollouts/rollout.yaml

@@ -0,0 +1,100 @@
+# The API, as a blue-green Rollout.
+#
+# Same shape as a Deployment, but the new version comes up alongside the old,
+# gets checked against Prometheus, and only then takes over the active Service.
+apiVersion: argoproj.io/v1alpha1
+kind: Rollout
+metadata:
+  name: nereus-api
+  labels:
+    app.kubernetes.io/name: nereus-api
+    app.kubernetes.io/part-of: nereus
+spec:
+  replicas: 2
+  revisionHistoryLimit: 3
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: nereus-api
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: nereus-api
+        app.kubernetes.io/part-of: nereus
+    spec:
+      imagePullSecrets:
+        - name: nereus-registry
+      securityContext:
+        # distroless nonroot.
+        runAsUser: 65532
+        runAsGroup: 65532
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: api
+          image: nereus-api:0.1.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 8080
+          envFrom:
+            - configMapRef:
+                name: nereus-api-config
+          env:
+            - name: DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: nereus-db
+                  key: DATABASE_URL
+          # /healthz ignores Postgres, so a database outage makes pods unready
+          # without restarting them.
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+            initialDelaySeconds: 3
+            periodSeconds: 10
+            timeoutSeconds: 2
+            failureThreshold: 3
+          # /readyz waits for migrations.
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: http
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 3
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              memory: 256Mi
+      volumes:
+        - name: tmp
+          emptyDir:
+            medium: Memory
+            sizeLimit: 16Mi
+  strategy:
+    blueGreen:
+      activeService: nereus-api-active
+      previewService: nereus-api-preview
+      # Auto-promote only if the analysis passes; a failure aborts instead.
+      autoPromotionEnabled: true
+      # Keep the old ReplicaSet warm so an abort falls straight back.
+      scaleDownDelaySeconds: 30
+      prePromotionAnalysis:
+        templates:
+          - templateName: nereus-api-error-rate
+        args:
+          - name: service
+            value: nereus-api-preview

deploy/secrets/README.md

@@ -0,0 +1,35 @@
+# Secrets
+
+`nereus-db` holds two keys, used by `postgres.yaml` and `rollout.yaml`:
+
+- `POSTGRES_PASSWORD`
+- `DATABASE_URL` — `postgres://nereus:<password>@nereus-postgres:5432/nereus?sslmode=disable`
+
+It is not in `base/kustomization.yaml`. Plaintext never lands in the repo, and
+gitleaks runs on every push.
+
+## dev (k3d)
+
+Create it directly, before `kubectl apply -k deploy/overlays/dev`:
+
+```fish
+set pw (openssl rand -hex 16)
+kubectl create secret generic nereus-db -n nereus \
+  --from-literal=POSTGRES_PASSWORD=$pw \
+  --from-literal=DATABASE_URL="postgres://nereus:$pw@nereus-postgres:5432/nereus?sslmode=disable"
+```
+
+Rotating means deleting the secret, the StatefulSet's PVC, and restarting —
+Postgres only reads `POSTGRES_PASSWORD` when it initialises the data directory.
+
+## prod (sealed)
+
+Write the plaintext to `nereus-db.plain.yaml` (gitignored), then:
+
+```fish
+kubeseal --format yaml < nereus-db.plain.yaml > nereus-db-sealed.yaml
+```
+
+Commit only `nereus-db-sealed.yaml` and add it to the prod overlay. The
+controller isn't installed yet — `kubectl get crd | grep sealed` comes back
+empty.