diff --git a/deploy/base/api-services.yaml b/deploy/base/api-services.yaml new file mode 100644 index 0000000..d64a1c2 --- /dev/null +++ b/deploy/base/api-services.yaml @@ -0,0 +1,41 @@ +# Active and preview Services for the blue-green Rollout. +# +# Argo Rollouts rewrites both selectors at runtime to pin them to the right +# ReplicaSet. What's below is just the starting state. +# +# Same app.kubernetes.io/name on both so one ServiceMonitor covers them, which +# is what gives the analysis query a `service` label to filter on. +--- +apiVersion: v1 +kind: Service +metadata: + name: nereus-api-active + labels: + app.kubernetes.io/name: nereus-api + app.kubernetes.io/part-of: nereus + app.kubernetes.io/component: active +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: nereus-api + ports: + - name: http + port: 8080 + targetPort: http +--- +apiVersion: v1 +kind: Service +metadata: + name: nereus-api-preview + labels: + app.kubernetes.io/name: nereus-api + app.kubernetes.io/part-of: nereus + app.kubernetes.io/component: preview +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: nereus-api + ports: + - name: http + port: 8080 + targetPort: http diff --git a/deploy/base/configmap.yaml b/deploy/base/configmap.yaml new file mode 100644 index 0000000..4e95227 --- /dev/null +++ b/deploy/base/configmap.yaml @@ -0,0 +1,18 @@ +# Non-secret API config. The DSN lives in the nereus-db Secret. +# +# Values must be quoted -- data is map[string]string, and an unquoted 8080 +# gets rejected as an integer. +apiVersion: v1 +kind: ConfigMap +metadata: + name: nereus-api-config + labels: + app.kubernetes.io/name: nereus-api + app.kubernetes.io/part-of: nereus +data: + PORT: "8080" + LOG_LEVEL: "info" + # main.go only enables tracing when this is set. + OTEL_EXPORTER_OTLP_ENDPOINT: "http://nereus-otel-collector.observability.svc.cluster.local:4317" + # Patched to a nonzero value on the preview to trigger the rollback demo. + CHAOS_ERROR_RATE: "0.0" diff --git a/deploy/base/ingress.yaml b/deploy/base/ingress.yaml new file mode 100644 index 0000000..07c22f0 --- /dev/null +++ b/deploy/base/ingress.yaml @@ -0,0 +1,40 @@ +# Traefik ingress. Reachable at http://localhost:8080 in the k3d lab. +# +# Active Service only -- the preview stays internal so real traffic can't hit +# an unpromoted version. /metrics is not exposed either. +# +# No host, so any Host header matches. A real hostname belongs in the prod +# overlay. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: nereus-api + labels: + app.kubernetes.io/name: nereus-api + app.kubernetes.io/part-of: nereus +spec: + ingressClassName: traefik + rules: + - http: + paths: + - path: /api/v1 + pathType: Prefix + backend: + service: + name: nereus-api-active + port: + name: http + - path: /healthz + pathType: Exact + backend: + service: + name: nereus-api-active + port: + name: http + - path: /readyz + pathType: Exact + backend: + service: + name: nereus-api-active + port: + name: http diff --git a/deploy/base/kustomization.yaml b/deploy/base/kustomization.yaml new file mode 100644 index 0000000..1d764ff --- /dev/null +++ b/deploy/base/kustomization.yaml @@ -0,0 +1,18 @@ +# Plain Kubernetes only -- no Rollouts CRDs, so this still applies on a +# cluster without Argo installed. +# +# The nereus-db Secret is not here; see deploy/secrets/README.md. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Sets the namespace on everything below, so the files don't repeat it. +namespace: nereus + +resources: + - namespace.yaml + - configmap.yaml + - postgres.yaml + - api-services.yaml + - servicemonitor.yaml + - loadgen.yaml + - ingress.yaml diff --git a/deploy/base/loadgen.yaml b/deploy/base/loadgen.yaml new file mode 100644 index 0000000..94780d7 --- /dev/null +++ b/deploy/base/loadgen.yaml @@ -0,0 +1,100 @@ +# Load generator. A plain Deployment -- nothing to promote, and it has to keep +# running straight through an API rollout. +# +# Points at the active Service so its traffic never lands on the preview. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nereus-loadgen + labels: + app.kubernetes.io/name: nereus-loadgen + app.kubernetes.io/part-of: nereus +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: nereus-loadgen + template: + metadata: + labels: + app.kubernetes.io/name: nereus-loadgen + app.kubernetes.io/part-of: nereus + spec: + imagePullSecrets: + - name: nereus-registry + securityContext: + # distroless nonroot. + runAsUser: 65532 + runAsGroup: 65532 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: loadgen + image: nereus-loadgen:0.1.0 + # Side-loaded with `k3d image import`; there's no registry. + imagePullPolicy: IfNotPresent + env: + - name: TARGET_URL + value: http://nereus-api-active.nereus.svc.cluster.local:8080 + - name: RPS + value: "5" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + memory: 128Mi +--- +# Keeps request samples flowing through the preview while analysis runs. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nereus-loadgen-preview + labels: + app.kubernetes.io/name: nereus-loadgen-preview + app.kubernetes.io/part-of: nereus +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: nereus-loadgen-preview + template: + metadata: + labels: + app.kubernetes.io/name: nereus-loadgen-preview + app.kubernetes.io/part-of: nereus + spec: + imagePullSecrets: + - name: nereus-registry + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: loadgen + image: nereus-loadgen:0.1.0 + imagePullPolicy: IfNotPresent + env: + - name: TARGET_URL + value: http://nereus-api-preview.nereus.svc.cluster.local:8080 + - name: RPS + value: "5" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + memory: 128Mi diff --git a/deploy/base/namespace.yaml b/deploy/base/namespace.yaml new file mode 100644 index 0000000..434efae --- /dev/null +++ b/deploy/base/namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: nereus + labels: + app.kubernetes.io/name: nereus + app.kubernetes.io/part-of: nereus diff --git a/deploy/base/postgres.yaml b/deploy/base/postgres.yaml new file mode 100644 index 0000000..e2b03da --- /dev/null +++ b/deploy/base/postgres.yaml @@ -0,0 +1,113 @@ +# Postgres 17, StatefulSet + headless Service. +# +# StatefulSet rather than Deployment for the stable pod name and a PVC that +# survives the pod. +--- +apiVersion: v1 +kind: Service +metadata: + name: nereus-postgres + labels: + app.kubernetes.io/name: nereus-postgres + app.kubernetes.io/part-of: nereus +spec: + clusterIP: None + selector: + app.kubernetes.io/name: nereus-postgres + ports: + - name: postgres + port: 5432 + targetPort: postgres +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nereus-postgres + labels: + app.kubernetes.io/name: nereus-postgres + app.kubernetes.io/part-of: nereus +spec: + serviceName: nereus-postgres + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: nereus-postgres + template: + metadata: + labels: + app.kubernetes.io/name: nereus-postgres + app.kubernetes.io/part-of: nereus + spec: + securityContext: + # postgres:17-alpine runs as uid 70. The Debian tags use 999. + runAsUser: 70 + runAsGroup: 70 + fsGroup: 70 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: postgres + image: postgres:17-alpine + ports: + - name: postgres + containerPort: 5432 + env: + - name: POSTGRES_DB + value: nereus + - name: POSTGRES_USER + value: nereus + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: nereus-db + key: POSTGRES_PASSWORD + # Subdirectory, or initdb trips over lost+found on the volume. + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + - name: runtime + mountPath: /var/run/postgresql + - name: tmp + mountPath: /tmp + readinessProbe: + exec: + command: ["pg_isready", "-U", "nereus", "-d", "nereus"] + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 6 + livenessProbe: + exec: + command: ["pg_isready", "-U", "nereus", "-d", "nereus"] + initialDelaySeconds: 30 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 1Gi + volumes: + - name: runtime + emptyDir: {} + - name: tmp + emptyDir: + sizeLimit: 64Mi + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 5Gi diff --git a/deploy/base/servicemonitor.yaml b/deploy/base/servicemonitor.yaml new file mode 100644 index 0000000..2b68dd8 --- /dev/null +++ b/deploy/base/servicemonitor.yaml @@ -0,0 +1,24 @@ +# Prometheus Operator CRD, not core Kubernetes. Works because the k3d values +# set serviceMonitorSelectorNilUsesHelmValues: false. +# +# Matches both Services, so each pod is scraped once per Service. That's the +# point -- the `service` label is how the analysis isolates the preview. +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: nereus-api + labels: + app.kubernetes.io/name: nereus-api + app.kubernetes.io/part-of: nereus +spec: + selector: + matchLabels: + app.kubernetes.io/name: nereus-api + namespaceSelector: + matchNames: + - nereus + endpoints: + - port: http + path: /metrics + interval: 15s + scrapeTimeout: 10s diff --git a/deploy/observability/kustomization.yaml b/deploy/observability/kustomization.yaml new file mode 100644 index 0000000..4c3bb12 --- /dev/null +++ b/deploy/observability/kustomization.yaml @@ -0,0 +1,9 @@ +# Lands in `observability`, alongside kube-prometheus-stack. +# kubectl apply -k deploy/observability +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: observability + +resources: + - otel-collector.yaml diff --git a/deploy/observability/otel-collector.yaml b/deploy/observability/otel-collector.yaml new file mode 100644 index 0000000..7504bba --- /dev/null +++ b/deploy/observability/otel-collector.yaml @@ -0,0 +1,109 @@ +# OTLP collector for the cluster. The API's OTEL_EXPORTER_OTLP_ENDPOINT +# points here. +# +# Traces only. The log pipeline in observability/otel-collector/config.yaml +# needs a DaemonSet with hostPath access to /var/log/pods plus RBAC, and Loki +# isn't in the cluster yet. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nereus-otel-collector + labels: + app.kubernetes.io/name: nereus-otel-collector +data: + config.yaml: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + + processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + batch: + timeout: 5s + + exporters: + otlp/tempo: + endpoint: tempo.observability.svc.cluster.local:4317 + tls: + insecure: true + # Keeps traces visible in `kubectl logs` until Tempo is deployed. + debug: + verbosity: basic + + service: + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/tempo, debug] +--- +apiVersion: v1 +kind: Service +metadata: + name: nereus-otel-collector + labels: + app.kubernetes.io/name: nereus-otel-collector +spec: + selector: + app.kubernetes.io/name: nereus-otel-collector + ports: + - name: otlp-grpc + port: 4317 + targetPort: otlp-grpc +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nereus-otel-collector + labels: + app.kubernetes.io/name: nereus-otel-collector +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: nereus-otel-collector + template: + metadata: + labels: + app.kubernetes.io/name: nereus-otel-collector + annotations: + # Restarts the pod when the config changes; the collector doesn't reload. + checksum/config: placeholder + spec: + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: otel-collector + image: otel/opentelemetry-collector-contrib:0.159.0 + args: ["--config=/conf/config.yaml"] + ports: + - name: otlp-grpc + containerPort: 4317 + volumeMounts: + - name: config + mountPath: /conf + readOnly: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 384Mi + volumes: + - name: config + configMap: + name: nereus-otel-collector diff --git a/deploy/overlays/dev/kustomization.yaml b/deploy/overlays/dev/kustomization.yaml new file mode 100644 index 0000000..164c8b8 --- /dev/null +++ b/deploy/overlays/dev/kustomization.yaml @@ -0,0 +1,30 @@ +# dev -- the local k3d lab. +# kubectl apply -k deploy/overlays/dev +# +# Needs the images imported and the nereus-db Secret created first. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: nereus + +resources: + - ../../base + - ../../rollouts + +patches: + # One replica is easier to watch swap over, and enough on a laptop. + - target: + kind: Rollout + name: nereus-api + patch: | + - op: replace + path: /spec/replicas + value: 1 + + - target: + kind: StatefulSet + name: nereus-postgres + patch: | + - op: replace + path: /spec/volumeClaimTemplates/0/spec/resources/requests/storage + value: 2Gi diff --git a/deploy/overlays/prod/kustomization.yaml b/deploy/overlays/prod/kustomization.yaml new file mode 100644 index 0000000..fdd2cf3 --- /dev/null +++ b/deploy/overlays/prod/kustomization.yaml @@ -0,0 +1,51 @@ +# prod overlay -- the two Fedora nodes. +# kubectl apply -k deploy/overlays/prod +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: nereus + +resources: + - ../../base + - ../../rollouts + +images: + - name: nereus-api + newName: git.fiwlabs.dev/fiwdev/nereus-api + newTag: 0.1.0 + - name: nereus-loadgen + newName: git.fiwlabs.dev/fiwdev/nereus-loadgen + newTag: 0.1.0 + +patches: + # 3 replicas so a node can go down mid-rollout. + - target: + kind: Rollout + name: nereus-api + patch: | + - op: replace + path: /spec/replicas + value: 3 + + # Spread the API across both nodes. + - target: + kind: Rollout + name: nereus-api + patch: | + - op: add + path: /spec/template/spec/topologySpreadConstraints + value: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: nereus-api + + - target: + kind: ConfigMap + name: nereus-api-config + patch: | + - op: replace + path: /data/LOG_LEVEL + value: "warn" diff --git a/deploy/rollouts/analysistemplate.yaml b/deploy/rollouts/analysistemplate.yaml new file mode 100644 index 0000000..1e3a9cf --- /dev/null +++ b/deploy/rollouts/analysistemplate.yaml @@ -0,0 +1,36 @@ +# The real error-rate gate, replacing the vector(0.0) harness in +# scripts/k3d/analysis-harness/. +# +# Runs before promotion. Every sample has to pass; one failure aborts and the +# active Service never moves. +apiVersion: argoproj.io/v1alpha1 +kind: AnalysisTemplate +metadata: + name: nereus-api-error-rate + labels: + app.kubernetes.io/name: nereus-api + app.kubernetes.io/part-of: nereus +spec: + args: + - name: service + metrics: + - name: error-rate + # Let the new pods take traffic first. + initialDelay: 30s + interval: 20s + count: 5 + failureLimit: 0 + # Empty result means no traffic yet -- not a failure. + successCondition: "len(result) == 0 || result[0] < 0.05" + provider: + prometheus: + address: http://kube-prometheus-stack-prometheus.observability.svc.cluster.local:9090 + # 5xx as a fraction of all responses on the preview. + # `or vector(0)` keeps the division defined when there are no errors. + query: | + ( + sum(rate(nereus_http_requests_total{service="{{args.service}}",status=~"5.."}[1m])) + or vector(0) + ) + / + sum(rate(nereus_http_requests_total{service="{{args.service}}"}[1m])) diff --git a/deploy/rollouts/kustomization.yaml b/deploy/rollouts/kustomization.yaml new file mode 100644 index 0000000..daf0db0 --- /dev/null +++ b/deploy/rollouts/kustomization.yaml @@ -0,0 +1,8 @@ +# Argo Rollouts resources, kept separate from base/ so the plain-Kubernetes +# manifests stay applicable on a cluster without the Rollouts CRDs installed. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - rollout.yaml + - analysistemplate.yaml diff --git a/deploy/rollouts/rollout.yaml b/deploy/rollouts/rollout.yaml new file mode 100644 index 0000000..d380bee --- /dev/null +++ b/deploy/rollouts/rollout.yaml @@ -0,0 +1,100 @@ +# The API, as a blue-green Rollout. +# +# Same shape as a Deployment, but the new version comes up alongside the old, +# gets checked against Prometheus, and only then takes over the active Service. +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: nereus-api + labels: + app.kubernetes.io/name: nereus-api + app.kubernetes.io/part-of: nereus +spec: + replicas: 2 + revisionHistoryLimit: 3 + selector: + matchLabels: + app.kubernetes.io/name: nereus-api + template: + metadata: + labels: + app.kubernetes.io/name: nereus-api + app.kubernetes.io/part-of: nereus + spec: + imagePullSecrets: + - name: nereus-registry + securityContext: + # distroless nonroot. + runAsUser: 65532 + runAsGroup: 65532 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: api + image: nereus-api:0.1.0 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 + envFrom: + - configMapRef: + name: nereus-api-config + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: nereus-db + key: DATABASE_URL + # /healthz ignores Postgres, so a database outage makes pods unready + # without restarting them. + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 3 + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + # /readyz waits for migrations. + readinessProbe: + httpGet: + path: /readyz + port: http + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + volumeMounts: + - name: tmp + mountPath: /tmp + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 256Mi + volumes: + - name: tmp + emptyDir: + medium: Memory + sizeLimit: 16Mi + strategy: + blueGreen: + activeService: nereus-api-active + previewService: nereus-api-preview + # Auto-promote only if the analysis passes; a failure aborts instead. + autoPromotionEnabled: true + # Keep the old ReplicaSet warm so an abort falls straight back. + scaleDownDelaySeconds: 30 + prePromotionAnalysis: + templates: + - templateName: nereus-api-error-rate + args: + - name: service + value: nereus-api-preview diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md new file mode 100644 index 0000000..0878768 --- /dev/null +++ b/deploy/secrets/README.md @@ -0,0 +1,35 @@ +# Secrets + +`nereus-db` holds two keys, used by `postgres.yaml` and `rollout.yaml`: + +- `POSTGRES_PASSWORD` +- `DATABASE_URL` — `postgres://nereus:@nereus-postgres:5432/nereus?sslmode=disable` + +It is not in `base/kustomization.yaml`. Plaintext never lands in the repo, and +gitleaks runs on every push. + +## dev (k3d) + +Create it directly, before `kubectl apply -k deploy/overlays/dev`: + +```fish +set pw (openssl rand -hex 16) +kubectl create secret generic nereus-db -n nereus \ + --from-literal=POSTGRES_PASSWORD=$pw \ + --from-literal=DATABASE_URL="postgres://nereus:$pw@nereus-postgres:5432/nereus?sslmode=disable" +``` + +Rotating means deleting the secret, the StatefulSet's PVC, and restarting — +Postgres only reads `POSTGRES_PASSWORD` when it initialises the data directory. + +## prod (sealed) + +Write the plaintext to `nereus-db.plain.yaml` (gitignored), then: + +```fish +kubeseal --format yaml < nereus-db.plain.yaml > nereus-db-sealed.yaml +``` + +Commit only `nereus-db-sealed.yaml` and add it to the prod overlay. The +controller isn't installed yet — `kubectl get crd | grep sealed` comes back +empty.