Compare commits
No commits in common. "12ac14f70917cb59a097fe4dd9c3dd9d690f9de5" and "a92af792585afa3560c1374818d3b72904ea031b" have entirely different histories.
12ac14f709
...
a92af79258
37 changed files with 42 additions and 1434 deletions
|
|
@ -18,7 +18,7 @@ jobs:
|
||||||
run: |
|
run: |
|
||||||
docker run --rm -v "$PWD:/repo" \
|
docker run --rm -v "$PWD:/repo" \
|
||||||
ghcr.io/gitleaks/gitleaks:v8.30.1 \
|
ghcr.io/gitleaks/gitleaks:v8.30.1 \
|
||||||
detect --source /repo --config /repo/.gitleaks.toml --redact -v
|
detect --source /repo --redact -v
|
||||||
|
|
||||||
- name: Test API
|
- name: Test API
|
||||||
run: |
|
run: |
|
||||||
|
|
@ -51,18 +51,6 @@ jobs:
|
||||||
docker run --rm -v "$PWD:/repo" -w /repo bash:5.3 \
|
docker run --rm -v "$PWD:/repo" -w /repo bash:5.3 \
|
||||||
bash -ec 'for file in scripts/k3d/lab.sh scripts/provision/bootstrap.sh scripts/provision/lab.sh; do bash -n "$file"; done'
|
bash -ec 'for file in scripts/k3d/lab.sh scripts/provision/bootstrap.sh scripts/provision/lab.sh; do bash -n "$file"; done'
|
||||||
|
|
||||||
- name: Validate Terraform
|
|
||||||
run: |
|
|
||||||
for stack in terraform/infra terraform/platform
|
|
||||||
do
|
|
||||||
docker run --rm -v "$PWD:/work" -w "/work/$stack" \
|
|
||||||
hashicorp/terraform:1.15.9 fmt -check
|
|
||||||
docker run --rm -v "$PWD:/work" -w "/work/$stack" \
|
|
||||||
hashicorp/terraform:1.15.9 init -backend=false -lockfile=readonly
|
|
||||||
docker run --rm -v "$PWD:/work" -w "/work/$stack" \
|
|
||||||
hashicorp/terraform:1.15.9 validate
|
|
||||||
done
|
|
||||||
|
|
||||||
- name: Validate Compose and Kubernetes configuration
|
- name: Validate Compose and Kubernetes configuration
|
||||||
env:
|
env:
|
||||||
CLUSTER_API_UPSTREAM: 192.0.2.1:8080
|
CLUSTER_API_UPSTREAM: 192.0.2.1:8080
|
||||||
|
|
@ -136,6 +124,9 @@ jobs:
|
||||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||||
needs: publish
|
needs: publish
|
||||||
runs-on: docker
|
runs-on: docker
|
||||||
|
env:
|
||||||
|
API_IMAGE: git.fiwlabs.dev/fiwdev/nereus-api:${{ github.sha }}
|
||||||
|
LOADGEN_IMAGE: git.fiwlabs.dev/fiwdev/nereus-loadgen:${{ github.sha }}
|
||||||
steps:
|
steps:
|
||||||
- name: Check out repository
|
- name: Check out repository
|
||||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
||||||
|
|
@ -169,26 +160,15 @@ jobs:
|
||||||
--from-file=.dockerconfigjson=/work/.ci/docker/config.json \
|
--from-file=.dockerconfigjson=/work/.ci/docker/config.json \
|
||||||
--dry-run=client -o yaml | kube apply -f -
|
--dry-run=client -o yaml | kube apply -f -
|
||||||
|
|
||||||
kube apply -k deploy/secrets
|
|
||||||
kube wait sealedsecret.bitnami.com/nereus-db --namespace nereus \
|
|
||||||
--for=condition=Synced --timeout=2m
|
|
||||||
kube get secret nereus-db --namespace nereus >/dev/null
|
kube get secret nereus-db --namespace nereus >/dev/null
|
||||||
kube apply -k deploy/observability
|
kube apply -k deploy/observability
|
||||||
|
kube apply -k deploy/overlays/prod
|
||||||
# Set the built tag before applying. Applying the overlay first would
|
kube patch rollout.argoproj.io nereus-api --namespace nereus --type json \
|
||||||
# roll out its pinned 0.1.0 tag and then roll out a second time on the
|
--patch "[{\"op\":\"replace\",\"path\":\"/spec/template/spec/containers/0/image\",\"value\":\"$API_IMAGE\"}]"
|
||||||
# patch, running the pre-promotion analysis against the wrong image.
|
kube set image deployment/nereus-loadgen --namespace nereus \
|
||||||
install -d .ci/release
|
"loadgen=$LOADGEN_IMAGE"
|
||||||
cat >.ci/release/kustomization.yaml <<'EOF'
|
kube set image deployment/nereus-loadgen-preview --namespace nereus \
|
||||||
resources:
|
"loadgen=$LOADGEN_IMAGE"
|
||||||
- ../../deploy/overlays/prod
|
|
||||||
images:
|
|
||||||
- name: git.fiwlabs.dev/fiwdev/nereus-api
|
|
||||||
newTag: ${{ github.sha }}
|
|
||||||
- name: git.fiwlabs.dev/fiwdev/nereus-loadgen
|
|
||||||
newTag: ${{ github.sha }}
|
|
||||||
EOF
|
|
||||||
kube apply -k .ci/release
|
|
||||||
|
|
||||||
kube rollout status deployment/nereus-loadgen --namespace nereus --timeout=5m
|
kube rollout status deployment/nereus-loadgen --namespace nereus --timeout=5m
|
||||||
kube rollout status deployment/nereus-loadgen-preview --namespace nereus --timeout=5m
|
kube rollout status deployment/nereus-loadgen-preview --namespace nereus --timeout=5m
|
||||||
|
|
@ -199,31 +179,3 @@ jobs:
|
||||||
- name: Remove cluster credentials
|
- name: Remove cluster credentials
|
||||||
if: always()
|
if: always()
|
||||||
run: rm -rf .ci
|
run: rm -rf .ci
|
||||||
|
|
||||||
mirror:
|
|
||||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main' && vars.MIRROR_ENABLED == 'true'
|
|
||||||
needs: deploy
|
|
||||||
runs-on: docker
|
|
||||||
steps:
|
|
||||||
- name: Check out repository
|
|
||||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
|
||||||
with:
|
|
||||||
fetch-depth: 0
|
|
||||||
|
|
||||||
- name: Push main to GitHub
|
|
||||||
env:
|
|
||||||
GITHUB_MIRROR_SSH_KEY: ${{ secrets.GITHUB_MIRROR_SSH_KEY }}
|
|
||||||
run: |
|
|
||||||
test -n "$GITHUB_MIRROR_SSH_KEY"
|
|
||||||
install -d -m 0700 .ci/mirror
|
|
||||||
printf '%s\n' "$GITHUB_MIRROR_SSH_KEY" >.ci/mirror/id_ed25519
|
|
||||||
chmod 0600 .ci/mirror/id_ed25519
|
|
||||||
cat >.ci/mirror/known_hosts <<'EOF'
|
|
||||||
github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
|
|
||||||
EOF
|
|
||||||
GIT_SSH_COMMAND="ssh -i $PWD/.ci/mirror/id_ed25519 -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$PWD/.ci/mirror/known_hosts" \
|
|
||||||
git push git@github.com:Fi3w0/Nereus.git HEAD:main
|
|
||||||
|
|
||||||
- name: Remove GitHub credentials
|
|
||||||
if: always()
|
|
||||||
run: rm -rf .ci
|
|
||||||
|
|
|
||||||
20
.github/workflows/ci.yml
vendored
20
.github/workflows/ci.yml
vendored
|
|
@ -38,7 +38,7 @@ jobs:
|
||||||
run: |
|
run: |
|
||||||
docker run --rm -v "$PWD:/repo" \
|
docker run --rm -v "$PWD:/repo" \
|
||||||
ghcr.io/gitleaks/gitleaks:v8.30.1 \
|
ghcr.io/gitleaks/gitleaks:v8.30.1 \
|
||||||
detect --source /repo --config /repo/.gitleaks.toml --redact -v
|
detect --source /repo --redact -v
|
||||||
|
|
||||||
go:
|
go:
|
||||||
name: vet, test, lint
|
name: vet, test, lint
|
||||||
|
|
@ -74,21 +74,3 @@ jobs:
|
||||||
if: steps.probe.outputs.ready == 'true'
|
if: steps.probe.outputs.ready == 'true'
|
||||||
with:
|
with:
|
||||||
working-directory: apps/api
|
working-directory: apps/api
|
||||||
|
|
||||||
terraform:
|
|
||||||
name: terraform
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v7
|
|
||||||
|
|
||||||
- name: format and validate
|
|
||||||
run: |
|
|
||||||
for stack in terraform/infra terraform/platform
|
|
||||||
do
|
|
||||||
docker run --rm -v "$PWD:/work" -w "/work/$stack" \
|
|
||||||
hashicorp/terraform:1.15.9 fmt -check
|
|
||||||
docker run --rm -v "$PWD:/work" -w "/work/$stack" \
|
|
||||||
hashicorp/terraform:1.15.9 init -backend=false -lockfile=readonly
|
|
||||||
docker run --rm -v "$PWD:/work" -w "/work/$stack" \
|
|
||||||
hashicorp/terraform:1.15.9 validate
|
|
||||||
done
|
|
||||||
|
|
|
||||||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -85,7 +85,6 @@ CLAUDE.local.md
|
||||||
*.tfstate.*
|
*.tfstate.*
|
||||||
*.tfvars
|
*.tfvars
|
||||||
!*.tfvars.example
|
!*.tfvars.example
|
||||||
terraform/**/inventory.yml
|
|
||||||
crash.log
|
crash.log
|
||||||
crash.*.log
|
crash.*.log
|
||||||
override.tf
|
override.tf
|
||||||
|
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
title = "Nereus gitleaks configuration"
|
|
||||||
|
|
||||||
[extend]
|
|
||||||
useDefault = true
|
|
||||||
|
|
||||||
[[allowlists]]
|
|
||||||
description = "Encrypted SealedSecret values are ciphertext, not plaintext credentials."
|
|
||||||
condition = "AND"
|
|
||||||
targetRules = ["generic-api-key"]
|
|
||||||
regexTarget = "line"
|
|
||||||
paths = ['''^deploy/secrets/[^/]+-sealed\.yaml$''']
|
|
||||||
regexes = ['''^\s+(POSTGRES_PASSWORD|DATABASE_URL):\s+Ag[A-Za-z0-9+/=]+\s*$''']
|
|
||||||
388
README.md
388
README.md
|
|
@ -1,388 +0,0 @@
|
||||||
# Nereus
|
|
||||||
|
|
||||||
Nereus is an ocean buoy telemetry API that exists so there is something real to
|
|
||||||
deploy. It stores buoys and their readings, exposes Prometheus metrics and
|
|
||||||
OpenTelemetry traces, and can be told to start failing on demand. The
|
|
||||||
application is deliberately small and boring. The project is everything wrapped
|
|
||||||
around it: custom images, a two-node k3s cluster, CI/CD on a self-hosted runner,
|
|
||||||
blue-green deploys that abort themselves when the new version is bad, and logs,
|
|
||||||
metrics and traces that actually join up.
|
|
||||||
|
|
||||||
This is my final DevOps project for Tokio School. The fictional company in the
|
|
||||||
brief is TechWave Solutions, which fits the water theme, so I ran with it.
|
|
||||||
|
|
||||||
**If you only run one thing, run the k3d lab.** It puts a real multi-node k3s
|
|
||||||
inside Docker and proves the automated rollback end to end on any OS with Docker
|
|
||||||
and 8 GB of RAM:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
scripts/k3d/lab.sh up
|
|
||||||
scripts/k3d/lab.sh prove
|
|
||||||
```
|
|
||||||
|
|
||||||
## What the API does
|
|
||||||
|
|
||||||
Go 1.26, `chi` for routing, `pgx/v5` against PostgreSQL 17, no ORM. Migrations
|
|
||||||
are plain SQL files applied on startup and written to be idempotent, so a pod
|
|
||||||
restart or a second replica does not break anything.
|
|
||||||
|
|
||||||
```
|
|
||||||
GET /healthz liveness, 200 whenever the process is up
|
|
||||||
GET /readyz readiness, 200 only once the database is reachable
|
|
||||||
GET /metrics Prometheus exposition
|
|
||||||
|
|
||||||
GET /api/v1/buoys list
|
|
||||||
POST /api/v1/buoys create
|
|
||||||
GET /api/v1/buoys/{id} fetch one
|
|
||||||
DELETE /api/v1/buoys/{id} delete
|
|
||||||
|
|
||||||
GET /api/v1/readings?buoy_id=&from=&to= list, paginated
|
|
||||||
POST /api/v1/readings create
|
|
||||||
GET /api/v1/readings/aggregate?window=1h deliberately expensive GROUP BY
|
|
||||||
```
|
|
||||||
|
|
||||||
The split between `/healthz` and `/readyz` is the part that matters for
|
|
||||||
Kubernetes. The service starts and serves `/healthz` even with the database
|
|
||||||
down, so a PostgreSQL outage makes pods unready instead of killing them in a
|
|
||||||
restart loop. `/api/v1/readings/aggregate` runs a real grouped query over a time
|
|
||||||
window and is not optimised on purpose, because the dashboards need something
|
|
||||||
that produces visible latency.
|
|
||||||
|
|
||||||
`CHAOS_ERROR_RATE` is a float between 0 and 1. Above zero, that fraction of
|
|
||||||
`/api/v1/*` requests return HTTP 500 with a JSON body. Health endpoints are never
|
|
||||||
affected. This is the entire mechanism for simulating a bad release, and it is
|
|
||||||
controlled by an environment variable rather than an admin endpoint so that
|
|
||||||
turning it on is a deployment, not a runtime toggle.
|
|
||||||
|
|
||||||
`apps/loadgen/` is a small Go binary that hits the API continuously with a
|
|
||||||
weighted mix of reads, writes and aggregates. It is not decoration. The rollback
|
|
||||||
analysis computes an error rate from Prometheus, and with no traffic there are
|
|
||||||
no samples and the analysis has nothing to decide on.
|
|
||||||
|
|
||||||
## Running it
|
|
||||||
|
|
||||||
There are three levels, and they exist because nobody grading this is going to
|
|
||||||
install KVM and download a Fedora image.
|
|
||||||
|
|
||||||
### Level 1: Docker Compose, about three minutes
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose up --build
|
|
||||||
```
|
|
||||||
|
|
||||||
API, PostgreSQL and an OTel Collector, with the API on `127.0.0.1:18080` and the
|
|
||||||
load generator already producing traffic. Add the observability overlay for
|
|
||||||
Grafana on `127.0.0.1:13000` and Prometheus on `127.0.0.1:19090`:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose -f compose.yaml -f observability/compose.yaml up -d
|
|
||||||
```
|
|
||||||
|
|
||||||
### Level 2: k3d, real Kubernetes, any OS with Docker
|
|
||||||
|
|
||||||
```bash
|
|
||||||
scripts/k3d/lab.sh up # one server, two agents, Argo Rollouts, kube-prometheus-stack
|
|
||||||
scripts/k3d/lab.sh prove # promote a healthy version, then abort a bad one
|
|
||||||
scripts/k3d/lab.sh destroy
|
|
||||||
```
|
|
||||||
|
|
||||||
`prove` is the interesting one. It drives a blue-green Rollout through a real
|
|
||||||
AnalysisRun that queries a real Prometheus over the network, first with a query
|
|
||||||
that returns a healthy value and then with one that returns a failing value, and
|
|
||||||
it fails loudly if the bad revision ever reaches the active Service.
|
|
||||||
|
|
||||||
Be clear about what is faked here: the harness in
|
|
||||||
`scripts/k3d/analysis-harness/` uses a throwaway nginx Rollout and a hardcoded
|
|
||||||
metric value. Argo Rollouts, the AnalysisRun, the Prometheus query and the abort
|
|
||||||
are all real. Only the number is synthetic. The real gate lives in
|
|
||||||
`deploy/rollouts/analysistemplate.yaml` and ran against the real API on the
|
|
||||||
Fedora cluster.
|
|
||||||
|
|
||||||
### Level 3: two Fedora 44 machines
|
|
||||||
|
|
||||||
This is my actual cluster. `scripts/provision/` holds an Ansible playbook that
|
|
||||||
turns two already-installed Fedora 44 hosts into a k3s server and agent:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cp scripts/provision/inventory.example.yml scripts/provision/inventory.yml
|
|
||||||
# edit addresses, CIDRs and SSH user
|
|
||||||
K3S_TOKEN="$(openssl rand -hex 32)" scripts/provision/bootstrap.sh scripts/provision/inventory.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
The playbook is idempotent and reruns clean. The token is passed through the
|
|
||||||
process environment and never written to the inventory or the repository.
|
|
||||||
|
|
||||||
`scripts/provision/lab.sh` builds the same two hosts as local QEMU VMs from the
|
|
||||||
Fedora 44 cloud image, with 2 vCPUs, 3 GiB of RAM and a 30 GiB thin disk each.
|
|
||||||
Those numbers are the lowest I actually tested, not a recommendation. VM state
|
|
||||||
and its SSH key live under `~/.local/state/nereus-lab`, outside the repo.
|
|
||||||
|
|
||||||
## How a bad deploy gets killed
|
|
||||||
|
|
||||||
This is the part I would defend first, so it is worth spelling out.
|
|
||||||
|
|
||||||
The API is an Argo Rollouts `Rollout`, not a Deployment. New pods come up
|
|
||||||
alongside the old ones and take no production traffic. `nereus-api-active` points
|
|
||||||
at the stable version and `nereus-api-preview` at the candidate. A second load
|
|
||||||
generator drives the preview Service, because an analysis that measures nothing
|
|
||||||
would treat "no samples" as success.
|
|
||||||
|
|
||||||
Before promotion, a `prePromotionAnalysis` runs this query against Prometheus:
|
|
||||||
|
|
||||||
```promql
|
|
||||||
(
|
|
||||||
sum(rate(nereus_http_requests_total{service="nereus-api-preview",status=~"5.."}[1m]))
|
|
||||||
or vector(0)
|
|
||||||
)
|
|
||||||
/
|
|
||||||
sum(rate(nereus_http_requests_total{service="nereus-api-preview"}[1m]))
|
|
||||||
```
|
|
||||||
|
|
||||||
Five samples, 20 seconds apart, after a 30 second delay to let the new pods take
|
|
||||||
traffic. The success condition is `len(result) == 0 || result[0] < 0.05`, and
|
|
||||||
`failureLimit: 0` means a single bad sample aborts. The `or vector(0)` keeps the
|
|
||||||
division defined when there are no errors at all, which is otherwise an empty
|
|
||||||
result rather than a zero.
|
|
||||||
|
|
||||||
When it aborts, the active Service selector never moves. `scaleDownDelaySeconds:
|
|
||||||
30` keeps the old ReplicaSet warm so the fallback is instant rather than a cold
|
|
||||||
start. From `docs/evidence.md`, the two AnalysisRuns from the mechanism proof:
|
|
||||||
|
|
||||||
```text
|
|
||||||
probe-746bbb94df-2-pre Successful vector(0.0)
|
|
||||||
probe-556d5b659b-3-pre Failed vector(1.0)
|
|
||||||
```
|
|
||||||
|
|
||||||
To trigger it for real, deploy a version with `CHAOS_ERROR_RATE` above 0.05 and
|
|
||||||
watch it refuse to promote.
|
|
||||||
|
|
||||||
## Architecture
|
|
||||||
|
|
||||||
```
|
|
||||||
Workstation (CachyOS)
|
|
||||||
└── libvirt/KVM
|
|
||||||
├── nereus-node1 Fedora 44, k3s server
|
|
||||||
└── nereus-node2 Fedora 44, k3s agent
|
|
||||||
|
|
||||||
Mini PC (always on)
|
|
||||||
├── Forgejo at git.fiwlabs.dev, behind Traefik with automatic TLS
|
|
||||||
├── Forgejo Runner, docker mode, same LAN as the cluster
|
|
||||||
├── Container registry, part of Forgejo
|
|
||||||
└── Static dashboard at nereus.fiwlabs.dev, nginx behind Traefik
|
|
||||||
|
|
||||||
Mirror to GitHub, which is the link I hand in.
|
|
||||||
```
|
|
||||||
|
|
||||||
Inside the cluster:
|
|
||||||
|
|
||||||
```
|
|
||||||
Traefik (ships with k3s)
|
|
||||||
└── Ingress → nereus-api-active
|
|
||||||
├── stable ReplicaSet (real traffic)
|
|
||||||
└── preview ReplicaSet (candidate, no traffic)
|
|
||||||
|
|
||||||
PostgreSQL StatefulSet on a local-path PVC
|
|
||||||
loadgen one Deployment against active, one against preview
|
|
||||||
|
|
||||||
observability/
|
|
||||||
kube-prometheus-stack Prometheus, Grafana, node-exporter, kube-state-metrics
|
|
||||||
OTel Collector Deployment, receives OTLP traces, exports to Tempo
|
|
||||||
OTel Collector DaemonSet, tails pod logs, exports to Loki
|
|
||||||
Tempo traces
|
|
||||||
Loki logs
|
|
||||||
```
|
|
||||||
|
|
||||||
The web dashboard runs on the Mini PC, not in the cluster, and that is
|
|
||||||
deliberate. The cluster is off most of the time. If the page lived inside it,
|
|
||||||
the domain would be dead 95% of the time and there would be nothing to show.
|
|
||||||
Traefik on the Mini PC serves the static files at `/` and proxies `/api/v1/*`,
|
|
||||||
`/healthz` and `/readyz` through to the cluster, so everything is same-origin
|
|
||||||
with no CORS and no hardcoded hostnames. With the cluster off, those paths return
|
|
||||||
502 and the page renders the last known values greyed out and keeps polling. An
|
|
||||||
unreachable API is a normal state for that page, not an error.
|
|
||||||
|
|
||||||
## Why I built it this way
|
|
||||||
|
|
||||||
**distroless, not alpine.** The final image is
|
|
||||||
`gcr.io/distroless/static:nonroot` with a statically linked binary copied in and
|
|
||||||
nothing else. No shell, no package manager, no busybox. If someone gets code
|
|
||||||
execution there is nothing to pivot with. It also happens to be tiny: the API
|
|
||||||
image is under 8 MB, with the exact figure recorded in `docs/evidence.md`.
|
|
||||||
|
|
||||||
**k3s, not full Kubernetes.** Same API, one binary, Traefik and a local-path
|
|
||||||
provisioner included, and it runs on two 3 GiB VMs. Nothing in this project uses
|
|
||||||
an API that k3s does not have.
|
|
||||||
|
|
||||||
**On-premise, not EKS or AKS.** The brief asks for a managed cloud cluster. I
|
|
||||||
did not do that, and the reason is cost and control rather than difficulty: I
|
|
||||||
have hardware sitting here and no interest in paying for a load balancer to prove
|
|
||||||
a point about Terraform. The manifests are plain Kubernetes and would apply to a
|
|
||||||
managed cluster unchanged. This is the deviation from the brief I expect to be
|
|
||||||
asked about, so it is stated here rather than buried.
|
|
||||||
|
|
||||||
**Forgejo Actions, not GitHub Actions.** The brief allows the tool of your
|
|
||||||
choice. The runner lives on the same LAN as the cluster, which means the deploy
|
|
||||||
job can reach the Kubernetes API without exposing it to the internet or
|
|
||||||
punching a hole through a firewall for a hosted runner. The registry is part of
|
|
||||||
the same Forgejo instance, so images never leave the network either. The GitHub
|
|
||||||
mirror still runs its own workflow so the repo a reviewer opens shows green
|
|
||||||
checks, and that workflow consumes no secrets by design.
|
|
||||||
|
|
||||||
**The cluster is ephemeral, on purpose.** I bring it up, record, and shut it
|
|
||||||
down. That is not a gap in the project, it is the evidence that the
|
|
||||||
infrastructure as code works. If the whole thing rebuilds from the repository
|
|
||||||
with a command, then the repository really is the source of truth.
|
|
||||||
|
|
||||||
**Everything runs non-root with a read-only root filesystem**, including the
|
|
||||||
pieces where that is inconvenient. The log collector was the awkward one. k3s
|
|
||||||
writes `/var/log/pods` as `0750 root:root` and each container log as `0640
|
|
||||||
root:root`, so a collector running as UID 10001 cannot even traverse the
|
|
||||||
directory. Adding the `DAC_READ_SEARCH` capability looks like the fix and does
|
|
||||||
nothing, because Kubernetes sets no ambient capabilities and the capability is
|
|
||||||
cleared on exec for a non-root user. The container ended up with
|
|
||||||
`CapEff: 0000000000000000`, matched no files, and reported no error at all. The
|
|
||||||
fix is `supplementalGroups: [0]`, since group root already has read on those
|
|
||||||
files. Non-root, no capabilities, read-only mount.
|
|
||||||
|
|
||||||
## Secrets and security
|
|
||||||
|
|
||||||
No secret is committed, and none is read by any tooling that logs. Gitleaks
|
|
||||||
scans both the working tree and the committed history on every push and a hit
|
|
||||||
fails the build. Trivy scans the release configuration and the built images for
|
|
||||||
HIGH and CRITICAL findings.
|
|
||||||
|
|
||||||
The `nereus-db` Secret holds the PostgreSQL password and DSN. Production gets
|
|
||||||
it from the encrypted `deploy/secrets/nereus-db-sealed.yaml`; the Sealed Secrets
|
|
||||||
controller is the only component that can decrypt it. The current Fedora
|
|
||||||
cluster was reset onto a fresh PVC and verified with that committed resource.
|
|
||||||
Development can still create an ephemeral Secret directly. See
|
|
||||||
`deploy/secrets/README.md` for both workflows.
|
|
||||||
|
|
||||||
The CI pipeline never uses a cluster-admin kubeconfig. It authenticates as a
|
|
||||||
`deployer` ServiceAccount with RBAC scoped to the `nereus` namespace, supplied
|
|
||||||
as an encrypted `KUBECONFIG_B64` repository secret, written to the workspace with
|
|
||||||
mode `0600` and deleted in an `always()` step. The registry pull secret is
|
|
||||||
created through a pipe so the generated manifest is never logged or written to
|
|
||||||
disk.
|
|
||||||
|
|
||||||
Containers drop all capabilities, disallow privilege escalation, run with a
|
|
||||||
read-only root filesystem and a `RuntimeDefault` seccomp profile, and mount
|
|
||||||
`tmpfs` where a writable path is genuinely needed. SELinux stays enforcing on
|
|
||||||
both Fedora nodes.
|
|
||||||
|
|
||||||
## Observability
|
|
||||||
|
|
||||||
Metrics, logs and traces come from the same request and can be walked between.
|
|
||||||
|
|
||||||
The API exports five metrics with fixed names, since the dashboards and the
|
|
||||||
rollback query depend on them literally:
|
|
||||||
|
|
||||||
```
|
|
||||||
nereus_http_requests_total{method, path, status, version} counter
|
|
||||||
nereus_http_request_duration_seconds{method, path, version} histogram
|
|
||||||
nereus_db_query_duration_seconds{operation} histogram
|
|
||||||
nereus_readings_ingested_total counter
|
|
||||||
nereus_buoys_active gauge
|
|
||||||
```
|
|
||||||
|
|
||||||
`path` is always the chi route template, `/api/v1/buoys/{id}` rather than the
|
|
||||||
resolved path, because resolved paths give unbounded cardinality and will
|
|
||||||
eventually take Prometheus down.
|
|
||||||
|
|
||||||
Traces go out over OTLP/gRPC to the collector Deployment and on to Tempo. Every
|
|
||||||
handler is a span and every database query is a child span. If
|
|
||||||
`OTEL_EXPORTER_OTLP_ENDPOINT` is unset, tracing is a no-op, so a missing
|
|
||||||
collector can never stop the service from starting.
|
|
||||||
|
|
||||||
Logs are structured JSON on stdout via `log/slog`. A DaemonSet collector tails
|
|
||||||
`/var/log/pods` on each node and ships to Loki. Every request logs one line
|
|
||||||
carrying its `trace_id`, and Grafana's Loki data source has a derived field that
|
|
||||||
turns that value into a link into Tempo, with the reverse link configured on the
|
|
||||||
Tempo side. A verified example, log line and the trace it resolves to, is in
|
|
||||||
`docs/evidence.md`.
|
|
||||||
|
|
||||||
Alert rules for error rate, latency, readiness failures and scrape failure are in
|
|
||||||
`observability/alerts/nereus.yaml`. cAdvisor and node-exporter metrics arrive
|
|
||||||
with kube-prometheus-stack rather than being installed separately.
|
|
||||||
|
|
||||||
## CI/CD
|
|
||||||
|
|
||||||
`.forgejo/workflows/pipeline.yml` runs on pushes and pull requests to `main`.
|
|
||||||
|
|
||||||
Verification runs the Go tests with the race detector, `go vet`, a pinned
|
|
||||||
`golangci-lint`, gitleaks over history and tree, a kustomize build of every
|
|
||||||
overlay, and a Trivy config scan. Everything runs as a plain container image
|
|
||||||
rather than a marketplace action, because Forgejo resolves actions from
|
|
||||||
`code.forgejo.org` and most third-party ones simply are not mirrored there.
|
|
||||||
|
|
||||||
A push to `main` then builds the three images, tags them with the immutable
|
|
||||||
commit SHA, scans them, pushes them to the Forgejo registry, and deploys the
|
|
||||||
production overlay. The deploy job waits on the Rollout, which means a failed
|
|
||||||
pre-promotion analysis fails the pipeline.
|
|
||||||
|
|
||||||
## Where each requirement from the brief lives
|
|
||||||
|
|
||||||
| Requirement | Where |
|
|
||||||
|---|---|
|
|
||||||
| Custom, optimised Docker images | `build/*.Dockerfile`, multi-stage into distroless, API under 8 MB |
|
|
||||||
| Docker Compose | `compose.yaml`, plus `observability/compose.yaml` and `build/compose.mini.yaml` |
|
|
||||||
| Kubernetes orchestration | k3s, two Fedora 44 nodes; k3d for a portable equivalent |
|
|
||||||
| Deployments, Services, Ingress, ConfigMaps, Secrets | `deploy/base/`, composed with kustomize overlays |
|
|
||||||
| Infrastructure as code | `terraform/infra/` for libvirt machines, `terraform/platform/` for cluster controllers, and `scripts/provision/` for host configuration |
|
|
||||||
| CI/CD pipeline | `.forgejo/workflows/pipeline.yml`, self-hosted runner |
|
|
||||||
| Container registry | Forgejo registry at `git.fiwlabs.dev`, TLS from Traefik |
|
|
||||||
| Secure credential handling | Gitleaks, Trivy, namespace-scoped `deployer` kubeconfig, and a committed encrypted `nereus-db` SealedSecret |
|
|
||||||
| Blue-green deployment | Argo Rollouts, `deploy/rollouts/rollout.yaml` |
|
|
||||||
| Automated rollback | `deploy/rollouts/analysistemplate.yaml`, Prometheus-driven abort |
|
|
||||||
| OpenTelemetry Collector | `deploy/observability/otel-collector.yaml` for traces, `otel-log-collector.yaml` for logs |
|
|
||||||
| Prometheus | kube-prometheus-stack, scraped through a ServiceMonitor |
|
|
||||||
| Grafana dashboards | `observability/dashboards/`, data sources in `deploy/observability/grafana-datasources.yaml` |
|
|
||||||
| Loki | `deploy/observability/loki.yaml` |
|
|
||||||
| cAdvisor and node-exporter | included with kube-prometheus-stack |
|
|
||||||
| Alerting | `observability/alerts/nereus.yaml` |
|
|
||||||
|
|
||||||
## What is not done
|
|
||||||
|
|
||||||
Stated plainly so it does not have to be discovered.
|
|
||||||
|
|
||||||
Discord routing is configured but disabled until an owner-provided webhook is
|
|
||||||
sealed. The Sealed Secrets controller key also needs an off-repository backup;
|
|
||||||
without it, a rebuilt cluster cannot decrypt the committed database resource.
|
|
||||||
|
|
||||||
The remaining acceptance gap is environmental: the reproducible fresh-VM path
|
|
||||||
has not been rerun on a second workstation with no dependencies installed.
|
|
||||||
After final sign-off, the local QEMU lab and packages installed only for it can
|
|
||||||
be removed.
|
|
||||||
|
|
||||||
## Fedora traps worth knowing
|
|
||||||
|
|
||||||
Three things broke the cluster in ways that took longer to find than to fix, all
|
|
||||||
of them handled by the playbook now.
|
|
||||||
|
|
||||||
| Symptom | Cause | Fix |
|
|
||||||
|---|---|---|
|
|
||||||
| Pods on different nodes cannot reach each other, DNS fails strangely | firewalld blocks VXLAN | open `8472/udp` between nodes |
|
|
||||||
| kubelet fails with permission errors | SELinux enforcing without the policy | `dnf install k3s-selinux` |
|
|
||||||
| k3s complains at startup | zram swap enabled | `systemctl disable --now zram-generator` |
|
|
||||||
|
|
||||||
## Repo layout
|
|
||||||
|
|
||||||
```
|
|
||||||
apps/api/ Go service
|
|
||||||
apps/loadgen/ traffic generator
|
|
||||||
apps/web/ static dashboard, runs on the Mini PC
|
|
||||||
build/ Dockerfiles and the Mini PC compose file
|
|
||||||
deploy/base/ kustomize base
|
|
||||||
deploy/overlays/ dev and prod
|
|
||||||
deploy/rollouts/ Rollout and AnalysisTemplate
|
|
||||||
deploy/observability/ Loki, Tempo, collectors, Grafana data sources
|
|
||||||
observability/ dashboards, alert rules, collector configs, local compose
|
|
||||||
scripts/k3d/ portable rollback lab
|
|
||||||
scripts/provision/ Ansible roles, bootstrap, QEMU lab
|
|
||||||
terraform/ libvirt machines, Ansible inventory and cluster-wide Helm releases
|
|
||||||
docs/ roadmap, evidence, CI/CD and Mini PC notes
|
|
||||||
```
|
|
||||||
|
|
||||||
`AGENTS.md` holds the rules the AI agents working in this repo had to follow.
|
|
||||||
`PLAN.md` is my own working plan, in Spanish, and it is a worklog rather than
|
|
||||||
documentation.
|
|
||||||
|
|
@ -1,35 +0,0 @@
|
||||||
# kube-prometheus-stack's Grafana sidecar loads labeled data sources.
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: nereus-grafana-datasources
|
|
||||||
labels:
|
|
||||||
grafana_datasource: "1"
|
|
||||||
data:
|
|
||||||
nereus.yaml: |
|
|
||||||
apiVersion: 1
|
|
||||||
datasources:
|
|
||||||
- name: Loki
|
|
||||||
uid: loki
|
|
||||||
type: loki
|
|
||||||
access: proxy
|
|
||||||
url: http://loki.observability.svc.cluster.local:3100
|
|
||||||
editable: false
|
|
||||||
jsonData:
|
|
||||||
derivedFields:
|
|
||||||
- name: TraceID
|
|
||||||
matcherRegex: '"trace_id":"([a-f0-9]{32})"'
|
|
||||||
datasourceUid: tempo
|
|
||||||
url: '$${__value.raw}'
|
|
||||||
- name: Tempo
|
|
||||||
uid: tempo
|
|
||||||
type: tempo
|
|
||||||
access: proxy
|
|
||||||
url: http://tempo.observability.svc.cluster.local:3200
|
|
||||||
editable: false
|
|
||||||
jsonData:
|
|
||||||
tracesToLogsV2:
|
|
||||||
datasourceUid: loki
|
|
||||||
spanStartTimeShift: -1m
|
|
||||||
spanEndTimeShift: 1m
|
|
||||||
filterByTraceID: true
|
|
||||||
|
|
@ -6,8 +6,4 @@ kind: Kustomization
|
||||||
namespace: observability
|
namespace: observability
|
||||||
|
|
||||||
resources:
|
resources:
|
||||||
- grafana-datasources.yaml
|
|
||||||
- loki.yaml
|
|
||||||
- otel-collector.yaml
|
- otel-collector.yaml
|
||||||
- otel-log-collector.yaml
|
|
||||||
- tempo.yaml
|
|
||||||
|
|
|
||||||
|
|
@ -1,113 +0,0 @@
|
||||||
# Single-process Loki for the two-node project cluster.
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: loki
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: loki
|
|
||||||
data:
|
|
||||||
loki.yaml: |
|
|
||||||
auth_enabled: false
|
|
||||||
|
|
||||||
server:
|
|
||||||
http_listen_port: 3100
|
|
||||||
|
|
||||||
common:
|
|
||||||
path_prefix: /var/loki
|
|
||||||
replication_factor: 1
|
|
||||||
ring:
|
|
||||||
kvstore:
|
|
||||||
store: inmemory
|
|
||||||
|
|
||||||
schema_config:
|
|
||||||
configs:
|
|
||||||
- from: 2024-01-01
|
|
||||||
store: tsdb
|
|
||||||
object_store: filesystem
|
|
||||||
schema: v13
|
|
||||||
index:
|
|
||||||
prefix: index_
|
|
||||||
period: 24h
|
|
||||||
|
|
||||||
storage_config:
|
|
||||||
filesystem:
|
|
||||||
directory: /var/loki/chunks
|
|
||||||
|
|
||||||
analytics:
|
|
||||||
reporting_enabled: false
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: loki
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: loki
|
|
||||||
spec:
|
|
||||||
selector:
|
|
||||||
app.kubernetes.io/name: loki
|
|
||||||
ports:
|
|
||||||
- name: http
|
|
||||||
port: 3100
|
|
||||||
targetPort: http
|
|
||||||
---
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: loki
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: loki
|
|
||||||
spec:
|
|
||||||
replicas: 1
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
app.kubernetes.io/name: loki
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: loki
|
|
||||||
spec:
|
|
||||||
automountServiceAccountToken: false
|
|
||||||
securityContext:
|
|
||||||
runAsUser: 10001
|
|
||||||
runAsGroup: 10001
|
|
||||||
runAsNonRoot: true
|
|
||||||
fsGroup: 10001
|
|
||||||
seccompProfile:
|
|
||||||
type: RuntimeDefault
|
|
||||||
containers:
|
|
||||||
- name: loki
|
|
||||||
image: grafana/loki:3.7.6
|
|
||||||
args: ["-config.file=/etc/loki/loki.yaml"]
|
|
||||||
ports:
|
|
||||||
- name: http
|
|
||||||
containerPort: 3100
|
|
||||||
readinessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /ready
|
|
||||||
port: http
|
|
||||||
initialDelaySeconds: 5
|
|
||||||
periodSeconds: 5
|
|
||||||
volumeMounts:
|
|
||||||
- name: config
|
|
||||||
mountPath: /etc/loki
|
|
||||||
readOnly: true
|
|
||||||
- name: data
|
|
||||||
mountPath: /var/loki
|
|
||||||
securityContext:
|
|
||||||
allowPrivilegeEscalation: false
|
|
||||||
capabilities:
|
|
||||||
drop: ["ALL"]
|
|
||||||
readOnlyRootFilesystem: true
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 25m
|
|
||||||
memory: 128Mi
|
|
||||||
limits:
|
|
||||||
memory: 384Mi
|
|
||||||
volumes:
|
|
||||||
- name: config
|
|
||||||
configMap:
|
|
||||||
name: loki
|
|
||||||
- name: data
|
|
||||||
emptyDir: {}
|
|
||||||
|
|
@ -1,8 +1,9 @@
|
||||||
# OTLP collector for the cluster. The API's OTEL_EXPORTER_OTLP_ENDPOINT
|
# OTLP collector for the cluster. The API's OTEL_EXPORTER_OTLP_ENDPOINT
|
||||||
# points here.
|
# points here.
|
||||||
#
|
#
|
||||||
# Pod logs use a separate node collector because this Deployment does not have
|
# Traces only. The log pipeline in observability/otel-collector/config.yaml
|
||||||
# access to the hosts' /var/log/pods directories.
|
# needs a DaemonSet with hostPath access to /var/log/pods plus RBAC, and Loki
|
||||||
|
# isn't in the cluster yet.
|
||||||
---
|
---
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: ConfigMap
|
kind: ConfigMap
|
||||||
|
|
|
||||||
|
|
@ -1,98 +0,0 @@
|
||||||
# One collector per node tails only Nereus container logs.
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: nereus-otel-log-collector
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: nereus-otel-log-collector
|
|
||||||
data:
|
|
||||||
config.yaml: |
|
|
||||||
receivers:
|
|
||||||
filelog/nereus:
|
|
||||||
include:
|
|
||||||
- /var/log/pods/nereus_*/*/*.log
|
|
||||||
include_file_path: true
|
|
||||||
start_at: beginning
|
|
||||||
operators:
|
|
||||||
- type: container
|
|
||||||
id: parse-container-log
|
|
||||||
|
|
||||||
processors:
|
|
||||||
memory_limiter:
|
|
||||||
check_interval: 1s
|
|
||||||
limit_mib: 96
|
|
||||||
batch:
|
|
||||||
timeout: 5s
|
|
||||||
# Loki's internal gRPC push limit is 4 MiB. start_at: beginning replays
|
|
||||||
# whole files after a restart, so an uncapped batch exceeds it and the
|
|
||||||
# exporter retries the same oversized request forever.
|
|
||||||
send_batch_size: 1000
|
|
||||||
send_batch_max_size: 2000
|
|
||||||
|
|
||||||
exporters:
|
|
||||||
otlphttp/loki:
|
|
||||||
endpoint: http://loki.observability.svc.cluster.local:3100/otlp
|
|
||||||
|
|
||||||
service:
|
|
||||||
pipelines:
|
|
||||||
logs:
|
|
||||||
receivers: [filelog/nereus]
|
|
||||||
processors: [memory_limiter, batch]
|
|
||||||
exporters: [otlphttp/loki]
|
|
||||||
---
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: DaemonSet
|
|
||||||
metadata:
|
|
||||||
name: nereus-otel-log-collector
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: nereus-otel-log-collector
|
|
||||||
spec:
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
app.kubernetes.io/name: nereus-otel-log-collector
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: nereus-otel-log-collector
|
|
||||||
spec:
|
|
||||||
automountServiceAccountToken: false
|
|
||||||
securityContext:
|
|
||||||
runAsUser: 10001
|
|
||||||
runAsGroup: 10001
|
|
||||||
runAsNonRoot: true
|
|
||||||
# k3s: /var/log/pods is 0750 root:root, container logs 0640 root:root.
|
|
||||||
# Group root has r-x / r--, so gid 0 is enough to tail them as non-root.
|
|
||||||
supplementalGroups: [0]
|
|
||||||
seccompProfile:
|
|
||||||
type: RuntimeDefault
|
|
||||||
containers:
|
|
||||||
- name: otel-collector
|
|
||||||
image: otel/opentelemetry-collector-contrib:0.159.0
|
|
||||||
args: ["--config=/conf/config.yaml"]
|
|
||||||
volumeMounts:
|
|
||||||
- name: config
|
|
||||||
mountPath: /conf
|
|
||||||
readOnly: true
|
|
||||||
- name: pod-logs
|
|
||||||
mountPath: /var/log/pods
|
|
||||||
readOnly: true
|
|
||||||
securityContext:
|
|
||||||
allowPrivilegeEscalation: false
|
|
||||||
capabilities:
|
|
||||||
drop: ["ALL"]
|
|
||||||
readOnlyRootFilesystem: true
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 20m
|
|
||||||
memory: 64Mi
|
|
||||||
limits:
|
|
||||||
memory: 128Mi
|
|
||||||
volumes:
|
|
||||||
- name: config
|
|
||||||
configMap:
|
|
||||||
name: nereus-otel-log-collector
|
|
||||||
- name: pod-logs
|
|
||||||
hostPath:
|
|
||||||
path: /var/log/pods
|
|
||||||
type: Directory
|
|
||||||
|
|
@ -1,107 +0,0 @@
|
||||||
# Single-process Tempo with local storage for short-lived lab traces.
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: tempo
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: tempo
|
|
||||||
data:
|
|
||||||
tempo.yaml: |
|
|
||||||
server:
|
|
||||||
http_listen_port: 3200
|
|
||||||
|
|
||||||
distributor:
|
|
||||||
receivers:
|
|
||||||
otlp:
|
|
||||||
protocols:
|
|
||||||
grpc:
|
|
||||||
endpoint: 0.0.0.0:4317
|
|
||||||
|
|
||||||
storage:
|
|
||||||
trace:
|
|
||||||
backend: local
|
|
||||||
wal:
|
|
||||||
path: /var/tempo/wal
|
|
||||||
local:
|
|
||||||
path: /var/tempo/traces
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: tempo
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: tempo
|
|
||||||
spec:
|
|
||||||
selector:
|
|
||||||
app.kubernetes.io/name: tempo
|
|
||||||
ports:
|
|
||||||
- name: http
|
|
||||||
port: 3200
|
|
||||||
targetPort: http
|
|
||||||
- name: otlp-grpc
|
|
||||||
port: 4317
|
|
||||||
targetPort: otlp-grpc
|
|
||||||
---
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: tempo
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: tempo
|
|
||||||
spec:
|
|
||||||
replicas: 1
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
app.kubernetes.io/name: tempo
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: tempo
|
|
||||||
spec:
|
|
||||||
automountServiceAccountToken: false
|
|
||||||
securityContext:
|
|
||||||
runAsUser: 10001
|
|
||||||
runAsGroup: 10001
|
|
||||||
runAsNonRoot: true
|
|
||||||
fsGroup: 10001
|
|
||||||
seccompProfile:
|
|
||||||
type: RuntimeDefault
|
|
||||||
containers:
|
|
||||||
- name: tempo
|
|
||||||
image: grafana/tempo:3.0.3
|
|
||||||
args: ["-config.file=/etc/tempo/tempo.yaml"]
|
|
||||||
ports:
|
|
||||||
- name: http
|
|
||||||
containerPort: 3200
|
|
||||||
- name: otlp-grpc
|
|
||||||
containerPort: 4317
|
|
||||||
readinessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /ready
|
|
||||||
port: http
|
|
||||||
initialDelaySeconds: 5
|
|
||||||
periodSeconds: 5
|
|
||||||
volumeMounts:
|
|
||||||
- name: config
|
|
||||||
mountPath: /etc/tempo
|
|
||||||
readOnly: true
|
|
||||||
- name: data
|
|
||||||
mountPath: /var/tempo
|
|
||||||
securityContext:
|
|
||||||
allowPrivilegeEscalation: false
|
|
||||||
capabilities:
|
|
||||||
drop: ["ALL"]
|
|
||||||
readOnlyRootFilesystem: true
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 25m
|
|
||||||
memory: 128Mi
|
|
||||||
limits:
|
|
||||||
memory: 384Mi
|
|
||||||
volumes:
|
|
||||||
- name: config
|
|
||||||
configMap:
|
|
||||||
name: tempo
|
|
||||||
- name: data
|
|
||||||
emptyDir: {}
|
|
||||||
|
|
@ -8,7 +8,6 @@ namespace: nereus
|
||||||
resources:
|
resources:
|
||||||
- ../../base
|
- ../../base
|
||||||
- ../../rollouts
|
- ../../rollouts
|
||||||
- ../../secrets
|
|
||||||
|
|
||||||
images:
|
images:
|
||||||
- name: nereus-api
|
- name: nereus-api
|
||||||
|
|
@ -42,3 +41,11 @@ patches:
|
||||||
labelSelector:
|
labelSelector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
app.kubernetes.io/name: nereus-api
|
app.kubernetes.io/name: nereus-api
|
||||||
|
|
||||||
|
- target:
|
||||||
|
kind: ConfigMap
|
||||||
|
name: nereus-api-config
|
||||||
|
patch: |
|
||||||
|
- op: replace
|
||||||
|
path: /data/LOG_LEVEL
|
||||||
|
value: "warn"
|
||||||
|
|
|
||||||
|
|
@ -24,50 +24,12 @@ Postgres only reads `POSTGRES_PASSWORD` when it initialises the data directory.
|
||||||
|
|
||||||
## prod (sealed)
|
## prod (sealed)
|
||||||
|
|
||||||
Install the controller through `terraform/platform` first. It is named
|
Write the plaintext to `nereus-db.plain.yaml` (gitignored), then:
|
||||||
`sealed-secrets-controller`, which is the default expected by `kubeseal`.
|
|
||||||
|
|
||||||
Create the database Secret locally and pipe it straight into `kubeseal`. The
|
```fish
|
||||||
plaintext exists only in the pipeline and the password variable:
|
kubeseal --format yaml < nereus-db.plain.yaml > nereus-db-sealed.yaml
|
||||||
|
|
||||||
```bash
|
|
||||||
db_password="$(openssl rand -hex 16)"
|
|
||||||
kubectl create secret generic nereus-db --namespace nereus \
|
|
||||||
--from-literal=POSTGRES_PASSWORD="$db_password" \
|
|
||||||
--from-literal=DATABASE_URL="postgres://nereus:${db_password}@nereus-postgres:5432/nereus?sslmode=disable" \
|
|
||||||
--dry-run=client -o yaml | \
|
|
||||||
kubeseal --format yaml >deploy/secrets/nereus-db-sealed.yaml
|
|
||||||
unset db_password
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Commit only `nereus-db-sealed.yaml` and add it to the prod overlay. Back up the
|
Commit only `nereus-db-sealed.yaml` and add it to the prod overlay. The
|
||||||
controller key outside the repository before relying on sealed secrets for
|
controller isn't installed yet — `kubectl get crd | grep sealed` comes back
|
||||||
recovery. Losing that key makes every committed `SealedSecret` undecryptable.
|
empty.
|
||||||
|
|
||||||
On a cluster that already has a manually created `nereus-db` Secret, do not
|
|
||||||
apply the sealed replacement in place. PostgreSQL keeps the password used when
|
|
||||||
its data directory was initialized. Cut over during a deliberate database
|
|
||||||
reset: remove the old Secret and disposable PVC, apply the `SealedSecret`, then
|
|
||||||
start PostgreSQL against the empty volume. Fresh clusters can apply the prod
|
|
||||||
overlay directly.
|
|
||||||
|
|
||||||
## Discord alert routing
|
|
||||||
|
|
||||||
Alertmanager reads its Discord webhook from the `nereus-discord` Secret as a
|
|
||||||
mounted file. Create the webhook in Discord, keep it in the shell environment,
|
|
||||||
and seal it without writing plaintext to disk:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
test -n "${DISCORD_WEBHOOK_URL:-}"
|
|
||||||
printf '%s' "$DISCORD_WEBHOOK_URL" | \
|
|
||||||
kubectl create secret generic nereus-discord --namespace observability \
|
|
||||||
--from-file=webhook-url=/dev/stdin --dry-run=client -o yaml | \
|
|
||||||
kubeseal --format yaml >deploy/secrets/nereus-discord-sealed.yaml
|
|
||||||
unset DISCORD_WEBHOOK_URL
|
|
||||||
```
|
|
||||||
|
|
||||||
Apply the sealed secret, confirm it produced the `nereus-discord` Secret, then
|
|
||||||
set `enable_discord = true` in the platform Terraform variables and apply
|
|
||||||
again. Discord routing defaults off so a missing owner-provided webhook cannot
|
|
||||||
stop Alertmanager from starting. The committed opt-in values route warning and
|
|
||||||
critical Nereus alerts to Discord and discard alerts that do not match.
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +0,0 @@
|
||||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
|
||||||
kind: Kustomization
|
|
||||||
|
|
||||||
resources:
|
|
||||||
- nereus-db-sealed.yaml
|
|
||||||
|
|
@ -1,14 +0,0 @@
|
||||||
---
|
|
||||||
apiVersion: bitnami.com/v1alpha1
|
|
||||||
kind: SealedSecret
|
|
||||||
metadata:
|
|
||||||
name: nereus-db
|
|
||||||
namespace: nereus
|
|
||||||
spec:
|
|
||||||
encryptedData:
|
|
||||||
DATABASE_URL: AgBYadb8AxzvkAIu3JumuybqEGw1rxowhl6RjAOBWl0talN1Fx1+VuMWhb2Thvl0ood8oDbY5fvGx0jD3zEctq3Vxl+22BEzFcea+KLdiOPbJ4tKoOSz3m8MCRbd95DhZwISChhxjBEwoZBdkKQ4rorDFJytx2cmhTlnIxlUKE+m3FHhYYZ/firF74EbhjP9Udk05Dg7COCyrlhyKEXvYv9md/e/PJe7LmMSFHudhV4k8lvg5Uv4helkTDis35GubdICfF1n4BefxIUbyV+uy8XcjU0RBh3lN3o03ZBV/OmL9n2aC1zJIRQZpl1PogWDf5QsUK/AUtlZdB8L1v06sWCk/U4IeQXNEo24HcwKDToL7yVjU8ykF5X5aGLEO+5ATdPKCnB2LsnMF3g1xDiw6p9q8IexLag3c7aUSHwDFmj8Q3j8ky8IgB2cGIb3QzHBFQkGrPvgTdxdeWPcQ+k2vIoOuBOkG6W3W0plJcN72i1/vPnF4ge9xFAzEe2a4Jmiq7gk0dwSZArFPesLeLRZBfTM776WGgrCpNv8PRl+f76KhxaP0hi0F85UClIR6z97H9WdtD/ZvE7gTCjAryz0SnB1yOt79cUlYYmxsqYkYPCmtdlVR+s6vzljr6uUOum3fHeRMzIO2w58g4vqGQF+Uwojk3vmjkUROncf+BRP1Hcd3pBy/OadBkxXInt27ezJcDyoH89HdmBukx7keXonUl3QbVN3ReHwqqAoqSGfwxtYYz5+puSdgzaDqS98r8PhooeNTJaEPvbmYtNMUiTkMv1ZsWgYXltnTZGdjweCJue1xRVIg3SCtWp5vODK3rJg
|
|
||||||
POSTGRES_PASSWORD: AgBtyba0ZixyNnSL5rmn9QdlSWaVHL9Nx29KMKI5lpgu0JLdvRgXbQkZWCYnFi+gLul4FU3eQVsqqvHPxisYe0O0gKj6POToP8dFT5snJWD5ikU0SZX6PWPp94VXrAeLlsYW33V1eoWXxKLbWjX33pCExGVwDoSu0pAEuixtuaUjtv32FsHMAd70SY2uPlcJWeHhi6696agBH9Q3/sBR0m0fikRF4OFz3OC77HQ7GDDkda71KU34uqHs9Cbz30ePmu+xys0HM41Pi3ahguXYg4J1jdwTrjyAde5cAo61efrFNaqRteXFNkDenh4T7QaRfAZbkNYGXQ6Gtt85B3h1AM9592Mo3yBzjuEsAXr5HevITwFM6KZhoK8nVCzmqqSBlNU6DkHyRJxE3q+QWUu17vuG4bAWrd01ua6NfbzFwZb1q4sfszMLp62W1DI3y3S4vg5z9Zhnb9Sa78uPAAR5OTqGx/zMSseJrJX8FXBkA/raMtXFbUn1j1JnLfxmdsOW3LrjhTm7SoMH1Hr4IRf4iYAGgpmak4a/2gSFw9fN77LJI8GlJ/qRN9GcE4mmaLw2gojb60nav+PW0iBRbdHxTcxYiwdyvc7WxFGOHqrkZkEYcVZ2B7lJQMD3uNwC0Xd5rIVpE+lwX4gHMBRbCI6+4OZMGzKnUOWqSsoKS4RptQruKe1b8sXRB4hQYpZ4qhewujM01NSamdz6ZW5Ttp7IlBA2s0yVbOlXQfesDXg77tQ0iQ==
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
name: nereus-db
|
|
||||||
namespace: nereus
|
|
||||||
|
|
@ -21,18 +21,9 @@ Forgejo UI:
|
||||||
- `REGISTRY_USERNAME`: account allowed to push the three Nereus packages.
|
- `REGISTRY_USERNAME`: account allowed to push the three Nereus packages.
|
||||||
- `REGISTRY_PASSWORD`: package-scoped token for that account.
|
- `REGISTRY_PASSWORD`: package-scoped token for that account.
|
||||||
- `KUBECONFIG_B64`: base64 encoding of a least-privilege deployment kubeconfig.
|
- `KUBECONFIG_B64`: base64 encoding of a least-privilege deployment kubeconfig.
|
||||||
- `GITHUB_MIRROR_SSH_KEY`: private half of a write-enabled deploy key for
|
|
||||||
`git@github.com:Fi3w0/Nereus.git`.
|
|
||||||
|
|
||||||
After registering that deploy key, set the Forgejo Actions repository variable
|
|
||||||
`MIRROR_ENABLED` to `true`. The mirror job stays skipped until both sides
|
|
||||||
are configured, so an absent GitHub credential cannot break production deploys.
|
|
||||||
|
|
||||||
The workflow never prints these values. It writes the kubeconfig to the
|
The workflow never prints these values. It writes the kubeconfig to the
|
||||||
ephemeral workspace with mode `0600`, removes it in an `always()` step, and
|
ephemeral workspace with mode `0600`, removes it in an `always()` step, and
|
||||||
creates the Kubernetes registry pull secret through a pipe so its generated
|
creates the Kubernetes registry pull secret through a pipe so its generated
|
||||||
manifest is not logged or committed. The deploy job applies the committed
|
manifest is not logged or committed. The pre-existing `nereus-db` Secret is
|
||||||
database SealedSecret and waits for the controller to synchronize it before
|
required and is only checked for presence.
|
||||||
starting the application rollout. After a successful production deployment,
|
|
||||||
the mirror job pushes `main` to GitHub over SSH and removes its temporary key in
|
|
||||||
an `always()` step.
|
|
||||||
|
|
|
||||||
|
|
@ -57,81 +57,6 @@ probe-556d5b659b-3-pre Failed vector(1.0)
|
||||||
After the failed analysis, the active Service selector remained on the healthy
|
After the failed analysis, the active Service selector remained on the healthy
|
||||||
revision `746bbb94df`.
|
revision `746bbb94df`.
|
||||||
|
|
||||||
## Log and trace correlation
|
|
||||||
|
|
||||||
Captured on 2026-08-27 from the two-node Fedora cluster. Loki and Tempo run in
|
|
||||||
the `observability` namespace, and a DaemonSet OTel Collector tails
|
|
||||||
`/var/log/pods` on both nodes.
|
|
||||||
|
|
||||||
The collector reads those files as non-root. k3s writes `/var/log/pods` as
|
|
||||||
`0750 root:root` and each container log as `0640 root:root`, so the pod runs
|
|
||||||
with `supplementalGroups: [0]` and keeps `runAsNonRoot: true`, `drop: ["ALL"]`,
|
|
||||||
a read-only root filesystem, and a read-only host mount. Granting
|
|
||||||
`DAC_READ_SEARCH` instead does not work: Kubernetes sets no ambient
|
|
||||||
capabilities, so the capability is cleared on exec and the container runs with
|
|
||||||
`CapEff: 0000000000000000`, matching no files and reporting no error.
|
|
||||||
|
|
||||||
Loki received logs from every Nereus container:
|
|
||||||
|
|
||||||
```text
|
|
||||||
k8s_container_name ["api", "loadgen", "postgres"]
|
|
||||||
k8s_namespace_name ["nereus"]
|
|
||||||
```
|
|
||||||
|
|
||||||
A single API request log line carries the trace identifier:
|
|
||||||
|
|
||||||
```text
|
|
||||||
{"time":"2026-08-27T19:21:24.107201Z","level":"INFO","msg":"request",
|
|
||||||
"method":"GET","path":"/api/v1/readings","status":200,"duration_ms":2,
|
|
||||||
"trace_id":"fd7812d8463298f50b4570374b03dfdb"}
|
|
||||||
```
|
|
||||||
|
|
||||||
Querying Tempo for that same identifier returned the matching trace, which is
|
|
||||||
the hop the provisioned Grafana data sources make in both directions:
|
|
||||||
|
|
||||||
```text
|
|
||||||
nereus-api http.request SPAN_KIND_SERVER
|
|
||||||
nereus-api list_readings SPAN_KIND_INTERNAL
|
|
||||||
```
|
|
||||||
|
|
||||||
Both data sources are provisioned by the labelled `nereus-grafana-datasources`
|
|
||||||
ConfigMap, which the kube-prometheus-stack Grafana sidecar loads as
|
|
||||||
`nereus.yaml`. The correlation requires the API at `LOG_LEVEL=info`, because the
|
|
||||||
per-request log line is the only one that carries `trace_id`.
|
|
||||||
|
|
||||||
Two collector defects were found and fixed on the way to this result, and both
|
|
||||||
belong in the report as corrective actions. The first is the capability problem
|
|
||||||
above. The second only appears after a restart: `start_at: beginning` makes the
|
|
||||||
receiver replay whole log files, the batch processor had no size cap, and Loki
|
|
||||||
refused the resulting push.
|
|
||||||
|
|
||||||
```text
|
|
||||||
HTTP 503, ResourceExhausted: grpc: received message larger than max
|
|
||||||
(4378796 vs. 4194304)
|
|
||||||
```
|
|
||||||
|
|
||||||
The exporter retries that request unchanged, so the backlog never drains.
|
|
||||||
Capping the batch with `send_batch_size: 1000` and `send_batch_max_size: 2000`
|
|
||||||
keeps every push under Loki's 4 MiB gRPC limit. What remains after a restart is
|
|
||||||
a `Permanent` rejection of replayed lines older than Loki's acceptance window,
|
|
||||||
which drops those lines rather than blocking, and leaves live traffic
|
|
||||||
unaffected.
|
|
||||||
|
|
||||||
## Sealed database cutover
|
|
||||||
|
|
||||||
Verified on 2026-08-28 against the two-node Fedora cluster. PostgreSQL was
|
|
||||||
stopped, its disposable PVC and manually managed Secret were removed, and the
|
|
||||||
committed `nereus-db` SealedSecret created the replacement Secret. The
|
|
||||||
controller reported `Synced=True`, the new 5 GiB PVC bound, PostgreSQL reached
|
|
||||||
one ready replica, three restarted API pods became ready, and both load
|
|
||||||
generators became healthy after reseeding the empty database. The first
|
|
||||||
analysis attempts correctly rejected stale reset errors. After one full clean
|
|
||||||
Prometheus window, AnalysisRun `nereus-api-75d884b4dd-19-pre` passed all five
|
|
||||||
measurements and promoted revision `75d884b4dd` to the active Service.
|
|
||||||
|
|
||||||
No plaintext credential was written to the repository, Terraform state, shell
|
|
||||||
output, or evidence.
|
|
||||||
|
|
||||||
## Presentation commands
|
## Presentation commands
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|
|
||||||
0
docs/memoria/.gitkeep
Normal file
0
docs/memoria/.gitkeep
Normal file
|
|
@ -1,6 +1,6 @@
|
||||||
# Nereus delivery roadmap
|
# Nereus delivery roadmap
|
||||||
|
|
||||||
Updated: 2026-08-28
|
Updated: 2026-08-26
|
||||||
|
|
||||||
`[x]` means locally verified. `[~]` means implemented but awaiting a real
|
`[x]` means locally verified. `[~]` means implemented but awaiting a real
|
||||||
integration environment. `[ ]` means not implemented. `[H]` is human-owned and
|
integration environment. `[ ]` means not implemented. `[H]` is human-owned and
|
||||||
|
|
@ -115,16 +115,15 @@ task. Mark a task `[x]` only after its stated verification passes.
|
||||||
|
|
||||||
- [x] Receive OTLP/gRPC traces in the Collector.
|
- [x] Receive OTLP/gRPC traces in the Collector.
|
||||||
- [x] Export traces from the Collector to Tempo.
|
- [x] Export traces from the Collector to Tempo.
|
||||||
- [x] Ship structured Kubernetes pod logs to Loki with the cluster Collector.
|
- [~] Ship structured Kubernetes pod logs to Loki with the cluster Collector.
|
||||||
- [x] Start Loki locally and verify its Grafana data source readiness.
|
- [x] Start Loki locally and verify its Grafana data source readiness.
|
||||||
- [x] Configure Prometheus to scrape `/metrics`.
|
- [x] Configure Prometheus to scrape `/metrics`.
|
||||||
- [x] Add request-rate, error-rate, and latency panels.
|
- [x] Add request-rate, error-rate, and latency panels.
|
||||||
- [x] Add database-operation latency panels.
|
- [x] Add database-operation latency panels.
|
||||||
- [x] Add readings-ingested and active-buoy panels.
|
- [x] Add readings-ingested and active-buoy panels.
|
||||||
- [x] Add a deployed-version dashboard variable.
|
- [x] Add a deployed-version dashboard variable.
|
||||||
- [x] Correlate logs and traces using `trace_id` after cluster log ingestion exists.
|
- [~] Correlate logs and traces using `trace_id` after cluster log ingestion exists.
|
||||||
- [x] Add API error-rate, latency, readiness, and scrape alerts.
|
- [x] Add API error-rate, latency, readiness, and scrape alerts.
|
||||||
- [~] Route warning and critical alerts to Discord; configuration validates and awaits the owner-provided sealed webhook.
|
|
||||||
- [x] Load the dashboard and evaluate all six PromQL queries.
|
- [x] Load the dashboard and evaluate all six PromQL queries.
|
||||||
- [x] Trigger chaos and distinguish the failing version in Grafana.
|
- [x] Trigger chaos and distinguish the failing version in Grafana.
|
||||||
|
|
||||||
|
|
@ -178,7 +177,7 @@ task. Mark a task `[x]` only after its stated verification passes.
|
||||||
- [x] Execute `lab.sh check` after a workstation reboot.
|
- [x] Execute `lab.sh check` after a workstation reboot.
|
||||||
- [x] Test recovery from an interrupted image download and validate the resumed QCOW2.
|
- [x] Test recovery from an interrupted image download and validate the resumed QCOW2.
|
||||||
- [x] Test recovery after node 2 is deleted manually and rejoin it with a fresh identity.
|
- [x] Test recovery after node 2 is deleted manually and rejoin it with a fresh identity.
|
||||||
- [~] Test on a workstation with no dependencies installed; the reproducible fresh-VM path is implemented and awaits a second host.
|
- [ ] Test on a workstation with no dependencies installed.
|
||||||
- [x] Decide whether an aarch64 image path is required; keep the lab x86_64-only.
|
- [x] Decide whether an aarch64 image path is required; keep the lab x86_64-only.
|
||||||
- [ ] After final sign-off, destroy the QEMU lab and remove packages installed only for it with Pacman's dependency-aware cleanup.
|
- [ ] After final sign-off, destroy the QEMU lab and remove packages installed only for it with Pacman's dependency-aware cleanup.
|
||||||
|
|
||||||
|
|
@ -199,8 +198,8 @@ task. Mark a task `[x]` only after its stated verification passes.
|
||||||
|
|
||||||
## Human-owned deployment and infrastructure
|
## Human-owned deployment and infrastructure
|
||||||
|
|
||||||
- [x] Define libvirt infrastructure and Helm platform stacks under `terraform/`.
|
- [H] Define infrastructure modules under `terraform/`.
|
||||||
- [x] Export stable node addresses to an Ansible inventory without provisioner hooks.
|
- [H] Export provisioned addresses for Ansible inventory.
|
||||||
- [H] Create Kubernetes base resources and overlays. Agent-authored 2026-08-21 at the owner's request; builds and server dry-runs pass, not applied.
|
- [H] Create Kubernetes base resources and overlays. Agent-authored 2026-08-21 at the owner's request; builds and server dry-runs pass, not applied.
|
||||||
- [H] Create API and load-generator workloads. Agent-authored 2026-08-21; not applied.
|
- [H] Create API and load-generator workloads. Agent-authored 2026-08-21; not applied.
|
||||||
- [H] Create active and preview Services. Agent-authored 2026-08-21; not applied.
|
- [H] Create active and preview Services. Agent-authored 2026-08-21; not applied.
|
||||||
|
|
@ -208,7 +207,7 @@ task. Mark a task `[x]` only after its stated verification passes.
|
||||||
- [H] Create the real error-rate AnalysisTemplate. Agent-authored 2026-08-21; PromQL never evaluated against real series.
|
- [H] Create the real error-rate AnalysisTemplate. Agent-authored 2026-08-21; PromQL never evaluated against real series.
|
||||||
- [H] Configure blue-green promotion and automated rollback. Agent-authored 2026-08-21; promotion and abort paths untested with the real API.
|
- [H] Configure blue-green promotion and automated rollback. Agent-authored 2026-08-21; promotion and abort paths untested with the real API.
|
||||||
- [H] Maintain Forgejo Actions and GitHub mirror workflows.
|
- [H] Maintain Forgejo Actions and GitHub mirror workflows.
|
||||||
- [x] Install Sealed Secrets, commit the encrypted database resource, and verify the fresh-volume cutover.
|
- [H] Create sealed secrets from off-repository plaintext inputs.
|
||||||
|
|
||||||
## Forgejo CI/CD
|
## Forgejo CI/CD
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -37,11 +37,10 @@ can later be managed by Ansible without rebuilding it.
|
||||||
## Terraform path
|
## Terraform path
|
||||||
|
|
||||||
Terraform is responsible only for creating machines, networks, and addresses.
|
Terraform is responsible only for creating machines, networks, and addresses.
|
||||||
`terraform/infra` creates two Fedora 44 libvirt machines and writes its resulting
|
After `terraform apply`, put its resulting addresses into `inventory.yml` and
|
||||||
addresses to `terraform/infra/inventory.yml`. Pass that file to `bootstrap.sh`
|
run this playbook. Keeping configuration out of provisioner hooks makes the
|
||||||
after `terraform apply`. Keeping configuration out of provisioner hooks makes
|
same Ansible workflow usable for physical hardware, VMs, and manually created
|
||||||
the same Ansible workflow usable for physical hardware, VMs, and manually
|
hosts.
|
||||||
created hosts.
|
|
||||||
|
|
||||||
## One-command local QEMU lab
|
## One-command local QEMU lab
|
||||||
|
|
||||||
|
|
|
||||||
44
terraform/infra/.terraform.lock.hcl
generated
44
terraform/infra/.terraform.lock.hcl
generated
|
|
@ -1,44 +0,0 @@
|
||||||
# This file is maintained automatically by "terraform init".
|
|
||||||
# Manual edits may be lost in future updates.
|
|
||||||
|
|
||||||
provider "registry.terraform.io/dmacvicar/libvirt" {
|
|
||||||
version = "0.8.3"
|
|
||||||
constraints = "0.8.3"
|
|
||||||
hashes = [
|
|
||||||
"h1:Tttxr3E9O75MM+dDmq5sYHQEw29PwtIj+XDj/5drdfE=",
|
|
||||||
"zh:06ff0169beafd1891dc5a30616983abd32004a4f570d1d3dbb5851d84bd1c007",
|
|
||||||
"zh:2dbdd726d0987cda73b56ecdfbcb98a67485e86a7a44aec976c0081b7239d89d",
|
|
||||||
"zh:2e195a7bbdfcc13c45460571a5ba848a5c1e746b477c8381058767560f0ac93b",
|
|
||||||
"zh:3952da13080018c5aec498b73e343c4c22ad884afb8c983138fb7255617aa991",
|
|
||||||
"zh:478841bcf57df938726ddb90f55c7953fad09db4f6348747519afe7fc84b403b",
|
|
||||||
"zh:53bce78b03a82c4782acfe1f32c2b46a68fa5fb2fb90d4a5392c90b436b44244",
|
|
||||||
"zh:5c157f23e9768c67cddf9e847a571adca441607cb5adfb96dbfdd626ceadf92c",
|
|
||||||
"zh:6bc78d631959fb695664966851308e140c38f3f5cf648dd89756320c2d91765d",
|
|
||||||
"zh:8605d7d6915190836802654920a8eea3d751ae437273c4f4476dc0ebb9167a1d",
|
|
||||||
"zh:8b66a22b97331c2a56aed092fd39152d06ad957fd4810aa3f0c4ade0f9b15755",
|
|
||||||
"zh:92586a47a04082f70bb33f722672127a287caeed109beaaca2668e2e1d6a9caf",
|
|
||||||
"zh:99a9ee414f5c4268e287660ce8edec2efcba1f79351f83791b64c7e5ab04f569",
|
|
||||||
"zh:b7cff09fe74b0eb63b5b9aa94de5b33dadbd006d6d5b9578ac476039ea20b062",
|
|
||||||
"zh:d4188a343ff32c0e03ff28c7e84abce0f43cad2fdbcd9046eaafc247429039ff",
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
provider "registry.terraform.io/hashicorp/local" {
|
|
||||||
version = "2.5.3"
|
|
||||||
constraints = "2.5.3"
|
|
||||||
hashes = [
|
|
||||||
"h1:1Nkh16jQJMp0EuDmvP/96f5Unnir0z12WyDuoR6HjMo=",
|
|
||||||
"zh:284d4b5b572eacd456e605e94372f740f6de27b71b4e1fd49b63745d8ecd4927",
|
|
||||||
"zh:40d9dfc9c549e406b5aab73c023aa485633c1b6b730c933d7bcc2fa67fd1ae6e",
|
|
||||||
"zh:6243509bb208656eb9dc17d3c525c89acdd27f08def427a0dce22d5db90a4c8b",
|
|
||||||
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
|
|
||||||
"zh:885d85869f927853b6fe330e235cd03c337ac3b933b0d9ae827ec32fa1fdcdbf",
|
|
||||||
"zh:bab66af51039bdfcccf85b25fe562cbba2f54f6b3812202f4873ade834ec201d",
|
|
||||||
"zh:c505ff1bf9442a889ac7dca3ac05a8ee6f852e0118dd9a61796a2f6ff4837f09",
|
|
||||||
"zh:d36c0b5770841ddb6eaf0499ba3de48e5d4fc99f4829b6ab66b0fab59b1aaf4f",
|
|
||||||
"zh:ddb6a407c7f3ec63efb4dad5f948b54f7f4434ee1a2607a49680d494b1776fe1",
|
|
||||||
"zh:e0dafdd4500bec23d3ff221e3a9b60621c5273e5df867bc59ef6b7e41f5c91f6",
|
|
||||||
"zh:ece8742fd2882a8fc9d6efd20e2590010d43db386b920b2a9c220cfecc18de47",
|
|
||||||
"zh:f4c6b3eb8f39105004cf720e202f04f57e3578441cfb76ca27611139bc116a82",
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
@ -1,19 +0,0 @@
|
||||||
# Libvirt infrastructure
|
|
||||||
|
|
||||||
This stack creates a dedicated NAT network and two Fedora 44 x86_64 machines.
|
|
||||||
It stops at the machine boundary. Ansible remains responsible for Fedora and
|
|
||||||
k3s configuration.
|
|
||||||
|
|
||||||
Keep real values in an untracked `terraform.tfvars` file:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cp terraform.tfvars.example terraform.tfvars
|
|
||||||
terraform init
|
|
||||||
terraform apply
|
|
||||||
K3S_TOKEN="$(openssl rand -hex 32)" \
|
|
||||||
../../scripts/provision/bootstrap.sh inventory.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
The generated `inventory.yml`, Terraform state, and variable file are ignored.
|
|
||||||
The SSH public key is safe to place in Terraform input; the matching private key
|
|
||||||
is referenced by absolute path and is never read into Terraform state.
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
#cloud-config
|
|
||||||
hostname: ${hostname}
|
|
||||||
manage_etc_hosts: true
|
|
||||||
users:
|
|
||||||
- name: fedora
|
|
||||||
groups: [wheel]
|
|
||||||
sudo: ALL=(ALL) NOPASSWD:ALL
|
|
||||||
shell: /bin/bash
|
|
||||||
ssh_authorized_keys:
|
|
||||||
- ${ssh_public_key}
|
|
||||||
ssh_pwauth: false
|
|
||||||
disable_root: true
|
|
||||||
|
|
@ -1,24 +0,0 @@
|
||||||
all:
|
|
||||||
vars:
|
|
||||||
ansible_user: fedora
|
|
||||||
ansible_ssh_private_key_file: ${private_key_file}
|
|
||||||
ansible_ssh_common_args: "-o StrictHostKeyChecking=accept-new"
|
|
||||||
k3s_version: v1.33.4+k3s1
|
|
||||||
k3s_cluster_cidr: 10.42.0.0/16
|
|
||||||
k3s_service_cidr: 10.43.0.0/16
|
|
||||||
k3s_node_cidr: ${node_cidr}
|
|
||||||
k3s_operator_cidrs:
|
|
||||||
%{ for cidr in operator_cidrs ~}
|
|
||||||
- ${cidr}
|
|
||||||
%{ endfor ~}
|
|
||||||
children:
|
|
||||||
k3s_server:
|
|
||||||
hosts:
|
|
||||||
node1:
|
|
||||||
ansible_host: ${server_address}
|
|
||||||
k3s_node_ip: ${server_address}
|
|
||||||
k3s_agent:
|
|
||||||
hosts:
|
|
||||||
node2:
|
|
||||||
ansible_host: ${agent_address}
|
|
||||||
k3s_node_ip: ${agent_address}
|
|
||||||
|
|
@ -1,97 +0,0 @@
|
||||||
locals {
|
|
||||||
nodes = {
|
|
||||||
nereus-node1 = {
|
|
||||||
address = var.node_addresses["nereus-node1"]
|
|
||||||
role = "server"
|
|
||||||
}
|
|
||||||
nereus-node2 = {
|
|
||||||
address = var.node_addresses["nereus-node2"]
|
|
||||||
role = "agent"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "libvirt_network" "nereus" {
|
|
||||||
name = "nereus"
|
|
||||||
mode = "nat"
|
|
||||||
domain = "nereus.test"
|
|
||||||
addresses = [var.network_cidr]
|
|
||||||
|
|
||||||
dhcp {
|
|
||||||
enabled = true
|
|
||||||
}
|
|
||||||
|
|
||||||
dns {
|
|
||||||
enabled = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "libvirt_volume" "fedora" {
|
|
||||||
name = "nereus-fedora-44-base.qcow2"
|
|
||||||
pool = "default"
|
|
||||||
source = var.fedora_image_url
|
|
||||||
format = "qcow2"
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "libvirt_volume" "node" {
|
|
||||||
for_each = local.nodes
|
|
||||||
name = "${each.key}.qcow2"
|
|
||||||
pool = "default"
|
|
||||||
base_volume_id = libvirt_volume.fedora.id
|
|
||||||
size = 30 * 1024 * 1024 * 1024
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "libvirt_cloudinit_disk" "node" {
|
|
||||||
for_each = local.nodes
|
|
||||||
name = "${each.key}-cloud-init.iso"
|
|
||||||
pool = "default"
|
|
||||||
user_data = templatefile("${path.module}/cloud-init.yaml.tftpl", {
|
|
||||||
hostname = each.key
|
|
||||||
ssh_public_key = var.ssh_public_key
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "libvirt_domain" "node" {
|
|
||||||
for_each = local.nodes
|
|
||||||
|
|
||||||
name = each.key
|
|
||||||
memory = 3072
|
|
||||||
vcpu = 2
|
|
||||||
autostart = true
|
|
||||||
cloudinit = libvirt_cloudinit_disk.node[each.key].id
|
|
||||||
|
|
||||||
disk {
|
|
||||||
volume_id = libvirt_volume.node[each.key].id
|
|
||||||
}
|
|
||||||
|
|
||||||
network_interface {
|
|
||||||
network_id = libvirt_network.nereus.id
|
|
||||||
hostname = each.key
|
|
||||||
addresses = [each.value.address]
|
|
||||||
wait_for_lease = true
|
|
||||||
}
|
|
||||||
|
|
||||||
console {
|
|
||||||
type = "pty"
|
|
||||||
target_type = "serial"
|
|
||||||
target_port = "0"
|
|
||||||
}
|
|
||||||
|
|
||||||
graphics {
|
|
||||||
type = "spice"
|
|
||||||
listen_type = "address"
|
|
||||||
autoport = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "local_sensitive_file" "inventory" {
|
|
||||||
filename = "${path.module}/inventory.yml"
|
|
||||||
file_permission = "0600"
|
|
||||||
content = templatefile("${path.module}/inventory.yml.tftpl", {
|
|
||||||
private_key_file = var.ssh_private_key_file
|
|
||||||
operator_cidrs = var.operator_cidrs
|
|
||||||
node_cidr = var.network_cidr
|
|
||||||
server_address = local.nodes["nereus-node1"].address
|
|
||||||
agent_address = local.nodes["nereus-node2"].address
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
@ -1,9 +0,0 @@
|
||||||
output "node_addresses" {
|
|
||||||
description = "Addresses passed to Ansible."
|
|
||||||
value = { for name, node in local.nodes : name => node.address }
|
|
||||||
}
|
|
||||||
|
|
||||||
output "ansible_inventory" {
|
|
||||||
description = "Generated inventory consumed by scripts/provision/bootstrap.sh."
|
|
||||||
value = local_sensitive_file.inventory.filename
|
|
||||||
}
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
ssh_public_key = "replace with one public key"
|
|
||||||
ssh_private_key_file = "/absolute/path/to/the/matching/private/key"
|
|
||||||
|
|
||||||
operator_cidrs = ["192.168.123.1/32"]
|
|
||||||
|
|
@ -1,42 +0,0 @@
|
||||||
variable "libvirt_uri" {
|
|
||||||
description = "Libvirt connection used to create the Fedora machines."
|
|
||||||
type = string
|
|
||||||
default = "qemu:///system"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "fedora_image_url" {
|
|
||||||
description = "Fedora 44 Cloud Base Generic x86_64 image URL."
|
|
||||||
type = string
|
|
||||||
default = "https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "ssh_public_key" {
|
|
||||||
description = "Public SSH key installed for the fedora user."
|
|
||||||
type = string
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "ssh_private_key_file" {
|
|
||||||
description = "Absolute private-key path written into the generated Ansible inventory."
|
|
||||||
type = string
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "operator_cidrs" {
|
|
||||||
description = "Networks allowed to administer k3s."
|
|
||||||
type = list(string)
|
|
||||||
default = ["192.168.123.1/32"]
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "network_cidr" {
|
|
||||||
description = "Dedicated libvirt network for the two nodes."
|
|
||||||
type = string
|
|
||||||
default = "192.168.123.0/24"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "node_addresses" {
|
|
||||||
description = "Stable addresses assigned to each node."
|
|
||||||
type = map(string)
|
|
||||||
default = {
|
|
||||||
nereus-node1 = "192.168.123.10"
|
|
||||||
nereus-node2 = "192.168.123.11"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,18 +0,0 @@
|
||||||
terraform {
|
|
||||||
required_version = ">= 1.10.0"
|
|
||||||
|
|
||||||
required_providers {
|
|
||||||
libvirt = {
|
|
||||||
source = "dmacvicar/libvirt"
|
|
||||||
version = "0.8.3"
|
|
||||||
}
|
|
||||||
local = {
|
|
||||||
source = "hashicorp/local"
|
|
||||||
version = "2.5.3"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
provider "libvirt" {
|
|
||||||
uri = var.libvirt_uri
|
|
||||||
}
|
|
||||||
22
terraform/platform/.terraform.lock.hcl
generated
22
terraform/platform/.terraform.lock.hcl
generated
|
|
@ -1,22 +0,0 @@
|
||||||
# This file is maintained automatically by "terraform init".
|
|
||||||
# Manual edits may be lost in future updates.
|
|
||||||
|
|
||||||
provider "registry.terraform.io/hashicorp/helm" {
|
|
||||||
version = "3.1.1"
|
|
||||||
constraints = "3.1.1"
|
|
||||||
hashes = [
|
|
||||||
"h1:5b2ojWKT0noujHiweCds37ZreRFRQLNaErdJLusJN88=",
|
|
||||||
"zh:1a6d5ce931708aec29d1f3d9e360c2a0c35ba5a54d03eeaff0ce3ca597cd0275",
|
|
||||||
"zh:3411919ba2a5941801e677f0fea08bdd0ae22ba3c9ce3309f55554699e06524a",
|
|
||||||
"zh:81b36138b8f2320dc7f877b50f9e38f4bc614affe68de885d322629dd0d16a29",
|
|
||||||
"zh:95a2a0a497a6082ee06f95b38bd0f0d6924a65722892a856cfd914c0d117f104",
|
|
||||||
"zh:9d3e78c2d1bb46508b972210ad706dd8c8b106f8b206ecf096cd211c54f46990",
|
|
||||||
"zh:a79139abf687387a6efdbbb04289a0a8e7eaca2bd91cdc0ce68ea4f3286c2c34",
|
|
||||||
"zh:aaa8784be125fbd50c48d84d6e171d3fb6ef84a221dbc5165c067ce05faab4c8",
|
|
||||||
"zh:afecd301f469975c9d8f350cc482fe656e082b6ab0f677d1a816c3c615837cc1",
|
|
||||||
"zh:c54c22b18d48ff9053d899d178d9ffef7d9d19785d9bf310a07d648b7aac075b",
|
|
||||||
"zh:db2eefd55aea48e73384a555c72bac3f7d428e24147bedb64e1a039398e5b903",
|
|
||||||
"zh:ee61666a233533fd2be971091cecc01650561f1585783c381b6f6e8a390198a4",
|
|
||||||
"zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
@ -1,20 +0,0 @@
|
||||||
# Cluster platform
|
|
||||||
|
|
||||||
This stack installs Argo Rollouts, Sealed Secrets, and kube-prometheus-stack on
|
|
||||||
an existing k3s cluster. Use it on a fresh cluster, or import matching Helm
|
|
||||||
releases before applying it to a cluster where they already exist.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cp terraform.tfvars.example terraform.tfvars
|
|
||||||
terraform init
|
|
||||||
terraform apply
|
|
||||||
```
|
|
||||||
|
|
||||||
The kubeconfig stays outside the repository. Terraform records only its path.
|
|
||||||
Application and observability manifests remain under `deploy/` and are applied
|
|
||||||
after the cluster-wide controllers are ready.
|
|
||||||
|
|
||||||
Discord routing is deliberately off by default so a missing webhook cannot
|
|
||||||
stop Alertmanager. After applying `nereus-discord-sealed.yaml` and confirming
|
|
||||||
that it produced the `nereus-discord` Secret, set `enable_discord = true` and
|
|
||||||
apply this stack again.
|
|
||||||
|
|
@ -1,23 +0,0 @@
|
||||||
alertmanager:
|
|
||||||
alertmanagerSpec:
|
|
||||||
secrets:
|
|
||||||
- nereus-discord
|
|
||||||
config:
|
|
||||||
global:
|
|
||||||
resolve_timeout: 5m
|
|
||||||
route:
|
|
||||||
receiver: discard
|
|
||||||
group_by: [alertname, namespace]
|
|
||||||
group_wait: 30s
|
|
||||||
group_interval: 5m
|
|
||||||
repeat_interval: 4h
|
|
||||||
routes:
|
|
||||||
- receiver: discord
|
|
||||||
matchers:
|
|
||||||
- severity=~"warning|critical"
|
|
||||||
receivers:
|
|
||||||
- name: discard
|
|
||||||
- name: discord
|
|
||||||
discord_configs:
|
|
||||||
- webhook_url_file: /etc/alertmanager/secrets/nereus-discord/webhook-url
|
|
||||||
send_resolved: true
|
|
||||||
|
|
@ -1,27 +0,0 @@
|
||||||
kubeEtcd:
|
|
||||||
enabled: false
|
|
||||||
kubeProxy:
|
|
||||||
enabled: false
|
|
||||||
|
|
||||||
prometheus:
|
|
||||||
prometheusSpec:
|
|
||||||
retention: 6h
|
|
||||||
serviceMonitorSelectorNilUsesHelmValues: false
|
|
||||||
podMonitorSelectorNilUsesHelmValues: false
|
|
||||||
ruleSelectorNilUsesHelmValues: false
|
|
||||||
storageSpec:
|
|
||||||
volumeClaimTemplate:
|
|
||||||
spec:
|
|
||||||
accessModes: [ReadWriteOnce]
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
storage: 5Gi
|
|
||||||
|
|
||||||
grafana:
|
|
||||||
defaultDashboardsTimezone: browser
|
|
||||||
|
|
||||||
alertmanager:
|
|
||||||
alertmanagerSpec:
|
|
||||||
resources:
|
|
||||||
requests: {cpu: 25m, memory: 64Mi}
|
|
||||||
limits: {memory: 256Mi}
|
|
||||||
|
|
@ -1,44 +0,0 @@
|
||||||
resource "helm_release" "argo_rollouts" {
|
|
||||||
name = "argo-rollouts"
|
|
||||||
repository = "https://argoproj.github.io/argo-helm"
|
|
||||||
chart = "argo-rollouts"
|
|
||||||
version = "2.41.1"
|
|
||||||
namespace = "argo-rollouts"
|
|
||||||
create_namespace = true
|
|
||||||
wait = true
|
|
||||||
timeout = 300
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "helm_release" "sealed_secrets" {
|
|
||||||
name = "sealed-secrets"
|
|
||||||
repository = "https://bitnami-labs.github.io/sealed-secrets"
|
|
||||||
chart = "sealed-secrets"
|
|
||||||
version = "2.19.3"
|
|
||||||
namespace = "kube-system"
|
|
||||||
create_namespace = false
|
|
||||||
wait = true
|
|
||||||
timeout = 300
|
|
||||||
|
|
||||||
values = [yamlencode({
|
|
||||||
fullnameOverride = "sealed-secrets-controller"
|
|
||||||
resources = {
|
|
||||||
requests = { cpu = "25m", memory = "64Mi" }
|
|
||||||
limits = { memory = "256Mi" }
|
|
||||||
}
|
|
||||||
})]
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "helm_release" "kube_prometheus_stack" {
|
|
||||||
name = "kube-prometheus-stack"
|
|
||||||
repository = "https://prometheus-community.github.io/helm-charts"
|
|
||||||
chart = "kube-prometheus-stack"
|
|
||||||
version = "88.5.2"
|
|
||||||
namespace = "observability"
|
|
||||||
create_namespace = true
|
|
||||||
wait = true
|
|
||||||
timeout = 600
|
|
||||||
values = concat(
|
|
||||||
[file("${path.module}/kube-prometheus-stack.values.yaml")],
|
|
||||||
var.enable_discord ? [file("${path.module}/alertmanager-discord.values.yaml")] : []
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
kubeconfig_path = "/absolute/path/to/kubeconfig"
|
|
||||||
enable_discord = false
|
|
||||||
|
|
@ -1,10 +0,0 @@
|
||||||
variable "kubeconfig_path" {
|
|
||||||
description = "Absolute path to the operator kubeconfig. Its contents never enter Terraform state."
|
|
||||||
type = string
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "enable_discord" {
|
|
||||||
description = "Enable Discord routing after the nereus-discord SealedSecret has created its Secret."
|
|
||||||
type = bool
|
|
||||||
default = false
|
|
||||||
}
|
|
||||||
|
|
@ -1,16 +0,0 @@
|
||||||
terraform {
|
|
||||||
required_version = ">= 1.10.0"
|
|
||||||
|
|
||||||
required_providers {
|
|
||||||
helm = {
|
|
||||||
source = "hashicorp/helm"
|
|
||||||
version = "3.1.1"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
provider "helm" {
|
|
||||||
kubernetes = {
|
|
||||||
config_path = var.kubeconfig_path
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Add table
Reference in a new issue