Nereus/scripts/k3d/lab.sh

#!/usr/bin/env bash

set -euo pipefail

readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
readonly CLUSTER_NAME="nereus"
readonly CONTEXT="k3d-${CLUSTER_NAME}"
readonly ARGO_CHART_VERSION="2.41.1"
readonly PROMETHEUS_CHART_VERSION="88.5.2"

require_commands() {
  local command_name
  for command_name in docker k3d helm kubectl; do
    command -v "${command_name}" >/dev/null || {
      echo "missing required command: ${command_name}" >&2
      exit 1
    }
  done
  docker info >/dev/null
}

use_context() {
  kubectl config use-context "${CONTEXT}" >/dev/null
}

cluster_exists() {
  k3d cluster get "${CLUSTER_NAME}" >/dev/null 2>&1
}

up() {
  require_commands
  if ! cluster_exists; then
    k3d cluster create --config "${REPO_ROOT}/scripts/k3d-nereus.yaml"
  else
    k3d cluster start "${CLUSTER_NAME}"
  fi
  use_context

  helm repo add argo https://argoproj.github.io/argo-helm --force-update
  helm repo add prometheus-community https://prometheus-community.github.io/helm-charts --force-update
  helm repo update

  helm upgrade --install argo-rollouts argo/argo-rollouts \
    --namespace argo-rollouts \
    --create-namespace \
    --version "${ARGO_CHART_VERSION}" \
    --wait \
    --timeout 5m

  helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack \
    --namespace observability \
    --create-namespace \
    --version "${PROMETHEUS_CHART_VERSION}" \
    --values "${SCRIPT_DIR}/kube-prometheus-stack.values.yaml" \
    --wait \
    --timeout 10m

  kubectl create namespace nereus --dry-run=client -o yaml | kubectl apply -f -
  kubectl apply -f "${SCRIPT_DIR}/analysis-harness/harness.yaml"
  check
}

check() {
  require_commands
  cluster_exists || {
    echo "cluster ${CLUSTER_NAME} does not exist; run $0 up" >&2
    exit 1
  }
  use_context

  kubectl wait node --all --for=condition=Ready --timeout=3m
  kubectl wait deployment --all --namespace argo-rollouts --for=condition=Available --timeout=3m
  kubectl wait deployment --all --namespace observability --for=condition=Available --timeout=5m
  kubectl get rollout probe --namespace nereus >/dev/null
  kubectl get analysistemplate error-rate --namespace nereus >/dev/null
  echo "k3d rollback lab is ready"
}

analysis_name() {
  kubectl get rollout probe --namespace nereus \
    -o jsonpath='{.status.blueGreen.prePromotionAnalysisRunStatus.name}'
}

wait_for_analysis() {
  local previous_name="$1"
  local expected_phase="$2"
  local analysis_run=""
  local phase=""
  local attempt

  for attempt in {1..90}; do
    analysis_run="$(analysis_name)"
    if [[ -n "${analysis_run}" && "${analysis_run}" != "${previous_name}" ]]; then
      phase="$(kubectl get analysisrun "${analysis_run}" --namespace nereus -o jsonpath='{.status.phase}')"
      if [[ "${phase}" == "${expected_phase}" ]]; then
        echo "${analysis_run} reached ${expected_phase}"
        return 0
      fi
      if [[ "${phase}" == "Error" || "${phase}" == "Inconclusive" ]]; then
        echo "${analysis_run} ended unexpectedly with ${phase}" >&2
        return 1
      fi
    fi
    sleep 2
  done

  echo "analysis did not reach ${expected_phase} within 180 seconds" >&2
  return 1
}

set_proof_revision() {
  local query="$1"
  local revision="$2"
  kubectl patch rollout probe --namespace nereus --type=merge --patch \
    "{\"spec\":{\"strategy\":{\"blueGreen\":{\"prePromotionAnalysis\":{\"args\":[{\"name\":\"query\",\"value\":\"${query}\"}]}}},\"template\":{\"metadata\":{\"annotations\":{\"nereus.fiwlabs.dev/proof\":\"${revision}\"}}}}}"
}

prove() {
  check

  local proof_id
  local previous_analysis
  local healthy_analysis
  local healthy_revision=""
  local active_revision=""
  local stable_revision=""
  local attempt
  proof_id="$(date -u +%Y%m%d%H%M%S)"

  previous_analysis="$(analysis_name)"
  set_proof_revision "vector(0.0)" "healthy-${proof_id}"
  wait_for_analysis "${previous_analysis}" Successful
  healthy_analysis="$(analysis_name)"

  for attempt in {1..30}; do
    healthy_revision="$(kubectl get rollout probe --namespace nereus -o jsonpath='{.status.stableRS}')"
    active_revision="$(kubectl get service probe-active --namespace nereus -o jsonpath='{.spec.selector.rollouts-pod-template-hash}')"
    if [[ -n "${healthy_revision}" && "${active_revision}" == "${healthy_revision}" ]]; then
      break
    fi
    sleep 2
  done
  [[ -n "${healthy_revision}" && "${active_revision}" == "${healthy_revision}" ]] || {
    echo "healthy revision was not promoted" >&2
    return 1
  }

  set_proof_revision "vector(1.0)" "failing-${proof_id}"
  wait_for_analysis "${healthy_analysis}" Failed
  stable_revision="$(kubectl get rollout probe --namespace nereus -o jsonpath='{.status.stableRS}')"
  active_revision="$(kubectl get service probe-active --namespace nereus -o jsonpath='{.spec.selector.rollouts-pod-template-hash}')"
  [[ "${stable_revision}" == "${healthy_revision}" && "${active_revision}" == "${healthy_revision}" ]] || {
    echo "failed revision replaced the active healthy revision" >&2
    return 1
  }

  set_proof_revision "vector(0.0)" "healthy-${proof_id}"
  echo "rollback proof passed; active revision stayed ${healthy_revision}"
}

destroy() {
  require_commands
  if cluster_exists; then
    k3d cluster delete "${CLUSTER_NAME}"
  else
    echo "cluster ${CLUSTER_NAME} does not exist"
  fi
}

case "${1:-}" in
  up) up ;;
  check) check ;;
  prove) prove ;;
  destroy) destroy ;;
  *)
    echo "usage: $0 {up|check|prove|destroy}" >&2
    exit 2
    ;;
esac