232 lines
10 KiB
Bash
Executable file
232 lines
10 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
action=${1:-up}
|
|
script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
|
|
state_root=${XDG_STATE_HOME:-"$HOME/.local/state"}/nereus-lab
|
|
image_root=/var/lib/libvirt/images/nereus-lab
|
|
inventory="$state_root/inventory.yml"
|
|
key="$state_root/id_ed25519"
|
|
known_hosts="$state_root/known_hosts"
|
|
token_file="$state_root/k3s-token"
|
|
fedora_release=44
|
|
nodes=(nereus-node1 nereus-node2)
|
|
created_nodes=()
|
|
|
|
log() { printf '[nereus-lab] %s\n' "$*"; }
|
|
fail() { printf '[nereus-lab] ERROR: %s\n' "$*" >&2; exit 1; }
|
|
|
|
install_dependencies() {
|
|
local missing=()
|
|
for tool in qemu-img virsh virt-install virt-customize cloud-localds ansible-playbook curl ssh-keygen openssl ping; do
|
|
command -v "$tool" >/dev/null 2>&1 || missing+=("$tool")
|
|
done
|
|
((${#missing[@]} == 0)) && return
|
|
[[ -r /etc/os-release ]] || fail "missing tools: ${missing[*]}"
|
|
# shellcheck disable=SC1091
|
|
. /etc/os-release
|
|
log "installing virtualization, cloud-init, and Ansible tooling"
|
|
case ${ID:-} in
|
|
fedora)
|
|
sudo dnf install -y @virtualization cloud-utils guestfs-tools ansible-core openssl iputils
|
|
;;
|
|
arch|cachyos)
|
|
sudo pacman -S --needed --noconfirm qemu-full libvirt virt-install cloud-image-utils guestfs-tools ansible-core openssl iputils dnsmasq
|
|
;;
|
|
*)
|
|
fail "automatic dependency installation does not support ${ID:-this host}; missing: ${missing[*]}"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
ensure_host() {
|
|
local uplink
|
|
install_dependencies
|
|
[[ $(uname -m) == x86_64 ]] || fail "the bundled Fedora image workflow currently supports x86_64 hosts"
|
|
[[ -e /dev/kvm ]] || fail "/dev/kvm is unavailable; enable CPU virtualization in firmware"
|
|
sudo systemctl enable --now libvirtd
|
|
if ! sudo virsh net-info default >/dev/null 2>&1; then
|
|
fail "libvirt's default network is missing"
|
|
fi
|
|
sudo virsh net-start default >/dev/null 2>&1 || true
|
|
sudo virsh net-autostart default >/dev/null
|
|
if command -v ufw >/dev/null 2>&1 && sudo ufw status | grep -q '^Status: active'; then
|
|
uplink=$(ip route show default | awk '/default/ {print $5; exit}')
|
|
[[ -n "$uplink" ]] || fail "could not determine the host's default network interface"
|
|
sudo ufw allow in on virbr0 to any port 53 proto udp >/dev/null
|
|
sudo ufw allow in on virbr0 to any port 53 proto tcp >/dev/null
|
|
sudo ufw allow in on virbr0 to any port 67 proto udp >/dev/null
|
|
sudo ufw route allow in on virbr0 out on virbr0 from 192.168.122.0/24 to 192.168.122.0/24 >/dev/null
|
|
sudo ufw route allow in on virbr0 out on "$uplink" from 192.168.122.0/24 >/dev/null
|
|
fi
|
|
mkdir -p "$state_root"
|
|
chmod 0700 "$state_root"
|
|
if [[ ! -f "$key" ]]; then
|
|
ssh-keygen -q -t ed25519 -N '' -f "$key"
|
|
fi
|
|
touch "$known_hosts"
|
|
chmod 0600 "$key" "$known_hosts"
|
|
if [[ ! -f "$token_file" ]]; then
|
|
umask 077
|
|
openssl rand -hex 32 >"$token_file"
|
|
fi
|
|
chmod 0600 "$token_file"
|
|
sudo install -d -m 0755 "$image_root"
|
|
}
|
|
|
|
image_url() {
|
|
local arch=x86_64 listing filename
|
|
listing="https://download.fedoraproject.org/pub/fedora/linux/releases/${fedora_release}/Cloud/${arch}/images/"
|
|
filename=$(curl -fsSL "$listing" | sed -nE "s/.*href=\"(Fedora-Cloud-Base-Generic-${fedora_release}-[0-9.]+\.${arch}\.qcow2)\".*/\1/p" | sort -V | tail -n1)
|
|
[[ -n "$filename" ]] || fail "could not discover the Fedora ${fedora_release} cloud image"
|
|
printf '%s%s\n' "$listing" "$filename"
|
|
}
|
|
|
|
ensure_base_image() {
|
|
local cached="$state_root/fedora-${fedora_release}-cloud-base.qcow2" source=${FEDORA_IMAGE:-}
|
|
if [[ ! -s "$cached" ]]; then
|
|
if [[ -n "$source" ]]; then
|
|
[[ -f "$source" ]] || fail "FEDORA_IMAGE does not exist: $source"
|
|
[[ $(basename "$source") == Fedora-Cloud-Base-Generic-* ]] || fail "FEDORA_IMAGE must be a Fedora Cloud Base Generic QCOW2 image"
|
|
qemu-img check -q "$source" || fail "FEDORA_IMAGE is not a valid QCOW2 image"
|
|
log "using local Fedora ${fedora_release} image: $source"
|
|
install -m 0644 "$source" "$cached"
|
|
else
|
|
log "downloading Fedora ${fedora_release} cloud image"
|
|
curl -fL --retry 3 --continue-at - -o "$cached.partial" "$(image_url)"
|
|
if ! qemu-img check -q "$cached.partial"; then
|
|
rm -f "$cached.partial"
|
|
fail "downloaded Fedora image is not a valid QCOW2 image"
|
|
fi
|
|
mv "$cached.partial" "$cached"
|
|
fi
|
|
fi
|
|
sudo install -m 0644 "$cached" "$image_root/fedora-${fedora_release}-cloud-base.qcow2"
|
|
}
|
|
|
|
node_mac() {
|
|
case $1 in
|
|
nereus-node1) printf '52:54:00:6e:01:01\n' ;;
|
|
nereus-node2) printf '52:54:00:6e:01:02\n' ;;
|
|
*) fail "no MAC address assigned for $1" ;;
|
|
esac
|
|
}
|
|
|
|
create_seed() {
|
|
local node=$1 seed="$state_root/${node}-seed.iso" user_data="$state_root/${node}-user-data"
|
|
{
|
|
printf '#cloud-config\nhostname: %s\nmanage_etc_hosts: true\nusers:\n' "$node"
|
|
printf ' - name: fedora\n groups: [wheel]\n sudo: ALL=(ALL) NOPASSWD:ALL\n shell: /bin/bash\n ssh_authorized_keys:\n'
|
|
printf ' - %s\n' "$(<"$key.pub")"
|
|
printf 'ssh_pwauth: false\npackages: [qemu-guest-agent]\nruncmd:\n - [systemctl, enable, --now, qemu-guest-agent]\n'
|
|
} >"$user_data"
|
|
chmod 0600 "$user_data"
|
|
cloud-localds --dsmode local --hostname "$node" "$seed" "$user_data"
|
|
sudo install -m 0644 "$seed" "$image_root/${node}-seed.iso"
|
|
}
|
|
|
|
install_network_service() {
|
|
local node=$1 disk=$2
|
|
sudo virt-customize -q -a "$disk" \
|
|
--copy-in "$script_dir/nereus-network.service:/etc/systemd/system" \
|
|
--copy-in "$script_dir/${node}.nmconnection:/etc/NetworkManager/system-connections" \
|
|
--chmod "0600:/etc/NetworkManager/system-connections/${node}.nmconnection" \
|
|
--run-command "chown root:root /etc/NetworkManager/system-connections/${node}.nmconnection" \
|
|
--run-command 'rm -f /etc/NetworkManager/system-connections/cloud-init-ens2.nmconnection' \
|
|
--run-command 'systemctl enable nereus-network.service'
|
|
}
|
|
|
|
create_vm() {
|
|
local node=$1 disk="$image_root/${node}.qcow2" mac
|
|
mac=$(node_mac "$node")
|
|
if sudo virsh dominfo "$node" >/dev/null 2>&1; then
|
|
if [[ $(sudo virsh domstate "$node") == "shut off" ]]; then
|
|
install_network_service "$node" "$disk"
|
|
fi
|
|
sudo virsh start "$node" >/dev/null 2>&1 || true
|
|
return
|
|
fi
|
|
log "creating $node"
|
|
created_nodes+=("$node")
|
|
ssh-keygen -q -f "$known_hosts" -R "$(node_ip "$node")" >/dev/null 2>&1 || true
|
|
create_seed "$node"
|
|
sudo qemu-img create -q -f qcow2 -F qcow2 -b "$image_root/fedora-${fedora_release}-cloud-base.qcow2" "$disk" 30G
|
|
install_network_service "$node" "$disk"
|
|
sudo virt-install --name "$node" --memory 3072 --vcpus 2 --import \
|
|
--disk "path=$disk,format=qcow2,bus=virtio" \
|
|
--disk "path=$image_root/${node}-seed.iso,format=raw,bus=virtio,readonly=on" \
|
|
--network "network=default,model=virtio,mac=$mac" --graphics none --noautoconsole \
|
|
--boot uefi --osinfo detect=on,require=off
|
|
}
|
|
|
|
node_ip() {
|
|
case $1 in
|
|
nereus-node1) printf '192.168.122.10\n' ;;
|
|
nereus-node2) printf '192.168.122.11\n' ;;
|
|
*) fail "no address assigned for $1" ;;
|
|
esac
|
|
}
|
|
|
|
wait_ssh() {
|
|
local ip=$1
|
|
for _ in $(seq 1 90); do
|
|
if ssh -i "$key" -o BatchMode=yes -o ConnectTimeout=2 -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$known_hosts" "fedora@$ip" true 2>/dev/null; then return; fi
|
|
sleep 2
|
|
done
|
|
fail "SSH did not become ready at $ip"
|
|
}
|
|
|
|
wait_ping() {
|
|
local node=$1 ip=$2
|
|
for _ in $(seq 1 60); do
|
|
ping -c 1 -W 1 "$ip" >/dev/null 2>&1 && return
|
|
sleep 2
|
|
done
|
|
fail "$node does not answer ping at $ip"
|
|
}
|
|
|
|
write_inventory() {
|
|
local ip1=$1 ip2=$2
|
|
umask 077
|
|
printf 'all:\n vars:\n ansible_user: fedora\n ansible_ssh_private_key_file: %s\n ansible_ssh_common_args: "-o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=%s"\n k3s_version: v1.33.4+k3s1\n k3s_cluster_cidr: 10.42.0.0/16\n k3s_service_cidr: 10.43.0.0/16\n k3s_node_cidr: 192.168.122.0/24\n k3s_operator_cidrs:\n - 192.168.122.1/32\n children:\n k3s_server:\n hosts:\n node1:\n ansible_host: %s\n k3s_node_ip: %s\n k3s_agent:\n hosts:\n node2:\n ansible_host: %s\n k3s_node_ip: %s\n' "$key" "$known_hosts" "$ip1" "$ip1" "$ip2" "$ip2" >"$inventory"
|
|
}
|
|
|
|
check_lab() {
|
|
[[ -f "$inventory" ]] || fail "run '$0 up' first"
|
|
log "checking reachability, services, ports, packages, and directories"
|
|
ansible all -i "$inventory" -m ansible.builtin.ping
|
|
ansible all -i "$inventory" -b -m ansible.builtin.shell -a 'for i in $(seq 1 60); do test -d /var/lib/rancher/k3s && command -v k3s >/dev/null && systemctl is-active --quiet firewalld && ss -lnt | grep -q ":10250 " && exit 0; sleep 2; done; exit 1'
|
|
ansible k3s_server -i "$inventory" -b -m ansible.builtin.shell -a 'for i in $(seq 1 60); do systemctl is-active --quiet k3s && ss -lnt | grep -q ":6443 " && exit 0; sleep 2; done; exit 1'
|
|
ansible k3s_agent -i "$inventory" -b -m ansible.builtin.shell -a 'for i in $(seq 1 60); do systemctl is-active --quiet k3s-agent && exit 0; sleep 2; done; exit 1'
|
|
ansible k3s_server -i "$inventory" -b -m ansible.builtin.command -a 'k3s kubectl get nodes -o wide'
|
|
ansible k3s_server -i "$inventory" -b -m ansible.builtin.command -a 'k3s kubectl wait --for=condition=Ready nodes --all --timeout=60s'
|
|
log "all checks passed"
|
|
}
|
|
|
|
case "$action" in
|
|
up)
|
|
ensure_host
|
|
ensure_base_image
|
|
for node in "${nodes[@]}"; do create_vm "$node"; done
|
|
ip1=$(node_ip "${nodes[0]}"); ip2=$(node_ip "${nodes[1]}")
|
|
log "node1=$ip1 node2=$ip2"
|
|
wait_ping node1 "$ip1"
|
|
wait_ping node2 "$ip2"
|
|
wait_ssh "$ip1"; wait_ssh "$ip2"
|
|
for node in "${created_nodes[@]}"; do
|
|
if [[ "$node" != "${nodes[0]}" ]]; then
|
|
ssh -i "$key" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$known_hosts" \
|
|
"fedora@$ip1" "sudo -n k3s kubectl delete node '$node' --ignore-not-found=true"
|
|
fi
|
|
done
|
|
write_inventory "$ip1" "$ip2"
|
|
K3S_TOKEN=$(<"$token_file") "$script_dir/bootstrap.sh" "$inventory"
|
|
check_lab
|
|
;;
|
|
check) check_lab ;;
|
|
stop) for node in "${nodes[@]}"; do sudo virsh shutdown "$node" >/dev/null 2>&1 || true; done ;;
|
|
destroy)
|
|
for node in "${nodes[@]}"; do sudo virsh destroy "$node" >/dev/null 2>&1 || true; sudo virsh undefine "$node" --remove-all-storage; done
|
|
;;
|
|
*) fail "usage: $0 [up|check|stop|destroy]" ;;
|
|
esac
|